diff --git a/apps/api/.env.example b/apps/api/.env.example index 2a2ebc2f..5c4065bf 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -13,6 +13,11 @@ # per-key ApiKeyThrottlerGuard (no code change needed). API_KEYS='dev-local=abc123' +# ── Logging (optional) ─────────────────────────────────────────────────────── +# LOG_HASH_KEY: key for the short hash that lets log lines about the same address be matched +# without storing it. Unset, a random key is generated per process. +# LOG_HASH_KEY= + # ── Rate limiting (optional; per-API-key) ──────────────────────────────────── # Defaults: 120 requests per 60000 ms. Override per environment if needed. # THROTTLE_LIMIT=120 diff --git a/apps/api/src/common/dto/error-response.dto.ts b/apps/api/src/common/dto/error-response.dto.ts index 4f0e809c..f847b45b 100644 --- a/apps/api/src/common/dto/error-response.dto.ts +++ b/apps/api/src/common/dto/error-response.dto.ts @@ -16,6 +16,12 @@ export class ErrorBodyDto { description: "Extra structured context, present only when there is more to say.", }) details?: unknown; + + @ApiPropertyOptional({ + description: + "The id of this request, also returned in the x-request-id header. Quote it when reporting a problem.", + }) + requestId?: string; } export class ErrorResponseDto { diff --git a/apps/api/src/common/json-logger.ts b/apps/api/src/common/json-logger.ts new file mode 100644 index 00000000..86dc94ef --- /dev/null +++ b/apps/api/src/common/json-logger.ts @@ -0,0 +1,79 @@ +import type { LoggerService } from "@nestjs/common"; +import { scrubValue } from "./log-privacy"; +import { currentRequestId, currentSensitiveLiterals } from "./request-context"; + +type Severity = "DEBUG" | "INFO" | "WARNING" | "ERROR" | "CRITICAL"; + +/** + * One JSON object per line on stdout, which Cloud Run forwards to Cloud Logging. `severity` is + * the field Cloud Logging reads to classify an entry; `message` becomes its text. + */ +export class JsonLogger implements LoggerService { + constructor(private readonly write: (line: string) => void = defaultWrite) {} + + log(message: unknown, ...params: unknown[]): void { + this.emit("INFO", message, params); + } + warn(message: unknown, ...params: unknown[]): void { + this.emit("WARNING", message, params); + } + error(message: unknown, ...params: unknown[]): void { + this.emit("ERROR", message, params); + } + fatal(message: unknown, ...params: unknown[]): void { + this.emit("CRITICAL", message, params); + } + debug(message: unknown, ...params: unknown[]): void { + this.emit("DEBUG", message, params); + } + verbose(message: unknown, ...params: unknown[]): void { + this.emit("DEBUG", message, params); + } + + private emit(severity: Severity, message: unknown, params: unknown[]): void { + const strings = params.filter((p): p is string => typeof p === "string"); + const isFailure = severity === "ERROR" || severity === "CRITICAL"; + // Nest appends the logger's context last; a failure may carry a stack before it. + const hasStack = isFailure && (strings.length > 1 || strings[0]?.includes("\n") === true); + const stack = hasStack ? strings[0] : undefined; + const context = hasStack && strings.length === 1 ? undefined : strings[strings.length - 1]; + + let text: string; + let stackText = stack; + let fields: Record = {}; + let trusted = false; + if (message instanceof Error) { + text = message.message; + stackText = stack ?? message.stack; + } else if (typeof message === "object" && message !== null) { + const { message: inner, ...rest } = message as Record; + text = typeof inner === "string" ? inner : ""; + fields = rest; + trusted = true; + } else { + text = String(message); + } + + // A structured record is written by our own code (the access line), so it keeps its values; + // free text and stacks can carry what a request supplied and lose the request's memo. + const literals = [...currentSensitiveLiterals()]; + const body = { + ...(scrubValue(fields, []) as Record), + message: scrubValue(text, trusted ? [] : literals), + context, + stack: scrubValue(stackText, literals), + }; + this.write( + JSON.stringify({ + ...body, + severity, + time: new Date().toISOString(), + requestId: currentRequestId(), + }) + ); + } +} + +function defaultWrite(line: string): void { + process.stdout.write(`${line}\n`); +} diff --git a/apps/api/src/common/log-privacy.ts b/apps/api/src/common/log-privacy.ts new file mode 100644 index 00000000..aeec9cdb --- /dev/null +++ b/apps/api/src/common/log-privacy.ts @@ -0,0 +1,53 @@ +import { createHmac, randomBytes } from "crypto"; +import { isCexAddress } from "@/lib/exchange-registry"; + +const SECRET = /(? = []): string { + let out = text; + for (const literal of literals) { + if (literal.length >= MIN_LITERAL_LENGTH) out = out.split(literal).join("[memo]"); + } + return out + .replace(SECRET, "[redacted-secret]") + .replace(BEARER, "Bearer [redacted]") + .replace(ADDRESS, redactAddress) + .replace(BLOB, "[redacted]"); +} + +/** `scrubText` over every string in a JSON-shaped value; object keys are left alone. */ +export function scrubValue(value: unknown, literals: readonly string[]): unknown { + if (typeof value === "string") return scrubText(value, literals); + if (Array.isArray(value)) return value.map((item) => scrubValue(item, literals)); + if (typeof value === "object" && value !== null) { + return Object.fromEntries( + Object.entries(value).map(([key, inner]) => [key, scrubValue(inner, literals)]) + ); + } + return value; +} diff --git a/apps/api/src/common/request-context.ts b/apps/api/src/common/request-context.ts new file mode 100644 index 00000000..2f4b7553 --- /dev/null +++ b/apps/api/src/common/request-context.ts @@ -0,0 +1,108 @@ +import { AsyncLocalStorage } from "async_hooks"; +import { randomUUID } from "crypto"; +import { Logger } from "@nestjs/common"; +import type { NextFunction, Request, RequestHandler, Response } from "express"; +import { xdr } from "@stellar/stellar-sdk"; +import type { AuthedRequest } from "@/auth/api-key.guard"; + +export const REQUEST_ID_HEADER = "x-request-id"; + +interface RequestContext { + requestId: string; + sensitive: Set; +} + +const storage = new AsyncLocalStorage(); + +export function currentRequestId(): string | undefined { + return storage.getStore()?.requestId; +} + +export function currentSensitiveLiterals(): Iterable { + return storage.getStore()?.sensitive ?? []; +} + +/** Marks a value the current request carried (a deposit memo) as never to be logged. */ +export function markSensitive(value: unknown): void { + if (typeof value === "string" && value.length > 0) storage.getStore()?.sensitive.add(value); +} + +function withRequestId(body: unknown, requestId: string): unknown { + if (typeof body !== "object" || body === null || !("error" in body)) return body; + const { error } = body as { error: unknown }; + if (typeof error !== "object" || error === null || Array.isArray(error)) return body; + return { ...body, error: { ...error, requestId } }; +} + +function severityMethod(status: number): "log" | "warn" | "error" { + if (status >= 500) return "error"; + if (status >= 400) return "warn"; + return "log"; +} + +/** + * Gives every request an id before anything that can answer it, sets the headers every + * response carries (`x-request-id`, `Cache-Control: no-store`), copies the id into any error + * envelope, and writes one access line when the response finishes. + * + * The id is always generated here: a caller-supplied one would let a client forge or inject + * into the log line it is meant to be matched against. The access line carries the matched route + * pattern, never the raw path, because paths hold account addresses. + */ +export function requestContextMiddleware(): RequestHandler { + const logger = new Logger("http"); + return (req: Request, res: Response, next: NextFunction): void => { + const requestId = randomUUID(); + const startedAt = process.hrtime.bigint(); + res.setHeader(REQUEST_ID_HEADER, requestId); + res.setHeader("Cache-Control", "no-store"); + + const json = res.json.bind(res); + res.json = ((body?: unknown) => + json(res.statusCode >= 400 ? withRequestId(body, requestId) : body)) as Response["json"]; + + res.on("finish", () => { + if (req.path === "/health") return; + const params = req.params as Record | undefined; + logger[severityMethod(res.statusCode)]({ + message: "request", + route: req.route ? `${req.baseUrl}${String(req.route.path)}` : "unrouted", + status: res.statusCode, + latencyMs: Math.round(Number(process.hrtime.bigint() - startedAt) / 1e6), + keyLabel: (req as AuthedRequest).apiKeyLabel, + network: params?.network, + }); + }); + + storage.run({ requestId, sensitive: new Set() }, next); + }; +} + +function memoOfEnvelope(signedXdr: string): string | undefined { + try { + const envelope = xdr.TransactionEnvelope.fromXDR(signedXdr, "base64"); + const tx = + envelope.switch().name === "envelopeTypeTxFeeBump" + ? envelope.feeBump().tx().innerTx().v1().tx() + : envelope.v1().tx(); + const memo = tx.memo(); + if (memo.switch().name === "memoText") return memo.text().toString("utf8"); + if (memo.switch().name === "memoId") return memo.id().toString(); + } catch { + return undefined; + } + return undefined; +} + +/** After body parsing: the memo is the one body value that identifies a deposit, so it is + * registered for removal from every line the rest of the request writes, whether it arrived as a + * field or inside a signed envelope. */ +export const registerRequestSecrets: RequestHandler = (req, _res, next) => { + const body: unknown = req.body; + if (typeof body === "object" && body !== null) { + const { memo, signedXdr } = body as { memo?: unknown; signedXdr?: unknown }; + markSensitive(memo); + if (typeof signedXdr === "string") markSensitive(memoOfEnvelope(signedXdr)); + } + next(); +}; diff --git a/apps/api/src/configure-app.ts b/apps/api/src/configure-app.ts index cc456bb6..1eb06915 100644 --- a/apps/api/src/configure-app.ts +++ b/apps/api/src/configure-app.ts @@ -1,9 +1,11 @@ -import type { INestApplication } from "@nestjs/common"; +import type { INestApplication, LoggerService } from "@nestjs/common"; import type { ErrorCode } from "@lumenwipe/types"; import { json } from "express"; -import type { ErrorRequestHandler, NextFunction, Request, RequestHandler, Response } from "express"; +import type { ErrorRequestHandler, RequestHandler } from "express"; import { RateLimiter } from "./auth/rate-limiter"; import { ErrorEnvelopeFilter } from "./common/error-envelope.filter"; +import { JsonLogger } from "./common/json-logger"; +import { registerRequestSecrets, requestContextMiddleware } from "./common/request-context"; export const JSON_BODY_LIMIT = "100kb"; @@ -40,7 +42,10 @@ const BODY_PARSER_ERRORS: Record { - res.setHeader("Cache-Control", "no-store"); - next(); - }); + app.useLogger(logger); + + // First, ahead of the rate limiter, body parsing and authentication, so the limiter's own 429, + // a 401 and a body error all carry x-request-id. Also sets Cache-Control: no-store: every + // response is dynamic and non-cacheable (account state, plans, unsigned XDR, mediator + // co-signatures), success or error. Order: request context, rate limiter, filters, body + // parser, body-error mapping, body normalization, secret registration. + app.use(requestContextMiddleware()); // Ahead of body parsing, routing and authentication, so the budget is spent and // reported on every response, including a 404 or a rejected body. @@ -111,4 +117,5 @@ export function configureApp(app: INestApplication): void { next(); }; app.use(normalizeBody); + app.use(registerRequestSecrets); } diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts index 8ac1695d..cc0f0332 100644 --- a/apps/api/src/main.ts +++ b/apps/api/src/main.ts @@ -4,6 +4,7 @@ import { Logger } from "@nestjs/common"; import { NestFactory } from "@nestjs/core"; import { SwaggerModule } from "@nestjs/swagger"; import { AppModule } from "./app.module"; +import { JsonLogger } from "./common/json-logger"; import { configureApp } from "./configure-app"; import { createOpenApiDocument } from "./openapi"; import { checkEnv, formatEnvFailure } from "./config/validate-env"; @@ -20,7 +21,7 @@ async function bootstrap(): Promise { process.exit(1); } - const app = await NestFactory.create(AppModule, { bodyParser: false }); + const app = await NestFactory.create(AppModule, { bodyParser: false, logger: new JsonLogger() }); const bootLogger = new Logger("bootstrap"); // Logged, not fatal: each of these disables a path rather than the service, and the operator // should learn it here rather than from a user hitting the disabled path. diff --git a/apps/api/tests/e2e/body-handling.e2e.test.ts b/apps/api/tests/e2e/body-handling.e2e.test.ts index c9d5e784..0f5fd8dc 100644 --- a/apps/api/tests/e2e/body-handling.e2e.test.ts +++ b/apps/api/tests/e2e/body-handling.e2e.test.ts @@ -4,6 +4,7 @@ import { Test } from "@nestjs/testing"; import request from "supertest"; import { Keypair, Networks, TransactionBuilder, Account, Operation } from "@stellar/stellar-sdk"; import { AppModule } from "@/app.module"; +import { JsonLogger } from "@/common/json-logger"; import { configureApp, JSON_BODY_LIMIT } from "@/configure-app"; const KEY = "e2e_test_key"; @@ -39,7 +40,7 @@ beforeAll(async () => { process.env.API_KEYS = `test=${KEY}`; const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile(); app = moduleRef.createNestApplication({ bodyParser: false }); - configureApp(app); + configureApp(app, new JsonLogger(() => undefined)); await app.init(); http = app.getHttpServer(); }); @@ -92,7 +93,7 @@ describe.each(POST_ROUTES)("POST %s", (route) => { const res = await post(route); expect(res.body.error.code).toBeString(); expect(res.body.error.code).not.toBe("internal_error"); - expect(spy).not.toHaveBeenCalled(); + expect(spy.mock.calls.filter(([message]) => typeof message === "string")).toEqual([]); spy.mockRestore(); }); }); diff --git a/apps/api/tests/e2e/contract.e2e.test.ts b/apps/api/tests/e2e/contract.e2e.test.ts index e77c48e4..3764beda 100644 --- a/apps/api/tests/e2e/contract.e2e.test.ts +++ b/apps/api/tests/e2e/contract.e2e.test.ts @@ -4,6 +4,7 @@ import { Test } from "@nestjs/testing"; import request from "supertest"; import { Keypair } from "@stellar/stellar-sdk"; import { AppModule } from "@/app.module"; +import { JsonLogger } from "@/common/json-logger"; import { configureApp } from "@/configure-app"; import { API_VERSION } from "@/openapi"; @@ -24,7 +25,7 @@ beforeAll(async () => { process.env.API_KEYS = `test=${KEY}`; const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile(); app = moduleRef.createNestApplication({ bodyParser: false }); - configureApp(app); + configureApp(app, new JsonLogger(() => undefined)); await app.init(); http = app.getHttpServer(); }); @@ -59,7 +60,7 @@ test("the service index at / is public and points at the docs", async () => { test("an authenticated route without a key is rejected 401", async () => { const res = await request(http).get("/testnet/account/NOPE"); expect(res.status).toBe(401); - expect(res.body).toEqual({ + expect(res.body).toMatchObject({ error: { code: "unauthorized", message: "A valid API key is required." }, }); }); @@ -67,7 +68,7 @@ test("an authenticated route without a key is rejected 401", async () => { test("an authenticated route with an unknown key is rejected 401", async () => { const res = await request(http).get("/testnet/account/NOPE").set("Authorization", "Bearer nope"); expect(res.status).toBe(401); - expect(res.body).toEqual({ + expect(res.body).toMatchObject({ error: { code: "unauthorized", message: "A valid API key is required." }, }); }); @@ -77,13 +78,15 @@ test("an authenticated route with an unknown key is rejected 401", async () => { test("close/plan rejects an invalid network with the v1 error shape", async () => { const res = await authPost("/v1/badnet/close/plan").send({}); expect(res.status).toBe(400); - expect(res.body).toEqual({ error: { code: "invalid_network", message: "Invalid network." } }); + expect(res.body).toMatchObject({ + error: { code: "invalid_network", message: "Invalid network." }, + }); }); test("close/plan rejects a missing source", async () => { const res = await authPost("/v1/testnet/close/plan").send({}); expect(res.status).toBe(400); - expect(res.body).toEqual({ + expect(res.body).toMatchObject({ error: { code: "invalid_source", message: "A valid source account (G...) is required." }, }); }); @@ -91,7 +94,9 @@ test("close/plan rejects a missing source", async () => { test("close/batch-plan rejects an invalid network with the v1 error shape", async () => { const res = await authPost("/v1/badnet/close/batch-plan").send({ addresses: [] }); expect(res.status).toBe(400); - expect(res.body).toEqual({ error: { code: "invalid_network", message: "Invalid network." } }); + expect(res.body).toMatchObject({ + error: { code: "invalid_network", message: "Invalid network." }, + }); }); test("close/batch-plan rejects an empty addresses array", async () => { @@ -133,7 +138,7 @@ test("close/batch-plan rejects an invalid destination", async () => { test("submit rejects a missing signedXdr", async () => { const res = await authPost("/v1/testnet/submit").send({}); expect(res.status).toBe(400); - expect(res.body).toEqual({ + expect(res.body).toMatchObject({ error: { code: "invalid_signed_xdr", message: "A signed transaction envelope (signedXdr) is required.", @@ -181,7 +186,7 @@ test("malformed JSON on a v1 endpoint returns the invalid_body contract", async .set("Content-Type", "application/json") .send('{ "source": '); expect(res.status).toBe(400); - expect(res.body).toEqual({ + expect(res.body).toMatchObject({ error: { code: "invalid_body", message: "Request body must be valid JSON." }, }); }); @@ -357,7 +362,7 @@ test("close/transactions rejects a text memo over 28 bytes with 422 (before any memo: "x".repeat(29), }); expect(res.status).toBe(422); - expect(res.body).toEqual({ + expect(res.body).toMatchObject({ error: { code: "invalid_memo", message: "A text memo must be at most 28 bytes." }, }); }); @@ -500,7 +505,9 @@ test("the stats feed is public and covers 365 days", async () => { test("stats reject an invalid network", async () => { const res = await request(http).get("/v1/badnet/stats"); expect(res.status).toBe(400); - expect(res.body).toEqual({ error: { code: "invalid_network", message: "Invalid network." } }); + expect(res.body).toMatchObject({ + error: { code: "invalid_network", message: "Invalid network." }, + }); }); test("recording a close requires an API key", async () => { diff --git a/apps/api/tests/e2e/log-privacy.e2e.test.ts b/apps/api/tests/e2e/log-privacy.e2e.test.ts new file mode 100644 index 00000000..8c826690 --- /dev/null +++ b/apps/api/tests/e2e/log-privacy.e2e.test.ts @@ -0,0 +1,152 @@ +import { afterAll, afterEach, beforeAll, expect, spyOn, test } from "bun:test"; +import type { INestApplication } from "@nestjs/common"; +import { Test } from "@nestjs/testing"; +import request from "supertest"; +import { + Account, + Keypair, + Memo, + Networks, + Operation, + TransactionBuilder, +} from "@stellar/stellar-sdk"; +import { AppModule } from "@/app.module"; +import { JsonLogger } from "@/common/json-logger"; +import { configureApp } from "@/configure-app"; +import * as accountPlan from "@/lib/close-api/account-plan"; +import * as readAccount from "@/lib/close-api/read-account"; +import { servedRegistry } from "@/lib/exchange-registry"; +import * as submitModule from "@/lib/stellar/submit"; + +const KEY = "e2e_test_key"; +const PLACEHOLDER_SECRET = `S${"A".repeat(55)}`; +const MEMO = "4815162342"; +const EXCHANGE = servedRegistry().entries.find((e) => e.requiresMediator && e.requiresMemo)!; +const SOURCE = Keypair.random().publicKey(); +const OTHER = Keypair.random().publicKey(); +const SIGNED_XDR = (() => { + const signer = Keypair.random(); + const tx = new TransactionBuilder(new Account(signer.publicKey(), "100"), { + fee: "100", + networkPassphrase: Networks.TESTNET, + }) + .addOperation(Operation.accountMerge({ destination: EXCHANGE.address })) + .addMemo(Memo.id(MEMO)) + .setTimeout(30) + .build(); + tx.sign(signer); + return tx.toXDR(); +})(); + +let app: INestApplication; +let http: ReturnType; +const written: string[] = []; + +beforeAll(async () => { + process.env.API_KEYS = `test=${KEY}`; + const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile(); + app = moduleRef.createNestApplication({ bodyParser: false }); + configureApp(app, new JsonLogger((line) => written.push(line))); + await app.init(); + http = app.getHttpServer(); +}); + +afterEach(() => { + spyOn(accountPlan, "buildAccountPlan").mockRestore(); + spyOn(readAccount, "readAccountState").mockRestore(); + spyOn(submitModule, "submitAndWait").mockRestore(); +}); + +afterAll(async () => { + await app.close(); +}); + +const post = (path: string, ip: string) => + request(http).post(path).set("Authorization", `Bearer ${KEY}`).set("X-Forwarded-For", ip); + +test("an exchange close that fails at every step logs no address, memo, secret or body", async () => { + const leak = `${SOURCE} -> ${EXCHANGE.address} memo ${MEMO} key ${PLACEHOLDER_SECRET} ${SIGNED_XDR}`; + spyOn(accountPlan, "buildAccountPlan").mockRejectedValue(new Error(leak)); + spyOn(readAccount, "readAccountState").mockRejectedValue(new Error(leak)); + spyOn(submitModule, "submitAndWait").mockRejectedValue(new Error(leak)); + + const plan = await post("/v1/testnet/close/plan", "203.0.113.20").send({ + source: SOURCE, + destination: EXCHANGE.address, + memo: MEMO, + }); + const transactions = await post("/v1/testnet/close/transactions", "203.0.113.21").send({ + source: SOURCE, + destination: EXCHANGE.address, + memo: MEMO, + }); + const submit = await post("/v1/testnet/submit", "203.0.113.22").send({ signedXdr: SIGNED_XDR }); + expect([plan.status, transactions.status, submit.status]).toEqual([500, 500, 502]); + for (const res of [plan, transactions, submit]) { + expect(JSON.stringify(res.body)).not.toContain(EXCHANGE.address); + expect(res.body.error.requestId).toBe(res.headers["x-request-id"]); + } + + const lines = written; + const records = lines.map((l) => JSON.parse(l) as Record); + const text = lines.join("\n"); + + expect(records.some((r) => r.severity === "ERROR")).toBe(true); + expect(text).toContain("[exchange]"); + expect(text).toContain("[memo]"); + expect(text).toContain(`${SOURCE.slice(0, 4)}...${SOURCE.slice(-4)}~`); + + expect(text).not.toMatch(/[GCM][A-Z2-7]{55}/); + expect(text).not.toMatch(/S[A-Z2-7]{55}/); + expect(text).not.toContain(EXCHANGE.address.slice(0, 8)); + expect(text).not.toContain(EXCHANGE.address.slice(-8)); + expect(text).not.toContain(`${EXCHANGE.address.slice(0, 4)}...${EXCHANGE.address.slice(-4)}`); + expect(text).not.toContain(MEMO); + expect(text).not.toContain(SIGNED_XDR); + expect(text).not.toContain(OTHER); +}); + +test("access lines name the route pattern and never the raw path", async () => { + written.length = 0; + await request(http) + .get(`/testnet/account/${OTHER}`) + .set("Authorization", `Bearer ${KEY}`) + .set("X-Forwarded-For", "203.0.113.23"); + const access = written + .map((l) => JSON.parse(l) as Record) + .find((r) => r.message === "request"); + expect(access).toMatchObject({ + route: "/:network/account/:address", + keyLabel: "test", + network: "testnet", + }); + expect(JSON.stringify(access)).not.toContain(OTHER); + expect(typeof access?.latencyMs).toBe("number"); + expect(access).not.toHaveProperty("ip"); +}); + +test.each(['","', '":"', '{"s', "severity", "message", "request", 'a\\b"c'])( + "the memo %p never breaks a log line or rewrites its keys", + async (memo) => { + written.length = 0; + spyOn(readAccount, "readAccountState").mockRejectedValue(new Error(`failed for memo ${memo}`)); + const res = await post("/v1/testnet/close/transactions", "203.0.113.30").send({ + source: SOURCE, + destination: EXCHANGE.address, + memo, + }); + expect(res.status).toBeGreaterThanOrEqual(400); + + const records = written.map((line) => JSON.parse(line) as Record); + expect(records.length).toBeGreaterThan(0); + for (const record of records) { + expect(Object.keys(record)).toEqual(expect.arrayContaining(["severity", "time", "message"])); + expect(["INFO", "WARNING", "ERROR"]).toContain(record.severity as string); + expect(typeof record.requestId).toBe("string"); + } + const failure = records.find((r) => r.context === "CloseController"); + expect(failure?.message).toBe("close/transactions failed"); + expect(String(failure?.stack ?? "")).not.toContain(memo); + expect(records.some((r) => r.message === "request" && r.status === res.status)).toBe(true); + } +); diff --git a/apps/api/tests/e2e/openapi-drift.e2e.test.ts b/apps/api/tests/e2e/openapi-drift.e2e.test.ts index d46d642e..5d54a574 100644 --- a/apps/api/tests/e2e/openapi-drift.e2e.test.ts +++ b/apps/api/tests/e2e/openapi-drift.e2e.test.ts @@ -6,6 +6,7 @@ import { Test } from "@nestjs/testing"; import SwaggerParser from "@apidevtools/swagger-parser"; import type { OpenAPIObject } from "@nestjs/swagger"; import { AppModule } from "@/app.module"; +import { JsonLogger } from "@/common/json-logger"; import { configureApp } from "@/configure-app"; import { ERROR_CODES } from "@/common/error-codes"; import { @@ -29,7 +30,7 @@ beforeAll(async () => { process.env.API_KEYS = "test=e2e_test_key"; const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile(); app = moduleRef.createNestApplication({ bodyParser: false }); - configureApp(app); + configureApp(app, new JsonLogger(() => undefined)); await app.init(); spec = createOpenApiDocument(app); generated = serializeOpenApiDocument(spec); diff --git a/apps/api/tests/e2e/openapi.e2e.test.ts b/apps/api/tests/e2e/openapi.e2e.test.ts index e05c4fff..1329b095 100644 --- a/apps/api/tests/e2e/openapi.e2e.test.ts +++ b/apps/api/tests/e2e/openapi.e2e.test.ts @@ -3,6 +3,7 @@ import type { INestApplication } from "@nestjs/common"; import { Test } from "@nestjs/testing"; import type { OpenAPIObject } from "@nestjs/swagger"; import { AppModule } from "@/app.module"; +import { JsonLogger } from "@/common/json-logger"; import { configureApp } from "@/configure-app"; import { createOpenApiDocument } from "@/openapi"; @@ -13,7 +14,7 @@ beforeAll(async () => { process.env.API_KEYS = "test=e2e_test_key"; const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile(); app = moduleRef.createNestApplication({ bodyParser: false }); - configureApp(app); + configureApp(app, new JsonLogger(() => undefined)); await app.init(); spec = createOpenApiDocument(app); }); diff --git a/apps/api/tests/e2e/param-validation.e2e.test.ts b/apps/api/tests/e2e/param-validation.e2e.test.ts index baa673d8..c11b8e2c 100644 --- a/apps/api/tests/e2e/param-validation.e2e.test.ts +++ b/apps/api/tests/e2e/param-validation.e2e.test.ts @@ -4,6 +4,7 @@ import { Test } from "@nestjs/testing"; import request from "supertest"; import { Keypair } from "@stellar/stellar-sdk"; import { AppModule } from "@/app.module"; +import { JsonLogger } from "@/common/json-logger"; import { configureApp } from "@/configure-app"; const KEY = "e2e_test_key"; @@ -15,7 +16,7 @@ beforeAll(async () => { process.env.API_KEYS = `test=${KEY}`; const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile(); app = moduleRef.createNestApplication({ bodyParser: false }); - configureApp(app); + configureApp(app, new JsonLogger(() => undefined)); await app.init(); }); @@ -35,7 +36,9 @@ for (const route of ADDRESS_ROUTES) { .get(route.path("nope", "x")) .set("Authorization", `Bearer ${KEY}`); expect(res.status).toBe(400); - expect(res.body).toEqual({ error: { code: "invalid_network", message: "Invalid network" } }); + expect(res.body).toMatchObject({ + error: { code: "invalid_network", message: "Invalid network" }, + }); }); test(`${route.path("{network}", "{address}")} rejects only the address on a valid network`, async () => { @@ -43,7 +46,7 @@ for (const route of ADDRESS_ROUTES) { .get(route.path("testnet", "x")) .set("Authorization", `Bearer ${KEY}`); expect(res.status).toBe(400); - expect(res.body).toEqual({ error: { code: "invalid_address", message: route.message } }); + expect(res.body).toMatchObject({ error: { code: "invalid_address", message: route.message } }); }); } @@ -52,5 +55,7 @@ test("a bad network on a close route keeps its trailing-period message", async ( .post("/v1/nope/close/plan") .set("Authorization", `Bearer ${KEY}`) .send({ source: ADDRESS }); - expect(res.body).toEqual({ error: { code: "invalid_network", message: "Invalid network." } }); + expect(res.body).toMatchObject({ + error: { code: "invalid_network", message: "Invalid network." }, + }); }); diff --git a/apps/api/tests/e2e/rate-limit-headers.e2e.test.ts b/apps/api/tests/e2e/rate-limit-headers.e2e.test.ts index a4eb6b66..cdf918d6 100644 --- a/apps/api/tests/e2e/rate-limit-headers.e2e.test.ts +++ b/apps/api/tests/e2e/rate-limit-headers.e2e.test.ts @@ -4,6 +4,7 @@ import { Test } from "@nestjs/testing"; import request from "supertest"; import { AppModule } from "@/app.module"; import { RateLimiter } from "@/auth/rate-limiter"; +import { JsonLogger } from "@/common/json-logger"; import { configureApp } from "@/configure-app"; const KEY = "e2e_test_key"; @@ -17,7 +18,7 @@ beforeAll(async () => { delete process.env.INTEGRATOR_SESSION_SECRET; const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile(); app = moduleRef.createNestApplication({ bodyParser: false }); - configureApp(app); + configureApp(app, new JsonLogger(() => undefined)); await app.init(); http = app.getHttpServer(); }); @@ -132,7 +133,7 @@ test("a 429 is the same whether the key is valid or not", async () => { } const res = await request(http).get("/testnet/account/G").set("Authorization", `Bearer ${key}`); expect(res.status).toBe(429); - bodies.push(res.body); + bodies.push({ ...res.body.error, requestId: undefined }); } expect(bodies[0]).toEqual(bodies[1]); }, 30_000); @@ -186,7 +187,7 @@ test("a limiter failure answers 500 and is non-cacheable", async () => { try { const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile(); const broken = moduleRef.createNestApplication({ bodyParser: false }); - configureApp(broken); + configureApp(broken, new JsonLogger(() => undefined)); await broken.init(); const res = await request(broken.getHttpServer()).get("/no/such/route"); expect(res.status).toBe(500); diff --git a/apps/api/tests/e2e/request-id.e2e.test.ts b/apps/api/tests/e2e/request-id.e2e.test.ts new file mode 100644 index 00000000..130c0da3 --- /dev/null +++ b/apps/api/tests/e2e/request-id.e2e.test.ts @@ -0,0 +1,113 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import type { INestApplication } from "@nestjs/common"; +import { Test } from "@nestjs/testing"; +import request from "supertest"; +import { AppModule } from "@/app.module"; +import { JsonLogger } from "@/common/json-logger"; +import { configureApp } from "@/configure-app"; + +const KEY = "e2e_test_key"; +const LIMIT = 120; +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +let app: INestApplication; +let http: ReturnType; + +beforeAll(async () => { + process.env.API_KEYS = `test=${KEY}`; + delete process.env.INTEGRATOR_SESSION_SECRET; + const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile(); + app = moduleRef.createNestApplication({ bodyParser: false }); + configureApp(app, new JsonLogger(() => undefined)); + await app.init(); + http = app.getHttpServer(); +}); + +afterAll(async () => { + await app.close(); +}); + +const ip = (address: string) => ({ "X-Forwarded-For": address }); + +function expectIdentified(res: request.Response): string { + const id = res.headers["x-request-id"]; + expect(id).toMatch(UUID); + expect(res.headers["cache-control"]).toBe("no-store"); + expect(res.headers["ratelimit-limit"]).toBe(String(LIMIT)); + expect(res.headers["x-ratelimit-limit"]).toBe(String(LIMIT)); + expect(res.body.error.requestId).toBe(id); + return id as string; +} + +test("a 401 carries the request id in the header and the envelope, with no-store and the budget", async () => { + const res = await request(http).get("/testnet/account/NOPE").set(ip("203.0.113.1")); + expect(res.status).toBe(401); + expect(res.body.error.message).toBe("A valid API key is required."); + expectIdentified(res); +}); + +test("a 429 from the limiter carries the request id, no-store and the budget headers", async () => { + const caller = ip("203.0.113.2"); + for (let i = 0; i < LIMIT; i++) await request(http).get("/testnet/account/G").set(caller); + const blocked = await request(http).get("/testnet/account/G").set(caller); + expect(blocked.status).toBe(429); + expect(blocked.headers["retry-after"]).toBeDefined(); + expectIdentified(blocked); +}, 20_000); + +test("404, a malformed body and an oversize body carry it too", async () => { + const unknown = await request(http) + .get("/no/such/route") + .set(ip("203.0.113.3")) + .set("Authorization", `Bearer ${KEY}`); + expect(unknown.status).toBe(404); + expectIdentified(unknown); + + const malformed = await request(http) + .post("/v1/testnet/close/plan") + .set(ip("203.0.113.4")) + .set("Authorization", `Bearer ${KEY}`) + .set("Content-Type", "application/json") + .send("{not json"); + expect(malformed.status).toBe(400); + expectIdentified(malformed); + + const oversize = await request(http) + .post("/v1/testnet/close/plan") + .set(ip("203.0.113.5")) + .set("Authorization", `Bearer ${KEY}`) + .set("Content-Type", "application/json") + .send(JSON.stringify({ pad: "x".repeat(200_000) })); + expect(oversize.status).toBe(413); + expectIdentified(oversize); +}); + +test("a controller error carries it, and a success carries the header without an envelope", async () => { + const invalid = await request(http) + .post("/v1/badnet/close/plan") + .set(ip("203.0.113.6")) + .set("Authorization", `Bearer ${KEY}`) + .send({}); + expect(invalid.status).toBe(400); + expect(invalid.body.error.code).toBe("invalid_network"); + expectIdentified(invalid); + + const ok = await request(http).get("/health").set(ip("203.0.113.7")); + expect(ok.status).toBe(200); + expect(ok.headers["x-request-id"]).toMatch(UUID); + expect(ok.headers["cache-control"]).toBe("no-store"); + expect(ok.body.error).toBeUndefined(); +}); + +test("every request gets its own id and a client-supplied one is ignored", async () => { + const a = await request(http).get("/health").set("X-Request-Id", "client-chosen-id"); + const b = await request(http).get("/health"); + expect(a.headers["x-request-id"]).toMatch(UUID); + expect(a.headers["x-request-id"]).not.toBe(b.headers["x-request-id"]); + const c = await request(http) + .get("/testnet/account/NOPE") + .set(ip("203.0.113.8")) + .set("X-Request-Id", "client-chosen-id"); + expect(c.body.error.requestId).toMatch(UUID); + expect(c.headers["x-request-id"]).toBe(c.body.error.requestId); +}); diff --git a/apps/api/tests/e2e/spec-errors.e2e.test.ts b/apps/api/tests/e2e/spec-errors.e2e.test.ts index 5975dfd4..76e7c58d 100644 --- a/apps/api/tests/e2e/spec-errors.e2e.test.ts +++ b/apps/api/tests/e2e/spec-errors.e2e.test.ts @@ -4,6 +4,7 @@ import type { OpenAPIObject } from "@nestjs/swagger"; import { Test } from "@nestjs/testing"; import request from "supertest"; import { AppModule } from "@/app.module"; +import { JsonLogger } from "@/common/json-logger"; import { configureApp } from "@/configure-app"; import { createOpenApiDocument } from "@/openapi"; @@ -27,7 +28,7 @@ beforeAll(async () => { delete process.env.INTEGRATOR_SESSION_SECRET; const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile(); app = moduleRef.createNestApplication({ bodyParser: false }); - configureApp(app); + configureApp(app, new JsonLogger(() => undefined)); await app.init(); http = app.getHttpServer(); spec = createOpenApiDocument(app); diff --git a/apps/api/tests/e2e/unauthenticated-throttle.e2e.test.ts b/apps/api/tests/e2e/unauthenticated-throttle.e2e.test.ts index b7cfdeb7..97e6cc43 100644 --- a/apps/api/tests/e2e/unauthenticated-throttle.e2e.test.ts +++ b/apps/api/tests/e2e/unauthenticated-throttle.e2e.test.ts @@ -3,6 +3,7 @@ import type { INestApplication } from "@nestjs/common"; import { Test } from "@nestjs/testing"; import request from "supertest"; import { AppModule } from "@/app.module"; +import { JsonLogger } from "@/common/json-logger"; import { configureApp } from "@/configure-app"; // Coverage for #59: a request with no API key (or a wrong one) used to be rejected with 401 @@ -26,7 +27,7 @@ beforeAll(async () => { process.env.API_KEYS = "test=real_key_unused_here"; const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile(); app = moduleRef.createNestApplication({ bodyParser: false }); - configureApp(app); + configureApp(app, new JsonLogger(() => undefined)); await app.init(); http = app.getHttpServer(); }); diff --git a/apps/api/tests/unit/json-logger.test.ts b/apps/api/tests/unit/json-logger.test.ts new file mode 100644 index 00000000..1ab531c2 --- /dev/null +++ b/apps/api/tests/unit/json-logger.test.ts @@ -0,0 +1,69 @@ +import { expect, test } from "bun:test"; +import { Keypair } from "@stellar/stellar-sdk"; +import { JsonLogger } from "@/common/json-logger"; +import { scrubText } from "@/common/log-privacy"; +import { servedRegistry } from "@/lib/exchange-registry"; + +const PLACEHOLDER_SECRET = `S${"A".repeat(55)}`; + +function capture(): { logger: JsonLogger; lines: string[] } { + const lines: string[] = []; + return { logger: new JsonLogger((line) => lines.push(line)), lines }; +} + +test("every line is one JSON object with a Cloud Logging severity", () => { + const { logger, lines } = capture(); + logger.log("started", "Boot"); + logger.warn("careful", "Boot"); + logger.error("failed", "Error: boom\n at x (y.ts:1:1)", "Boot"); + logger.debug("detail"); + logger.fatal("dead"); + const parsed = lines.map((l) => JSON.parse(l) as Record); + expect(parsed.map((p) => p.severity)).toEqual(["INFO", "WARNING", "ERROR", "DEBUG", "CRITICAL"]); + expect(parsed[0]).toMatchObject({ message: "started", context: "Boot" }); + expect(parsed[2]).toMatchObject({ context: "Boot", stack: "Error: boom\n at x (y.ts:1:1)" }); + expect(lines.every((l) => !l.includes("\n"))).toBe(true); +}); + +test("structured fields are merged and cannot override the severity", () => { + const { logger, lines } = capture(); + logger.log({ message: "request", status: 200, severity: "ERROR" }); + expect(JSON.parse(lines[0]!)).toMatchObject({ + message: "request", + status: 200, + severity: "INFO", + }); +}); + +test("an address is cut to four and four characters plus a stable keyed hash", () => { + const address = Keypair.random().publicKey(); + const other = Keypair.random().publicKey(); + const out = scrubText(`read ${address} then ${address} and ${other}`); + expect(out).not.toContain(address); + const shown = out.match(/G[A-Z2-7]{3}\.\.\.[A-Z2-7]{4}~[0-9a-f]{8}/g) ?? []; + expect(shown).toHaveLength(3); + expect(shown[0]!.startsWith(`${address.slice(0, 4)}...${address.slice(-4)}~`)).toBe(true); + expect(shown[1]).toBe(shown[0]!); + expect(shown[2]).not.toBe(shown[0]!); +}); + +test("an exchange deposit address becomes a fixed token with no prefix suffix or hash", () => { + const exchange = servedRegistry().entries[0]!.address; + const out = scrubText(`destination ${exchange}`); + expect(out).toBe("destination [exchange]"); +}); + +test("secret-shaped strings, bearer tokens and XDR-sized blobs never survive", () => { + const out = scrubText( + `key ${PLACEHOLDER_SECRET} Authorization: Bearer abc123 xdr ${"QUFB".repeat(60)}` + ); + expect(out).not.toContain(PLACEHOLDER_SECRET); + expect(out).not.toContain("abc123"); + expect(out).not.toContain("QUFB"); + expect(out).toContain("[redacted-secret]"); +}); + +test("literals the request carried are removed, escaped or not", () => { + expect(scrubText('memo "a\\b" and a\\b', ["a\\b"])).toBe('memo "[memo]" and [memo]'); + expect(scrubText("memo 12", ["12"])).toBe("memo 12"); +}); diff --git a/apps/web/components/execution/ExecutionWizard.tsx b/apps/web/components/execution/ExecutionWizard.tsx index bec26cc0..afbb1355 100644 --- a/apps/web/components/execution/ExecutionWizard.tsx +++ b/apps/web/components/execution/ExecutionWizard.tsx @@ -33,6 +33,7 @@ export default function ExecutionWizard({ network }: ExecutionWizardProps) { const mediatorRequired = useDemolishStore((s) => s.mediatorRequired); const phase = useDemolishStore((s) => s.phase); const lastError = useDemolishStore((s) => s.lastError); + const lastErrorReference = useDemolishStore((s) => s.lastErrorReference); const accountState = useDemolishStore((s) => s.accountState); const { run, progressStatus, signatureStatus, submitPreAuthTransaction } = useCloseExecution(); @@ -394,7 +395,14 @@ export default function ExecutionWizard({ network }: ExecutionWizardProps) { className="flex items-start gap-2.5 rounded-lg border border-destructive/30 bg-destructive/10 px-3 py-2.5 text-sm text-white/70 outline-none" > - {lastError ?? "The close could not be completed."} + + {lastError ?? "The close could not be completed."} + {lastErrorReference && ( + + Reference: {lastErrorReference} + + )} +