feat(api): give every request an id and log under a written privacy policy

apps/api/.env.example

@@ -13,6 +13,11 @@
 # per-key ApiKeyThrottlerGuard (no code change needed).
 API_KEYS='dev-local=abc123'
 
+# ── Logging (optional) ───────────────────────────────────────────────────────
+# LOG_HASH_KEY: key for the short hash that lets log lines about the same address be matched
+# without storing it. Unset, a random key is generated per process.
+# LOG_HASH_KEY=
+
 # ── Rate limiting (optional; per-API-key) ────────────────────────────────────
 # Defaults: 120 requests per 60000 ms. Override per environment if needed.
 # THROTTLE_LIMIT=120

apps/api/src/common/dto/error-response.dto.ts

@@ -16,6 +16,12 @@ export class ErrorBodyDto {
     description: "Extra structured context, present only when there is more to say.",
   })
   details?: unknown;
+
+  @ApiPropertyOptional({
+    description:
+      "The id of this request, also returned in the x-request-id header. Quote it when reporting a problem.",
+  })
+  requestId?: string;
 }
 
 export class ErrorResponseDto {

apps/api/src/common/json-logger.ts

@@ -0,0 +1,79 @@
+import type { LoggerService } from "@nestjs/common";
+import { scrubValue } from "./log-privacy";
+import { currentRequestId, currentSensitiveLiterals } from "./request-context";
+
+type Severity = "DEBUG" | "INFO" | "WARNING" | "ERROR" | "CRITICAL";
+
+/**
+ * One JSON object per line on stdout, which Cloud Run forwards to Cloud Logging. `severity` is
+ * the field Cloud Logging reads to classify an entry; `message` becomes its text.
+ */
+export class JsonLogger implements LoggerService {
+  constructor(private readonly write: (line: string) => void = defaultWrite) {}
+
+  log(message: unknown, ...params: unknown[]): void {
+    this.emit("INFO", message, params);
+  }
+  warn(message: unknown, ...params: unknown[]): void {
+    this.emit("WARNING", message, params);
+  }
+  error(message: unknown, ...params: unknown[]): void {
+    this.emit("ERROR", message, params);
+  }
+  fatal(message: unknown, ...params: unknown[]): void {
+    this.emit("CRITICAL", message, params);
+  }
+  debug(message: unknown, ...params: unknown[]): void {
+    this.emit("DEBUG", message, params);
+  }
+  verbose(message: unknown, ...params: unknown[]): void {
+    this.emit("DEBUG", message, params);
+  }
+
+  private emit(severity: Severity, message: unknown, params: unknown[]): void {
+    const strings = params.filter((p): p is string => typeof p === "string");
+    const isFailure = severity === "ERROR" || severity === "CRITICAL";
+    // Nest appends the logger's context last; a failure may carry a stack before it.
+    const hasStack = isFailure && (strings.length > 1 || strings[0]?.includes("\n") === true);
+    const stack = hasStack ? strings[0] : undefined;
+    const context = hasStack && strings.length === 1 ? undefined : strings[strings.length - 1];
+
+    let text: string;
+    let stackText = stack;
+    let fields: Record<string, unknown> = {};
+    let trusted = false;
+    if (message instanceof Error) {
+      text = message.message;
+      stackText = stack ?? message.stack;
+    } else if (typeof message === "object" && message !== null) {
+      const { message: inner, ...rest } = message as Record<string, unknown>;
+      text = typeof inner === "string" ? inner : "";
+      fields = rest;
+      trusted = true;
+    } else {
+      text = String(message);
+    }
+
+    // A structured record is written by our own code (the access line), so it keeps its values;
+    // free text and stacks can carry what a request supplied and lose the request's memo.
+    const literals = [...currentSensitiveLiterals()];
+    const body = {
+      ...(scrubValue(fields, []) as Record<string, unknown>),
+      message: scrubValue(text, trusted ? [] : literals),
+      context,
+      stack: scrubValue(stackText, literals),
+    };
+    this.write(
+      JSON.stringify({
+        ...body,
+        severity,
+        time: new Date().toISOString(),
+        requestId: currentRequestId(),
+      })
+    );
+  }
+}
+
+function defaultWrite(line: string): void {
+  process.stdout.write(`${line}\n`);
+}

apps/api/src/common/log-privacy.ts

@@ -0,0 +1,53 @@
+import { createHmac, randomBytes } from "crypto";
+import { isCexAddress } from "@/lib/exchange-registry";
+
+const SECRET = /(?<![A-Z2-7])S[A-Z2-7]{55}(?![A-Z2-7])/g;
+const ADDRESS = /(?<![A-Z2-7])(?:M[A-Z2-7]{68}|[GC][A-Z2-7]{55})(?![A-Z2-7])/g;
+const BEARER = /Bearer\s+\S+/gi;
+const BLOB = /[A-Za-z0-9+/]{100,}={0,2}/g;
+
+const MIN_LITERAL_LENGTH = 3;
+
+const hashKey = process.env.LOG_HASH_KEY || randomBytes(32).toString("hex");
+
+function correlationHash(address: string): string {
+  return createHmac("sha256", hashKey).update(address).digest("hex").slice(0, 8);
+}
+
+function redactAddress(address: string): string {
+  if (isCexAddress(address)) return "[exchange]";
+  return `${address.slice(0, 4)}...${address.slice(-4)}~${correlationHash(address)}`;
+}
+
+/**
+ * The log privacy policy (docs/reference/logging-and-privacy.mdx), applied to the text of every
+ * value the logger writes: whatever a call site interpolates, no secret-shaped string, full
+ * address, token or XDR-sized blob leaves the process. `literals` are values the request itself
+ * carried that must never appear, such as a deposit memo.
+ *
+ * It runs on values before they are serialized, never on the JSON line: a user-chosen memo such
+ * as `","` or `severity` would otherwise rewrite the line's own structure.
+ */
+export function scrubText(text: string, literals: Iterable<string> = []): string {
+  let out = text;
+  for (const literal of literals) {
+    if (literal.length >= MIN_LITERAL_LENGTH) out = out.split(literal).join("[memo]");
+  }
+  return out
+    .replace(SECRET, "[redacted-secret]")
+    .replace(BEARER, "Bearer [redacted]")
+    .replace(ADDRESS, redactAddress)
+    .replace(BLOB, "[redacted]");
+}
+
+/** `scrubText` over every string in a JSON-shaped value; object keys are left alone. */
+export function scrubValue(value: unknown, literals: readonly string[]): unknown {
+  if (typeof value === "string") return scrubText(value, literals);
+  if (Array.isArray(value)) return value.map((item) => scrubValue(item, literals));
+  if (typeof value === "object" && value !== null) {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, inner]) => [key, scrubValue(inner, literals)])
+    );
+  }
+  return value;
+}

apps/api/src/common/request-context.ts

@@ -0,0 +1,108 @@
+import { AsyncLocalStorage } from "async_hooks";
+import { randomUUID } from "crypto";
+import { Logger } from "@nestjs/common";
+import type { NextFunction, Request, RequestHandler, Response } from "express";
+import { xdr } from "@stellar/stellar-sdk";
+import type { AuthedRequest } from "@/auth/api-key.guard";
+
+export const REQUEST_ID_HEADER = "x-request-id";
+
+interface RequestContext {
+  requestId: string;
+  sensitive: Set<string>;
+}
+
+const storage = new AsyncLocalStorage<RequestContext>();
+
+export function currentRequestId(): string | undefined {
+  return storage.getStore()?.requestId;
+}
+
+export function currentSensitiveLiterals(): Iterable<string> {
+  return storage.getStore()?.sensitive ?? [];
+}
+
+/** Marks a value the current request carried (a deposit memo) as never to be logged. */
+export function markSensitive(value: unknown): void {
+  if (typeof value === "string" && value.length > 0) storage.getStore()?.sensitive.add(value);
+}
+
+function withRequestId(body: unknown, requestId: string): unknown {
+  if (typeof body !== "object" || body === null || !("error" in body)) return body;
+  const { error } = body as { error: unknown };
+  if (typeof error !== "object" || error === null || Array.isArray(error)) return body;
+  return { ...body, error: { ...error, requestId } };
+}
+
+function severityMethod(status: number): "log" | "warn" | "error" {
+  if (status >= 500) return "error";
+  if (status >= 400) return "warn";
+  return "log";
+}
+
+/**
+ * Gives every request an id before anything that can answer it, sets the headers every
+ * response carries (`x-request-id`, `Cache-Control: no-store`), copies the id into any error
+ * envelope, and writes one access line when the response finishes.
+ *
+ * The id is always generated here: a caller-supplied one would let a client forge or inject
+ * into the log line it is meant to be matched against. The access line carries the matched route
+ * pattern, never the raw path, because paths hold account addresses.
+ */
+export function requestContextMiddleware(): RequestHandler {
+  const logger = new Logger("http");
+  return (req: Request, res: Response, next: NextFunction): void => {
+    const requestId = randomUUID();
+    const startedAt = process.hrtime.bigint();
+    res.setHeader(REQUEST_ID_HEADER, requestId);
+    res.setHeader("Cache-Control", "no-store");
+
+    const json = res.json.bind(res);
+    res.json = ((body?: unknown) =>
+      json(res.statusCode >= 400 ? withRequestId(body, requestId) : body)) as Response["json"];
+
+    res.on("finish", () => {
+      if (req.path === "/health") return;
+      const params = req.params as Record<string, string | undefined> | undefined;
+      logger[severityMethod(res.statusCode)]({
+        message: "request",
+        route: req.route ? `${req.baseUrl}${String(req.route.path)}` : "unrouted",
+        status: res.statusCode,
+        latencyMs: Math.round(Number(process.hrtime.bigint() - startedAt) / 1e6),
+        keyLabel: (req as AuthedRequest).apiKeyLabel,
+        network: params?.network,
+      });
+    });
+
+    storage.run({ requestId, sensitive: new Set() }, next);
+  };
+}
+
+function memoOfEnvelope(signedXdr: string): string | undefined {
+  try {
+    const envelope = xdr.TransactionEnvelope.fromXDR(signedXdr, "base64");
+    const tx =
+      envelope.switch().name === "envelopeTypeTxFeeBump"
+        ? envelope.feeBump().tx().innerTx().v1().tx()
+        : envelope.v1().tx();
+    const memo = tx.memo();
+    if (memo.switch().name === "memoText") return memo.text().toString("utf8");
+    if (memo.switch().name === "memoId") return memo.id().toString();
+  } catch {
+    return undefined;
+  }
+  return undefined;
+}
+
+/** After body parsing: the memo is the one body value that identifies a deposit, so it is
+ *  registered for removal from every line the rest of the request writes, whether it arrived as a
+ *  field or inside a signed envelope. */
+export const registerRequestSecrets: RequestHandler = (req, _res, next) => {
+  const body: unknown = req.body;
+  if (typeof body === "object" && body !== null) {
+    const { memo, signedXdr } = body as { memo?: unknown; signedXdr?: unknown };
+    markSensitive(memo);
+    if (typeof signedXdr === "string") markSensitive(memoOfEnvelope(signedXdr));
+  }
+  next();
+};

apps/api/src/configure-app.ts

@@ -1,9 +1,11 @@
-import type { INestApplication } from "@nestjs/common";
+import type { INestApplication, LoggerService } from "@nestjs/common";
 import type { ErrorCode } from "@lumenwipe/types";
 import { json } from "express";
-import type { ErrorRequestHandler, NextFunction, Request, RequestHandler, Response } from "express";
+import type { ErrorRequestHandler, RequestHandler } from "express";
 import { RateLimiter } from "./auth/rate-limiter";
 import { ErrorEnvelopeFilter } from "./common/error-envelope.filter";
+import { JsonLogger } from "./common/json-logger";
+import { registerRequestSecrets, requestContextMiddleware } from "./common/request-context";
 
 export const JSON_BODY_LIMIT = "100kb";
 
@@ -40,7 +42,10 @@ const BODY_PARSER_ERRORS: Record<string, { status: number; code: ErrorCode; mess
  * we own the JSON error contract: a malformed body must return the original
  * routes' shape, not Nest's default `{ statusCode, message, error }`.
  */
-export function configureApp(app: INestApplication): void {
+export function configureApp(
+  app: INestApplication,
+  logger: LoggerService = new JsonLogger()
+): void {
   // Cloud Run terminates TLS and proxies every request through its own frontend - without this,
   // Express's req.ip reports that proxy's address for every request, not the real caller's, so
   // RateLimiter's per-IP fallback (unauthenticated requests, which carry no API key to
@@ -49,13 +54,14 @@ export function configureApp(app: INestApplication): void {
   // trusting it to report the real client in X-Forwarded-For is safe here.
   app.getHttpAdapter().getInstance().set("trust proxy", true);
 
-  // Every response is dynamic and non-cacheable (account state, plans, unsigned
-  // XDR, mediator co-signatures) - no client, proxy, or CDN should store any of
-  // it, success or error. First, so the limiter's own 429 and 500 carry it too.
-  app.use((_req: Request, res: Response, next: NextFunction) => {
-    res.setHeader("Cache-Control", "no-store");
-    next();
-  });
+  app.useLogger(logger);
+
+  // First, ahead of the rate limiter, body parsing and authentication, so the limiter's own 429,
+  // a 401 and a body error all carry x-request-id. Also sets Cache-Control: no-store: every
+  // response is dynamic and non-cacheable (account state, plans, unsigned XDR, mediator
+  // co-signatures), success or error. Order: request context, rate limiter, filters, body
+  // parser, body-error mapping, body normalization, secret registration.
+  app.use(requestContextMiddleware());
 
   // Ahead of body parsing, routing and authentication, so the budget is spent and
   // reported on every response, including a 404 or a rejected body.
@@ -111,4 +117,5 @@ export function configureApp(app: INestApplication): void {
     next();
   };
   app.use(normalizeBody);
+  app.use(registerRequestSecrets);
 }

apps/api/src/main.ts

@@ -4,6 +4,7 @@ import { Logger } from "@nestjs/common";
 import { NestFactory } from "@nestjs/core";
 import { SwaggerModule } from "@nestjs/swagger";
 import { AppModule } from "./app.module";
+import { JsonLogger } from "./common/json-logger";
 import { configureApp } from "./configure-app";
 import { createOpenApiDocument } from "./openapi";
 import { checkEnv, formatEnvFailure } from "./config/validate-env";
@@ -20,7 +21,7 @@ async function bootstrap(): Promise<void> {
     process.exit(1);
   }
 
-  const app = await NestFactory.create(AppModule, { bodyParser: false });
+  const app = await NestFactory.create(AppModule, { bodyParser: false, logger: new JsonLogger() });
   const bootLogger = new Logger("bootstrap");
   // Logged, not fatal: each of these disables a path rather than the service, and the operator
   // should learn it here rather than from a user hitting the disabled path.

apps/api/tests/e2e/body-handling.e2e.test.ts

@@ -4,6 +4,7 @@ import { Test } from "@nestjs/testing";
 import request from "supertest";
 import { Keypair, Networks, TransactionBuilder, Account, Operation } from "@stellar/stellar-sdk";
 import { AppModule } from "@/app.module";
+import { JsonLogger } from "@/common/json-logger";
 import { configureApp, JSON_BODY_LIMIT } from "@/configure-app";
 
 const KEY = "e2e_test_key";
@@ -39,7 +40,7 @@ beforeAll(async () => {
   process.env.API_KEYS = `test=${KEY}`;
   const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile();
   app = moduleRef.createNestApplication({ bodyParser: false });
-  configureApp(app);
+  configureApp(app, new JsonLogger(() => undefined));
   await app.init();
   http = app.getHttpServer();
 });
@@ -92,7 +93,7 @@ describe.each(POST_ROUTES)("POST %s", (route) => {
     const res = await post(route);
     expect(res.body.error.code).toBeString();
     expect(res.body.error.code).not.toBe("internal_error");
-    expect(spy).not.toHaveBeenCalled();
+    expect(spy.mock.calls.filter(([message]) => typeof message === "string")).toEqual([]);
     spy.mockRestore();
   });
 });