test(testkit): TEMP CI diagnostics for multi-node non-convergence

The two CI-only failures (ParallelMultiNodeSpec ×4, LeaseMajority) are
non-convergence on GitHub's hosted runners — NOT reproducible on any
local hardware (they pass at 2 vCPU even under the near-full suite), and
timeouts are ruled out (90s still times out).  Since it can't be
debugged locally, this adds temporary, rate-limited diagnostics so a
single CI run reveals the failure mode:

  - MultiNodeBroker: frame counters (recv / delivered / dropped-by-reason).
  - ParallelMultiNodeSpec.awaitMembers: logs the observed member view +
    broker counters over time, tagged [MNS-DEBUG].
  - lease-majority.test.ts: per-node total/up member counts + self status
    across the arbitration poll, tagged [LEASE-DEBUG].

Locally these show gossip starting ~3s after spawn then converging in
~200ms; on CI they'll show whether broker.recv stays 0 (workers never
gossip / port wiring dead), rpcErr climbs (workers wedged), or frames
flow but convergence stalls.  All logging is temporary — revert once the
GitHub-runner non-convergence is root-caused.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: pathosDev

src/testkit/ParallelMultiNodeSpec.ts

@@ -281,10 +281,37 @@ export class ParallelMultiNodeSpec {
   async awaitMembers(
     role: string, expectedCount: number, timeoutMs: number = this.settings.awaitTimeoutMs,
   ): Promise<void> {
+    // TEMP diagnostic (#flaky-ci): log the observed member view + broker
+    // frame counters over time, so a CI-only non-convergence is fully
+    // debuggable from the workflow log.  Rate-limited (on-change + every
+    // 3s).  Remove once the GitHub-runner non-convergence is root-caused.
+    const startedAt = Date.now();
+    let lastLog = 0; let lastUp = -2; let polls = 0; let rpcErrs = 0;
     await this.awaitCondition(
       async () => {
-        const members = await this.getMembers(role);
-        return members.filter((m) => m.status === 'up').length === expectedCount;
+        polls++;
+        let up = -1; let statuses = '';
+        try {
+          const members = await this.getMembers(role);
+          up = members.filter((m) => m.status === 'up').length;
+          statuses = members.map((m) => `${m.address}=${m.status}`).join(' ');
+        } catch (e) {
+          rpcErrs++; statuses = `RPC-ERR:${(e as Error).message}`;
+        }
+        const now = Date.now();
+        if (up !== lastUp || now - lastLog > 3_000) {
+          const s = this.broker.stats;
+          // eslint-disable-next-line no-console
+          console.error(
+            `[MNS-DEBUG] awaitMembers(${role} want=${expectedCount}) `
+            + `t=${now - startedAt}ms up=${up} polls=${polls} rpcErr=${rpcErrs} `
+            + `broker{recv=${s.recv} deliv=${s.delivered} senderGone=${s.dropSenderGone} `
+            + `part=${s.dropPartition} noTgt=${s.dropNoTarget} stop=${s.dropStopped}} `
+            + `view[${statuses}]`,
+          );
+          lastLog = now; lastUp = up;
+        }
+        return up === expectedCount;
       },
       `awaitMembers(${role}, expected=${expectedCount})`,
       timeoutMs,

src/testkit/internal/MultiNodeBroker.ts

@@ -32,6 +32,17 @@ export class MultiNodeBroker {
   private readonly blocked = new Set<string>();
   private stopped = false;
 
+  /**
+   * TEMP diagnostic counters (#flaky-ci) — frame flow through the broker.
+   * If `recv` stays ~0 the workers never gossip (worker→broker transport
+   * dead); if `recv` climbs but `delivered` lags, routing is dropping
+   * frames.  Read by ParallelMultiNodeSpec's awaitMembers debug log.
+   */
+  readonly stats = {
+    recv: 0, delivered: 0,
+    dropStopped: 0, dropSenderGone: 0, dropPartition: 0, dropNoTarget: 0,
+  };
+
   register(address: NodeAddress, port: PortLike): void {
     const key = address.toString();
     if (this.ports.has(key)) {
@@ -80,13 +91,15 @@ export class MultiNodeBroker {
   /* -------------------------------- internals ------------------------- */
 
   private onMessage(sourceKey: string, env: BrokeredMessage): void {
-    if (this.stopped) return;
-    if (!this.ports.has(sourceKey)) return;     // sender was unregistered
+    this.stats.recv++;
+    if (this.stopped) { this.stats.dropStopped++; return; }
+    if (!this.ports.has(sourceKey)) { this.stats.dropSenderGone++; return; }     // sender was unregistered
     const targetAddr = NodeAddressCtor.fromJSON(env.to);
     const targetKey = targetAddr.toString();
-    if (this.blocked.has(`${sourceKey}→${targetKey}`)) return;  // partition
+    if (this.blocked.has(`${sourceKey}→${targetKey}`)) { this.stats.dropPartition++; return; }  // partition
     const target = this.ports.get(targetKey);
-    if (!target) return;                        // unknown destination
+    if (!target) { this.stats.dropNoTarget++; return; }                        // unknown destination
+    this.stats.delivered++;
     target.postMessage(env);
   }
 }

tests/multi-node/lease-majority.test.ts

@@ -100,12 +100,27 @@ describe('LeaseMajority — end-to-end split-brain', () => {
       // running this inside the full suite, where it blew past a 10s
       // budget (the original flake).  The loop still breaks the instant
       // one side dies, so the happy path stays quick.
+      const startedAt = Date.now();
+      let lastLog = 0;
       const deadline = Date.now() + 25_000;
       let leftAlive: string[] = [];
       let rightAlive: string[] = [];
       while (Date.now() < deadline) {
         leftAlive = ['a', 'b'].filter(isClusterAlive);
         rightAlive = ['c', 'd'].filter(isClusterAlive);
+        // TEMP diagnostic (#flaky-ci): per-node view over time so a
+        // CI-only non-convergence is debuggable from the workflow log.
+        const now = Date.now();
+        if (now - lastLog > 3_000) {
+          const snap = ['a', 'b', 'c', 'd'].map((r) => {
+            const c = spec.clusterFor(r);
+            const self = c.getMembers().find((m) => m.address.equals(c.selfAddress));
+            return `${r}{tot=${c.getMembers().length} up=${c.upMembers().length} self=${self?.status ?? '?'}}`;
+          }).join(' ');
+          // eslint-disable-next-line no-console
+          console.error(`[LEASE-DEBUG] t=${now - startedAt}ms leftAlive=[${leftAlive}] rightAlive=[${rightAlive}] ${snap}`);
+          lastLog = now;
+        }
         // One side fully self-downed → arbitration has converged.
         if (leftAlive.length === 0 || rightAlive.length === 0) break;
         await Bun.sleep(50);