Reduce merge queue flakiness (#2484)

* ci: reduce merge queue flakiness

Contain hanging .NET test processes while preserving diagnostics, make Java native artifacts stable across rerun attempts, retry transient package downloads, and allow slower Rust proxy startup on loaded runners.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix(test): prevent replay proxy teardown hangs

Force-close active proxy connections and upstream requests during shutdown so fixture disposal cannot wait forever after a streaming test. Cover pending requests and streaming responses with regression tests.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* ci: preserve existing .NET timeout budgets

Keep the existing job and hang-detection timeouts while retaining diagnostic output for genuine failures.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* test: run harness regression tests in CI

Execute the harness suite once in the Node workflow and include it in the repository-wide test target so proxy shutdown coverage gates changes.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* test: isolate replay writes from CI environment

Replay proxy unit tests verify snapshot serialization, so preserve and clear the ambient GITHUB_ACTIONS flag around each test instead of suppressing their output files in CI.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: SteveSandersonMS

.github/workflows/dotnet-sdk-tests.yml

@@ -175,7 +175,16 @@ jobs:
           # whole suite normally finishes in about five.
           # The validation job performs the full analyzer-enabled SDK build.
           # Build only test-consumed frameworks here and do not repeat analyzers.
-          args=(--no-restore -v n --blame-hang --blame-hang-timeout 10m --blame-hang-dump-type none -p:RunAnalyzers=false)
+          args=(
+            --no-restore
+            -v n
+            --blame-hang
+            --blame-hang-timeout 10m
+            --blame-hang-dump-type none
+            --logger "trx;LogFilePrefix=test-results"
+            --results-directory "$GITHUB_WORKSPACE/dotnet/TestResults"
+            -p:RunAnalyzers=false
+          )
 
           filter="$DOTNET_TEST_FILTER"
           if [[ "$DOTNET_TEST_SHARD" != "full" ]]; then
@@ -217,3 +226,12 @@ jobs:
             args+=(--filter "$filter")
           fi
           dotnet test test/GitHub.Copilot.SDK.Test.csproj "${args[@]}"
+
+      - name: Upload .NET test diagnostics
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: dotnet-test-diagnostics-${{ matrix.os }}-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.shard }}-${{ github.run_attempt }}
+          path: dotnet/TestResults/
+          if-no-files-found: warn
+          retention-days: 7

.github/workflows/java-sdk-tests.yml

@@ -7,6 +7,11 @@ on:
 permissions:
   contents: read
 
+env:
+  MAVEN_OPTS: >-
+    -Daether.connector.http.retryHandler.count=3
+    -Daether.connector.http.retryHandler.serviceUnavailable=429,502,503
+
 jobs:
   java-sdk-inprocess:
     name: "Java SDK InProcess Tests (${{ matrix.classifier }})"
@@ -121,11 +126,12 @@ jobs:
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: java-native-publication-linux-arm64-${{ github.run_id }}-${{ github.run_attempt }}
+          name: java-native-publication-linux-arm64-${{ github.run_id }}
           path: |
             java/copilot-native/target/copilot-sdk-java-runtime-${{ steps.build.outputs.version }}-linux-arm64.jar
             java/copilot-native/target/linux-arm64-${{ steps.build.outputs.version }}.sha256
           if-no-files-found: error
+          overwrite: true
           retention-days: 1
 
   java-native-publication-windows:
@@ -178,11 +184,12 @@ jobs:
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: java-native-publication-win32-x64-${{ github.run_id }}-${{ github.run_attempt }}
+          name: java-native-publication-win32-x64-${{ github.run_id }}
           path: |
             java/copilot-native/target/copilot-sdk-java-runtime-${{ steps.build.outputs.version }}-win32-x64.jar
             java/copilot-native/target/win32-x64-${{ steps.build.outputs.version }}.sha256
           if-no-files-found: error
+          overwrite: true
           retention-days: 1
 
   java-native-publication-windows-arm64:
@@ -235,11 +242,12 @@ jobs:
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: java-native-publication-win32-arm64-${{ github.run_id }}-${{ github.run_attempt }}
+          name: java-native-publication-win32-arm64-${{ github.run_id }}
           path: |
             java/copilot-native/target/copilot-sdk-java-runtime-${{ steps.build.outputs.version }}-win32-arm64.jar
             java/copilot-native/target/win32-arm64-${{ steps.build.outputs.version }}.sha256
           if-no-files-found: error
+          overwrite: true
           retention-days: 1
 
   java-native-publication-darwin:
@@ -293,11 +301,12 @@ jobs:
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: java-native-publication-darwin-arm64-${{ github.run_id }}-${{ github.run_attempt }}
+          name: java-native-publication-darwin-arm64-${{ github.run_id }}
           path: |
             java/copilot-native/target/copilot-sdk-java-runtime-${{ steps.build.outputs.version }}-darwin-arm64.jar
             java/copilot-native/target/darwin-arm64-${{ steps.build.outputs.version }}.sha256
           if-no-files-found: error
+          overwrite: true
           retention-days: 1
 
   java-native-publication-assembly:
@@ -333,22 +342,22 @@ jobs:
 
       - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
-          name: java-native-publication-linux-arm64-${{ github.run_id }}-${{ github.run_attempt }}
+          name: java-native-publication-linux-arm64-${{ github.run_id }}
           path: ${{ github.workspace }}/java/native-publication-input/linux-arm64
 
       - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
-          name: java-native-publication-win32-x64-${{ github.run_id }}-${{ github.run_attempt }}
+          name: java-native-publication-win32-x64-${{ github.run_id }}
           path: ${{ github.workspace }}/java/native-publication-input/windows
 
       - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
-          name: java-native-publication-win32-arm64-${{ github.run_id }}-${{ github.run_attempt }}
+          name: java-native-publication-win32-arm64-${{ github.run_id }}
           path: ${{ github.workspace }}/java/native-publication-input/windows-arm64
 
       - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
-          name: java-native-publication-darwin-arm64-${{ github.run_id }}-${{ github.run_attempt }}
+          name: java-native-publication-darwin-arm64-${{ github.run_id }}
           path: ${{ github.workspace }}/java/native-publication-input/darwin
 
       - name: Verify native inputs and deploy the complete local release

.github/workflows/nodejs-sdk-tests.yml

@@ -53,6 +53,11 @@ jobs:
         working-directory: ./test/harness
         run: npm ci --ignore-scripts
 
+      - name: Run test harness tests
+        if: runner.os == 'Linux' && matrix.transport == 'default'
+        working-directory: ./test/harness
+        run: npm test
+
       - name: Warm up PowerShell
         if: runner.os == 'Windows'
         run: pwsh.exe -Command "Write-Host 'PowerShell ready'"

.github/workflows/python-sdk-tests.yml

@@ -50,7 +50,7 @@ jobs:
 
       - name: Install Node.js dependencies (for CLI in tests)
         working-directory: ./nodejs
-        run: npm ci --ignore-scripts
+        run: npm ci --ignore-scripts --fetch-retries=4 --fetch-retry-mintimeout=10000 --fetch-retry-maxtimeout=60000
 
       - name: Run ruff format check
         run: uv run ruff format --check .
@@ -63,7 +63,7 @@ jobs:
 
       - name: Install test harness dependencies
         working-directory: ./test/harness
-        run: npm ci --ignore-scripts
+        run: npm ci --ignore-scripts --fetch-retries=4 --fetch-retry-mintimeout=10000 --fetch-retry-maxtimeout=60000
 
       - name: Warm up PowerShell
         if: runner.os == 'Windows'

justfile

@@ -9,7 +9,7 @@ format: format-go format-python format-nodejs format-dotnet format-rust
 lint: lint-go lint-python lint-nodejs lint-dotnet lint-rust
 
 # Run tests for all languages
-test: test-go test-python test-nodejs test-dotnet test-rust test-corrections
+test: test-go test-python test-nodejs test-dotnet test-rust test-harness test-corrections
 
 # Format Go code
 format-go:
@@ -66,6 +66,11 @@ test-nodejs:
     @echo "=== Testing Node.js code ==="
     @cd nodejs && npm test
 
+# Run test harness tests
+test-harness:
+    @echo "=== Testing test harness ==="
+    @cd test/harness && npm test
+
 # Test .NET code
 test-dotnet:
     @echo "=== Testing .NET code ==="
@@ -168,4 +173,3 @@ validate-docs-go:
 validate-docs-cs:
     @echo "=== Validating C# documentation ==="
     @cd scripts/docs-validation && npm run validate:cs
-

rust/tests/e2e/support.rs

@@ -31,6 +31,7 @@ static SHARED_E2E_RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(||
         .expect("create shared E2E runtime")
 });
 const SHARED_E2E_CLEANUP_TIMEOUT: Duration = Duration::from_secs(10);
+const PROXY_STARTUP_TIMEOUT: Duration = Duration::from_secs(30);
 
 pub const DEFAULT_TEST_TOKEN: &str = "rust-e2e-token";
 
@@ -1274,7 +1275,7 @@ impl CapiProxy {
             }
         });
         let re = regex::Regex::new(r"Listening: (http://[^\s]+)\s+(\{.*\})$").unwrap();
-        let deadline = Instant::now() + SHARED_E2E_CLEANUP_TIMEOUT;
+        let deadline = Instant::now() + PROXY_STARTUP_TIMEOUT;
         while let Some(remaining) = deadline.checked_duration_since(Instant::now()) {
             let line = match line_rx.recv_timeout(remaining) {
                 Ok(Ok(line)) => line,
@@ -1341,7 +1342,7 @@ impl CapiProxy {
 
         kill_and_wait_child(&mut child);
         Err(std::io::Error::other(format!(
-            "timed out after {SHARED_E2E_CLEANUP_TIMEOUT:?} waiting for proxy startup"
+            "timed out after {PROXY_STARTUP_TIMEOUT:?} waiting for proxy startup"
         )))
     }
 

test/harness/capturingHttpProxy.test.ts

@@ -10,9 +10,21 @@ describe("Capturing HTTP Proxy", () => {
   let proxy: CapturingHttpProxy;
   let testServer: http.Server;
   let testServerAddress: string;
+  let onHangingRequest: (() => void) | undefined;
+  let onStreamingResponse: (() => void) | undefined;
 
   beforeEach(async () => {
     testServer = http.createServer((req, res) => {
+      if (req.url === "/hang") {
+        onHangingRequest?.();
+        return;
+      }
+      if (req.url === "/stream") {
+        res.writeHead(200, { "content-type": "text/plain" });
+        res.write("started");
+        onStreamingResponse?.();
+        return;
+      }
       res.writeHead(200, { "content-type": "application/json" });
       res.end(JSON.stringify({ message: "Hello", path: req.url }));
     });
@@ -71,4 +83,34 @@ describe("Capturing HTTP Proxy", () => {
       } as CapturedExchange,
     ]);
   });
+
+  test("stops while a proxied request is still active", async () => {
+    proxy = new CapturingHttpProxy(testServerAddress);
+    const proxyUrl = await proxy.start();
+    const requestStarted = new Promise<void>((resolve) => {
+      onHangingRequest = resolve;
+    });
+    const request = fetch(`${proxyUrl}/hang`).catch(() => undefined);
+    await requestStarted;
+
+    await proxy.stop();
+
+    await request;
+  });
+
+  test("stops while a proxied response is still streaming", async () => {
+    proxy = new CapturingHttpProxy(testServerAddress);
+    const proxyUrl = await proxy.start();
+    const responseStarted = new Promise<void>((resolve) => {
+      onStreamingResponse = resolve;
+    });
+    const responsePromise = fetch(`${proxyUrl}/stream`);
+    await responseStarted;
+    const response = await responsePromise;
+    const body = response.text().catch(() => undefined);
+
+    await proxy.stop();
+
+    await body;
+  });
 });

test/harness/capturingHttpProxy.ts

@@ -10,7 +10,10 @@ import https from "https";
  */
 export class CapturingHttpProxy {
   private readonly capturedExchanges: CapturedExchange[] = [];
+  private readonly activeRequests = new Set<http.ClientRequest>();
+  private readonly activeResponses = new Set<http.IncomingMessage>();
   private server?: http.Server;
+  private stopPromise?: Promise<void>;
 
   constructor(private targetUrl: string) {}
 
@@ -90,6 +93,10 @@ export class CapturingHttpProxy {
             res.end();
           },
           onError: (err) => {
+            if (!this.server) {
+              res.destroy();
+              return;
+            }
             console.error("Error in proxying request:", err);
             const endTime = Date.now();
             const formattedError =
@@ -130,24 +137,58 @@ export class CapturingHttpProxy {
   }
 
   async stop(): Promise<void> {
-    if (this.server) {
-      return new Promise((resolve, reject) => {
-        this.server!.close((err) => {
+    if (this.stopPromise) {
+      return this.stopPromise;
+    }
+
+    const server = this.server;
+    if (!server) {
+      return;
+    }
+
+    this.server = undefined;
+    this.stopPromise = (async () => {
+      const closed = new Promise<void>((resolve, reject) => {
+        server.close((err) => {
           if (err) {
             reject(err);
           } else {
             resolve();
           }
         });
       });
-    }
+
+      // server.close() waits for active connections. A replayed streaming request
+      // can otherwise wedge fixture teardown after its test has already passed.
+      server.closeAllConnections();
+      for (const response of this.activeResponses) {
+        response.destroy();
+      }
+      this.activeResponses.clear();
+      for (const request of this.activeRequests) {
+        request.destroy();
+      }
+      this.activeRequests.clear();
+
+      await closed;
+    })();
+    return this.stopPromise;
   }
 
   performRequest(options: PerformRequestOptions): void {
+    if (this.stopPromise) {
+      options.onError(new Error("Proxy is stopping"));
+      return;
+    }
+
     const protocol = options.isHttps ? https : http;
     const upstreamRequest = protocol.request(
       options.requestOptions,
       (upstreamResponse) => {
+        this.activeResponses.add(upstreamResponse);
+        upstreamResponse.once("close", () => {
+          this.activeResponses.delete(upstreamResponse);
+        });
         options.onResponseStart(
           upstreamResponse.statusCode || 500,
           upstreamResponse.headers,
@@ -157,6 +198,10 @@ export class CapturingHttpProxy {
       },
     );
 
+    this.activeRequests.add(upstreamRequest);
+    upstreamRequest.once("close", () => {
+      this.activeRequests.delete(upstreamRequest);
+    });
     upstreamRequest.on("error", options.onError);
 
     if (options.body) {

test/harness/replayingCapiProxy.test.ts

@@ -24,13 +24,21 @@ import { ShellConfig } from "./util";
 describe("ReplayingCapiProxy", () => {
   let tempDir: string;
   let workDir: string;
+  let githubActions: string | undefined;
 
   beforeEach(async () => {
+    githubActions = process.env.GITHUB_ACTIONS;
+    delete process.env.GITHUB_ACTIONS;
     tempDir = await mkdtemp(path.join(os.tmpdir(), "capi-proxy-test-"));
     workDir = path.join(tempDir, "work");
   });
 
   afterEach(async () => {
+    if (githubActions === undefined) {
+      delete process.env.GITHUB_ACTIONS;
+    } else {
+      process.env.GITHUB_ACTIONS = githubActions;
+    }
     await rm(tempDir, { recursive: true, force: true });
   });