fix: Send Prometheus host credentials as a Basic auth header in the proxy

proxyToPrometheus passed the connection host to fetch with its userinfo,
which fetch rejects, so every PromQL query to a host like
http://user:pass@prometheus:9090 failed with a 502. The ClickHouse
Prometheus API probe had the same problem. Both now use the header that
the MCP label lookup sends.

Author: karl-power

.changeset/prometheus-proxy-basic-auth.md

@@ -0,0 +1,7 @@
+---
+'@hyperdx/api': patch
+---
+
+fix: Authenticate to Prometheus connections whose host includes credentials
+
+A Prometheus connection host such as `http://user:pass@prometheus:9090` made every PromQL query fail with "Failed to reach Prometheus", because `fetch` rejects URLs that carry credentials. The credentials are now sent as a Basic auth header.

packages/api/src/controllers/promql.ts

@@ -50,7 +50,7 @@ export function joinPrometheusUpstreamUrl(
  * Removes `user:pass@` from `url` and returns it as a Basic auth header. fetch
  * rejects a URL that carries credentials, so they cannot stay in the host.
  */
-function takeBasicAuthHeader(url: URL): Record<string, string> {
+export function takeBasicAuthHeader(url: URL): Record<string, string> {
   if (!url.username && !url.password) return {};
   const credentials = `${decodeURIComponent(url.username)}:${decodeURIComponent(url.password)}`;
   url.username = '';

packages/api/src/routers/api/__tests__/prometheus.int.test.ts

@@ -517,10 +517,9 @@ describe('prometheus router', () => {
     });
 
     // Every other userinfo test asserts what's redacted for *display* (the
-    // error bodies below). The actual outgoing `fetch` target must still
-    // carry real credentials, or authenticated Prometheus connections would
-    // silently stop authenticating.
-    it('still sends real credentials to the actual upstream fetch call', async () => {
+    // error bodies below). The outgoing request must still carry the real
+    // credentials, as a header: fetch rejects a URL that includes them.
+    it('sends host credentials to the upstream as a Basic auth header', async () => {
       const { agent, team } = await getLoggedInAgent(server);
       const source = await seedPrometheusSource(
         team._id,
@@ -545,7 +544,11 @@ describe('prometheus router', () => {
         })
         .expect(200);
 
-      expect(mockFetch.mock.calls[0][0] as string).toContain('user:pw@');
+      const [target, init] = mockFetch.mock.calls[0];
+      expect(new URL(String(target)).username).toBe('');
+      expect(init?.headers).toEqual({
+        Authorization: `Basic ${Buffer.from('user:pw').toString('base64')}`,
+      });
     });
 
     it('leaves a host-pinned param untouched when the request never references it', async () => {

packages/api/src/routers/api/prometheus.ts

@@ -18,7 +18,10 @@ import z from 'zod';
 
 import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
-import { joinPrometheusUpstreamUrl } from '@/controllers/promql';
+import {
+  joinPrometheusUpstreamUrl,
+  takeBasicAuthHeader,
+} from '@/controllers/promql';
 import { getSource } from '@/controllers/sources';
 import {
   connectionSupportsPrometheusHttpApi,
@@ -346,7 +349,10 @@ async function clickhouseServesPrometheusHttpApi(
     );
     url.searchParams.set('query', 'up');
     const resp = await fetch(url, {
-      headers: clickhouseAuthHeaders(connection),
+      headers: {
+        ...takeBasicAuthHeader(url),
+        ...clickhouseAuthHeaders(connection),
+      },
       signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
     });
     return typeof JSON.parse(await resp.text())?.status === 'string';
@@ -424,6 +430,9 @@ async function proxyToPrometheus(
       url.searchParams.set(k, v);
     }
   }
+  // fetch rejects a URL that carries credentials, so `user:pw@` in the host
+  // becomes a Basic auth header instead.
+  const authHeader = takeBasicAuthHeader(url);
   const target = url.toString();
 
   // A connection host may carry basic-auth credentials (`http://user:pw@host`)
@@ -433,14 +442,13 @@ async function proxyToPrometheus(
   // the browser. `url.origin` never includes userinfo (WHATWG URL spec), and
   // dropping the query/hash entirely -- rather than trying to redact only the
   // secret-shaped parts of it -- means there's no query key to ever miss.
-  // `target` itself is unaffected and still carries everything needed to
-  // authenticate.
+  // `target` still carries the query string, and `authHeader` the userinfo.
   const redactedTarget = `${url.origin}${url.pathname}`;
 
   let upstreamResp: Response;
   try {
     upstreamResp = await fetch(target, {
-      headers,
+      headers: { ...authHeader, ...headers },
       signal: AbortSignal.timeout(PROMETHEUS_PROXY_TIMEOUT_MS),
     });
   } catch (err) {