fix(api): resolve a vanished alert group with its values

.changeset/alert-resolve-vanished-group-values.md

@@ -0,0 +1,5 @@
+---
+'@hyperdx/api': patch
+---
+
+fix: A grouped alert whose group vanishes from the data now resolves with the group's values in `{{attributes.*}}` (alert title and message) and `{{var.*}}` (webhook body)

docs/alert-webhook-template-variables.md

@@ -21,7 +21,7 @@ a HyperDX alert without parsing the human-readable message body.
 | `{{thresholdMax}}` | number | Upper bound of a `between`/`outside` condition. Always unset for every other comparator, even if the alert was once a range, so guard it (see below). |
 | `{{value}}` | number | The value that triggered or resolved the alert. |
 | `{{groupKey}}` | string | The breaching group, for a grouped alert. |
-| `{{var}}` | object | The breaching group's values by group-by column or SQL alias. Use one value at a time, e.g. `{{var.customer_id}}`, or `{{var.[service.name]}}` for a name with dots. Empty when not grouped, and on the resolve of a group that has no rows left. |
+| `{{var}}` | object | The breaching group's values by group-by column or SQL alias. Use one value at a time, e.g. `{{var.customer_id}}`, or `{{var.[service.name]}}` for a name with dots. Empty when not grouped. |
 | `{{sourceQuery}}` | string | The condition behind the alert — search expression or SQL (see below). |
 | `{{teamId}}` | string | Team the alert belongs to. |
 | `{{note}}` | string | The alert's freeform note — commonly a runbook link. |

packages/api/src/models/alertHistory.ts

@@ -32,6 +32,25 @@ export interface IAlertHistoryAnalytics {
   notificationTargets?: AlertNotificationTargetTiming[];
 }
 
+/** One group-by column and its value, stored as a pair. */
+export type GroupValue = { key: string; value: string };
+
+export const attributesToGroupValues = (
+  attributes: Record<string, string>,
+): GroupValue[] | undefined => {
+  const entries = Object.entries(attributes);
+  return entries.length > 0
+    ? entries.map(([key, value]) => ({ key, value }))
+    : undefined;
+};
+
+export const groupValuesToAttributes = (
+  groupValues: GroupValue[] | null | undefined,
+): Record<string, string> | undefined =>
+  groupValues?.length
+    ? Object.fromEntries(groupValues.map(v => [v.key, v.value]))
+    : undefined;
+
 export interface IAlertHistory {
   alert: ObjectId;
   counts: number;
@@ -40,6 +59,13 @@ export interface IAlertHistory {
   lastValues: { startTime: Date; count: number }[];
   group?: string; // For group-by alerts, stores the group identifier
   fired?: boolean;
+  /**
+   * Group-by values of a group-by alert's group. Kept so a group that later
+   * vanishes from the data still resolves with its values, which the joined
+   * `group` string cannot be parsed back into. Stored as pairs because column
+   * names may contain dots, which Mongo would read as nested paths in a key.
+   */
+  groupValues?: GroupValue[];
   /**
    * Errors recorded for this evaluation window. Present on ERROR-state rows
    * (query/processing failures where no normal history is written) and on
@@ -85,6 +111,17 @@ const AlertHistorySchema = new Schema<IAlertHistory>({
     type: Boolean,
     required: false,
   },
+  groupValues: {
+    type: [
+      {
+        _id: false,
+        key: { type: String, required: true },
+        value: { type: String },
+      },
+    ],
+    required: false,
+    default: undefined,
+  },
   errors: {
     type: [
       {

packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts

@@ -2036,7 +2036,7 @@ describe('checkAlerts', () => {
     const setupSavedSearchAlertTest = async ({
       webhookSettings,
     }: Partial<{
-      webhookSettings: IWebhook;
+      webhookSettings: Partial<IWebhook>;
     }> = {}) => {
       const team = await createTeam({ name: 'My Team' });
 
@@ -5807,6 +5807,208 @@ describe('checkAlerts', () => {
       expect(renderedBody).toContain('Group: "ServiceName:web"');
     });
 
+    // Fires a grouped raw SQL alert for one service, then evaluates again once
+    // the only event has left the window so the group has no rows.
+    const fireThenVanish = async ({
+      alias,
+      serviceName = 'web',
+      setup,
+      name,
+      message,
+    }: {
+      alias: string;
+      serviceName?: string;
+      setup?: Parameters<typeof setupSavedSearchAlertTest>[0];
+      name?: string;
+      message?: string;
+    }) => {
+      const { team, webhook, connection, teamWebhooksById, clickhouseClient } =
+        await setupSavedSearchAlertTest(setup);
+
+      const firedAt = new Date('2023-11-16T22:12:00.000Z');
+      const eventMs = firedAt.getTime() - ms('5m');
+
+      await bulkInsertLogs(
+        [1, 2].map(i => ({
+          ServiceName: serviceName,
+          Timestamp: new Date(eventMs),
+          SeverityText: 'error',
+          Body: `error ${i}`,
+        })),
+      );
+
+      const dashboard = await new Dashboard({
+        name: 'Raw SQL Vanished Group Dashboard',
+        team: team._id,
+        tiles: [
+          {
+            id: 'rawsql-vanished',
+            x: 0,
+            y: 0,
+            w: 6,
+            h: 4,
+            config: {
+              configType: 'sql',
+              displayType: 'line',
+              sqlTemplate: `
+        SELECT
+          toStartOfInterval(Timestamp, INTERVAL {intervalSeconds:Int64} second) AS ts,
+          ServiceName AS "${alias}",
+          count() AS cnt
+        FROM default.otel_logs
+        WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64})
+          AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})
+        GROUP BY ts, "${alias}"
+        ORDER BY ts`,
+              connection: connection.id,
+            },
+          },
+        ],
+      }).save();
+
+      const tile = dashboard.tiles?.find(t => t.id === 'rawsql-vanished');
+      if (!tile) throw new Error('tile not found');
+
+      const details = await createAlertDetails(
+        team,
+        undefined,
+        {
+          source: AlertSource.TILE,
+          channel: {
+            type: 'webhook',
+            webhookId: webhook._id.toString(),
+          },
+          interval: '5m',
+          thresholdType: AlertThresholdType.ABOVE,
+          threshold: 2,
+          dashboardId: dashboard.id,
+          tileId: 'rawsql-vanished',
+          name,
+          message,
+        },
+        {
+          taskType: AlertTaskType.TILE,
+          tile,
+          dashboard,
+        },
+      );
+
+      const evaluateAt = (at: Date) =>
+        processAlertAtTime(
+          at,
+          details,
+          clickhouseClient,
+          connection,
+          alertProvider,
+          teamWebhooksById,
+        );
+
+      await evaluateAt(firedAt);
+      expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT');
+
+      const firedRow = await AlertHistory.findOne({
+        alert: details.alert.id,
+        group: `${alias}:${serviceName}`,
+      }).lean();
+
+      return {
+        details,
+        firedRow,
+        resolve: async () => {
+          await evaluateAt(new Date(firedAt.getTime() + ms('10m')));
+          expect((await Alert.findById(details.alert.id))!.state).toBe('OK');
+        },
+      };
+    };
+
+    // Alert titles and messages read the nested `attributes` view, so a dotted
+    // alias is addressed by its path; `var` exists only in webhook bodies.
+    it.each([
+      ['ServiceName', 'attributes.ServiceName'],
+      ['service.name', 'attributes.service.name'],
+    ])(
+      'TILE alert (raw SQL) - a vanished group keyed by %s persists and resolves with its values',
+      async (alias, ref) => {
+        const { firedRow, resolve } = await fireThenVanish({
+          alias,
+          name: `Errors for {{${ref}}}`,
+          message: `Affected service: {{${ref}}}`,
+        });
+        expect(firedRow?.groupValues).toEqual([{ key: alias, value: 'web' }]);
+
+        await resolve();
+
+        expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2);
+        const [, resolvePayload] = jest.mocked(slack.postMessageToWebhook).mock
+          .calls[1];
+        expect(resolvePayload.text).toContain('Errors for web');
+        expect(JSON.stringify(resolvePayload.blocks)).toContain(
+          'Affected service: web',
+        );
+      },
+    );
+
+    it('TILE alert (raw SQL) - a vanished group resolves with its values in a generic webhook body', async () => {
+      const fetchMock = jest.fn().mockResolvedValue({
+        ok: true,
+        text: jest.fn().mockResolvedValue(''),
+      });
+      global.fetch = fetchMock;
+
+      const { resolve } = await fireThenVanish({
+        alias: 'service.name',
+        setup: {
+          webhookSettings: {
+            service: WebhookService.Generic,
+            url: 'https://webhook.site/123',
+            name: 'Generic Webhook',
+            body: JSON.stringify({
+              state: '{{state}}',
+              service: '{{var.[service.name]}}',
+            }),
+          },
+        },
+      });
+      await resolve();
+
+      expect(fetchMock).toHaveBeenCalledTimes(2);
+      const resolveBody = JSON.parse(fetchMock.mock.calls[1][1].body);
+      expect(resolveBody).toEqual({ state: 'OK', service: 'web' });
+    });
+
+    it('TILE alert (raw SQL) - a vanished group with an empty value persists and resolves', async () => {
+      const { firedRow, resolve } = await fireThenVanish({
+        alias: 'ServiceName',
+        serviceName: '',
+      });
+      expect(firedRow?.groupValues).toEqual([
+        { key: 'ServiceName', value: '' },
+      ]);
+
+      await resolve();
+
+      expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2);
+    });
+
+    it('TILE alert (raw SQL) - a previous ALERT row without groupValues still resolves with empty attributes', async () => {
+      const { details, resolve } = await fireThenVanish({
+        alias: 'ServiceName',
+        name: 'Errors for {{attributes.ServiceName}}',
+      });
+      await AlertHistory.updateMany(
+        { alert: details.alert.id },
+        { $unset: { groupValues: 1 } },
+      );
+
+      await resolve();
+
+      expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2);
+      const [, resolvePayload] = jest.mocked(slack.postMessageToWebhook).mock
+        .calls[1];
+      expect(resolvePayload.text).toContain('Errors for');
+      expect(resolvePayload.text).not.toContain('web');
+    });
+
     it('TILE alert (raw SQL) - alert is evaluated using the last numeric column', async () => {
       const { team, webhook, connection, teamWebhooksById, clickhouseClient } =
         await setupSavedSearchAlertTest();

packages/api/src/tasks/checkAlerts/index.ts

@@ -60,6 +60,9 @@ import { serializeError } from 'serialize-error';
 import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory';
 import { AlertState, IAlert, IAlertError } from '@/models/alert';
 import AlertHistory, {
+  attributesToGroupValues,
+  GroupValue,
+  groupValuesToAttributes,
   IAlertHistory,
   IAlertHistoryAnalytics,
 } from '@/models/alertHistory';
@@ -1284,7 +1287,6 @@ export const processAlert = async (
       string,
       { value: number; attributes: Record<string, string>; startTime: Date }
     >();
-    // ponytail: a group that vanished from the data resolves without its values.
     const groupAttributes = new Map<string, Record<string, string>>();
 
     // Helper to get or create history for a group
@@ -1471,7 +1473,12 @@ export const processAlert = async (
           group: groupKey,
           totalCount: lastValue?.count || 0,
           startTime: lastValue?.startTime || nowInMinsRoundDown,
-          attributes: groupAttributes.get(groupKey),
+          // A group with no rows left has no current values; use the ones
+          // recorded when it last reported. The aggregation yields null for
+          // rows that never stored any, which would skip the `{}` default.
+          attributes:
+            groupAttributes.get(groupKey) ??
+            groupValuesToAttributes(previousHistory?.groupValues),
         });
       }
     };
@@ -1648,7 +1655,6 @@ export const processAlert = async (
 
       for (const [groupKey, evaluation] of bucketEvaluations.entries()) {
         const history = getOrCreateHistory(groupKey);
-
         if (evaluation.exceeds) {
           history.counts += 1;
           if (shouldFireBasedOnConsecutiveWindows(groupKey)) {
@@ -1712,6 +1718,18 @@ export const processAlert = async (
       getOrCreateHistory('');
     }
 
+    // A resolve reads values only from a previous ALERT or PENDING row.
+    for (const [groupKey, history] of histories.entries()) {
+      if (
+        history.state === AlertState.ALERT ||
+        history.state === AlertState.PENDING
+      ) {
+        history.groupValues = attributesToGroupValues(
+          groupAttributes.get(groupKey) ?? {},
+        );
+      }
+    }
+
     // Check for state transitions and send notifications
     for (const [groupKey, history] of histories.entries()) {
       const previousKey = computeHistoryMapKey(alert.id, groupKey);
@@ -1819,6 +1837,7 @@ export interface AggregatedAlertHistory {
   state: AlertState;
   group?: string;
   fired?: boolean;
+  groupValues?: GroupValue[] | null;
 }
 
 /**
@@ -1880,6 +1899,7 @@ export const getPreviousAlertHistories = async (
               createdAt: { $first: '$createdAt' },
               state: { $first: '$state' },
               fired: { $first: '$fired' },
+              groupValues: { $first: '$groupValues' },
             },
           },
           {
@@ -1889,6 +1909,7 @@ export const getPreviousAlertHistories = async (
               state: 1,
               group: '$_id.group',
               fired: 1,
+              groupValues: 1,
             },
           },
         ]);