fix(patterns): mask dates before Drain

.changeset/patterns-mask-dated-lines.md

@@ -0,0 +1,17 @@
+---
+'@hyperdx/common-utils': patch
+'@hyperdx/api': patch
+'@hyperdx/cli': patch
+---
+
+fix: MCP `event_patterns`, `emerging_signals` and search denoise, and the CLI,
+no longer split the same log line by its date. Before mining, the Drain miner
+masks the date of a timestamp (weekday, day, month, year) to `<DATE>`, so dated
+lines from different days, months or years get the same template instead of one
+"new" and one "gone". Times and ids are not masked. Within one day lines group
+as before, except that klog lines with different severity letters are no longer
+merged. No mask changes a line's token count: the `event_patterns` drill-down
+predicate matches `<DATE>` as one or more non-whitespace characters, also inside
+a token, and search denoise masks bodies the same way before matching. Masking
+instructions get an optional `perWord` flag (default off, as in drain3). The web
+app's Event patterns view uses its own drain3 config and is unchanged.

packages/api/src/mcp/tools/query/__tests__/denoise.test.ts

@@ -98,6 +98,35 @@ describe('buildNoisyBodyMatcher', () => {
     expect(isNoisy('something unseen')).toBe(false);
   });
 
+  it('removes dated and id-bearing members of noisy clusters mined with the default masks', () => {
+    const noisy = Array.from(
+      { length: 40 },
+      (_, i) =>
+        `2026-11-0${3 + (i % 2)} 06:12:${String(i % 60).padStart(2, '0')},345 INFO heartbeat trace_id=4bf92f3577b34da6a3ce929d0e0e47${String(i).padStart(2, '0')} ok`,
+    );
+    const rare = '2026-11-04 06:13:00,001 ERROR payment failed for order 42';
+    const { patterns } = minePatterns(
+      [...noisy, rare].map(body => ({ body })),
+      {
+        totalCount: noisy.length + 1,
+        startDate: new Date(0),
+        endDate: new Date(1000),
+        trendBuckets: 0,
+        maxSamples: noisy.length + 1,
+        getBody: r => r.body,
+        getTimestamp: () => 0,
+      },
+    );
+    const heartbeat = patterns.find(p => p.pattern.includes('heartbeat'));
+    expect(heartbeat?.pattern).toContain('<DATE>');
+    const isNoisy = buildNoisyBodyMatcher(
+      patterns.map(p => p.pattern),
+      new Set([heartbeat!.pattern]),
+    );
+    for (const body of noisy) expect(isNoisy(body)).toBe(true);
+    expect(isNoisy(rare)).toBe(false);
+  });
+
   it('prefers the noisy template on a tie', () => {
     const isNoisy = buildNoisyBodyMatcher(
       ['a <*> c', 'a b <*>'],

packages/api/src/mcp/tools/query/__tests__/patternDrilldown.test.ts

@@ -97,6 +97,23 @@ describe('templateToRegex', () => {
     expect(matches('id <*> ok', 'id 日本 語 ok')).toBe(false);
   });
 
+  it('a masked date is one word, also inside a token', () => {
+    expect(
+      matches('wrote backup_<DATE>.tar', 'wrote backup_2026-11-04.tar'),
+    ).toBe(true);
+    expect(matches('wrote backup_<DATE>.tar', 'wrote backup_.tar')).toBe(false);
+    expect(
+      matches('wrote backup_<DATE>.tar', 'wrote backup_2026-11-04.tar.gz'),
+    ).toBe(false);
+    expect(
+      matches('I<DATE> <*> sync done', 'I1104 06:12:12.345678 sync done'),
+    ).toBe(true);
+    expect(
+      matches('<DATE> <*> INFO x', '2026-11-04\n06:12:12,345 INFO x'),
+    ).toBe(true);
+    expect(matches('<DATE> <*> INFO x', '2026-11-04 INFO x')).toBe(false);
+  });
+
   it('turns an empty template into a whitespace-only match', () => {
     expect(matches('', '')).toBe(true);
     expect(matches('', '  \n')).toBe(true);
@@ -145,6 +162,73 @@ describe('templateToRegex', () => {
       }
     }
   });
+  it('matches the samples of patterns mined from dated lines with the default masks', () => {
+    const messages = [
+      ...[1, 2, 3].map(
+        n =>
+          `2026-11-0${n} 06:12:1${n},345 INFO [main] c.e.Job - Sync ok for 10${n}`,
+      ),
+      ...[4, 5].map(n => `[0${n}/Nov/2026:06:12:12 +0000] "GET /health" 200`),
+      ...[
+        '4bf92f3577b34da6a3ce929d0e0e4736',
+        '00f067aa0ba902b7a3ce929d0e0e4736',
+      ].map(id => `exported span trace_id=${id} status=ok`),
+      ...[3, 4].map(n => `Nov  ${n} 06:12:12 host sshd[42]: session opened`),
+    ];
+    const { patterns } = minePatterns(
+      messages.map(body => ({ body })),
+      {
+        totalCount: messages.length,
+        startDate: new Date(0),
+        endDate: new Date(1000),
+        trendBuckets: 0,
+        maxSamples: messages.length,
+        getBody: r => r.body,
+        getTimestamp: () => 0,
+      },
+    );
+    expect(patterns).toHaveLength(4);
+    for (const p of patterns) {
+      const re = toJsRegex(templateToRegex(p.pattern));
+      for (const s of p.samples) expect(re.test(s.body)).toBe(true);
+      for (const o of patterns.filter(o => o !== p).flatMap(o => o.samples)) {
+        expect(re.test(o.body)).toBe(false);
+      }
+    }
+  });
+});
+
+describe('templateToRegex with mined templates', () => {
+  it('matches every sample when a masked timestamp and a word share a position', () => {
+    const messages = [
+      'Started job at 2026-11-04 06:12:12',
+      'Started job at boot',
+      'Started job at shutdown',
+      'Started job at 2026-11-04T06:12:13Z',
+    ];
+    const { patterns } = minePatterns(
+      messages.map(body => ({ body })),
+      {
+        totalCount: messages.length,
+        startDate: new Date(0),
+        endDate: new Date(1000),
+        trendBuckets: 0,
+        maxSamples: messages.length,
+        getBody: r => r.body,
+        getTimestamp: () => 0,
+      },
+    );
+    // The space-separated timestamp is two words, so it can't share a cluster
+    // with the one-word lines; the one-word timestamp can, as <*>.
+    expect(patterns.map(p => p.pattern).sort()).toEqual([
+      'Started job at <*>',
+      'Started job at <DATE> 06:12:12',
+    ]);
+    for (const p of patterns) {
+      const re = toJsRegex(templateToRegex(p.pattern));
+      for (const sample of p.samples) expect(re.test(sample.body)).toBe(true);
+    }
+  });
 });
 
 describe('buildPatternPredicate', () => {

packages/api/src/mcp/tools/query/denoise.ts

@@ -1,6 +1,7 @@
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils';
 import {
+  createPatternMasker,
   DENOISE_NOISE_THRESHOLD,
   DENOISE_SAMPLE_SIZE,
   flattenBody,
@@ -270,8 +271,10 @@ export function buildNoisyBodyMatcher(
       noisy: noisyTemplates.has(template),
     };
   });
+  // Templates hold masks (<DATE>, <*>); mask the body the same way to compare.
+  const masker = createPatternMasker();
   return body => {
-    const tokens = tokenize(flattenBody(body));
+    const tokens = tokenize(masker.mask(flattenBody(body)));
     let best: (typeof candidates)[number] | undefined;
     for (const c of candidates) {
       const fits =

packages/api/src/mcp/tools/query/patternDrilldown.ts

@@ -1,13 +1,17 @@
 import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { renderWhereExpressionStr } from '@hyperdx/common-utils/dist/core/renderChartConfig';
 import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils';
+import { DATE_MASK } from '@hyperdx/common-utils/dist/drain';
 import type {
   ChartVariable,
   UseTextIndex,
 } from '@hyperdx/common-utils/dist/types';
 import { substituteVariables } from '@hyperdx/common-utils/dist/variables';
 
 const PATTERN_WILDCARD = '<*>';
+// The date mask applied before mining (common-utils drain/config.ts). Unlike
+// <*>, it can also be part of a token.
+const DATE_TOKEN = `<${DATE_MASK}>`;
 
 /**
  * Templates longer than this get no predicate: the regex (and the SQL that
@@ -19,7 +23,9 @@ export const MAX_PREDICATE_TEMPLATE_LENGTH = 4000;
 export const PATTERN_MATCH_SEMANTICS =
   'whereSnippet matches values with the same whitespace-separated token count as the ' +
   'template, the same literal tokens (case-sensitive) in the same positions, and exactly ' +
-  'one non-whitespace token at each <*>. Every sampled member of the cluster matches. ' +
+  `one non-whitespace token at each <*>, and one or more non-whitespace characters at each ${DATE_TOKEN} ` +
+  `(also inside a token, e.g. backup_${DATE_TOKEN}.tar). ` +
+  'Every sampled member of the cluster matches. ' +
   'It can also match unsampled events whose shape fits, including ones Drain would have ' +
   'put in a more specific cluster, so the drill-down is a superset of cluster membership.';
 
@@ -39,7 +45,13 @@ function escapeRe2Literal(token: string): string {
 export function templateToRegex(template: string): string {
   const tokens = template.split(/\s+/).filter(t => t.length > 0);
   const body = tokens
-    .map(t => (t === PATTERN_WILDCARD ? `${NON_WS}+` : escapeRe2Literal(t)))
+    // <DATE> is one or more non-whitespace characters, as a whole token or
+    // inside one (`backup_<DATE>.tar`).
+    .map(t =>
+      t === PATTERN_WILDCARD
+        ? `${NON_WS}+`
+        : t.split(DATE_TOKEN).map(escapeRe2Literal).join(`${NON_WS}+`),
+    )
     .join(`${WS}+`);
   return `^${WS}*${body}${WS}*$`;
 }