emerging_signals reports dated log lines as both "new" and "gone": Drain keeps the literal date, and each window is mined separately #3544

// Repro: emerging_signals keys Drain templates by text, and dated log lines keep
// the literal date in the template, so the same pattern is "emerging" AND "disappeared".
// Run: npx tsx repro.mts <hyperdx-checkout>/packages/common-utils/src/drain
import { pathToFileURL } from 'node:url';

const drainDir = process.argv[2];
const { TemplateMiner } = await import(pathToFileURL(`${drainDir}/template-miner.ts`).href);
const { TemplateMinerConfig } = await import(pathToFileURL(`${drainDir}/config.ts`).href);

// Same as normalizeTemplate() in packages/api/src/mcp/tools/query/runEventPatterns.ts
const normalizeTemplate = (p: string) =>
  p.replace(/<\*>/g, '\u0001').replace(/\s+/g, ' ').trim().toLowerCase();

const lines = (day: string) =>
  Array.from({ length: 200 }, (_, i) => [
    `${day} 06:${String(i % 60).padStart(2, '0')}:12.345 ERROR [worker-${i % 7}] c.e.SyncJob - Sync failed for account ${1000 + i}: HTTP 500`,
    `${day} 06:${String(i % 60).padStart(2, '0')}:13.001 INFO  [worker-${i % 7}] c.e.SyncJob - Sync finished in ${i * 3} ms`,
  ]).flat();

function mine(rows: string[], masking: { pattern: string; maskWith: string }[]) {
  const config = new TemplateMinerConfig(); // what minePatterns() uses today: no masking instructions
  config.maskingInstructions = masking;
  const miner = new TemplateMiner(config);
  const templates = new Map<number, string>();
  for (const r of rows) {
    const res = miner.addLogMessage(r);
    templates.set(res.clusterId, res.templateMined);
  }
  return new Set([...templates.values()].map(normalizeTemplate));
}

for (const [label, masking] of [
  ['today (no masking)', []],
  ['with a date/time mask', [{ pattern: '\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?', maskWith: 'TS' }]],
] as const) {
  const baseline = mine(lines('2026-11-03'), [...masking]);
  const current = mine(lines('2026-11-04'), [...masking]);
  const emerging = [...current].filter(t => !baseline.has(t));
  const gone = [...baseline].filter(t => !current.has(t));
  console.log(`${label}: ${current.size} templates; emerging=${emerging.length}, disappeared=${gone.length}`);
  for (const t of emerging) console.log('  emerging:', t.replace(/\u0001/g, '<*>'));
}

// Same day, no dates involved: each window is mined by its own Drain tree, so a token
// that happened to be constant in one window stays literal there and becomes <*> in the other.
const sameDay = (hosts: string[]) => Array.from({ length: 200 }, (_, i) =>
  `upstream connect error host=${hosts[i % hosts.length]} retry=${i % 3} reason=timeout`);
{
  const baseline = mine(sameDay(['db-1']), []);
  const current = mine(sameDay(['db-1', 'db-2', 'db-3']), []);
  const emerging = [...current].filter(t => !baseline.has(t));
  const gone = [...baseline].filter(t => !current.has(t));
  console.log(`same day, separate trees: emerging=${emerging.length}, disappeared=${gone.length}`);
  for (const t of [...emerging.map(e => 'emerging: ' + e), ...gone.map(g => 'gone:     ' + g)]) console.log('  ' + t.replace(/\u0001/g, '<*>'));
}