diff --git a/.changeset/patterns-mask-dated-lines.md b/.changeset/patterns-mask-dated-lines.md new file mode 100644 index 0000000000..3d9f7a032a --- /dev/null +++ b/.changeset/patterns-mask-dated-lines.md @@ -0,0 +1,17 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/api': patch +'@hyperdx/cli': patch +--- + +fix: MCP `event_patterns`, `emerging_signals` and search denoise, and the CLI, +no longer split the same log line by its date. Before mining, the Drain miner +masks the date of a timestamp (weekday, day, month, year) to ``, so dated +lines from different days, months or years get the same template instead of one +"new" and one "gone". Times and ids are not masked. Within one day lines group +as before, except that klog lines with different severity letters are no longer +merged. No mask changes a line's token count: the `event_patterns` drill-down +predicate matches `` as one or more non-whitespace characters, also inside +a token, and search denoise masks bodies the same way before matching. Masking +instructions get an optional `perWord` flag (default off, as in drain3). The web +app's Event patterns view uses its own drain3 config and is unchanged. diff --git a/packages/api/src/mcp/tools/query/__tests__/denoise.test.ts b/packages/api/src/mcp/tools/query/__tests__/denoise.test.ts index 20393f6d4a..c8fe16fd32 100644 --- a/packages/api/src/mcp/tools/query/__tests__/denoise.test.ts +++ b/packages/api/src/mcp/tools/query/__tests__/denoise.test.ts @@ -98,6 +98,35 @@ describe('buildNoisyBodyMatcher', () => { expect(isNoisy('something unseen')).toBe(false); }); + it('removes dated and id-bearing members of noisy clusters mined with the default masks', () => { + const noisy = Array.from( + { length: 40 }, + (_, i) => + `2026-11-0${3 + (i % 2)} 06:12:${String(i % 60).padStart(2, '0')},345 INFO heartbeat trace_id=4bf92f3577b34da6a3ce929d0e0e47${String(i).padStart(2, '0')} ok`, + ); + const rare = '2026-11-04 06:13:00,001 ERROR payment failed for order 42'; + const { patterns } = minePatterns( + [...noisy, rare].map(body => ({ body })), + { + totalCount: noisy.length + 1, + startDate: new Date(0), + endDate: new Date(1000), + trendBuckets: 0, + maxSamples: noisy.length + 1, + getBody: r => r.body, + getTimestamp: () => 0, + }, + ); + const heartbeat = patterns.find(p => p.pattern.includes('heartbeat')); + expect(heartbeat?.pattern).toContain(''); + const isNoisy = buildNoisyBodyMatcher( + patterns.map(p => p.pattern), + new Set([heartbeat!.pattern]), + ); + for (const body of noisy) expect(isNoisy(body)).toBe(true); + expect(isNoisy(rare)).toBe(false); + }); + it('prefers the noisy template on a tie', () => { const isNoisy = buildNoisyBodyMatcher( ['a <*> c', 'a b <*>'], diff --git a/packages/api/src/mcp/tools/query/__tests__/patternDrilldown.test.ts b/packages/api/src/mcp/tools/query/__tests__/patternDrilldown.test.ts index ce69e41d69..e69af049f7 100644 --- a/packages/api/src/mcp/tools/query/__tests__/patternDrilldown.test.ts +++ b/packages/api/src/mcp/tools/query/__tests__/patternDrilldown.test.ts @@ -97,6 +97,23 @@ describe('templateToRegex', () => { expect(matches('id <*> ok', 'id 日本 語 ok')).toBe(false); }); + it('a masked date is one word, also inside a token', () => { + expect( + matches('wrote backup_.tar', 'wrote backup_2026-11-04.tar'), + ).toBe(true); + expect(matches('wrote backup_.tar', 'wrote backup_.tar')).toBe(false); + expect( + matches('wrote backup_.tar', 'wrote backup_2026-11-04.tar.gz'), + ).toBe(false); + expect( + matches('I <*> sync done', 'I1104 06:12:12.345678 sync done'), + ).toBe(true); + expect( + matches(' <*> INFO x', '2026-11-04\n06:12:12,345 INFO x'), + ).toBe(true); + expect(matches(' <*> INFO x', '2026-11-04 INFO x')).toBe(false); + }); + it('turns an empty template into a whitespace-only match', () => { expect(matches('', '')).toBe(true); expect(matches('', ' \n')).toBe(true); @@ -145,6 +162,73 @@ describe('templateToRegex', () => { } } }); + it('matches the samples of patterns mined from dated lines with the default masks', () => { + const messages = [ + ...[1, 2, 3].map( + n => + `2026-11-0${n} 06:12:1${n},345 INFO [main] c.e.Job - Sync ok for 10${n}`, + ), + ...[4, 5].map(n => `[0${n}/Nov/2026:06:12:12 +0000] "GET /health" 200`), + ...[ + '4bf92f3577b34da6a3ce929d0e0e4736', + '00f067aa0ba902b7a3ce929d0e0e4736', + ].map(id => `exported span trace_id=${id} status=ok`), + ...[3, 4].map(n => `Nov ${n} 06:12:12 host sshd[42]: session opened`), + ]; + const { patterns } = minePatterns( + messages.map(body => ({ body })), + { + totalCount: messages.length, + startDate: new Date(0), + endDate: new Date(1000), + trendBuckets: 0, + maxSamples: messages.length, + getBody: r => r.body, + getTimestamp: () => 0, + }, + ); + expect(patterns).toHaveLength(4); + for (const p of patterns) { + const re = toJsRegex(templateToRegex(p.pattern)); + for (const s of p.samples) expect(re.test(s.body)).toBe(true); + for (const o of patterns.filter(o => o !== p).flatMap(o => o.samples)) { + expect(re.test(o.body)).toBe(false); + } + } + }); +}); + +describe('templateToRegex with mined templates', () => { + it('matches every sample when a masked timestamp and a word share a position', () => { + const messages = [ + 'Started job at 2026-11-04 06:12:12', + 'Started job at boot', + 'Started job at shutdown', + 'Started job at 2026-11-04T06:12:13Z', + ]; + const { patterns } = minePatterns( + messages.map(body => ({ body })), + { + totalCount: messages.length, + startDate: new Date(0), + endDate: new Date(1000), + trendBuckets: 0, + maxSamples: messages.length, + getBody: r => r.body, + getTimestamp: () => 0, + }, + ); + // The space-separated timestamp is two words, so it can't share a cluster + // with the one-word lines; the one-word timestamp can, as <*>. + expect(patterns.map(p => p.pattern).sort()).toEqual([ + 'Started job at <*>', + 'Started job at 06:12:12', + ]); + for (const p of patterns) { + const re = toJsRegex(templateToRegex(p.pattern)); + for (const sample of p.samples) expect(re.test(sample.body)).toBe(true); + } + }); }); describe('buildPatternPredicate', () => { diff --git a/packages/api/src/mcp/tools/query/denoise.ts b/packages/api/src/mcp/tools/query/denoise.ts index 231cad2401..24bcbbb0ce 100644 --- a/packages/api/src/mcp/tools/query/denoise.ts +++ b/packages/api/src/mcp/tools/query/denoise.ts @@ -1,6 +1,7 @@ import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils'; import { + createPatternMasker, DENOISE_NOISE_THRESHOLD, DENOISE_SAMPLE_SIZE, flattenBody, @@ -270,8 +271,10 @@ export function buildNoisyBodyMatcher( noisy: noisyTemplates.has(template), }; }); + // Templates hold masks (, <*>); mask the body the same way to compare. + const masker = createPatternMasker(); return body => { - const tokens = tokenize(flattenBody(body)); + const tokens = tokenize(masker.mask(flattenBody(body))); let best: (typeof candidates)[number] | undefined; for (const c of candidates) { const fits = diff --git a/packages/api/src/mcp/tools/query/patternDrilldown.ts b/packages/api/src/mcp/tools/query/patternDrilldown.ts index 5e85cb6c00..24f8bbdc84 100644 --- a/packages/api/src/mcp/tools/query/patternDrilldown.ts +++ b/packages/api/src/mcp/tools/query/patternDrilldown.ts @@ -1,6 +1,7 @@ import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; import { renderWhereExpressionStr } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; +import { DATE_MASK } from '@hyperdx/common-utils/dist/drain'; import type { ChartVariable, UseTextIndex, @@ -8,6 +9,9 @@ import type { import { substituteVariables } from '@hyperdx/common-utils/dist/variables'; const PATTERN_WILDCARD = '<*>'; +// The date mask applied before mining (common-utils drain/config.ts). Unlike +// <*>, it can also be part of a token. +const DATE_TOKEN = `<${DATE_MASK}>`; /** * Templates longer than this get no predicate: the regex (and the SQL that @@ -19,7 +23,9 @@ export const MAX_PREDICATE_TEMPLATE_LENGTH = 4000; export const PATTERN_MATCH_SEMANTICS = 'whereSnippet matches values with the same whitespace-separated token count as the ' + 'template, the same literal tokens (case-sensitive) in the same positions, and exactly ' + - 'one non-whitespace token at each <*>. Every sampled member of the cluster matches. ' + + `one non-whitespace token at each <*>, and one or more non-whitespace characters at each ${DATE_TOKEN} ` + + `(also inside a token, e.g. backup_${DATE_TOKEN}.tar). ` + + 'Every sampled member of the cluster matches. ' + 'It can also match unsampled events whose shape fits, including ones Drain would have ' + 'put in a more specific cluster, so the drill-down is a superset of cluster membership.'; @@ -39,7 +45,13 @@ function escapeRe2Literal(token: string): string { export function templateToRegex(template: string): string { const tokens = template.split(/\s+/).filter(t => t.length > 0); const body = tokens - .map(t => (t === PATTERN_WILDCARD ? `${NON_WS}+` : escapeRe2Literal(t))) + // is one or more non-whitespace characters, as a whole token or + // inside one (`backup_.tar`). + .map(t => + t === PATTERN_WILDCARD + ? `${NON_WS}+` + : t.split(DATE_TOKEN).map(escapeRe2Literal).join(`${NON_WS}+`), + ) .join(`${WS}+`); return `^${WS}*${body}${WS}*$`; } diff --git a/packages/common-utils/src/__tests__/drain-masking.test.ts b/packages/common-utils/src/__tests__/drain-masking.test.ts new file mode 100644 index 0000000000..f79b05fc9d --- /dev/null +++ b/packages/common-utils/src/__tests__/drain-masking.test.ts @@ -0,0 +1,256 @@ +import { + DEFAULT_MASKING_INSTRUCTIONS, + TemplateMinerConfig, +} from '@/drain/config'; +import { LogMasker } from '@/drain/masking'; +import { minePatterns } from '@/drain/mine-patterns'; +import { TemplateMiner } from '@/drain/template-miner'; + +type Row = { body: string }; + +const mask = (s: string) => + LogMasker.fromConfig(new TemplateMinerConfig()).mask(s); + +const mine = (bodies: string[]) => + minePatterns( + bodies.map(body => ({ body })), + { + totalCount: bodies.length, + startDate: new Date('2026-11-03T00:00:00Z'), + endDate: new Date('2026-11-05T00:00:00Z'), + trendBuckets: 1, + getBody: r => r.body, + getTimestamp: () => null, + }, + ).patterns; + +const pad = (n: number) => String(n).padStart(2, '0'); + +// One day of mixed logs: Java/log4j, access logs, syslog, klog, ISO-8601, +// HTTP, JavaScript and java.util.logging dates. As in real logs, every line +// has its own time and id. +// `day` is the day of November 2026; `weekday` its name. +const dayOfLogs = (day: number, weekday: string) => { + let n = 0; + const t = () => { + n++; + return `${pad(6 + Math.floor(n / 60))}:${pad(n % 60)}:${pad((n * 7) % 60)}`; + }; + const hex = () => + `${pad(n % 100)}bf92f3577b34da6a3ce929d0e0e47${pad(n % 100)}`; + return Array.from({ length: 12 }, (_, i) => [ + `2026-11-${pad(day)} ${t()},345 ${hex()} INFO started`, + `2026-11-${pad(day)} ${t()},345 ${hex()} ERROR crashed`, + `2026-11-${pad(day)} ${t()} ok`, + `2026-11-${pad(day)} ${t()} queue_depth=${i}`, + `[2026-11-${pad(day)} ${t()},345][pool-1-thread-${i}] INFO request handled ok`, + `[${pad(day)}/Nov/2026:${t()} +0000] "GET /a" 200`, + `[${pad(day)}/Nov/2026:${t()} +0000] "POST /b" 500`, + `Nov ${String(day).padStart(2, ' ')} ${t()} host sshd[${100 + i}]: session opened`, + `I11${pad(day)} ${t()}.345678 1 main.go:42] sync done`, + `E11${pad(day)} ${t()}.345678 1 main.go:42] sync failed`, + `2026-11-${pad(day)}T${t()}Z request ok`, + `Date: ${weekday}, ${pad(day)} Nov 2026 ${t()} GMT`, + `${weekday} Nov ${pad(day)} 2026 ${t()} GMT+0000 (Coordinated Universal Time) tick`, + `Nov ${day}, 2026 ${t().replace(/^0/, '')} AM org.example.Main run`, + `Started job at ${i % 2 ? 'boot' : 'shutdown'}`, + ]).flat(); +}; + +// Groups of line indices that a miner put in the same cluster. +const clusterGroups = (config: TemplateMinerConfig, lines: string[]) => { + const miner = new TemplateMiner(config); + const ids = lines.map(line => miner.addLogMessage(line).clusterId); + const groups = new Map(); + ids.forEach((id, i) => groups.set(id, [...(groups.get(id) ?? []), i])); + return [...groups.values()].map(g => g.join(',')).sort(); +}; + +describe('default masking', () => { + it.each([ + ['2026-11-04T06:12:12.345Z started', 'T06:12:12.345Z started'], + ['2026-11-04 06:12:12,345 INFO x', ' 06:12:12,345 INFO x'], + [ + 'run_id=scheduled__2026-11-04T06:00:00+00:00', + 'run_id=scheduled__T06:00:00+00:00', + ], + [ + 'Date: Wed, 04 Nov 2026 06:12:12 GMT', + 'Date: 06:12:12 GMT', + ], + ['[04/Nov/2026:06:12:12 +0000] "GET /"', '[:06:12:12 +0000] "GET /"'], + ['04-Nov-2026 06:12:12.345 INFO main', ' 06:12:12.345 INFO main'], + ['Nov 4 06:12:12 host sshd', ' 06:12:12 host sshd'], + ['I1104 06:12:12.345678 1 main.go', 'I 06:12:12.345678 1 main.go'], + ['I1104\t06:12:12.345678 1 main.go', 'I\t06:12:12.345678 1 main.go'], + [ + '26/11/04 16:09:18 WARN KafkaOffsetReader: retry', + ' 16:09:18 WARN KafkaOffsetReader: retry', + ], + ['11/04/2026 06:12:12 x', ' 06:12:12 x'], + ['Nov 4 06:12:12 2026-11-04 x', ' 06:12:12 x'], + [ + 'Wed Nov 4 06:12:12 +0000 2026 x', + ' 06:12:12 +0000 x', + ], + ['04.11.2026 06:12:12 x', ' 06:12:12 x'], + [ + '04 Nov 2026 06:12:12,345 INFO job done', + ' 06:12:12,345 INFO job done', + ], + ['partition 2026-11-04 loaded', 'partition loaded'], + ['wrote backup_2026-11-04.tar', 'wrote backup_.tar'], + [ + 'Wed Nov 4 06:12:12 2026 job done', + ' 06:12:12 job done', + ], + [ + 'at Wed Nov 04 06:12:12 UTC 2026 done', + 'at 06:12:12 UTC done', + ], + ['2026/11/04 06:12:12 server started', ' 06:12:12 server started'], + [ + 'Wed Nov 04 2026 06:12:12 GMT+0000 (Coordinated Universal Time) tick', + ' 06:12:12 GMT+0000 (Coordinated Universal Time) tick', + ], + [ + 'Nov 4, 2026 6:12:12 PM org.example.Main run', + ' 6:12:12 PM org.example.Main run', + ], + [ + '[2026-11-04 06:12:12,345][pool-1-thread-3] INFO ok', + '[ 06:12:12,345][pool-1-thread-3] INFO ok', + ], + ])('masks %s', (input, expected) => { + expect(mask(input)).toBe(expected); + }); + + it.each([ + 'version v1.2.3 build 2026', + 'flags 0x1f and 0xdeadbeef', + 'commit abc1234 on main', + 'took 12.345 ms at 10:30', + 'Sun is shining on 4 hosts', + 'Meeting on Mon at noon', + 'sent 5 Mar items ok', + 'F1234 not a klog date', + '06:12:12 GET 2048 bytes', + 'trace 4bf92f3577b34da6a3ce929d0e0e4736 ok', + 'req 123e4567-e89b-12d3-a456-426614174000 done', + 'release 10.11.2026 notes', + 'part 1234-56-78 x', + 'ip 10.11.12.13 06:12:12 up', + ])('leaves %s alone', input => { + expect(mask(input)).toBe(input); + }); + + it('masks a drain3-style instruction once per match unless perWord is set', () => { + const config = new TemplateMinerConfig(); + config.maskingInstructions = [{ pattern: '\\d+ ms', maskWith: 'DUR' }]; + expect(LogMasker.fromConfig(config).mask('took 12 ms')).toBe('took '); + config.maskingInstructions = [ + { pattern: '\\d+ ms', maskWith: 'DUR', perWord: true }, + ]; + expect(LogMasker.fromConfig(config).mask('took 12 ms')).toBe( + 'took ', + ); + }); + + it('never changes the word count of a line', () => { + for (const line of dayOfLogs(4, 'Wed')) { + expect(mask(line).trim().split(/\s+/)).toHaveLength( + line.trim().split(/\s+/).length, + ); + } + }); + + it('stays linear on long runs of whitespace', () => { + // A lookbehind that ends in \s+ rescans the run at every position: 128 KiB + // took seconds that way. + const run = ' '.repeat(1 << 17); + const start = performance.now(); + expect(mask(`Nov 4 06:12:12${run}2026`)).toBe( + ` 06:12:12${run}`, + ); + expect(performance.now() - start).toBeLessThan(1000); + }); + + it('lets extractParameters read a per-word masked date back', () => { + const miner = new TemplateMiner(); + const line = 'Date: Wed, 04 Nov 2026 06:12:12 GMT'; + const { templateMined } = miner.addLogMessage(line); + expect( + miner.extractParameters(templateMined, line)?.map(p => p.value), + ).toEqual(['Wed,', '04', 'Nov', '2026']); + }); + + it('is on by default', () => { + expect(new TemplateMinerConfig().maskingInstructions).toEqual( + DEFAULT_MASKING_INSTRUCTIONS, + ); + }); + + it('is off for a drain3 JSON config that lists no masks, as in drain3', () => { + expect(TemplateMinerConfig.fromJSON({}).maskingInstructions).toEqual([]); + }); + + it('takes the masks a drain3 JSON config lists', () => { + expect( + TemplateMinerConfig.fromJSON({ + masking_instructions: [{ regex_pattern: '\\d+', mask_with: 'NUM' }], + }).maskingInstructions, + ).toEqual([{ pattern: '\\d+', maskWith: 'NUM' }]); + }); +}); + +describe('patterns with the default masks', () => { + it('cluster one day of logs as without masks', () => { + const unmasked = new TemplateMinerConfig(); + unmasked.maskingInstructions = []; + const isKlog = (line: string) => /^[IWEF]\d{4} /.test(line); + const lines = dayOfLogs(4, 'Wed').filter(line => !isKlog(line)); + expect(clusterGroups(new TemplateMinerConfig(), lines)).toEqual( + clusterGroups(unmasked, lines), + ); + }); + + it('keeps klog severities apart, which unmasked mining merges', () => { + // Unmasked, I1104 and E1104 both route as numbers in Drain's prefix tree; + // masked, I and E route by their severity letter. + const lines = dayOfLogs(4, 'Wed').filter(line => /^[IE]\d{4} /.test(line)); + expect( + mine(lines) + .map(p => p.pattern) + .sort(), + ).toEqual([ + 'E <*> 1 main.go:42] sync failed', + 'I <*> 1 main.go:42] sync done', + ]); + }); + + it('give the last day of a month and the first of the next the same templates', () => { + const templates = (day: string, weekday: string, rfcDay: string) => + mine([ + `${day} 06:12:11 host sshd[41]: session opened`, + `Date: ${weekday}, ${rfcDay} 2026 06:12:11 GMT`, + `${weekday} ${day} 06:12:11 2026 job done`, + `${rfcDay} 2026 06:12:11,345 INFO job done`, + ]) + .map(p => p.pattern) + .sort(); + expect(templates('Oct 31', 'Sat', '31 Oct')).toEqual( + templates('Nov 1', 'Sun', '01 Nov'), + ); + }); + + it('give the logs of two different days the same templates', () => { + const templates = (lines: string[]) => + mine(lines) + .map(p => p.pattern) + .sort(); + const day1 = templates(dayOfLogs(3, 'Tue')); + const day2 = templates(dayOfLogs(4, 'Wed')); + expect(day1).toEqual(day2); + expect(day1.join('\n')).not.toMatch(/2026|Nov|Tue|Wed|[IE]110/); + }); +}); diff --git a/packages/common-utils/src/drain/config.ts b/packages/common-utils/src/drain/config.ts index 0624e5fb72..d31e20b448 100644 --- a/packages/common-utils/src/drain/config.ts +++ b/packages/common-utils/src/drain/config.ts @@ -1,8 +1,75 @@ export interface MaskingInstructionConfig { pattern: string; maskWith: string; + /** Mask each word of a match separately (default: the whole match once). */ + perWord?: boolean; } +const MONTH = '(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)'; +const WEEKDAY = '(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)'; +const DAY = '(?:0?[1-9]|[12]\\d|3[01])'; +const TIME = '\\d{1,2}:\\d{2}:\\d{2}'; +const NOT_WORD_BEFORE = '(?` + * in a template. + */ +export const DATE_MASK = 'DATE'; + +/** + * Masks the date of a timestamp (weekday, day, month, year), so the same + * message gets the same template on any day. Every line of a day shares its + * date and `` matches itself the same way, so within one day lines group + * as without masks. Times and ids are left alone: they differ between lines on + * any day, and Drain already treats them as variables. No mask changes a + * line's token count. + */ +export const DEFAULT_MASKING_INSTRUCTIONS: MaskingInstructionConfig[] = [ + // 2026-11-04, 2026/11/04, also before an ISO-8601 time: 2026-11-04T06:12:12Z + { + pattern: `${NOT_WORD_BEFORE}\\d{4}[-/](?:0[1-9]|1[0-2])[-/](?:0[1-9]|[12]\\d|3[01])(?:(?=T\\d)|${NOT_WORD_AFTER})`, + maskWith: DATE_MASK, + }, + // Month-name dates before a time: (Wed, )04 Nov 2026 + { + pattern: `${NOT_WORD_BEFORE}(?:${WEEKDAY},?\\s+)?${DAY}\\s+${MONTH}\\s+\\d{4}(?=\\s+${TIME})`, + maskWith: DATE_MASK, + perWord: true, + }, + // The year after a ctime / Java Date time: Nov 4 06:12:12 [UTC] 2026. + // Before the next mask, which rewrites the month and day it looks for. The + // lookahead runs the lookbehind only where a year starts, keeping it linear + // on long runs of whitespace. + { + pattern: `(?=(?:19|20)\\d{2}${NOT_WORD_AFTER})(?<=${MONTH}\\s+${DAY}\\s+${TIME}(?:[.,]\\d+)?(?:\\s+(?:[A-Z]{2,5}|[+-]\\d{4}))?\\s+)(?:19|20)\\d{2}`, + maskWith: DATE_MASK, + }, + // (Wed )Nov 4, and with the year first: Wed Nov 04 2026 (JavaScript), + // Nov 4, 2026 (java.util.logging) + { + pattern: `${NOT_WORD_BEFORE}(?:${WEEKDAY}\\s+)?${MONTH}\\s+${DAY}(?:,?\\s+\\d{4})?(?=\\s+${TIME})`, + maskWith: DATE_MASK, + perWord: true, + }, + // Access log and Tomcat: [04/Nov/2026:06:12:12, 04-Nov-2026 06:12:12 + { + pattern: `${NOT_WORD_BEFORE}\\d{2}(?:/${MONTH}/|-${MONTH}-)\\d{4}(?=:\\d|\\s+${TIME})`, + maskWith: DATE_MASK, + }, + // dd/MM/yyyy, MM/dd/yyyy, dd.MM.yyyy and yy/MM/dd before a time + { + pattern: `(?'; - maskingInstructions: MaskingInstructionConfig[] = []; + maskingInstructions: MaskingInstructionConfig[] = [ + ...DEFAULT_MASKING_INSTRUCTIONS, + ]; parametrizeNumericTokens: boolean = true; parameterExtractionCacheCapacity: number = 3000; static fromJSON(json: Record): TemplateMinerConfig { const config = new TemplateMinerConfig(); + // drain3 applies no masks unless the JSON lists them. + config.maskingInstructions = []; if (typeof json.drain_depth === 'number') config.drainDepth = json.drain_depth; if (typeof json.drain_sim_th === 'number') diff --git a/packages/common-utils/src/drain/index.ts b/packages/common-utils/src/drain/index.ts index f735377f06..018896a127 100644 --- a/packages/common-utils/src/drain/index.ts +++ b/packages/common-utils/src/drain/index.ts @@ -1,5 +1,5 @@ export type { MaskingInstructionConfig } from './config'; -export { TemplateMinerConfig } from './config'; +export { DATE_MASK, TemplateMinerConfig } from './config'; export { Drain } from './drain'; export { LogCluster } from './log-cluster'; export { LruCache } from './lru-cache'; @@ -11,6 +11,7 @@ export type { TrendBucket, } from './mine-patterns'; export { + createPatternMasker, DENOISE_NOISE_THRESHOLD, DENOISE_SAMPLE_SIZE, flattenBody, diff --git a/packages/common-utils/src/drain/masking.ts b/packages/common-utils/src/drain/masking.ts index dd578322d3..a2f17503a6 100644 --- a/packages/common-utils/src/drain/masking.ts +++ b/packages/common-utils/src/drain/masking.ts @@ -1,12 +1,16 @@ +import type { TemplateMinerConfig } from './config'; + export class MaskingInstruction { private regex: RegExp; private _pattern: string; maskWith: string; + perWord: boolean; - constructor(pattern: string, maskWith: string) { + constructor(pattern: string, maskWith: string, perWord = false) { this._pattern = pattern; this.regex = new RegExp(pattern, 'g'); this.maskWith = maskWith; + this.perWord = perWord; } get pattern(): string { @@ -16,7 +20,11 @@ export class MaskingInstruction { mask(content: string, maskPrefix: string, maskSuffix: string): string { const replacement = maskPrefix + this.maskWith + maskSuffix; this.regex.lastIndex = 0; - return content.replace(this.regex, replacement); + if (!this.perWord) return content.replace(this.regex, replacement); + // One mask per word of the match, so the line keeps its token count. + return content.replace(this.regex, match => + match.replace(/\S+/g, replacement), + ); } } @@ -42,6 +50,16 @@ export class LogMasker { } } + static fromConfig(config: TemplateMinerConfig): LogMasker { + return new LogMasker( + config.maskingInstructions.map( + mi => new MaskingInstruction(mi.pattern, mi.maskWith, mi.perWord), + ), + config.maskPrefix, + config.maskSuffix, + ); + } + mask(content: string): string { let result = content; for (const mi of this._maskingInstructions) { diff --git a/packages/common-utils/src/drain/mine-patterns.ts b/packages/common-utils/src/drain/mine-patterns.ts index f5f663962d..1aa5962f42 100644 --- a/packages/common-utils/src/drain/mine-patterns.ts +++ b/packages/common-utils/src/drain/mine-patterns.ts @@ -6,6 +6,7 @@ import { } from '@/core/utils'; import { TemplateMinerConfig } from './config'; +import { LogMasker } from './masking'; import { TemplateMiner } from './template-miner'; // ─── Denoise constants ─────────────────────────────────────────────────────── @@ -17,6 +18,14 @@ export const DENOISE_SAMPLE_SIZE = 10_000; /** Patterns matching more than this fraction of sampled events are "noisy". */ export const DENOISE_NOISE_THRESHOLD = 0.1; +/** + * Masks text the way minePatterns does before Drain, so raw text can be + * compared token by token with a mined template. + */ +export function createPatternMasker(): LogMasker { + return LogMasker.fromConfig(new TemplateMinerConfig()); +} + // ─── Body normalization ────────────────────────────────────────────────────── /** Collapse newlines and runs of whitespace into single spaces. */ diff --git a/packages/common-utils/src/drain/template-miner.ts b/packages/common-utils/src/drain/template-miner.ts index b718cba507..76e731dd46 100644 --- a/packages/common-utils/src/drain/template-miner.ts +++ b/packages/common-utils/src/drain/template-miner.ts @@ -2,7 +2,7 @@ import { TemplateMinerConfig } from './config'; import { Drain } from './drain'; import { LogCluster } from './log-cluster'; import { LruCache } from './lru-cache'; -import { LogMasker, MaskingInstruction } from './masking'; +import { LogMasker } from './masking'; export interface ExtractedParameter { value: string; @@ -30,15 +30,7 @@ export class TemplateMiner { const cfg = config ?? new TemplateMinerConfig(); const paramStr = cfg.maskPrefix + '*' + cfg.maskSuffix; - const maskingInstructions = cfg.maskingInstructions.map( - mi => new MaskingInstruction(mi.pattern, mi.maskWith), - ); - - this.masker = new LogMasker( - maskingInstructions, - cfg.maskPrefix, - cfg.maskSuffix, - ); + this.masker = LogMasker.fromConfig(cfg); this.drain = new Drain( cfg.drainDepth, @@ -136,7 +128,8 @@ export class TemplateMiner { if (exactMatching && maskName !== '*') { const instructions = this.masker.instructionsByMaskName(maskName); for (const mi of instructions) { - allowedPatterns.push(mi.pattern); + // A per-word mask stands for one word of its pattern's match. + allowedPatterns.push(mi.perWord ? '\\S+' : mi.pattern); } } if (!exactMatching || maskName === '*') {