From cafedabce0a558a7043075d75a0808f1ee318881 Mon Sep 17 00:00:00 2001 From: Bryan Font Date: Mon, 13 Jul 2026 19:45:56 -0400 Subject: [PATCH 1/2] Profile and optimize lineage accounting --- .../Generated/CodexParserHash.generated.swift | 2 +- .../Providers/Codex/CodexLineageEngine.swift | 31 +++++- .../Providers/Codex/CodexLineageLedger.swift | 97 +++++++++++++------ .../Codex/CodexLineageTwoPassDiscovery.swift | 33 ++++--- .../Vendored/CostUsage/CostUsageJsonl.swift | 6 +- .../Vendored/CostUsage/CostUsageScanner.swift | 54 ++++++++++- .../CodexLineageEngineTests.swift | 20 ++++ .../CodexLineageLocalValidationTests.swift | 9 ++ 8 files changed, 201 insertions(+), 51 deletions(-) diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index 66e60596f4..cc3fe322ff 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "5caa3e1eda239d03" + static let value = "f0c3db302ce7c1bb" } diff --git a/Sources/CodexBarCore/Providers/Codex/CodexLineageEngine.swift b/Sources/CodexBarCore/Providers/Codex/CodexLineageEngine.swift index 22bd558301..bf8cb4f41d 100644 --- a/Sources/CodexBarCore/Providers/Codex/CodexLineageEngine.swift +++ b/Sources/CodexBarCore/Providers/Codex/CodexLineageEngine.swift @@ -54,6 +54,9 @@ enum CodexLineageEngine { let observationCount: Int let peakFamilyObservationCount: Int let peakAcceptedFingerprintCount: Int + let documentLoadMilliseconds: Int64 + let familyReconciliationMilliseconds: Int64 + let compositionMilliseconds: Int64 } struct Result: Equatable, Sendable { @@ -189,6 +192,8 @@ enum CodexLineageEngine { var reused = 0 var peakLoadedObservations = 0 var peakAccepted = 0 + var documentLoadDuration = Duration.zero + var familyReconciliationDuration = Duration.zero for family in families.sorted(by: { $0.stableID < $1.stableID }) { try checkCancellation?() let cacheFingerprint = Self.cacheFingerprint( @@ -203,6 +208,7 @@ enum CodexLineageEngine { var documents: [CodexLineageLedger.Document] = [] documents.reserveCapacity(family.descriptors.count) var loadedObservations = 0 + let loadStarted = ContinuousClock.now for descriptor in family.descriptors { try checkCancellation?() let document: CodexLineageLedger.Document = if let loadDocument { @@ -215,12 +221,15 @@ enum CodexLineageEngine { loadedObservations += document.observations.count documents.append(document) } + documentLoadDuration += loadStarted.duration(to: .now) peakLoadedObservations = max(peakLoadedObservations, loadedObservations) + let reconciliationStarted = ContinuousClock.now let conservative = try CodexLineageLedger.reconcileConservatively( documents: documents, unresolvedParents: family.unresolvedParents, localTimeZone: localTimeZone, checkCancellation: checkCancellation) + familyReconciliationDuration += reconciliationStarted.duration(to: .now) let quality = conservative.families.first?.quality ?? .primary let result = FamilyResult( stableID: family.stableID, @@ -233,7 +242,9 @@ enum CodexLineageEngine { peakAccepted = max(peakAccepted, result.report.acceptedObservationCount) } results.sort { $0.stableID < $1.stableID } + let compositionStarted = ContinuousClock.now let report = try Self.compose(results.map(\.report), checkCancellation: checkCancellation) + let compositionDuration = compositionStarted.duration(to: .now) let candidate = Cache( algorithmVersion: Self.algorithmVersion, familiesByInputFingerprint: Dictionary(uniqueKeysWithValues: results.map { @@ -250,7 +261,10 @@ enum CodexLineageEngine { reusedFamilyCount: reused, observationCount: families.reduce(0) { $0 + $1.observationCount }, peakFamilyObservationCount: peakLoadedObservations, - peakAcceptedFingerprintCount: peakAccepted)) + peakAcceptedFingerprintCount: peakAccepted, + documentLoadMilliseconds: Self.milliseconds(documentLoadDuration), + familyReconciliationMilliseconds: Self.milliseconds(familyReconciliationDuration), + compositionMilliseconds: Self.milliseconds(compositionDuration))) } static func reconcile( @@ -267,6 +281,7 @@ enum CodexLineageEngine { var recomputed = 0 var reused = 0 var peakAccepted = 0 + var familyReconciliationDuration = Duration.zero for family in families.sorted(by: { $0.stableID < $1.stableID }) { try checkCancellation?() @@ -282,11 +297,13 @@ enum CodexLineageEngine { peakAccepted = max(peakAccepted, cached.report.acceptedObservationCount) continue } + let reconciliationStarted = ContinuousClock.now let conservative = try CodexLineageLedger.reconcileConservatively( documents: family.documents, unresolvedParents: family.unresolvedParents, localTimeZone: localTimeZone, checkCancellation: checkCancellation) + familyReconciliationDuration += reconciliationStarted.duration(to: .now) let quality = conservative.families.first?.quality ?? .primary let familyFingerprint = Self.familyFingerprint(input: cacheFingerprint, quality: quality) let result = FamilyResult( @@ -301,7 +318,9 @@ enum CodexLineageEngine { } results.sort { $0.stableID < $1.stableID } + let compositionStarted = ContinuousClock.now let report = try Self.compose(results.map(\.report), checkCancellation: checkCancellation) + let compositionDuration = compositionStarted.duration(to: .now) let candidate = Cache( algorithmVersion: Self.algorithmVersion, familiesByInputFingerprint: Dictionary(uniqueKeysWithValues: results.map { @@ -318,7 +337,10 @@ enum CodexLineageEngine { reusedFamilyCount: reused, observationCount: families.reduce(0) { $0 + $1.observationCount }, peakFamilyObservationCount: families.map(\.observationCount).max() ?? 0, - peakAcceptedFingerprintCount: peakAccepted)) + peakAcceptedFingerprintCount: peakAccepted, + documentLoadMilliseconds: 0, + familyReconciliationMilliseconds: Self.milliseconds(familyReconciliationDuration), + compositionMilliseconds: Self.milliseconds(compositionDuration))) } static func publish( @@ -550,6 +572,11 @@ enum CodexLineageEngine { scopeID + "\u{0}" + self.canonical(value) } + private static func milliseconds(_ duration: Duration) -> Int64 { + let components = duration.components + return components.seconds * 1000 + Int64(components.attoseconds / 1_000_000_000_000_000) + } + private struct DisjointSet { var parents: [String: String] = [:] diff --git a/Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift b/Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift index 8fc8e3df9d..98c93a0910 100644 --- a/Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift +++ b/Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift @@ -127,6 +127,19 @@ enum CodexLineageLedger { documents: [Document], localTimeZone: TimeZone, checkCancellation: CostUsageScanner.CancellationCheck? = nil) throws -> Report + { + try self.reconcile( + documents: documents, + localTimeZone: localTimeZone, + checkCancellation: checkCancellation, + validatedDates: [:]) + } + + private static func reconcile( + documents: [Document], + localTimeZone: TimeZone, + checkCancellation: CostUsageScanner.CancellationCheck?, + validatedDates: [String: Date]) throws -> Report { var graph = DisjointSet() for document in documents { @@ -142,42 +155,51 @@ enum CodexLineageLedger { } } - var acceptedByComponent: [String: [Fingerprint: AcceptedObservation]] = [:] - var physicalObservationCount = 0 + var documentsByComponent: [String: [Document]] = [:] for document in documents { try checkCancellation?() let componentID = graph.find(Self.scoped(document.ownerID, document: document)) - var accepted = acceptedByComponent[componentID] ?? [:] - for observation in document.observations { - try checkCancellation?() - physicalObservationCount += 1 - let date = try Self.date(from: observation.timestamp) - let fingerprint = Fingerprint(last: observation.last, total: observation.total) - if let existing = accepted[fingerprint] { - if existing.date < date { - continue - } - if existing.date == date, - !Self.shouldPreferModel(observation.model, over: existing.model) - { - continue - } - } - accepted[fingerprint] = AcceptedObservation( - date: date, - model: observation.model, - last: observation.last) - } - acceptedByComponent[componentID] = accepted + documentsByComponent[componentID, default: []].append(document) } + var physicalObservationCount = 0 + var parsedDates = validatedDates var utcDays: [String: Totals] = [:] var localDays: [String: Totals] = [:] var utcRows: [DailyRowKey: DailyRowValue] = [:] var localRows: [DailyRowKey: DailyRowValue] = [:] var acceptedObservationCount = 0 - for accepted in acceptedByComponent.values { + for componentDocuments in documentsByComponent.values { try checkCancellation?() + var accepted: [Fingerprint: AcceptedObservation] = [:] + for document in componentDocuments { + for observation in document.observations { + try checkCancellation?() + physicalObservationCount += 1 + let date: Date + if let cached = parsedDates[observation.timestamp] { + date = cached + } else { + date = try Self.date(from: observation.timestamp) + parsedDates[observation.timestamp] = date + } + let fingerprint = Fingerprint(last: observation.last, total: observation.total) + if let existing = accepted[fingerprint] { + if existing.date < date { + continue + } + if existing.date == date, + !Self.shouldPreferModel(observation.model, over: existing.model) + { + continue + } + } + accepted[fingerprint] = AcceptedObservation( + date: date, + model: observation.model, + last: observation.last) + } + } acceptedObservationCount += accepted.count for observation in accepted.values { try checkCancellation?() @@ -230,9 +252,10 @@ enum CodexLineageLedger { var primaryDocuments: [Document] = [] var containedDocuments: [Document] = [] var families: [FamilyDisposition] = [] + var validatedDates: [String: Date] = [:] for familyDocuments in documentsByFamily.values { try checkCancellation?() - let reasons = Self.containmentReasons(in: familyDocuments) + let reasons = Self.containmentReasons(in: familyDocuments, validatedDates: &validatedDates) let owners = Set(familyDocuments.map(\.ownerID)) let unresolved = familyDocuments.contains { document in guard let parent = Self.nonEmpty(document.parentSessionID) else { return false } @@ -263,7 +286,8 @@ enum CodexLineageLedger { primary: Self.reconcile( documents: primaryDocuments, localTimeZone: localTimeZone, - checkCancellation: checkCancellation), + checkCancellation: checkCancellation, + validatedDates: validatedDates), families: families, containedDocuments: containedDocuments) } @@ -336,14 +360,25 @@ enum CodexLineageLedger { ].joined(separator: "\u{0}") } - private static func containmentReasons(in documents: [Document]) -> Set { + private static func containmentReasons( + in documents: [Document], + validatedDates: inout [String: Date]) -> Set + { var reasons: Set = [] if documents.contains(where: { $0.incompleteObservationCount > 0 }) { reasons.insert(.incompleteObservation) } - if documents.flatMap(\.observations) - .contains(where: { CostUsageScanner.dateFromTimestamp($0.timestamp) == nil }) - { + var hasMalformedTimestamp = false + timestampValidation: for document in documents { + for observation in document.observations where validatedDates[observation.timestamp] == nil { + guard let date = CostUsageScanner.dateFromTimestamp(observation.timestamp) else { + hasMalformedTimestamp = true + break timestampValidation + } + validatedDates[observation.timestamp] = date + } + } + if hasMalformedTimestamp { reasons.insert(.malformedTimestamp) } let ownerGroups = Dictionary(grouping: documents, by: \.ownerID) diff --git a/Sources/CodexBarCore/Providers/Codex/CodexLineageTwoPassDiscovery.swift b/Sources/CodexBarCore/Providers/Codex/CodexLineageTwoPassDiscovery.swift index 16edd16396..52c0b213e2 100644 --- a/Sources/CodexBarCore/Providers/Codex/CodexLineageTwoPassDiscovery.swift +++ b/Sources/CodexBarCore/Providers/Codex/CodexLineageTwoPassDiscovery.swift @@ -109,19 +109,19 @@ enum CodexLineageTwoPassDiscovery { // Validate both sides of the parse. A post-parse signature alone can bless a // mixed read when a rollout is replaced while the parser has the file open. let initialSignature = try Self.signature(fileURL: fileURL, checkCancellation: checkCancellation) - let summary = try CostUsageScanner.parseCodexLineageDocumentSummary( + let parsed = try CostUsageScanner.parseCodexLineageDocumentSummaryWithSHA256( fileURL: fileURL, checkCancellation: checkCancellation) - let finalSignature = try Self.signature(fileURL: fileURL, checkCancellation: checkCancellation) + let finalSignature = try Self.signatureMetadata(fileURL: fileURL, contentSHA256: parsed.sha256) guard initialSignature == finalSignature else { throw DiscoveryError.fileChangedDuringScan } return Descriptor( fileURL: fileURL, - ownerID: summary.ownerID, - metadataSessionID: summary.metadataSessionID, - parentSessionID: summary.parentSessionID, - scopeID: summary.scopeID, - incompleteObservationCount: summary.incompleteObservationCount, - observationCount: summary.observationCount, + ownerID: parsed.summary.ownerID, + metadataSessionID: parsed.summary.metadataSessionID, + parentSessionID: parsed.summary.parentSessionID, + scopeID: parsed.summary.scopeID, + incompleteObservationCount: parsed.summary.incompleteObservationCount, + observationCount: parsed.summary.observationCount, signature: finalSignature) } @@ -129,16 +129,13 @@ enum CodexLineageTwoPassDiscovery { _ descriptor: Descriptor, checkCancellation: CostUsageScanner.CancellationCheck? = nil) throws -> CodexLineageLedger.Document { - guard try self.signature(fileURL: descriptor.fileURL, checkCancellation: checkCancellation) == descriptor - .signature - else { throw DiscoveryError.fileChangedDuringScan } - let document = try CostUsageScanner.parseCodexLineageDocument( + let parsed = try CostUsageScanner.parseCodexLineageDocumentWithSHA256( fileURL: descriptor.fileURL, checkCancellation: checkCancellation) - guard try Self.signature(fileURL: descriptor.fileURL, checkCancellation: checkCancellation) == descriptor + guard try Self.signatureMetadata(fileURL: descriptor.fileURL, contentSHA256: parsed.sha256) == descriptor .signature else { throw DiscoveryError.fileChangedDuringScan } - return document + return parsed.document } private static func signature( @@ -162,6 +159,14 @@ enum CodexLineageTwoPassDiscovery { contentSHA256: digest) } + private static func signatureMetadata(fileURL: URL, contentSHA256: String) throws -> FileSignature { + let values = try fileURL.resourceValues(forKeys: [.fileSizeKey, .contentModificationDateKey]) + return FileSignature( + size: Int64(values.fileSize ?? 0), + modifiedMilliseconds: Int64((values.contentModificationDate?.timeIntervalSince1970 ?? 0) * 1000), + contentSHA256: contentSHA256) + } + private struct ScopedIdentity: Equatable, Hashable { let scopeID: String let sessionID: String diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift index 35f58b446a..ec886847ac 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift @@ -31,6 +31,7 @@ enum CostUsageJsonl { maxLineBytes: Int, prefixBytes: Int, checkCancellation: (() throws -> Void)? = nil, + onChunk: ((Data) -> Void)? = nil, onLine: (Line) -> Void) throws -> Int64 { @@ -81,6 +82,7 @@ enum CostUsageJsonl { } try checkCancellation?() + onChunk?(chunk) bytesRead += Int64(chunk.count) chunk.withUnsafeBytes { rawBuffer in guard let base = rawBuffer.bindMemory(to: UInt8.self).baseAddress else { return } @@ -100,7 +102,9 @@ enum CostUsageJsonl { } return false } - if reachedEOF { break } + if reachedEOF { + break + } try checkCancellation?() } diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index 3964cac3e6..8f045b0a77 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -1703,8 +1703,10 @@ enum CostUsageScanner { private static func parseCodexTokenSnapshots( fileURL: URL, retainEvidence: Bool = true, + retainSnapshots: Bool = true, suppressScanErrors: Bool = true, - checkCancellation: CancellationCheck? = nil) throws -> CodexParsedTokenEvidence + checkCancellation: CancellationCheck? = nil, + onChunk: ((Data) -> Void)? = nil) throws -> CodexParsedTokenEvidence { var sessionId: String? var forkedFromId: String? @@ -1737,7 +1739,7 @@ enum CostUsageScanner { if last == nil || total == nil { incompleteObservationCount += 1 } - if retainEvidence { + if retainEvidence, retainSnapshots { let counted = accumulator.apply(last: last, total: total) snapshots.append(CodexTimestampedTotals( timestamp: timestamp, @@ -1764,6 +1766,7 @@ enum CostUsageScanner { maxLineBytes: 512 * 1024, prefixBytes: 512 * 1024, checkCancellation: checkCancellation, + onChunk: onChunk, onLine: { line in guard !line.bytes.isEmpty else { return } if line.wasTruncated { @@ -1889,6 +1892,7 @@ enum CostUsageScanner { { let parsed = try Self.parseCodexTokenSnapshots( fileURL: fileURL, + retainSnapshots: false, suppressScanErrors: false, checkCancellation: checkCancellation) return CodexLineageLedger.Document( @@ -1900,6 +1904,27 @@ enum CostUsageScanner { incompleteObservationCount: parsed.incompleteObservationCount) } + static func parseCodexLineageDocumentWithSHA256( + fileURL: URL, + checkCancellation: CancellationCheck? = nil) throws -> (document: CodexLineageLedger.Document, sha256: String) + { + var hasher = SHA256() + let parsed = try Self.parseCodexTokenSnapshots( + fileURL: fileURL, + retainSnapshots: false, + suppressScanErrors: false, + checkCancellation: checkCancellation, + onChunk: { hasher.update(data: $0) }) + let document = CodexLineageLedger.Document( + ownerID: Self.codexRolloutOwnerID(fileURL: fileURL) ?? parsed.sessionId ?? fileURL.standardizedFileURL.path, + metadataSessionID: parsed.sessionId, + parentSessionID: parsed.forkedFromId, + observations: parsed.observations, + scopeID: Self.codexLineageScopeID(fileURL: fileURL), + incompleteObservationCount: parsed.incompleteObservationCount) + return (document, Self.hexDigest(hasher.finalize())) + } + static func parseCodexLineageDocumentSummary( fileURL: URL, checkCancellation: CancellationCheck? = nil) throws -> CodexLineageDocumentSummary @@ -1918,6 +1943,31 @@ enum CostUsageScanner { observationCount: parsed.observationCount) } + static func parseCodexLineageDocumentSummaryWithSHA256( + fileURL: URL, + checkCancellation: CancellationCheck? = nil) throws -> (summary: CodexLineageDocumentSummary, sha256: String) + { + var hasher = SHA256() + let parsed = try Self.parseCodexTokenSnapshots( + fileURL: fileURL, + retainEvidence: false, + suppressScanErrors: false, + checkCancellation: checkCancellation, + onChunk: { hasher.update(data: $0) }) + let summary = CodexLineageDocumentSummary( + ownerID: Self.codexRolloutOwnerID(fileURL: fileURL) ?? parsed.sessionId ?? fileURL.standardizedFileURL.path, + metadataSessionID: parsed.sessionId, + parentSessionID: parsed.forkedFromId, + scopeID: Self.codexLineageScopeID(fileURL: fileURL), + incompleteObservationCount: parsed.incompleteObservationCount, + observationCount: parsed.observationCount) + return (summary, Self.hexDigest(hasher.finalize())) + } + + private static func hexDigest(_ digest: some Sequence) -> String { + digest.map { String(format: "%02x", $0) }.joined() + } + static func codexLineageScopeID(fileURL: URL) -> String { let standardized = fileURL.standardizedFileURL let components = standardized.pathComponents diff --git a/Tests/CodexBarTests/CodexLineageEngineTests.swift b/Tests/CodexBarTests/CodexLineageEngineTests.swift index 2ad505de2e..5a8ebe6c39 100644 --- a/Tests/CodexBarTests/CodexLineageEngineTests.swift +++ b/Tests/CodexBarTests/CodexLineageEngineTests.swift @@ -156,6 +156,26 @@ struct CodexLineageEngineTests { #expect(result.diagnostics.peakAcceptedFingerprintCount <= 2000) } + @Test(.timeLimit(.minutes(2))) + func `duplicate heavy multi million observation family keeps one accepted fingerprint`() throws { + let repeated = Array(repeating: Self.observation(input: 1, total: 1), count: 100_000) + let documents = (0..<20).map { index in + Self.document( + owner: "owner-\(index)", + parent: index == 0 ? nil : "owner-0", + observations: repeated) + } + let families = try CodexLineageEngine.prepareFamilies(documents: documents) + + let result = try CodexLineageEngine.reconcile(families: families, localTimeZone: .gmt) + + #expect(result.diagnostics.observationCount == 2_000_000) + #expect(result.diagnostics.peakFamilyObservationCount == 2_000_000) + #expect(result.diagnostics.peakAcceptedFingerprintCount == 1) + #expect(result.report.acceptedObservationCount == 1) + #expect(result.report.duplicateObservationCount == 1_999_999) + } + @Test func `cancellation propagates during preparation and reconciliation without a candidate`() throws { let documents = (0..<20).map { index in diff --git a/Tests/CodexBarTests/CodexLineageLocalValidationTests.swift b/Tests/CodexBarTests/CodexLineageLocalValidationTests.swift index 7c6c02feb8..85ef010101 100644 --- a/Tests/CodexBarTests/CodexLineageLocalValidationTests.swift +++ b/Tests/CodexBarTests/CodexLineageLocalValidationTests.swift @@ -182,13 +182,17 @@ struct CodexLineageLocalValidationTests { let lineageStarted = ContinuousClock.now Self.progress("discovery-start") + let discoveryStarted = ContinuousClock.now let discovery = try CodexLineageTwoPassDiscovery.discover( includedFiles: included, roots: [sessions, archived]) + let discoveryDuration = discoveryStarted.duration(to: .now) Self.progress("discovery-ready") + let preparationStarted = ContinuousClock.now let families = try CodexLineageEngine.prepareDescriptorFamilies( descriptors: discovery.descriptors, unresolvedParents: discovery.unresolvedParents) + let preparationDuration = preparationStarted.duration(to: .now) Self.progress("families-ready") let lineage = try CodexLineageEngine.reconcileStreaming(families: families, localTimeZone: .gmt) Self.progress("lineage-ready") @@ -307,6 +311,11 @@ struct CodexLineageLocalValidationTests { "performance": [ "legacyMilliseconds": Self.milliseconds(legacyDuration), "lineageMilliseconds": Self.milliseconds(lineageDuration), + "discoveryMilliseconds": Self.milliseconds(discoveryDuration), + "preparationMilliseconds": Self.milliseconds(preparationDuration), + "documentLoadMilliseconds": lineage.diagnostics.documentLoadMilliseconds, + "familyReconciliationMilliseconds": lineage.diagnostics.familyReconciliationMilliseconds, + "compositionMilliseconds": lineage.diagnostics.compositionMilliseconds, ], "ordinaryDayDivergenceCount": ordinaryDivergenceCount, "resetEpochDiagnostics": [ From 62600d9408651356f57e5e7c6fe6b2bca19007f0 Mon Sep 17 00:00:00 2001 From: Bryan Font Date: Mon, 13 Jul 2026 20:36:21 -0400 Subject: [PATCH 2/2] Stabilize lineage validation compilation --- .../CodexLineageLocalValidationTests.swift | 131 +++++++++--------- 1 file changed, 68 insertions(+), 63 deletions(-) diff --git a/Tests/CodexBarTests/CodexLineageLocalValidationTests.swift b/Tests/CodexBarTests/CodexLineageLocalValidationTests.swift index 85ef010101..a81a454426 100644 --- a/Tests/CodexBarTests/CodexLineageLocalValidationTests.swift +++ b/Tests/CodexBarTests/CodexLineageLocalValidationTests.swift @@ -272,71 +272,76 @@ struct CodexLineageLocalValidationTests { return abs((selectedTokens[day] ?? 0) - legacyTokens) > Int(Double(legacyTokens) * 0.01) } + let dayRows: [[String: Any]] = Self.referenceDays.map { day in + let result = classification.days.first { $0.day == day } + return [ + "day": day, + "legacy": legacy[day] ?? 0, + "primary": primary[day] ?? 0, + "contained": contained[day] ?? 0, + "selected": selectedTokens[day] ?? 0, + "shadow": directReports[CodexLineageAccountingMode.shadow.rawValue]?[day] ?? 0, + "scannerLineage": directReports[CodexLineageAccountingMode.lineage.rawValue]?[day] ?? 0, + "reference": references[day]?.tokens ?? 0, + "finalized": references[day]?.finalized ?? false, + "classification": result?.classification.rawValue ?? "missing", + ] as [String: Any] + } + let ordinaryDayRows: [[String: Any]] = Self.ordinaryDays.map { day in + [ + "day": day, + "legacy": legacy[day] ?? 0, + "selected": selectedTokens[day] ?? 0, + ] as [String: Any] + } + let coverage: [String: Any] = [ + "documents": discovery.descriptors.count, + "families": lineage.diagnostics.familyCount, + "containedFamilies": containedFamilyCount, + "selectableContainedFamilies": selectableContainedFamilies.count, + "containedObservations": containedObservationCount, + "containmentReasons": containmentReasons, + "referencedParents": discovery.referencedParentDocumentCount, + "unresolvedParents": discovery.unresolvedParents.count, + "observations": lineage.diagnostics.observationCount, + "peakFamilyObservations": lineage.diagnostics.peakFamilyObservationCount, + "duplicateObservations": lineage.report.duplicateObservationCount, + ] + let performance: [String: Any] = [ + "legacyMilliseconds": Self.milliseconds(legacyDuration), + "lineageMilliseconds": Self.milliseconds(lineageDuration), + "discoveryMilliseconds": Self.milliseconds(discoveryDuration), + "preparationMilliseconds": Self.milliseconds(preparationDuration), + "documentLoadMilliseconds": lineage.diagnostics.documentLoadMilliseconds, + "familyReconciliationMilliseconds": lineage.diagnostics.familyReconciliationMilliseconds, + "compositionMilliseconds": lineage.diagnostics.compositionMilliseconds, + ] + let resetDiagnostics: [String: Any] = [ + "strongResetBoundaries": resetEpochDiagnostics?.strongResetBoundaryCount ?? 0, + "mixedRegressions": resetEpochDiagnostics?.mixedRegressionCount ?? 0, + "postResetRepeatedFingerprints": resetEpochDiagnostics?.postResetRepeatedFingerprintCount ?? 0, + "sameOwnerRepeats": resetEpochDiagnostics?.sameOwnerRepeatCount ?? 0, + "crossOwnerRepeats": resetEpochDiagnostics?.crossOwnerRepeatCount ?? 0, + "estimatedSuppressedTokens": resetEpochDiagnostics.map { + $0.estimatedSuppressed.input + $0.estimatedSuppressed.output + } ?? 0, + "estimatedSuppressedUTC": resetEpochDiagnostics?.estimatedSuppressedUTC.mapValues { + $0.input + $0.output + } ?? [:], + "sameOwnerEstimatedSuppressedTokens": resetEpochDiagnostics.map { + $0.sameOwnerEstimatedSuppressed.input + $0.sameOwnerEstimatedSuppressed.output + } ?? 0, + "sameOwnerEstimatedSuppressedUTC": resetEpochDiagnostics?.sameOwnerEstimatedSuppressedUTC.mapValues { + $0.input + $0.output + } ?? [:], + ] let output: [String: Any] = [ - "days": Self.referenceDays.map { day in - let result = classification.days.first { $0.day == day } - return [ - "day": day, - "legacy": legacy[day] ?? 0, - "primary": primary[day] ?? 0, - "contained": contained[day] ?? 0, - "selected": selectedTokens[day] ?? 0, - "shadow": directReports[CodexLineageAccountingMode.shadow.rawValue]?[day] ?? 0, - "scannerLineage": directReports[CodexLineageAccountingMode.lineage.rawValue]?[day] ?? 0, - "reference": references[day]?.tokens ?? 0, - "finalized": references[day]?.finalized ?? false, - "classification": result?.classification.rawValue ?? "missing", - ] as [String: Any] - }, - "ordinaryDays": Self.ordinaryDays.map { day in - [ - "day": day, - "legacy": legacy[day] ?? 0, - "selected": selectedTokens[day] ?? 0, - ] as [String: Any] - }, - "coverage": [ - "documents": discovery.descriptors.count, - "families": lineage.diagnostics.familyCount, - "containedFamilies": containedFamilyCount, - "selectableContainedFamilies": selectableContainedFamilies.count, - "containedObservations": containedObservationCount, - "containmentReasons": containmentReasons, - "referencedParents": discovery.referencedParentDocumentCount, - "unresolvedParents": discovery.unresolvedParents.count, - "observations": lineage.diagnostics.observationCount, - "peakFamilyObservations": lineage.diagnostics.peakFamilyObservationCount, - "duplicateObservations": lineage.report.duplicateObservationCount, - ], - "performance": [ - "legacyMilliseconds": Self.milliseconds(legacyDuration), - "lineageMilliseconds": Self.milliseconds(lineageDuration), - "discoveryMilliseconds": Self.milliseconds(discoveryDuration), - "preparationMilliseconds": Self.milliseconds(preparationDuration), - "documentLoadMilliseconds": lineage.diagnostics.documentLoadMilliseconds, - "familyReconciliationMilliseconds": lineage.diagnostics.familyReconciliationMilliseconds, - "compositionMilliseconds": lineage.diagnostics.compositionMilliseconds, - ], + "days": dayRows, + "ordinaryDays": ordinaryDayRows, + "coverage": coverage, + "performance": performance, "ordinaryDayDivergenceCount": ordinaryDivergenceCount, - "resetEpochDiagnostics": [ - "strongResetBoundaries": resetEpochDiagnostics?.strongResetBoundaryCount ?? 0, - "mixedRegressions": resetEpochDiagnostics?.mixedRegressionCount ?? 0, - "postResetRepeatedFingerprints": resetEpochDiagnostics?.postResetRepeatedFingerprintCount ?? 0, - "sameOwnerRepeats": resetEpochDiagnostics?.sameOwnerRepeatCount ?? 0, - "crossOwnerRepeats": resetEpochDiagnostics?.crossOwnerRepeatCount ?? 0, - "estimatedSuppressedTokens": resetEpochDiagnostics.map { - $0.estimatedSuppressed.input + $0.estimatedSuppressed.output - } ?? 0, - "estimatedSuppressedUTC": resetEpochDiagnostics?.estimatedSuppressedUTC.mapValues { - $0.input + $0.output - } ?? [:], - "sameOwnerEstimatedSuppressedTokens": resetEpochDiagnostics.map { - $0.sameOwnerEstimatedSuppressed.input + $0.sameOwnerEstimatedSuppressed.output - } ?? 0, - "sameOwnerEstimatedSuppressedUTC": resetEpochDiagnostics?.sameOwnerEstimatedSuppressedUTC.mapValues { - $0.input + $0.output - } ?? [:], - ], + "resetEpochDiagnostics": resetDiagnostics, "aggregateImproved": classification.improvesAggregateError, // This replay is one validation artifact, not permission to remove the rollback path. "supportsLegacyRemoval": false,