Skip to content

Feature/Fix: Profile and optimize shadow lineage accounting #44

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
iam-brain wants to merge 2 commits into
base: iam-brain/lineage-branch-frontier-diagnostics
Choose a base branch
from
Open

Feature/Fix: Profile and optimize shadow lineage accounting #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cafedab
Profile and optimize lineage accounting
iam-brain Jul 13, 2026
62600d9
Stabilize lineage validation compilation
iam-brain Jul 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Sources/CodexBarCore/Generated/CodexParserHash.generated.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand.

enum CodexParserHash {
static let value = "5caa3e1eda239d03"
static let value = "f0c3db302ce7c1bb"
}
31 changes: 29 additions & 2 deletions Sources/CodexBarCore/Providers/Codex/CodexLineageEngine.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ enum CodexLineageEngine {
let observationCount: Int
let peakFamilyObservationCount: Int
let peakAcceptedFingerprintCount: Int
let documentLoadMilliseconds: Int64
let familyReconciliationMilliseconds: Int64
let compositionMilliseconds: Int64
}

struct Result: Equatable, Sendable {
Expand Down Expand Up @@ -189,6 +192,8 @@ enum CodexLineageEngine {
var reused = 0
var peakLoadedObservations = 0
var peakAccepted = 0
var documentLoadDuration = Duration.zero
var familyReconciliationDuration = Duration.zero
for family in families.sorted(by: { $0.stableID < $1.stableID }) {
try checkCancellation?()
let cacheFingerprint = Self.cacheFingerprint(
Expand All @@ -203,6 +208,7 @@ enum CodexLineageEngine {
var documents: [CodexLineageLedger.Document] = []
documents.reserveCapacity(family.descriptors.count)
var loadedObservations = 0
let loadStarted = ContinuousClock.now
for descriptor in family.descriptors {
try checkCancellation?()
let document: CodexLineageLedger.Document = if let loadDocument {
Expand All @@ -215,12 +221,15 @@ enum CodexLineageEngine {
loadedObservations += document.observations.count
documents.append(document)
}
documentLoadDuration += loadStarted.duration(to: .now)
peakLoadedObservations = max(peakLoadedObservations, loadedObservations)
let reconciliationStarted = ContinuousClock.now
let conservative = try CodexLineageLedger.reconcileConservatively(
documents: documents,
unresolvedParents: family.unresolvedParents,
localTimeZone: localTimeZone,
checkCancellation: checkCancellation)
familyReconciliationDuration += reconciliationStarted.duration(to: .now)
let quality = conservative.families.first?.quality ?? .primary
let result = FamilyResult(
stableID: family.stableID,
Expand All @@ -233,7 +242,9 @@ enum CodexLineageEngine {
peakAccepted = max(peakAccepted, result.report.acceptedObservationCount)
}
results.sort { $0.stableID < $1.stableID }
let compositionStarted = ContinuousClock.now
let report = try Self.compose(results.map(\.report), checkCancellation: checkCancellation)
let compositionDuration = compositionStarted.duration(to: .now)
let candidate = Cache(
algorithmVersion: Self.algorithmVersion,
familiesByInputFingerprint: Dictionary(uniqueKeysWithValues: results.map {
Expand All @@ -250,7 +261,10 @@ enum CodexLineageEngine {
reusedFamilyCount: reused,
observationCount: families.reduce(0) { $0 + $1.observationCount },
peakFamilyObservationCount: peakLoadedObservations,
peakAcceptedFingerprintCount: peakAccepted))
peakAcceptedFingerprintCount: peakAccepted,
documentLoadMilliseconds: Self.milliseconds(documentLoadDuration),
familyReconciliationMilliseconds: Self.milliseconds(familyReconciliationDuration),
compositionMilliseconds: Self.milliseconds(compositionDuration)))
}

static func reconcile(
Expand All @@ -267,6 +281,7 @@ enum CodexLineageEngine {
var recomputed = 0
var reused = 0
var peakAccepted = 0
var familyReconciliationDuration = Duration.zero

for family in families.sorted(by: { $0.stableID < $1.stableID }) {
try checkCancellation?()
Expand All @@ -282,11 +297,13 @@ enum CodexLineageEngine {
peakAccepted = max(peakAccepted, cached.report.acceptedObservationCount)
continue
}
let reconciliationStarted = ContinuousClock.now
let conservative = try CodexLineageLedger.reconcileConservatively(
documents: family.documents,
unresolvedParents: family.unresolvedParents,
localTimeZone: localTimeZone,
checkCancellation: checkCancellation)
familyReconciliationDuration += reconciliationStarted.duration(to: .now)
let quality = conservative.families.first?.quality ?? .primary
let familyFingerprint = Self.familyFingerprint(input: cacheFingerprint, quality: quality)
let result = FamilyResult(
Expand All @@ -301,7 +318,9 @@ enum CodexLineageEngine {
}

results.sort { $0.stableID < $1.stableID }
let compositionStarted = ContinuousClock.now
let report = try Self.compose(results.map(\.report), checkCancellation: checkCancellation)
let compositionDuration = compositionStarted.duration(to: .now)
let candidate = Cache(
algorithmVersion: Self.algorithmVersion,
familiesByInputFingerprint: Dictionary(uniqueKeysWithValues: results.map {
Expand All @@ -318,7 +337,10 @@ enum CodexLineageEngine {
reusedFamilyCount: reused,
observationCount: families.reduce(0) { $0 + $1.observationCount },
peakFamilyObservationCount: families.map(\.observationCount).max() ?? 0,
peakAcceptedFingerprintCount: peakAccepted))
peakAcceptedFingerprintCount: peakAccepted,
documentLoadMilliseconds: 0,
familyReconciliationMilliseconds: Self.milliseconds(familyReconciliationDuration),
compositionMilliseconds: Self.milliseconds(compositionDuration)))
}

static func publish(
Expand Down Expand Up @@ -550,6 +572,11 @@ enum CodexLineageEngine {
scopeID + "\u{0}" + self.canonical(value)
}

private static func milliseconds(_ duration: Duration) -> Int64 {
let components = duration.components
return components.seconds * 1000 + Int64(components.attoseconds / 1_000_000_000_000_000)
}

private struct DisjointSet {
var parents: [String: String] = [:]

Expand Down
97 changes: 66 additions & 31 deletions Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,19 @@ enum CodexLineageLedger {
documents: [Document],
localTimeZone: TimeZone,
checkCancellation: CostUsageScanner.CancellationCheck? = nil) throws -> Report
{
try self.reconcile(
documents: documents,
localTimeZone: localTimeZone,
checkCancellation: checkCancellation,
validatedDates: [:])
}

private static func reconcile(
documents: [Document],
localTimeZone: TimeZone,
checkCancellation: CostUsageScanner.CancellationCheck?,
validatedDates: [String: Date]) throws -> Report
{
var graph = DisjointSet()
for document in documents {
Expand All @@ -142,42 +155,51 @@ enum CodexLineageLedger {
}
}

var acceptedByComponent: [String: [Fingerprint: AcceptedObservation]] = [:]
var physicalObservationCount = 0
var documentsByComponent: [String: [Document]] = [:]
for document in documents {
try checkCancellation?()
let componentID = graph.find(Self.scoped(document.ownerID, document: document))
var accepted = acceptedByComponent[componentID] ?? [:]
for observation in document.observations {
try checkCancellation?()
physicalObservationCount += 1
let date = try Self.date(from: observation.timestamp)
let fingerprint = Fingerprint(last: observation.last, total: observation.total)
if let existing = accepted[fingerprint] {
if existing.date < date {
continue
}
if existing.date == date,
!Self.shouldPreferModel(observation.model, over: existing.model)
{
continue
}
}
accepted[fingerprint] = AcceptedObservation(
date: date,
model: observation.model,
last: observation.last)
}
acceptedByComponent[componentID] = accepted
documentsByComponent[componentID, default: []].append(document)
}

var physicalObservationCount = 0
var parsedDates = validatedDates
var utcDays: [String: Totals] = [:]
var localDays: [String: Totals] = [:]
var utcRows: [DailyRowKey: DailyRowValue] = [:]
var localRows: [DailyRowKey: DailyRowValue] = [:]
var acceptedObservationCount = 0
for accepted in acceptedByComponent.values {
for componentDocuments in documentsByComponent.values {
try checkCancellation?()
var accepted: [Fingerprint: AcceptedObservation] = [:]
for document in componentDocuments {
for observation in document.observations {
try checkCancellation?()
physicalObservationCount += 1
let date: Date
if let cached = parsedDates[observation.timestamp] {
date = cached
} else {
date = try Self.date(from: observation.timestamp)
parsedDates[observation.timestamp] = date
}
let fingerprint = Fingerprint(last: observation.last, total: observation.total)
if let existing = accepted[fingerprint] {
if existing.date < date {
continue
}
if existing.date == date,
!Self.shouldPreferModel(observation.model, over: existing.model)
{
continue
}
}
accepted[fingerprint] = AcceptedObservation(
date: date,
model: observation.model,
last: observation.last)
}
}
acceptedObservationCount += accepted.count
for observation in accepted.values {
try checkCancellation?()
Expand Down Expand Up @@ -230,9 +252,10 @@ enum CodexLineageLedger {
var primaryDocuments: [Document] = []
var containedDocuments: [Document] = []
var families: [FamilyDisposition] = []
var validatedDates: [String: Date] = [:]
for familyDocuments in documentsByFamily.values {
try checkCancellation?()
let reasons = Self.containmentReasons(in: familyDocuments)
let reasons = Self.containmentReasons(in: familyDocuments, validatedDates: &validatedDates)
let owners = Set(familyDocuments.map(\.ownerID))
let unresolved = familyDocuments.contains { document in
guard let parent = Self.nonEmpty(document.parentSessionID) else { return false }
Expand Down Expand Up @@ -263,7 +286,8 @@ enum CodexLineageLedger {
primary: Self.reconcile(
documents: primaryDocuments,
localTimeZone: localTimeZone,
checkCancellation: checkCancellation),
checkCancellation: checkCancellation,
validatedDates: validatedDates),
families: families,
containedDocuments: containedDocuments)
}
Expand Down Expand Up @@ -336,14 +360,25 @@ enum CodexLineageLedger {
].joined(separator: "\u{0}")
}

private static func containmentReasons(in documents: [Document]) -> Set<ContainmentReason> {
private static func containmentReasons(
in documents: [Document],
validatedDates: inout [String: Date]) -> Set<ContainmentReason>
{
var reasons: Set<ContainmentReason> = []
if documents.contains(where: { $0.incompleteObservationCount > 0 }) {
reasons.insert(.incompleteObservation)
}
if documents.flatMap(\.observations)
.contains(where: { CostUsageScanner.dateFromTimestamp($0.timestamp) == nil })
{
var hasMalformedTimestamp = false
timestampValidation: for document in documents {
for observation in document.observations where validatedDates[observation.timestamp] == nil {
guard let date = CostUsageScanner.dateFromTimestamp(observation.timestamp) else {
hasMalformedTimestamp = true
break timestampValidation
}
validatedDates[observation.timestamp] = date
}
}
if hasMalformedTimestamp {
reasons.insert(.malformedTimestamp)
}
let ownerGroups = Dictionary(grouping: documents, by: \.ownerID)
Expand Down
33 changes: 19 additions & 14 deletions Sources/CodexBarCore/Providers/Codex/CodexLineageTwoPassDiscovery.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,36 +109,33 @@ enum CodexLineageTwoPassDiscovery {
// Validate both sides of the parse. A post-parse signature alone can bless a
// mixed read when a rollout is replaced while the parser has the file open.
let initialSignature = try Self.signature(fileURL: fileURL, checkCancellation: checkCancellation)
let summary = try CostUsageScanner.parseCodexLineageDocumentSummary(
let parsed = try CostUsageScanner.parseCodexLineageDocumentSummaryWithSHA256(
fileURL: fileURL,
checkCancellation: checkCancellation)
let finalSignature = try Self.signature(fileURL: fileURL, checkCancellation: checkCancellation)
let finalSignature = try Self.signatureMetadata(fileURL: fileURL, contentSHA256: parsed.sha256)
guard initialSignature == finalSignature else { throw DiscoveryError.fileChangedDuringScan }
return Descriptor(
fileURL: fileURL,
ownerID: summary.ownerID,
metadataSessionID: summary.metadataSessionID,
parentSessionID: summary.parentSessionID,
scopeID: summary.scopeID,
incompleteObservationCount: summary.incompleteObservationCount,
observationCount: summary.observationCount,
ownerID: parsed.summary.ownerID,
metadataSessionID: parsed.summary.metadataSessionID,
parentSessionID: parsed.summary.parentSessionID,
scopeID: parsed.summary.scopeID,
incompleteObservationCount: parsed.summary.incompleteObservationCount,
observationCount: parsed.summary.observationCount,
signature: finalSignature)
}

static func loadDocument(
_ descriptor: Descriptor,
checkCancellation: CostUsageScanner.CancellationCheck? = nil) throws -> CodexLineageLedger.Document
{
guard try self.signature(fileURL: descriptor.fileURL, checkCancellation: checkCancellation) == descriptor
.signature
else { throw DiscoveryError.fileChangedDuringScan }
let document = try CostUsageScanner.parseCodexLineageDocument(
let parsed = try CostUsageScanner.parseCodexLineageDocumentWithSHA256(
fileURL: descriptor.fileURL,
checkCancellation: checkCancellation)
guard try Self.signature(fileURL: descriptor.fileURL, checkCancellation: checkCancellation) == descriptor
guard try Self.signatureMetadata(fileURL: descriptor.fileURL, contentSHA256: parsed.sha256) == descriptor
.signature
else { throw DiscoveryError.fileChangedDuringScan }
return document
return parsed.document
}

private static func signature(
Expand All @@ -162,6 +159,14 @@ enum CodexLineageTwoPassDiscovery {
contentSHA256: digest)
}

private static func signatureMetadata(fileURL: URL, contentSHA256: String) throws -> FileSignature {
let values = try fileURL.resourceValues(forKeys: [.fileSizeKey, .contentModificationDateKey])
return FileSignature(
size: Int64(values.fileSize ?? 0),
modifiedMilliseconds: Int64((values.contentModificationDate?.timeIntervalSince1970 ?? 0) * 1000),
contentSHA256: contentSHA256)
}

private struct ScopedIdentity: Equatable, Hashable {
let scopeID: String
let sessionID: String
Expand Down
6 changes: 5 additions & 1 deletion Sources/CodexBarCore/Vendored/CostUsage/CostUsageJsonl.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ enum CostUsageJsonl {
maxLineBytes: Int,
prefixBytes: Int,
checkCancellation: (() throws -> Void)? = nil,
onChunk: ((Data) -> Void)? = nil,
onLine: (Line) -> Void) throws
-> Int64
{
Expand Down Expand Up @@ -81,6 +82,7 @@ enum CostUsageJsonl {
}

try checkCancellation?()
onChunk?(chunk)
bytesRead += Int64(chunk.count)
chunk.withUnsafeBytes { rawBuffer in
guard let base = rawBuffer.bindMemory(to: UInt8.self).baseAddress else { return }
Expand All @@ -100,7 +102,9 @@ enum CostUsageJsonl {
}
return false
}
if reachedEOF { break }
if reachedEOF {
break
}
try checkCancellation?()
}

Expand Down
Loading