-
Notifications
You must be signed in to change notification settings - Fork 106
fix(pi): contain historian failures and busy-starved lease releases #412
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,8 @@ | ||
| /// <reference types="bun-types" /> | ||
|
|
||
| import { describe, expect, it, mock } from "bun:test"; | ||
| import { tmpdir } from "node:os"; | ||
| import { join } from "node:path"; | ||
| import { | ||
| acquireCompartmentLease, | ||
| releaseCompartmentLease, | ||
|
|
@@ -39,6 +41,14 @@ function createDb(): Database { | |
| return db; | ||
| } | ||
|
|
||
| function createFileDb(name: string): Database { | ||
| const path = join(tmpdir(), `${name}-${crypto.randomUUID()}.db`); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: Each execution leaves the file-backed test database in the shared temporary directory. Track the generated path and remove the database artifacts in Prompt for AI agents |
||
| const db = new Database(path); | ||
| initializeDatabase(db); | ||
| runMigrations(db); | ||
| return db; | ||
| } | ||
|
|
||
| function branch(count: number) { | ||
| return Array.from({ length: count }, (_, index) => ({ | ||
| id: `m-${index + 1}`, | ||
|
|
@@ -276,6 +286,54 @@ describe("Pi /ctx-wrapup", () => { | |
| } | ||
| }); | ||
|
|
||
| it("survives a busy-database lease release failure without rejecting", async () => { | ||
| // Cross-connection lock contention requires a shared file; two :memory: | ||
| // databases never contend. Mirrors the production shape: a second | ||
| // connection (omp session, dreamer, FTS indexer) holds BEGIN IMMEDIATE | ||
| // past busy_timeout while wrapup releases its compartment lease. | ||
| const db = createFileDb("pi-wrapup-busy-release"); | ||
| try { | ||
| const sessionId = "pi-wrapup-busy-release"; | ||
| const runPiHistorianForWrapup = mock(async (args) => { | ||
| const start = getLastCompartmentEndMessage(db, sessionId) + 1; | ||
| const end = Math.min( | ||
| args.boundarySnapshot.eligibleEndOrdinal - 1, | ||
| start + 2, | ||
| ); | ||
| appendRange(db, sessionId, start, end); | ||
| args.onPublished?.(); | ||
| // Simulate the production contention window: another process holds | ||
| // the writer lock past busy_timeout while wrapup releases its lease. | ||
| const blocker = new Database(db.filename); | ||
| blocker.exec("PRAGMA busy_timeout=1; BEGIN IMMEDIATE;"); | ||
| try { | ||
| // Shrink the release connection's busy wait so the expected | ||
| // SQLITE_BUSY lands well inside bun's 5s default test timeout | ||
| // (initializeDatabase sets 5000ms; plain `bun test` in CI has | ||
| // no override — project constraint #5917). | ||
| db.exec("PRAGMA busy_timeout=250"); | ||
| releaseCompartmentLease(db, sessionId, "stale-holder"); | ||
| } finally { | ||
| blocker.exec("ROLLBACK"); | ||
| blocker.close(); | ||
| } | ||
| }); | ||
|
|
||
| const result = await runPiWrapup( | ||
| pi().api, | ||
| deps(db, { runPiHistorianForWrapup }), | ||
| ctx(sessionId, 8), | ||
| sessionId, | ||
| 2, | ||
| ); | ||
|
|
||
| expect(result).toContain("## Magic Wrapup"); | ||
| expect(getWrapupInProgressState(db, sessionId)).toBeNull(); | ||
| } finally { | ||
| closeQuietly(db); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: Delete the temporary SQLite file after closing the database; this test creates a UUID-named file under Prompt for AI agents |
||
| } | ||
| }); | ||
|
|
||
| it("signals deferred history and materialization after a wrapup publish", async () => { | ||
| const db = createDb(); | ||
| try { | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -398,6 +398,7 @@ export async function runPiWrapup( | |||||||
| ); | ||||||||
| } | ||||||||
| }, COMPARTMENT_LEASE_RENEWAL_MS); | ||||||||
| let chunkFailure: string | null = null; | ||||||||
| try { | ||||||||
| const runHistorian = deps.runPiHistorianForWrapup ?? runPiHistorian; | ||||||||
| await runHistorian({ | ||||||||
|
|
@@ -451,11 +452,28 @@ export async function runPiWrapup( | |||||||
| signalPiDeferredMaterialization(sessionId); | ||||||||
| }, | ||||||||
| }); | ||||||||
| } catch (err) { | ||||||||
| // Parity with OpenCode: runCompartmentAgent carries its own .catch, | ||||||||
| // so its wrapup finally only sees settled promises. Pi lacks that | ||||||||
| // inner layer; record the failure instead of letting a busy-starved | ||||||||
| // release (or any historian throw) escape the command handler. | ||||||||
| chunkFailure = err instanceof Error ? err.message : String(err); | ||||||||
| } finally { | ||||||||
| clearInterval(leaseRenewal); | ||||||||
| releaseCompartmentLease(deps.db, sessionId, leaseHolder); | ||||||||
| try { | ||||||||
| releaseCompartmentLease(deps.db, sessionId, leaseHolder); | ||||||||
| } catch (err) { | ||||||||
| // Best-effort teardown: the lease row expires via its own TTL. | ||||||||
| console.warn( | ||||||||
| `[magic-context][pi] /ctx-wrapup compartment lease release failed for ${sessionId} (expires via TTL): ${err instanceof Error ? err.message : String(err)}`, | ||||||||
| ); | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| if (chunkFailure) { | ||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When the historian throws an error with an empty message,
Suggested change
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: When a historian rejects with an empty error message, Prompt for AI agents |
||||||||
| failure = `historian chunk failed (${chunkFailure}); wrapped up through message ${lastEnd}. Run /ctx-wrapup again to continue.`; | ||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: When a historian publishes the chunk and then Prompt for AI agents
Suggested change
|
||||||||
| break; | ||||||||
| } | ||||||||
| const afterEnd = getLastCompartmentEndMessage(deps.db, sessionId); | ||||||||
| if (afterEnd <= lastEnd) { | ||||||||
| failure = `No forward progress after chunk ${chunkIndex}; wrapped up through message ${lastEnd}. Run /ctx-wrapup again to continue.`; | ||||||||
|
|
||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3640,7 +3640,15 @@ function spawnPiHistorianRun(args: { | |
| // Close the cross-process check/lease race: /ctx-wrapup may have published | ||
| // its marker after the first check but before this process won the lease. | ||
| sessionLog(sessionId, "historian skipped: /ctx-wrapup became active"); | ||
| releaseCompartmentLease(db, sessionId, holderId); | ||
| try { | ||
| releaseCompartmentLease(db, sessionId, holderId); | ||
| } catch (err) { | ||
| // Same best-effort contract as the finally-block release below. | ||
| sessionLog( | ||
| sessionId, | ||
| `historian lease release failed (expires via TTL): ${err instanceof Error ? err.message : String(err)}`, | ||
| ); | ||
| } | ||
| return; | ||
| } | ||
| const renewal = startPiCompartmentLeaseRenewal(db, sessionId, holderId); | ||
|
|
@@ -3726,15 +3734,32 @@ function spawnPiHistorianRun(args: { | |
| }); | ||
| } finally { | ||
| clearInterval(renewal); | ||
| releaseCompartmentLease(db, sessionId, holderId); | ||
| } | ||
| })().finally(() => { | ||
| inFlightHistorian.delete(sessionId); | ||
| unregister(); | ||
| if (isContextHandlerSessionActive(sessionId)) { | ||
| historian.onStatusChange?.(ctx, sessionId); | ||
| try { | ||
| releaseCompartmentLease(db, sessionId, holderId); | ||
| } catch (err) { | ||
| // Best-effort teardown: the lease row expires via its own TTL, so a | ||
| // busy_timeout-starved DELETE must not escape and reject this promise. | ||
| sessionLog( | ||
| sessionId, | ||
| `historian lease release failed (expires via TTL): ${err instanceof Error ? err.message : String(err)}`, | ||
| ); | ||
| } | ||
| } | ||
| }); | ||
| })() | ||
| .catch((err) => { | ||
| // Parity with OpenCode's startCompartmentAgent .catch: the historian run | ||
| // promise is parked in inFlightHistorian for emergency/shutdown awaits, | ||
| // so an unhandled rejection here crashes the host via | ||
| // processTicksAndRejections instead of surfacing as a session log. | ||
| sessionLog(sessionId, "historian run failed:", err); | ||
| }) | ||
| .finally(() => { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: When Prompt for AI agents |
||
| inFlightHistorian.delete(sessionId); | ||
| unregister(); | ||
| if (isContextHandlerSessionActive(sessionId)) { | ||
| historian.onStatusChange?.(ctx, sessionId); | ||
| } | ||
| }); | ||
| inFlightHistorian.set(sessionId, runPromise); | ||
| historian.onStatusChange?.(ctx, sessionId); | ||
| } | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Each execution creates a UUID-named SQLite file under the system temporary directory, but teardown only closes the database handle. Repeated local and CI runs therefore leave the generated
pi-wrapup-busy-release-*.dbfiles behind.