diff --git a/packages/core/src/accounts.ts b/packages/core/src/accounts.ts index bc8b648d..a1ceeb30 100644 --- a/packages/core/src/accounts.ts +++ b/packages/core/src/accounts.ts @@ -13,6 +13,7 @@ import { } from './constants.ts' import { type LogLevel, log, logger } from './logger.ts' import { isTransientNetworkError } from './network-errors.ts' +import { tokenFingerprint } from './token-fingerprint.ts' const setRefreshLockRenewalTimeout = globalThis.setTimeout.bind(globalThis) const clearRefreshLockRenewalTimeout = globalThis.clearTimeout.bind(globalThis) @@ -114,6 +115,8 @@ export type AccountOperationError = { retryCount?: number accountIdentity?: string tokenHash?: string + /** Fingerprint of the refresh token that produced this error. */ + refreshTokenFingerprint?: string /** * HTTP status of the underlying refresh/quota failure, when known. Lets * consumers distinguish a permanently-dead token (400 invalid_grant → @@ -244,6 +247,7 @@ export type AccountStorage = { intervalMinutes?: number refreshBeforeExpiryMinutes?: number mainLastRefreshError?: AccountOperationError + mainRefreshErrorClearedAt?: number mainRefreshLeaseId?: string mainRefreshLeaseUntil?: number mainRefreshLeaseTokenHash?: string @@ -364,6 +368,7 @@ export type AccountRuntimeState = { quotaErrorGeneration?: number quotaErrorClearedAt?: number lastRefreshError?: AccountOperationError + refreshErrorClearedAt?: number refreshLeaseId?: string refreshLeaseUntil?: number refreshLeaseTokenHash?: string @@ -615,6 +620,11 @@ function normalizeOperationError( : undefined, tokenHash: typeof value.tokenHash === 'string' ? value.tokenHash : undefined, + refreshTokenFingerprint: + typeof value.refreshTokenFingerprint === 'string' && + value.refreshTokenFingerprint.trim() + ? value.refreshTokenFingerprint.trim() + : undefined, // Preserve the dead-token discriminators across save/load. Without these, // a retry-exhausted transient (permanent=false, 24h backoff) would lose its // flag on reload and the 24h-delay heuristic would wrongly re-classify it @@ -939,6 +949,15 @@ export function mergeMainQuotaErrorClearedAt( return Math.max(existing, incoming) } +export function mergeMainRefreshErrorClearedAt( + existing: number | undefined, + incoming: number | undefined, +): number | undefined { + if (incoming === undefined || !Number.isFinite(incoming)) return existing + if (existing === undefined || !Number.isFinite(existing)) return incoming + return Math.max(existing, incoming) +} + function accountCredentialTimestamp(value: Record): number { return Math.max( numericField(value.lastRefreshedAt), @@ -1026,6 +1045,12 @@ function mergeConfigAndState( mainState.quotaErrorClearedAt >= 0 ? mainState.quotaErrorClearedAt : undefined + const mainRefreshErrorClearedAt = + typeof mainState?.refreshErrorClearedAt === 'number' && + Number.isFinite(mainState.refreshErrorClearedAt) && + mainState.refreshErrorClearedAt >= 0 + ? mainState.refreshErrorClearedAt + : undefined const configQuotaError = quotaConfig.mainLastQuotaApiError const mainLastQuotaApiError = mainState?.lastQuotaApiError ?? @@ -1035,6 +1060,15 @@ function mergeConfigAndState( configQuotaError.checkedAt <= mainQuotaErrorClearedAt ? undefined : configQuotaError) + const configRefreshError = refreshConfig.mainLastRefreshError + const mainLastRefreshError = + mainState?.lastRefreshError ?? + (mainRefreshErrorClearedAt !== undefined && + isRecord(configRefreshError) && + typeof configRefreshError.checkedAt === 'number' && + configRefreshError.checkedAt <= mainRefreshErrorClearedAt + ? undefined + : configRefreshError) const accounts = Array.isArray(configValue.accounts) ? configValue.accounts.map((account) => { @@ -1059,7 +1093,8 @@ function mergeConfigAndState( }, refresh: objectWithDefinedEntries({ ...refreshConfig, - mainLastRefreshError: mainRefreshSource.lastRefreshError, + mainLastRefreshError, + mainRefreshErrorClearedAt: mainRefreshSource.refreshErrorClearedAt, mainRefreshLeaseId: mainRefreshSource.refreshLeaseId, mainRefreshLeaseUntil: mainRefreshSource.refreshLeaseUntil, mainRefreshLeaseTokenHash: mainRefreshSource.refreshLeaseTokenHash, @@ -1752,7 +1787,12 @@ function applyMainQuotaStatePatch( ) { state.main.quotaErrorGeneration = incomingGeneration } - } else if (incomingError) { + } else if ( + incomingError && + typeof incomingError.checkedAt === 'number' && + Number.isFinite(incomingError.checkedAt) && + incomingError.checkedAt > existingObservedAt + ) { const acceptsByObservation = incomingErrorObservedAt !== undefined && incomingErrorObservedAt > existingObservedAt @@ -1827,7 +1867,32 @@ function applyMainRefreshStatePatch( storage: AccountStorage, ) { state.main = state.main ?? {} - state.main.lastRefreshError = storage.refresh?.mainLastRefreshError + const incomingError = storage.refresh?.mainLastRefreshError + const incomingClearedAt = storage.refresh?.mainRefreshErrorClearedAt + const existingErrorObservedAt = state.main.lastRefreshError?.checkedAt + const existingClearedAt = state.main.refreshErrorClearedAt + const existingObservedAt = Math.max( + existingErrorObservedAt ?? 0, + existingClearedAt ?? 0, + ) + if ( + incomingError === undefined && + incomingClearedAt !== undefined && + incomingClearedAt >= existingObservedAt + ) { + state.main.lastRefreshError = undefined + state.main.refreshErrorClearedAt = mergeMainRefreshErrorClearedAt( + existingClearedAt, + incomingClearedAt, + ) + } else if ( + incomingError && + typeof incomingError.checkedAt === 'number' && + Number.isFinite(incomingError.checkedAt) && + incomingError.checkedAt > existingObservedAt + ) { + state.main.lastRefreshError = incomingError + } state.main.refreshLeaseId = storage.refresh?.mainRefreshLeaseId state.main.refreshLeaseUntil = storage.refresh?.mainRefreshLeaseUntil state.main.refreshLeaseTokenHash = storage.refresh?.mainRefreshLeaseTokenHash @@ -2757,10 +2822,12 @@ export function buildRefreshOperationError(input: { error: unknown now: number accountIdentity: string | undefined + refreshTokenFingerprint?: string previous?: AccountOperationError }): AccountOperationError { const previousRetryCount = - input.previous?.accountIdentity === input.accountIdentity + input.previous?.accountIdentity === input.accountIdentity && + input.previous?.refreshTokenFingerprint === input.refreshTokenFingerprint ? (input.previous?.retryCount ?? 0) : 0 const retryCount = previousRetryCount + 1 @@ -2804,6 +2871,7 @@ export function buildRefreshOperationError(input: { nextRetryAt: input.now + delay, retryCount, accountIdentity: input.accountIdentity, + refreshTokenFingerprint: input.refreshTokenFingerprint, status, permanent: status === 400 && isInvalidGrant, } @@ -2855,10 +2923,18 @@ export function refreshBackoffActive( error: AccountOperationError | undefined, accountIdentity: string | undefined, now: number, + currentRefreshTokenFingerprint: string | undefined, ) { if (!error) return false const retryAt = effectiveRefreshRetryAt(error) if (!retryAt || retryAt <= now) return false + if ( + error.refreshTokenFingerprint && + currentRefreshTokenFingerprint && + error.refreshTokenFingerprint !== currentRefreshTokenFingerprint + ) { + return false + } if (!error.accountIdentity) return true if (!accountIdentity) return true return error.accountIdentity === accountIdentity @@ -3696,6 +3772,7 @@ function recordRefreshError( error, now, accountIdentity: account.id, + refreshTokenFingerprint: tokenFingerprint(account.refresh), previous: account.lastRefreshError, }) } @@ -3870,7 +3947,12 @@ export class FallbackAccountManager { const refreshError = next.lastRefreshError if ( refreshError && - refreshBackoffActive(refreshError, next.id, this.now()) + refreshBackoffActive( + refreshError, + next.id, + this.now(), + tokenFingerprint(next.refresh), + ) ) { throw createRefreshBackoffActiveError(refreshError, this.now()) } @@ -3935,7 +4017,12 @@ export class FallbackAccountManager { } } else if ( !failClosedOnUnknownQuota(storage) && - !refreshBackoffActive(next.lastRefreshError, next.id, this.now()) && + !refreshBackoffActive( + next.lastRefreshError, + next.id, + this.now(), + tokenFingerprint(next.refresh), + ) && quotaSnapshotPassesModelScope(next.quota, options.modelId) ) { usable.push(next) @@ -3994,7 +4081,12 @@ export class FallbackAccountManager { ) continue if ( - refreshBackoffActive(account.lastRefreshError, account.id, this.now()) + refreshBackoffActive( + account.lastRefreshError, + account.id, + this.now(), + tokenFingerprint(account.refresh), + ) ) { // Backoff skips are steady-state while a fallback account is waiting for // its next retry. Logging every background tick from every OpenCode @@ -4040,7 +4132,12 @@ export class FallbackAccountManager { !this.isFallbackAccountVaultServed(next.id, storage) ) { if ( - refreshBackoffActive(next.lastRefreshError, next.id, this.now()) + refreshBackoffActive( + next.lastRefreshError, + next.id, + this.now(), + tokenFingerprint(next.refresh), + ) ) { continue } @@ -4090,7 +4187,12 @@ export class FallbackAccountManager { const refreshError = next.lastRefreshError if ( refreshError && - refreshBackoffActive(refreshError, next.id, this.now()) + refreshBackoffActive( + refreshError, + next.id, + this.now(), + tokenFingerprint(next.refresh), + ) ) { throw createRefreshBackoffActiveError(refreshError, this.now()) } @@ -4197,7 +4299,12 @@ export class FallbackAccountManager { const refreshError = latestAccount.lastRefreshError if ( refreshError && - refreshBackoffActive(refreshError, latestAccount.id, this.now()) + refreshBackoffActive( + refreshError, + latestAccount.id, + this.now(), + tokenFingerprint(latestAccount.refresh), + ) ) { updateStoredAccount(storage, latestAccount) throw createRefreshBackoffActiveError(refreshError, this.now()) diff --git a/packages/core/src/commands/account.ts b/packages/core/src/commands/account.ts index c2c9c077..fd2c5623 100644 --- a/packages/core/src/commands/account.ts +++ b/packages/core/src/commands/account.ts @@ -12,6 +12,7 @@ export type AccountCommandAction = | { type: 'remove'; id: string } | { type: 'move-up'; id: string } | { type: 'move-down'; id: string } + | { type: 'reset-backoff' } | { type: 'add-apikey' apiKey: string @@ -37,6 +38,7 @@ export function parseAccountCommandAction( if (action === 'remove' && rest) return { type: 'remove', id: rest } if (action === 'move-up' && rest) return { type: 'move-up', id: rest } if (action === 'move-down' && rest) return { type: 'move-down', id: rest } + if (action === 'reset-backoff' && !rest) return { type: 'reset-backoff' } if (action === 'add-apikey' && rest) { let remaining = rest @@ -163,6 +165,7 @@ const USAGE_TEXT = [ ' /claude-account remove Remove a fallback account', ' /claude-account move-up Move a fallback account up', ' /claude-account move-down Move a fallback account down', + ' /claude-account reset-backoff Clear main OAuth refresh and quota backoff', ' /claude-account add-apikey Add an API key fallback account', ' /claude-account add-oauth-start Start OAuth device flow', ' /claude-account add-oauth-finish Complete OAuth flow', @@ -176,7 +179,7 @@ export function executeAccountCommand(input: { text: string updated?: { id: string - action: 'enable' | 'disable' | 'remove' | 'reorder' + action: 'enable' | 'disable' | 'remove' | 'reorder' | 'reset-backoff' enabled?: boolean previousOrder?: string[] newOrder?: string[] @@ -218,6 +221,12 @@ export function executeAccountCommand(input: { if (action.type === 'add-oauth-finish') { return { text: 'add-oauth-finish' } } + if (action.type === 'reset-backoff') { + return { + text: 'Main OAuth refresh and quota backoff cleared.', + updated: { id: 'main', action: 'reset-backoff' }, + } + } const id = action.id if (id === mainId) { diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 84224887..5822c771 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -115,6 +115,7 @@ import { mergeAnthropicBetas, mergeHeaderQuotaForPersistence, mergeMainQuotaErrorClearedAt, + mergeMainRefreshErrorClearedAt, normalizeQuotaHeaders, type OAuthAccount, type OAuthQuotaSnapshot, @@ -1993,6 +1994,7 @@ const anthropicAuthPlugin = async ( account.lastRefreshError, account.id, claustrumNow(), + account.refresh ? tokenFingerprint(account.refresh) : undefined, ) ) { return @@ -2842,6 +2844,7 @@ const anthropicAuthPlugin = async ( activeId?: string route: string mainAccessToken?: string + mainRefreshToken?: string routingAuthoritative?: boolean useHydratedProfiles?: boolean skipFallbackQuotaSeed?: boolean @@ -2912,6 +2915,9 @@ const anthropicAuthPlugin = async ( mainRefreshError, mainAccountId ?? storage?.mainAccountId, Date.now(), + options.mainRefreshToken + ? tokenFingerprint(options.mainRefreshToken) + : undefined, ) : false, refreshBackoffUntil: mainRefreshError?.nextRetryAt, @@ -2943,6 +2949,7 @@ const anthropicAuthPlugin = async ( account.lastRefreshError, account.id, Date.now(), + tokenFingerprint(account.refresh), ) && isPermanentRefreshError(account.lastRefreshError), vaultReauth: @@ -3058,6 +3065,7 @@ const anthropicAuthPlugin = async ( activeId: lastSidebarRouting.activeId, route: lastSidebarRouting.route, mainAccessToken: access, + mainRefreshToken: undefined, routingAuthoritative: false, }) } @@ -3121,14 +3129,27 @@ const anthropicAuthPlugin = async ( return storage?.refresh?.enabled !== false } - async function clearStaleMainRefreshError(accountIdentity?: string) { + async function clearStaleMainRefreshError( + accountIdentity?: string, + currentRefreshTokenFingerprint?: string, + ) { if (!accountIdentity) return const storage = await loadAccounts(accountStoragePath) const error = storage?.refresh?.mainLastRefreshError if (!storage?.refresh || !error) return - if (!error.accountIdentity || error.accountIdentity === accountIdentity) - return + const identityChanged = + Boolean(error.accountIdentity) && + error.accountIdentity !== accountIdentity + const refreshTokenChanged = + Boolean(error.refreshTokenFingerprint) && + Boolean(currentRefreshTokenFingerprint) && + error.refreshTokenFingerprint !== currentRefreshTokenFingerprint + if (!identityChanged && !refreshTokenChanged) return storage.refresh.mainLastRefreshError = undefined + storage.refresh.mainRefreshErrorClearedAt = mergeMainRefreshErrorClearedAt( + storage.refresh.mainRefreshErrorClearedAt, + Date.now(), + ) if (!mainQuotaIdentityAccessToken?.startsWith('sk-ant-oat')) { const quotaError = storage.quota?.mainLastQuotaApiError if (quotaError && quotaError.accountIdentity !== mainQuotaAccountId) { @@ -3243,6 +3264,7 @@ const anthropicAuthPlugin = async ( activeId: lastSidebarRouting.activeId, route: lastSidebarRouting.route, mainAccessToken: auth.access, + mainRefreshToken: auth.refresh, routingAuthoritative: false, }) } catch { @@ -3273,6 +3295,7 @@ const anthropicAuthPlugin = async ( activeId: lastSidebarRouting.activeId, route: lastSidebarRouting.route, mainAccessToken: auth.access, + mainRefreshToken: auth.refresh, routingAuthoritative: false, }) @@ -3503,6 +3526,7 @@ const anthropicAuthPlugin = async ( activeId: lastSidebarRouting.activeId, route: lastSidebarRouting.route, mainAccessToken: auth.access, + mainRefreshToken: auth.refresh, routingAuthoritative: false, }) } @@ -3940,6 +3964,45 @@ const anthropicAuthPlugin = async ( id: updatedId, label: account?.label, }) + } else if (result.updated.action === 'reset-backoff') { + const mainIdentity = mainAccountId ?? storage?.mainAccountId + storage = storage ?? createEmptyStorage() + storage.refresh = storage.refresh ?? {} + storage.refresh.mainLastRefreshError = undefined + storage.refresh.mainRefreshErrorClearedAt = + mergeMainRefreshErrorClearedAt( + storage.refresh.mainRefreshErrorClearedAt, + Date.now(), + ) + const quotaError = storage.quota?.mainLastQuotaApiError + if ( + !quotaError?.accountIdentity || + !mainIdentity || + quotaError.accountIdentity === mainIdentity + ) { + const clearedGeneration = quotaManager.clearMainBackoff() + storage.quota = storage.quota ?? {} + storage.quota.mainLastQuotaApiError = undefined + storage.quota.mainQuotaErrorGeneration = Math.max( + storage.quota.mainQuotaErrorGeneration ?? 0, + clearedGeneration ?? quotaManager.getMainQuotaErrorGeneration(), + ) + storage.quota.mainQuotaErrorClearedAt = mergeMainQuotaErrorClearedAt( + storage.quota.mainQuotaErrorClearedAt, + quotaManager.getMainQuotaErrorClearedAt() ?? Date.now(), + ) + } + await saveAccountState(storage, accountStoragePath, { + mainRefresh: true, + mainQuota: true, + }) + logger.info( + 'commands', + 'main OAuth refresh and quota backoff cleared', + { + mainIdentity, + }, + ) } const updatedStorage = await loadAccounts(accountStoragePath) @@ -3950,6 +4013,7 @@ const anthropicAuthPlugin = async ( activeId: lastSidebarRouting.activeId, route: lastSidebarRouting.route, mainAccessToken: auth.access, + mainRefreshToken: auth.refresh, routingAuthoritative: false, }) } catch { @@ -4614,6 +4678,7 @@ const anthropicAuthPlugin = async ( mainError, mainAccountId ?? storage?.mainAccountId, Date.now(), + tokenFingerprint(freshAuth.refresh), ) : false, retryCount: mainError?.retryCount, @@ -4625,6 +4690,7 @@ const anthropicAuthPlugin = async ( mainError, mainAccountId ?? storage?.mainAccountId, Date.now(), + tokenFingerprint(freshAuth.refresh), ) ) { log( @@ -4791,6 +4857,8 @@ const anthropicAuthPlugin = async ( now: Date.now(), accountIdentity: mainAccountId ?? storage.mainAccountId, + refreshTokenFingerprint: + tokenFingerprint(failedRefreshToken), previous: storage.refresh.mainLastRefreshError, }) }) @@ -4852,6 +4920,9 @@ const anthropicAuthPlugin = async ( if (latestAuth.type !== 'oauth') return await clearStaleMainRefreshError( mainAccountId ?? storage?.mainAccountId, + latestAuth.refresh + ? tokenFingerprint(latestAuth.refresh) + : undefined, ) if (!latestAuth.expires) return const expiresInMs = latestAuth.expires - Date.now() @@ -4868,6 +4939,9 @@ const anthropicAuthPlugin = async ( storage?.refresh?.mainLastRefreshError, mainAccountId ?? storage?.mainAccountId, Date.now(), + latestAuth.refresh + ? tokenFingerprint(latestAuth.refresh) + : undefined, ) ) { log( @@ -5761,6 +5835,7 @@ const anthropicAuthPlugin = async ( account.lastRefreshError, account.id, Date.now(), + tokenFingerprint(account.refresh), ) && storageArg?.quota?.failClosedOnUnknownQuota !== true && !quotaSnapshotHasStandardWindows(getFallbackQuota(account)) @@ -5795,6 +5870,7 @@ const anthropicAuthPlugin = async ( async function buildStickyOAuthRoutes(input: { storage: AccountStorage | null mainAccessToken: string + mainRefreshToken?: string requestedModelId?: string mainQuotaIdentity?: MainQuotaIdentityBinding }) { @@ -5909,6 +5985,13 @@ const anthropicAuthPlugin = async ( refreshError, accountIdentity, Date.now(), + route.id === STICKY_ROUTING_MAIN_ACCOUNT_ID + ? input.mainRefreshToken + ? tokenFingerprint(input.mainRefreshToken) + : undefined + : route.account?.refresh + ? tokenFingerprint(route.account.refresh) + : undefined, ) && (route.id === STICKY_ROUTING_MAIN_ACCOUNT_ID || !usableIds.has(route.id)) @@ -5963,6 +6046,13 @@ const anthropicAuthPlugin = async ( refreshError, accountIdentity, Date.now(), + route.id === STICKY_ROUTING_MAIN_ACCOUNT_ID + ? input.mainRefreshToken + ? tokenFingerprint(input.mainRefreshToken) + : undefined + : route.account?.refresh + ? tokenFingerprint(route.account.refresh) + : undefined, ) && (route.id === STICKY_ROUTING_MAIN_ACCOUNT_ID || !usableIds.has(route.id))) @@ -6508,7 +6598,10 @@ const anthropicAuthPlugin = async ( } requestMainQuotaIdentity = resolution } - await clearStaleMainRefreshError(mainAccountId) + await clearStaleMainRefreshError( + mainAccountId, + auth.refresh ? tokenFingerprint(auth.refresh) : undefined, + ) const loadStart = nowMs() const storage = await loadAccounts() trace.mark('load_storage', { ms: roundMs(nowMs() - loadStart) }) @@ -6603,6 +6696,7 @@ const anthropicAuthPlugin = async ( let stickyRoutes = await buildStickyOAuthRoutes({ storage, mainAccessToken: auth.access, + mainRefreshToken: auth.refresh, requestedModelId: routingModelId, mainQuotaIdentity: requestMainQuotaIdentity, }) @@ -6919,6 +7013,7 @@ const anthropicAuthPlugin = async ( stickyRoutes = await buildStickyOAuthRoutes({ storage: stickyRoutes.storage, mainAccessToken: auth.access, + mainRefreshToken: auth.refresh, requestedModelId: routingModelId, mainQuotaIdentity: requestMainQuotaIdentity, }) @@ -7065,6 +7160,7 @@ const anthropicAuthPlugin = async ( mainRefreshError, mainAccountId ?? refreshStorage?.mainAccountId, Date.now(), + auth.refresh ? tokenFingerprint(auth.refresh) : undefined, ) ) { log('[refresh] opencode main oauth request skipped backoff', { diff --git a/packages/opencode/src/tests/accounts.test.ts b/packages/opencode/src/tests/accounts.test.ts index 8cf26500..3d65a6cf 100644 --- a/packages/opencode/src/tests/accounts.test.ts +++ b/packages/opencode/src/tests/accounts.test.ts @@ -374,6 +374,54 @@ describe('cross-process main quota backoff', () => { managerB.updateStorage(finalStorage) expect(managerB.isBackedOff()).toBe(false) }) + + test('a stale main refresh error cannot resurrect a refresh clear', async () => { + const error = { + message: 'invalid_grant', + checkedAt: 900_000, + nextRetryAt: 1_060_000, + retryCount: 1, + accountIdentity: 'main-slot', + permanent: true, + } + await saveAccounts( + { + ...baseStorage(), + mainAccountId: 'main-slot', + refresh: { ...baseStorage().refresh, mainLastRefreshError: error }, + }, + accountPath, + ) + const staleWriterStorage = (await loadAccounts(accountPath))! + const clearWriterStorage = (await loadAccounts(accountPath))! + clearWriterStorage.refresh = { + ...clearWriterStorage.refresh, + mainLastRefreshError: undefined, + mainRefreshErrorClearedAt: 1_000_200, + } + await saveAccountState(clearWriterStorage, accountPath, { + mainRefresh: true, + }) + + await saveAccountState(staleWriterStorage, accountPath, { + mainRefresh: true, + }) + expect( + (await loadAccounts(accountPath))?.refresh?.mainLastRefreshError, + ).toBeUndefined() + + staleWriterStorage.refresh = { + ...staleWriterStorage.refresh, + mainLastRefreshError: { ...error, checkedAt: 1_000_300 }, + } + await saveAccountState(staleWriterStorage, accountPath, { + mainRefresh: true, + }) + expect( + (await loadAccounts(accountPath))?.refresh?.mainLastRefreshError + ?.checkedAt, + ).toBe(1_000_300) + }) }) beforeEach(async () => { @@ -3677,28 +3725,114 @@ describe('FallbackAccountManager', () => { expect(expectOAuthAccount(saved?.accounts[0]).refresh).toBe('new-refresh') }) - test('stable identity backoff survives refresh-token rotation and releases after expiry', () => { + test('background fallback refresh retries after a permanent backoff belongs to an older refresh token', async () => { + const now = Date.now() + const storage = baseStorage() + storage.accounts.push({ + id: 'fallback-rotated', + type: 'oauth', + authLineageId: 'fallback-lineage', + access: 'old-access', + refresh: 'current-refresh', + expires: now - 1, + lastRefreshError: { + message: 'Claude OAuth refresh failed: 400 — invalid_grant', + checkedAt: now - 1_000, + nextRetryAt: now + 24 * 60 * 60_000, + retryCount: 1, + accountIdentity: 'fallback-rotated', + refreshTokenFingerprint: tokenFingerprint('failed-refresh'), + status: 400, + permanent: true, + }, + }) + await saveAccounts(storage) + let tokenRefreshCalls = 0 + const fetchImpl = mock((input: Request | string) => { + if (String(input).includes('/v1/oauth/token')) { + tokenRefreshCalls += 1 + return Promise.resolve( + Response.json({ + access_token: 'refreshed-access', + refresh_token: 'refreshed-refresh', + expires_in: 8 * 60 * 60, + }), + ) + } + return Promise.resolve(new Response(null, { status: 200 })) + }) as unknown as typeof fetch + const manager = new FallbackAccountManager({ + fetchImpl, + now: () => now, + }) + + await manager.refreshDueAccounts() + + expect(tokenRefreshCalls).toBe(1) + }) + + test('refresh backoff releases only when the refresh token fingerprint changes', () => { const first = buildRefreshOperationError({ error: new ClaudeOAuthRefreshError(429, 'rate limited'), now: 1_000, accountIdentity: 'fallback-1', - refreshToken: 'old-refresh', + refreshTokenFingerprint: tokenFingerprint('old-refresh'), } as never) - const second = buildRefreshOperationError({ + + expect( + refreshBackoffActive( + first, + 'fallback-1', + (first.nextRetryAt ?? 0) - 1, + tokenFingerprint('rotated-refresh'), + ), + ).toBe(false) + expect( + refreshBackoffActive( + first, + 'fallback-1', + (first.nextRetryAt ?? 0) - 1, + tokenFingerprint('old-refresh'), + ), + ).toBe(true) + }) + + test('refresh retry counts reset when the refresh token fingerprint changes', () => { + const first = buildRefreshOperationError({ error: new ClaudeOAuthRefreshError(429, 'rate limited'), - now: first.nextRetryAt ?? 2_000, + now: 1_000, accountIdentity: 'fallback-1', - refreshToken: 'rotated-refresh', - previous: first, - } as never) + refreshTokenFingerprint: tokenFingerprint('old-refresh'), + }) + const rotated = buildRefreshOperationError({ + error: new ClaudeOAuthRefreshError(429, 'rate limited'), + now: 2_000, + accountIdentity: 'fallback-1', + refreshTokenFingerprint: tokenFingerprint('new-refresh'), + previous: { ...first, retryCount: 6 }, + }) + + expect(rotated.retryCount).toBe(1) + }) + + test('fingerprint-less permanent errors hold through token rotation on the same identity', () => { + const legacy = { + message: 'Claude OAuth refresh failed: 400 — invalid_grant', + checkedAt: 1_000, + nextRetryAt: 1_000 + 24 * 60 * 60_000, + retryCount: 1, + accountIdentity: 'fallback-1', + permanent: true, + } - expect(second.retryCount).toBe(2) expect( - refreshBackoffActive(second, 'fallback-1', (second.nextRetryAt ?? 0) - 1), + refreshBackoffActive( + legacy, + 'fallback-1', + legacy.nextRetryAt - 1, + tokenFingerprint('rotated-refresh'), + ), ).toBe(true) - expect( - refreshBackoffActive(second, 'fallback-1', second.nextRetryAt ?? 0), - ).toBe(false) }) test('stable identity backoff does not transfer to a different account', () => { @@ -3718,7 +3852,12 @@ describe('FallbackAccountManager', () => { expect(other.retryCount).toBe(1) expect( - refreshBackoffActive(other, 'fallback-1', (other.nextRetryAt ?? 0) - 1), + refreshBackoffActive( + other, + 'fallback-1', + (other.nextRetryAt ?? 0) - 1, + undefined, + ), ).toBe(false) }) @@ -3731,7 +3870,12 @@ describe('FallbackAccountManager', () => { } as never) expect( - refreshBackoffActive(error, undefined, (error.nextRetryAt ?? 0) - 1), + refreshBackoffActive( + error, + undefined, + (error.nextRetryAt ?? 0) - 1, + undefined, + ), ).toBe(true) }) @@ -3759,6 +3903,7 @@ describe('FallbackAccountManager', () => { persistedLongBackoff, 'fallback-1', 1_000 + 5 * 60_000 - 1, + undefined, ), ).toBe(true) expect( @@ -3766,6 +3911,7 @@ describe('FallbackAccountManager', () => { persistedLongBackoff, 'fallback-1', 1_000 + 5 * 60_000, + undefined, ), ).toBe(false) @@ -3872,7 +4018,12 @@ describe('FallbackAccountManager', () => { expect(upgraded.accountIdentity).toBe('fallback-1') expect(upgraded.tokenHash).toBeUndefined() expect( - refreshBackoffActive(legacy, 'fallback-1', (legacy.nextRetryAt ?? 0) - 1), + refreshBackoffActive( + legacy, + 'fallback-1', + (legacy.nextRetryAt ?? 0) - 1, + undefined, + ), ).toBe(true) }) diff --git a/packages/opencode/src/tests/index.test.ts b/packages/opencode/src/tests/index.test.ts index e87f0a69..3531aa57 100644 --- a/packages/opencode/src/tests/index.test.ts +++ b/packages/opencode/src/tests/index.test.ts @@ -11708,6 +11708,149 @@ describe('auth.loader', () => { }) }) + test('background refresh retries after a permanent main backoff belongs to an older refresh token', async () => { + const now = Date.now() + await useTempAccountFile( + createFallbackStorage({ + accounts: [], + mainAccountId: 'main-account-id', + quota: { enabled: false }, + refresh: { + enabled: true, + refreshBeforeExpiryMinutes: 30, + mainLastRefreshError: { + message: 'Claude OAuth refresh failed: 400 — invalid_grant', + checkedAt: now - 1_000, + nextRetryAt: now + 24 * 60 * 60_000, + retryCount: 1, + accountIdentity: 'main-account-id', + refreshTokenFingerprint: tokenFingerprint('failed-refresh'), + status: 400, + permanent: true, + }, + }, + }), + ) + const intervalHandlers: Array<() => void> = [] + const setIntervalMock = mock((handler: () => void) => { + intervalHandlers.push(handler) + return { unref() {} } + }) as unknown as typeof setInterval + let tokenRefreshCalls = 0 + globalThis.fetch = mock((input: any) => { + if (extractUrl(input).includes('/v1/oauth/token')) { + tokenRefreshCalls += 1 + return Promise.resolve( + Response.json({ + refresh_token: 'refreshed-refresh', + access_token: 'refreshed-access', + expires_in: 8 * 60 * 60, + }), + ) + } + return Promise.resolve(new Response(null, { status: 200 })) + }) as unknown as typeof fetch + + const mockClient = createMockClient() + const plugin = await getPlugin(mockClient, undefined, { + setInterval: setIntervalMock, + clearInterval: mock(() => {}) as unknown as typeof clearInterval, + }) + await plugin.auth.loader( + () => + Promise.resolve({ + type: 'oauth' as const, + access: 'current-access', + refresh: 'current-refresh', + expires: now + 5 * 60_000, + }), + { models: {} }, + ) + + for (const handler of intervalHandlers) handler() + await waitForMockCall(mockClient.auth.set) + + expect(tokenRefreshCalls).toBe(1) + expect(mockClient.auth.set).toHaveBeenCalledTimes(1) + expect( + (await loadAccounts())?.refresh?.mainLastRefreshError, + ).toBeUndefined() + }) + + test('reset-backoff clears a legacy main latch before the next background refresh', async () => { + const now = Date.now() + await useTempAccountFile( + createFallbackStorage({ + accounts: [], + mainAccountId: 'main-account-id', + quota: { enabled: false }, + refresh: { + enabled: true, + refreshBeforeExpiryMinutes: 30, + mainLastRefreshError: { + message: 'Claude OAuth refresh failed: 400 — invalid_grant', + checkedAt: now - 1_000, + nextRetryAt: now + 24 * 60 * 60_000, + retryCount: 1, + accountIdentity: 'main-account-id', + status: 400, + permanent: true, + }, + }, + }), + ) + const intervalHandlers: Array<() => void> = [] + const setIntervalMock = mock((handler: () => void) => { + intervalHandlers.push(handler) + return { unref() {} } + }) as unknown as typeof setInterval + let tokenRefreshCalls = 0 + globalThis.fetch = mock((input: any) => { + if (extractUrl(input).includes('/v1/oauth/token')) { + tokenRefreshCalls += 1 + return Promise.resolve( + Response.json({ + refresh_token: 'refreshed-refresh', + access_token: 'refreshed-access', + expires_in: 8 * 60 * 60, + }), + ) + } + return Promise.resolve(new Response(null, { status: 200 })) + }) as unknown as typeof fetch + + const mockClient = createMockClient() + const plugin = await getPlugin(mockClient, undefined, { + setInterval: setIntervalMock, + clearInterval: mock(() => {}) as unknown as typeof clearInterval, + }) + await plugin.auth.loader( + () => + Promise.resolve({ + type: 'oauth' as const, + access: 'current-access', + refresh: 'current-refresh', + expires: now + 5 * 60_000, + }), + { models: {} }, + ) + + await expectHandledCommandResponse( + plugin['command.execute.before']({ + command: 'claude-account', + arguments: 'reset-backoff', + sessionID: 'session-1', + }), + ) + for (const handler of intervalHandlers) handler() + await waitForMockCall(mockClient.auth.set) + + expect(tokenRefreshCalls).toBe(1) + expect( + (await loadAccounts())?.refresh?.mainLastRefreshError, + ).toBeUndefined() + }) + test('background refresh uses a four-hour minimum window for main oauth', async () => { await useTempAccountFile( createFallbackStorage({ diff --git a/packages/pi/src/commands.ts b/packages/pi/src/commands.ts index 4273c746..890bfd1a 100644 --- a/packages/pi/src/commands.ts +++ b/packages/pi/src/commands.ts @@ -28,6 +28,8 @@ import { isFastModePersistentlyEnabled, isPrimePersistentlyEnabled, loadAccounts, + mergeMainQuotaErrorClearedAt, + mergeMainRefreshErrorClearedAt, parseAccountCommandAction, parseCache1hCommandAction, parseCacheKeepCommandAction, @@ -37,6 +39,7 @@ import { parseRoutingCommandAction, removeAccountPersistent, reorderAccountsPersistent, + saveAccountState, setAccountEnabledPersistent, setCache1hPersistentEnabled, setCache1hPersistentMode, @@ -299,6 +302,40 @@ export function registerCommands(pi: ExtensionAPI) { if (newOrder) { await reorderAccountsPersistent(newOrder, path) } + } else if (mutationAction === 'reset-backoff') { + const nextStorage = storage ?? createEmptyStorage() + nextStorage.refresh = nextStorage.refresh ?? {} + nextStorage.refresh.mainLastRefreshError = undefined + nextStorage.refresh.mainRefreshErrorClearedAt = + mergeMainRefreshErrorClearedAt( + nextStorage.refresh.mainRefreshErrorClearedAt, + Date.now(), + ) + const mainIdentity = nextStorage.mainAccountId + const quotaError = nextStorage.quota?.mainLastQuotaApiError + if ( + !quotaError?.accountIdentity || + !mainIdentity || + quotaError.accountIdentity === mainIdentity + ) { + nextStorage.quota = nextStorage.quota ?? {} + nextStorage.quota.mainLastQuotaApiError = undefined + const nextQuotaErrorGeneration = + (nextStorage.quota.mainQuotaErrorGeneration ?? 0) + 1 + nextStorage.quota.mainQuotaErrorGeneration = Math.max( + nextStorage.quota.mainQuotaErrorGeneration ?? 0, + nextQuotaErrorGeneration, + ) + nextStorage.quota.mainQuotaErrorClearedAt = + mergeMainQuotaErrorClearedAt( + nextStorage.quota.mainQuotaErrorClearedAt, + Date.now(), + ) + } + await saveAccountState(nextStorage, path, { + mainRefresh: true, + mainQuota: true, + }) } notify(ctx, result.text) diff --git a/packages/pi/src/stream.ts b/packages/pi/src/stream.ts index f00364af..2b6b9981 100644 --- a/packages/pi/src/stream.ts +++ b/packages/pi/src/stream.ts @@ -52,6 +52,7 @@ import { stickyRetryAfterWithJitter, stickyRouteFamilyForModel, THINKING_BINDING_CONTROLS_BETA, + tokenFingerprint, usesMidConversationOutputConfig, } from '@cortexkit/anthropic-auth-core' import { @@ -749,6 +750,7 @@ async function executeWithFallback(options: { configured.lastRefreshError, configured.id, Date.now(), + tokenFingerprint(configured.refresh), ) ) continue diff --git a/packages/pi/src/tests/commands.test.ts b/packages/pi/src/tests/commands.test.ts index bc5b4349..63e3b92c 100644 --- a/packages/pi/src/tests/commands.test.ts +++ b/packages/pi/src/tests/commands.test.ts @@ -209,6 +209,49 @@ describe('claude-account persistence', () => { expect(storage.accounts[0].enabled).toBe(true) }) + test('reset-backoff clears and persists main refresh and quota errors', async () => { + await writeFile( + accountPath, + JSON.stringify({ + version: 1, + mainAccountId: 'main-account-id', + refresh: { + mainLastRefreshError: { + message: 'invalid_grant', + checkedAt: 1, + nextRetryAt: 2, + accountIdentity: 'main-account-id', + permanent: true, + }, + }, + quota: { + mainLastQuotaApiError: { + message: 'quota unavailable', + checkedAt: 1, + nextRetryAt: 2, + accountIdentity: 'main-account-id', + }, + }, + accounts: [], + }), + 'utf8', + ) + const { pi, commands } = mockPi() + registerCommands(pi) + const handler = commands.get('claude-account')?.handler + expect(handler).toBeDefined() + + const { ctx, notified } = mockNotify() + await handler!('reset-backoff', ctx) + + expect(notified[0]).toContain( + 'Main OAuth refresh and quota backoff cleared.', + ) + const state = JSON.parse(await readFile(statePath, 'utf8')) + expect(state.main.lastRefreshError).toBeUndefined() + expect(state.main.lastQuotaApiError).toBeUndefined() + }) + test('status is display-only (no mutation)', async () => { const original = { version: 1,