Skip to content

fix: clear a permanent main refresh latch when the refresh token rotates #195

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ualtinok merged 1 commit into from
Sep 2, 2026
Merged

fix: clear a permanent main refresh latch when the refresh token rotates #195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 117 additions & 10 deletions packages/core/src/accounts.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
} from './constants.ts'
import { type LogLevel, log, logger } from './logger.ts'
import { isTransientNetworkError } from './network-errors.ts'
import { tokenFingerprint } from './token-fingerprint.ts'

const setRefreshLockRenewalTimeout = globalThis.setTimeout.bind(globalThis)
const clearRefreshLockRenewalTimeout = globalThis.clearTimeout.bind(globalThis)
Expand Down Expand Up @@ -114,6 +115,8 @@ export type AccountOperationError = {
retryCount?: number
accountIdentity?: string
tokenHash?: string
/** Fingerprint of the refresh token that produced this error. */
refreshTokenFingerprint?: string
/**
* HTTP status of the underlying refresh/quota failure, when known. Lets
* consumers distinguish a permanently-dead token (400 invalid_grant →
Expand Down Expand Up @@ -244,6 +247,7 @@ export type AccountStorage = {
intervalMinutes?: number
refreshBeforeExpiryMinutes?: number
mainLastRefreshError?: AccountOperationError
mainRefreshErrorClearedAt?: number
mainRefreshLeaseId?: string
mainRefreshLeaseUntil?: number
mainRefreshLeaseTokenHash?: string
Expand Down Expand Up @@ -364,6 +368,7 @@ export type AccountRuntimeState = {
quotaErrorGeneration?: number
quotaErrorClearedAt?: number
lastRefreshError?: AccountOperationError
refreshErrorClearedAt?: number
refreshLeaseId?: string
refreshLeaseUntil?: number
refreshLeaseTokenHash?: string
Expand Down Expand Up @@ -615,6 +620,11 @@ function normalizeOperationError(
: undefined,
tokenHash:
typeof value.tokenHash === 'string' ? value.tokenHash : undefined,
refreshTokenFingerprint:
typeof value.refreshTokenFingerprint === 'string' &&
value.refreshTokenFingerprint.trim()
? value.refreshTokenFingerprint.trim()
: undefined,
// Preserve the dead-token discriminators across save/load. Without these,
// a retry-exhausted transient (permanent=false, 24h backoff) would lose its
// flag on reload and the 24h-delay heuristic would wrongly re-classify it
Expand Down Expand Up @@ -939,6 +949,15 @@ export function mergeMainQuotaErrorClearedAt(
return Math.max(existing, incoming)
}

export function mergeMainRefreshErrorClearedAt(
existing: number | undefined,
incoming: number | undefined,
): number | undefined {
if (incoming === undefined || !Number.isFinite(incoming)) return existing
if (existing === undefined || !Number.isFinite(existing)) return incoming
return Math.max(existing, incoming)
}

function accountCredentialTimestamp(value: Record<string, unknown>): number {
return Math.max(
numericField(value.lastRefreshedAt),
Expand Down Expand Up @@ -1026,6 +1045,12 @@ function mergeConfigAndState(
mainState.quotaErrorClearedAt >= 0
? mainState.quotaErrorClearedAt
: undefined
const mainRefreshErrorClearedAt =
typeof mainState?.refreshErrorClearedAt === 'number' &&
Number.isFinite(mainState.refreshErrorClearedAt) &&
mainState.refreshErrorClearedAt >= 0
? mainState.refreshErrorClearedAt
: undefined
const configQuotaError = quotaConfig.mainLastQuotaApiError
const mainLastQuotaApiError =
mainState?.lastQuotaApiError ??
Expand All @@ -1035,6 +1060,15 @@ function mergeConfigAndState(
configQuotaError.checkedAt <= mainQuotaErrorClearedAt
? undefined
: configQuotaError)
const configRefreshError = refreshConfig.mainLastRefreshError
const mainLastRefreshError =
mainState?.lastRefreshError ??
(mainRefreshErrorClearedAt !== undefined &&
isRecord(configRefreshError) &&
typeof configRefreshError.checkedAt === 'number' &&
configRefreshError.checkedAt <= mainRefreshErrorClearedAt
? undefined
: configRefreshError)

const accounts = Array.isArray(configValue.accounts)
? configValue.accounts.map((account) => {
Expand All @@ -1059,7 +1093,8 @@ function mergeConfigAndState(
},
refresh: objectWithDefinedEntries({
...refreshConfig,
mainLastRefreshError: mainRefreshSource.lastRefreshError,
mainLastRefreshError,
mainRefreshErrorClearedAt: mainRefreshSource.refreshErrorClearedAt,
mainRefreshLeaseId: mainRefreshSource.refreshLeaseId,
mainRefreshLeaseUntil: mainRefreshSource.refreshLeaseUntil,
mainRefreshLeaseTokenHash: mainRefreshSource.refreshLeaseTokenHash,
Expand Down Expand Up @@ -1752,7 +1787,12 @@ function applyMainQuotaStatePatch(
) {
state.main.quotaErrorGeneration = incomingGeneration
}
} else if (incomingError) {
} else if (
incomingError &&
typeof incomingError.checkedAt === 'number' &&
Number.isFinite(incomingError.checkedAt) &&
incomingError.checkedAt > existingObservedAt
) {
const acceptsByObservation =
incomingErrorObservedAt !== undefined &&
incomingErrorObservedAt > existingObservedAt
Expand Down Expand Up @@ -1827,7 +1867,32 @@ function applyMainRefreshStatePatch(
storage: AccountStorage,
) {
state.main = state.main ?? {}
state.main.lastRefreshError = storage.refresh?.mainLastRefreshError
const incomingError = storage.refresh?.mainLastRefreshError
const incomingClearedAt = storage.refresh?.mainRefreshErrorClearedAt
const existingErrorObservedAt = state.main.lastRefreshError?.checkedAt
const existingClearedAt = state.main.refreshErrorClearedAt
const existingObservedAt = Math.max(
existingErrorObservedAt ?? 0,
existingClearedAt ?? 0,
)
if (
incomingError === undefined &&
incomingClearedAt !== undefined &&
incomingClearedAt >= existingObservedAt
) {
state.main.lastRefreshError = undefined
state.main.refreshErrorClearedAt = mergeMainRefreshErrorClearedAt(
existingClearedAt,
incomingClearedAt,
)
} else if (
incomingError &&
typeof incomingError.checkedAt === 'number' &&
Number.isFinite(incomingError.checkedAt) &&
incomingError.checkedAt > existingObservedAt
) {
state.main.lastRefreshError = incomingError
}
state.main.refreshLeaseId = storage.refresh?.mainRefreshLeaseId
state.main.refreshLeaseUntil = storage.refresh?.mainRefreshLeaseUntil
state.main.refreshLeaseTokenHash = storage.refresh?.mainRefreshLeaseTokenHash
Expand Down Expand Up @@ -2757,10 +2822,12 @@ export function buildRefreshOperationError(input: {
error: unknown
now: number
accountIdentity: string | undefined
refreshTokenFingerprint?: string
previous?: AccountOperationError
}): AccountOperationError {
const previousRetryCount =
input.previous?.accountIdentity === input.accountIdentity
input.previous?.accountIdentity === input.accountIdentity &&
input.previous?.refreshTokenFingerprint === input.refreshTokenFingerprint
? (input.previous?.retryCount ?? 0)
: 0
const retryCount = previousRetryCount + 1
Expand Down Expand Up @@ -2804,6 +2871,7 @@ export function buildRefreshOperationError(input: {
nextRetryAt: input.now + delay,
retryCount,
accountIdentity: input.accountIdentity,
refreshTokenFingerprint: input.refreshTokenFingerprint,
Comment thread
iceteaSA marked this conversation as resolved.
status,
permanent: status === 400 && isInvalidGrant,
}
Expand Down Expand Up @@ -2855,10 +2923,18 @@ export function refreshBackoffActive(
error: AccountOperationError | undefined,
accountIdentity: string | undefined,
now: number,
currentRefreshTokenFingerprint: string | undefined,
) {
if (!error) return false
const retryAt = effectiveRefreshRetryAt(error)
if (!retryAt || retryAt <= now) return false
if (
error.refreshTokenFingerprint &&
currentRefreshTokenFingerprint &&
error.refreshTokenFingerprint !== currentRefreshTokenFingerprint
) {
return false
}
if (!error.accountIdentity) return true
if (!accountIdentity) return true
return error.accountIdentity === accountIdentity
Expand Down Expand Up @@ -3696,6 +3772,7 @@ function recordRefreshError(
error,
now,
accountIdentity: account.id,
refreshTokenFingerprint: tokenFingerprint(account.refresh),
previous: account.lastRefreshError,
})
}
Expand Down Expand Up @@ -3870,7 +3947,12 @@ export class FallbackAccountManager {
const refreshError = next.lastRefreshError
if (
refreshError &&
refreshBackoffActive(refreshError, next.id, this.now())
refreshBackoffActive(
refreshError,
next.id,
this.now(),
tokenFingerprint(next.refresh),
)
) {
throw createRefreshBackoffActiveError(refreshError, this.now())
}
Expand Down Expand Up @@ -3935,7 +4017,12 @@ export class FallbackAccountManager {
}
} else if (
!failClosedOnUnknownQuota(storage) &&
!refreshBackoffActive(next.lastRefreshError, next.id, this.now()) &&
!refreshBackoffActive(
next.lastRefreshError,
next.id,
this.now(),
tokenFingerprint(next.refresh),
) &&
quotaSnapshotPassesModelScope(next.quota, options.modelId)
) {
usable.push(next)
Expand Down Expand Up @@ -3994,7 +4081,12 @@ export class FallbackAccountManager {
)
continue
if (
refreshBackoffActive(account.lastRefreshError, account.id, this.now())
refreshBackoffActive(
account.lastRefreshError,
account.id,
this.now(),
tokenFingerprint(account.refresh),
)
) {
// Backoff skips are steady-state while a fallback account is waiting for
// its next retry. Logging every background tick from every OpenCode
Expand Down Expand Up @@ -4040,7 +4132,12 @@ export class FallbackAccountManager {
!this.isFallbackAccountVaultServed(next.id, storage)
) {
if (
refreshBackoffActive(next.lastRefreshError, next.id, this.now())
refreshBackoffActive(
next.lastRefreshError,
next.id,
this.now(),
tokenFingerprint(next.refresh),
)
) {
continue
}
Expand Down Expand Up @@ -4090,7 +4187,12 @@ export class FallbackAccountManager {
const refreshError = next.lastRefreshError
if (
refreshError &&
refreshBackoffActive(refreshError, next.id, this.now())
refreshBackoffActive(
refreshError,
next.id,
this.now(),
tokenFingerprint(next.refresh),
)
) {
throw createRefreshBackoffActiveError(refreshError, this.now())
}
Expand Down Expand Up @@ -4197,7 +4299,12 @@ export class FallbackAccountManager {
const refreshError = latestAccount.lastRefreshError
if (
refreshError &&
refreshBackoffActive(refreshError, latestAccount.id, this.now())
refreshBackoffActive(
refreshError,
latestAccount.id,
this.now(),
tokenFingerprint(latestAccount.refresh),
)
) {
updateStoredAccount(storage, latestAccount)
throw createRefreshBackoffActiveError(refreshError, this.now())
Expand Down
11 changes: 10 additions & 1 deletion packages/core/src/commands/account.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ export type AccountCommandAction =
| { type: 'remove'; id: string }
| { type: 'move-up'; id: string }
| { type: 'move-down'; id: string }
| { type: 'reset-backoff' }
| {
type: 'add-apikey'
apiKey: string
Expand All @@ -37,6 +38,7 @@ export function parseAccountCommandAction(
if (action === 'remove' && rest) return { type: 'remove', id: rest }
if (action === 'move-up' && rest) return { type: 'move-up', id: rest }
if (action === 'move-down' && rest) return { type: 'move-down', id: rest }
if (action === 'reset-backoff' && !rest) return { type: 'reset-backoff' }

if (action === 'add-apikey' && rest) {
let remaining = rest
Expand Down Expand Up @@ -163,6 +165,7 @@ const USAGE_TEXT = [
' /claude-account remove <id> Remove a fallback account',
' /claude-account move-up <id> Move a fallback account up',
' /claude-account move-down <id> Move a fallback account down',
' /claude-account reset-backoff Clear main OAuth refresh and quota backoff',
' /claude-account add-apikey <key> Add an API key fallback account',
' /claude-account add-oauth-start Start OAuth device flow',
' /claude-account add-oauth-finish <code> Complete OAuth flow',
Expand All @@ -176,7 +179,7 @@ export function executeAccountCommand(input: {
text: string
updated?: {
id: string
action: 'enable' | 'disable' | 'remove' | 'reorder'
action: 'enable' | 'disable' | 'remove' | 'reorder' | 'reset-backoff'
enabled?: boolean
previousOrder?: string[]
newOrder?: string[]
Expand Down Expand Up @@ -218,6 +221,12 @@ export function executeAccountCommand(input: {
if (action.type === 'add-oauth-finish') {
return { text: 'add-oauth-finish' }
}
if (action.type === 'reset-backoff') {
return {
text: 'Main OAuth refresh and quota backoff cleared.',
updated: { id: 'main', action: 'reset-backoff' },
}
}

const id = action.id
if (id === mainId) {
Expand Down
Loading