|
@@ -233,6 +233,11 @@ export class ForegroundFallbackManager {
|
|
|
/** sessionID → consecutive 429 count for the current model.
|
|
/** sessionID → consecutive 429 count for the current model.
|
|
|
* Reset on model swap or session deletion. */
|
|
* Reset on model swap or session deletion. */
|
|
|
private readonly sessionRetries = new Map<string, number>();
|
|
private readonly sessionRetries = new Map<string, number>();
|
|
|
|
|
+ /** sessionID → chain-exhaustion stage:
|
|
|
|
|
+ * 0 = not exhausted; 1 = chain exhausted once, reset to sticky fallback
|
|
|
|
|
+ * (one retry chance); 2 = exhausted again, aborted — stop intervening.
|
|
|
|
|
+ * Reset to 0 on successful responses or session deletion. */
|
|
|
|
|
+ private readonly chainExhaustion = new Map<string, number>();
|
|
|
|
|
|
|
|
/** Exposed for task-session-manager: prevents idle reconciliation
|
|
/** Exposed for task-session-manager: prevents idle reconciliation
|
|
|
* while a fallback abort/re-prompt is in flight for this session. */
|
|
* while a fallback abort/re-prompt is in flight for this session. */
|
|
@@ -291,6 +296,7 @@ export class ForegroundFallbackManager {
|
|
|
this.lastTrigger.delete(id);
|
|
this.lastTrigger.delete(id);
|
|
|
this.lastTriggerModel.delete(id);
|
|
this.lastTriggerModel.delete(id);
|
|
|
this.sessionRetries.delete(id);
|
|
this.sessionRetries.delete(id);
|
|
|
|
|
+ this.chainExhaustion.delete(id);
|
|
|
});
|
|
});
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
@@ -334,6 +340,7 @@ export class ForegroundFallbackManager {
|
|
|
} else {
|
|
} else {
|
|
|
// Successful response: clear retry count so recovery is not forgotten.
|
|
// Successful response: clear retry count so recovery is not forgotten.
|
|
|
this.sessionRetries.delete(sessionID);
|
|
this.sessionRetries.delete(sessionID);
|
|
|
|
|
+ this.chainExhaustion.delete(sessionID);
|
|
|
}
|
|
}
|
|
|
break;
|
|
break;
|
|
|
}
|
|
}
|
|
@@ -406,6 +413,7 @@ export class ForegroundFallbackManager {
|
|
|
if (this.isRecoveredStatus(props.status?.type)) {
|
|
if (this.isRecoveredStatus(props.status?.type)) {
|
|
|
// Recovered/terminal status: clear retry count.
|
|
// Recovered/terminal status: clear retry count.
|
|
|
this.sessionRetries.delete(sessionID);
|
|
this.sessionRetries.delete(sessionID);
|
|
|
|
|
+ this.chainExhaustion.delete(sessionID);
|
|
|
}
|
|
}
|
|
|
// Note: do NOT clear sessionRetries here on non-rate-limit statuses.
|
|
// Note: do NOT clear sessionRetries here on non-rate-limit statuses.
|
|
|
// Abort events triggered by our own fallback carry non-rate-limit
|
|
// Abort events triggered by our own fallback carry non-rate-limit
|
|
@@ -554,6 +562,10 @@ export class ForegroundFallbackManager {
|
|
|
|
|
|
|
|
private async execFallback(sessionID: string): Promise<void> {
|
|
private async execFallback(sessionID: string): Promise<void> {
|
|
|
try {
|
|
try {
|
|
|
|
|
+ // After the chain has been exhausted twice (reset retry failed and we
|
|
|
|
|
+ // aborted), do not intervene again for this session: re-entering would
|
|
|
|
|
+ // keep aborting in a loop. Surface errors to the user instead.
|
|
|
|
|
+ if (this.chainExhaustion.get(sessionID) === 2) return;
|
|
|
let currentModel = this.sessionModel.get(sessionID);
|
|
let currentModel = this.sessionModel.get(sessionID);
|
|
|
const agentName = this.sessionAgent.get(sessionID);
|
|
const agentName = this.sessionAgent.get(sessionID);
|
|
|
const chain = this.resolveChain(agentName, currentModel);
|
|
const chain = this.resolveChain(agentName, currentModel);
|
|
@@ -579,11 +591,29 @@ export class ForegroundFallbackManager {
|
|
|
let nextModel = chain.find((m) => !tried.has(m));
|
|
let nextModel = chain.find((m) => !tried.has(m));
|
|
|
if (!nextModel) {
|
|
if (!nextModel) {
|
|
|
if (chain.length > 1) {
|
|
if (chain.length > 1) {
|
|
|
- // Chain exhausted but we have fallbacks: reset tried set and
|
|
|
|
|
- // stick to the deepest fallback model so we stop re-trying the
|
|
|
|
|
- // dead primary model on every subsequent message.
|
|
|
|
|
|
|
+ // Chain exhausted but we have fallbacks: on the first exhaustion
|
|
|
|
|
+ // reset the tried set and stick to the deepest fallback model so
|
|
|
|
|
+ // we stop re-trying the dead primary model on every subsequent
|
|
|
|
|
+ // message. If the sticky fallback itself fails afterwards (second
|
|
|
|
|
+ // exhaustion), abort once and stop intervening — otherwise the
|
|
|
|
|
+ // reset re-prompt would loop forever on a fully dead chain.
|
|
|
const primary = chain[0];
|
|
const primary = chain[0];
|
|
|
const stickyFallback = chain[chain.length - 1];
|
|
const stickyFallback = chain[chain.length - 1];
|
|
|
|
|
+ if ((this.chainExhaustion.get(sessionID) ?? 0) >= 1) {
|
|
|
|
|
+ this.chainExhaustion.set(sessionID, 2);
|
|
|
|
|
+ log(
|
|
|
|
|
+ '[foreground-fallback] chain exhausted after re-fallback, aborting',
|
|
|
|
|
+ {
|
|
|
|
|
+ sessionID,
|
|
|
|
|
+ agentName,
|
|
|
|
|
+ currentModel,
|
|
|
|
|
+ tried: [...tried],
|
|
|
|
|
+ },
|
|
|
|
|
+ );
|
|
|
|
|
+ await abortSessionWithTimeout(getClient(this.input), sessionID);
|
|
|
|
|
+ return;
|
|
|
|
|
+ }
|
|
|
|
|
+ this.chainExhaustion.set(sessionID, 1);
|
|
|
log('[foreground-fallback] resetting tried set for re-fallback', {
|
|
log('[foreground-fallback] resetting tried set for re-fallback', {
|
|
|
sessionID,
|
|
sessionID,
|
|
|
agentName,
|
|
agentName,
|
|
@@ -597,6 +627,7 @@ export class ForegroundFallbackManager {
|
|
|
this.sessionTried.set(sessionID, tried);
|
|
this.sessionTried.set(sessionID, tried);
|
|
|
nextModel = stickyFallback;
|
|
nextModel = stickyFallback;
|
|
|
} else {
|
|
} else {
|
|
|
|
|
+ this.chainExhaustion.set(sessionID, 2);
|
|
|
log('[foreground-fallback] fallback chain exhausted, aborting', {
|
|
log('[foreground-fallback] fallback chain exhausted, aborting', {
|
|
|
sessionID,
|
|
sessionID,
|
|
|
agentName,
|
|
agentName,
|