Merge pull request #608 from alvinunreal/task/607-auto-resume-timeout-subagents

Recover timed-out subagent sessions

src/hooks/task-session-manager/codemap.md

@@ -28,7 +28,9 @@ All modules depend on `BackgroundJobBoard` from `src/utils/background-job-board.
    - Intercepts `task` tool calls on managed sessions
    - Generates a task label from `description`/`prompt` via `deriveTaskSessionLabel`
    - Creates a `PendingTaskCall` record with call ID, parent session ID, agent type, and label
-   - Resolves reusable task IDs from the job board; if found, updates the task ID and marks it as used
+   - Resolves reusable task IDs from the job board; completed/reconciled jobs
+     are reusable by alias, while timed-out running jobs become recoverable
+     only after a live busy signal confirms they are safe to resume
    - If no reusable task exists, allows fresh task creation
 
 2. **Task Launch (`tool.execute.after`)**
@@ -95,4 +97,4 @@ The original monolithic module was split to improve:
 - **Maintainability**: Changes to one concern (e.g., context tracking) do not affect unrelated logic.
 - **Scalability**: Capped data structures prevent unbounded memory growth.
 
-Each submodule adheres to the **Single Responsibility Principle** while collaborating through the facade to provide a cohesive user experience.
+Each submodule adheres to the **Single Responsibility Principle** while collaborating through the facade to provide a cohesive user experience.

src/hooks/task-session-manager/index.test.ts

@@ -245,6 +245,182 @@ describe('task-session-manager hook', () => {
     );
   });
 
+  test('reuses timed-out running aliases after live busy recovery', async () => {
+    const board = new BackgroundJobBoard();
+    const { hook } = createHook({ backgroundJobBoard: board });
+
+    await hook['tool.execute.before'](
+      { tool: 'task', sessionID: 'parent-1', callID: 'call-1' },
+      {
+        args: {
+          subagent_type: 'explorer',
+          description: 'map timed out session',
+        },
+      },
+    );
+    await hook['tool.execute.after'](
+      { tool: 'task', sessionID: 'parent-1', callID: 'call-1' },
+      {
+        output: [
+          'task_id: child-1',
+          'state: running',
+          '',
+          '<task_result>',
+          'Timed out after 120000ms while waiting for task completion.',
+          '</task_result>',
+        ].join('\n'),
+      },
+    );
+
+    expect(
+      board.resolveRecoverable('parent-1', 'exp-1', 'explorer')?.taskID,
+    ).toBeUndefined();
+
+    await hook.event({
+      event: {
+        type: 'session.status',
+        properties: {
+          sessionID: 'child-1',
+          status: { type: 'busy' },
+        },
+      },
+    });
+
+    expect(
+      board.resolveRecoverable('parent-1', 'exp-1', 'explorer')?.taskID,
+    ).toBe('child-1');
+
+    const resume = {
+      args: { subagent_type: 'explorer', task_id: 'exp-1' },
+    };
+    await hook['tool.execute.before'](
+      { tool: 'task', sessionID: 'parent-1', callID: 'resume-1' },
+      resume,
+    );
+
+    expect(resume.args.task_id).toBe('child-1');
+    expect(board.get('child-1')).toMatchObject({
+      state: 'running',
+      timedOut: false,
+      recoverableAfterLiveBusy: true,
+    });
+  });
+
+  test('does not bypass live busy recovery gate for known raw session ids', async () => {
+    const board = new BackgroundJobBoard();
+    const { hook } = createHook({ backgroundJobBoard: board });
+
+    await hook['tool.execute.before'](
+      { tool: 'task', sessionID: 'parent-1', callID: 'call-1' },
+      {
+        args: {
+          subagent_type: 'explorer',
+          description: 'map timed out session',
+        },
+      },
+    );
+    await hook['tool.execute.after'](
+      { tool: 'task', sessionID: 'parent-1', callID: 'call-1' },
+      {
+        output: [
+          'task_id: ses_timeout',
+          'state: running',
+          '',
+          '<task_result>',
+          'Timed out after 120000ms while waiting for task completion.',
+          '</task_result>',
+        ].join('\n'),
+      },
+    );
+
+    const resumeBeforeLiveBusy = {
+      args: { subagent_type: 'explorer', task_id: 'ses_timeout' },
+    };
+    await hook['tool.execute.before'](
+      { tool: 'task', sessionID: 'parent-1', callID: 'resume-1' },
+      resumeBeforeLiveBusy,
+    );
+
+    expect(resumeBeforeLiveBusy.args.task_id).toBeUndefined();
+
+    await hook.event({
+      event: {
+        type: 'session.status',
+        properties: {
+          sessionID: 'ses_timeout',
+          status: { type: 'busy' },
+        },
+      },
+    });
+
+    const resumeAfterLiveBusy = {
+      args: { subagent_type: 'explorer', task_id: 'ses_timeout' },
+    };
+    await hook['tool.execute.before'](
+      { tool: 'task', sessionID: 'parent-1', callID: 'resume-2' },
+      resumeAfterLiveBusy,
+    );
+
+    expect(resumeAfterLiveBusy.args.task_id).toBe('ses_timeout');
+  });
+
+  test('busy timeout recovery clears timeout overlay from prompt', async () => {
+    const board = new BackgroundJobBoard();
+    const { hook } = createHook({ backgroundJobBoard: board });
+
+    await hook['tool.execute.before'](
+      { tool: 'task', sessionID: 'parent-1', callID: 'call-1' },
+      {
+        args: {
+          subagent_type: 'explorer',
+          description: 'recover timed out child',
+        },
+      },
+    );
+    await hook['tool.execute.after'](
+      { tool: 'task', sessionID: 'parent-1', callID: 'call-1' },
+      {
+        output: [
+          'task_id: child-1',
+          'state: running',
+          '',
+          '<task_result>',
+          'Timed out after 120000ms while waiting for task completion.',
+          '</task_result>',
+        ].join('\n'),
+      },
+    );
+
+    const beforeMessages = createMessages('parent-1', 'before busy');
+    await hook['experimental.chat.messages.transform']({}, beforeMessages);
+    expect(beforeMessages.messages[0].parts[0].text).toContain(
+      'running, timed out',
+    );
+
+    await hook.event({
+      event: {
+        type: 'session.status',
+        properties: {
+          sessionID: 'child-1',
+          status: { type: 'busy' },
+        },
+      },
+    });
+
+    expect(board.get('child-1')).toMatchObject({
+      state: 'running',
+      timedOut: false,
+      recoverableAfterLiveBusy: true,
+      statusUncertain: false,
+    });
+
+    const afterMessages = createMessages('parent-1', 'after busy');
+    await hook['experimental.chat.messages.transform']({}, afterMessages);
+    expect(afterMessages.messages[0].parts[0].text).not.toContain(
+      'running, timed out',
+    );
+  });
+
   test('updates background job board from injected completion messages', async () => {
     const board = new BackgroundJobBoard();
     const { hook } = createHook({ backgroundJobBoard: board });

src/hooks/task-session-manager/index.ts

@@ -334,13 +334,28 @@ export function createTaskSessionManagerHook(
       }
 
       const requested = args.task_id.trim();
-      const remembered = backgroundJobBoard.resolveReusable(
-        input.sessionID,
-        requested,
-        agentType,
-      );
+      const remembered =
+        backgroundJobBoard.resolveReusable(
+          input.sessionID,
+          requested,
+          agentType,
+        ) ??
+        backgroundJobBoard.resolveRecoverable(
+          input.sessionID,
+          requested,
+          agentType,
+        );
 
       if (!remembered) {
+        const knownManagedTask = backgroundJobBoard.resolve(
+          input.sessionID,
+          requested,
+        );
+        if (knownManagedTask) {
+          delete args.task_id;
+          return;
+        }
+
         if (RAW_SESSION_ID_PATTERN.test(requested)) {
           pendingCall.resumedTaskId = requested;
           pendingCallTracker.add(pendingCall);

src/multiplexer/session-manager.test.ts

@@ -410,6 +410,72 @@ describe('MultiplexerSessionManager', () => {
       expect(mockMultiplexer.closePane).not.toHaveBeenCalled();
     });
 
+    test('timed out running jobs still close after safe recovery and completion', async () => {
+      const ctx = createMockContext();
+      const board = new BackgroundJobBoard();
+      board.registerLaunch({
+        taskID: 'timedout-child',
+        parentSessionID: 'parent-1',
+        agent: 'explorer',
+      });
+      board.updateStatus({
+        taskID: 'timedout-child',
+        state: 'running',
+        timedOut: true,
+        now: 100,
+      });
+      mockMultiplexer.spawnPane.mockResolvedValue({
+        success: true,
+        paneId: 'p-timedout-child',
+      });
+      const manager = new MultiplexerSessionManager(
+        ctx,
+        defaultMultiplexerConfig,
+        board,
+      );
+      board.setTerminalStateListener((taskID) => {
+        void manager.retryDeferredIdleClose(taskID);
+      });
+
+      await manager.onSessionCreated({
+        type: 'session.created',
+        properties: {
+          info: { id: 'timedout-child', parentID: 'parent-1' },
+        },
+      });
+
+      await manager.onSessionStatus({
+        type: 'session.status',
+        properties: {
+          sessionID: 'timedout-child',
+          status: { type: 'idle' },
+        },
+      });
+
+      expect(mockMultiplexer.closePane).not.toHaveBeenCalled();
+
+      await manager.onSessionStatus({
+        type: 'session.status',
+        properties: {
+          sessionID: 'timedout-child',
+          status: { type: 'busy' },
+        },
+      });
+
+      expect(mockMultiplexer.closePane).not.toHaveBeenCalled();
+
+      board.updateStatus({
+        taskID: 'timedout-child',
+        state: 'completed',
+        resultSummary: 'done',
+      });
+      await Promise.resolve();
+
+      expect(mockMultiplexer.closePane).toHaveBeenCalledWith(
+        'p-timedout-child',
+      );
+    });
+
     test('deferred idle closes retry on terminal status updates', async () => {
       for (const state of ['completed', 'error', 'cancelled'] as const) {
         resetMultiplexerSessionManagerState();

src/multiplexer/session-manager.ts

@@ -283,9 +283,10 @@ export class MultiplexerSessionManager {
     }
 
     if (statusType) {
-      this.deferredIdleCloses.delete(sessionId);
-
-      if (statusType !== 'busy') return;
+      if (statusType !== 'busy') {
+        this.deferredIdleCloses.delete(sessionId);
+        return;
+      }
 
       log('[multiplexer-session-manager] session busy event received', {
         instanceId: this.instanceId,

src/utils/background-job-board.test.ts

@@ -198,7 +198,7 @@ describe('BackgroundJobBoard', () => {
     expect(board.resolveReusable('parent-1', 'ses_error')).toBeUndefined();
   });
 
-  test('prompt tells orchestrator to reuse completed sessions only', () => {
+  test('prompt distinguishes reusable and recoverable sessions', () => {
     const board = new BackgroundJobBoard();
     board.registerLaunch({
       taskID: 'ses_1',
@@ -208,10 +208,24 @@ describe('BackgroundJobBoard', () => {
     });
     board.updateStatus({ taskID: 'ses_1', state: 'completed' });
     board.markReconciled('ses_1');
+    board.registerLaunch({
+      taskID: 'ses_2',
+      parentSessionID: 'parent-1',
+      agent: 'explorer',
+      description: 'recover timed out task',
+    });
+    board.updateStatus({ taskID: 'ses_2', state: 'running', timedOut: true });
 
-    expect(board.formatForPrompt('parent-1')).toContain(
-      'Reuse only completed sessions',
+    const prompt = board.formatForPrompt('parent-1');
+
+    expect(prompt).toContain(
+      'Completed or reconciled sessions are reusable by alias',
+    );
+    expect(prompt).toContain(
+      'Timed-out running sessions are recoverable by alias for safe resume after a live busy signal.',
     );
+    expect(prompt).toContain('Cancelled or errored sessions are not reusable.');
+    expect(prompt).toContain('exp-1 / ses_2 / explorer / running, timed out');
   });
 
   test('does not reconcile running jobs', () => {
@@ -604,6 +618,71 @@ describe('BackgroundJobBoard', () => {
     expect(updated?.completedAt).toBeDefined();
   });
 
+  test('live busy recovery clears timeout state on running jobs', () => {
+    const board = new BackgroundJobBoard();
+    board.registerLaunch({
+      taskID: 'ses_1',
+      parentSessionID: 'parent-1',
+      agent: 'explorer',
+      now: 100,
+    });
+    board.updateStatus({
+      taskID: 'ses_1',
+      state: 'running',
+      timedOut: true,
+      statusUncertain: true,
+      now: 150,
+    });
+
+    const updated = board.markRunningFromLiveSession('ses_1', 200);
+
+    expect(updated).toMatchObject({
+      state: 'running',
+      timedOut: false,
+      recoverableAfterLiveBusy: true,
+      statusUncertain: false,
+      lastLiveBusyAt: 200,
+      updatedAt: 200,
+      alias: 'exp-1',
+    });
+  });
+
+  test('resolves timed-out running jobs only after live busy recovery', () => {
+    const board = new BackgroundJobBoard();
+    board.registerLaunch({
+      taskID: 'ses_1',
+      parentSessionID: 'parent-1',
+      agent: 'explorer',
+    });
+    board.updateStatus({
+      taskID: 'ses_1',
+      state: 'running',
+      timedOut: true,
+    });
+
+    expect(
+      board.resolveReusable('parent-1', 'exp-1', 'explorer'),
+    ).toBeUndefined();
+    expect(
+      board.resolveRecoverable('parent-1', 'exp-1', 'explorer'),
+    ).toBeUndefined();
+
+    board.markRunningFromLiveSession('ses_1', 200);
+
+    expect(
+      board.resolveRecoverable('parent-1', 'exp-1', 'explorer'),
+    ).toMatchObject({
+      taskID: 'ses_1',
+      state: 'running',
+      timedOut: false,
+      recoverableAfterLiveBusy: true,
+      lastLiveBusyAt: 200,
+    });
+    expect(
+      board.resolveRecoverable('parent-1', 'exp-1', 'oracle'),
+    ).toBeUndefined();
+  });
+
   test('stale status updates cannot reopen already reconciled jobs', () => {
     const board = new BackgroundJobBoard();
     board.registerLaunch({

src/utils/background-job-board.ts

@@ -17,6 +17,7 @@ export interface BackgroundJobRecord {
   objective?: string;
   state: BackgroundJobState;
   timedOut: boolean;
+  recoverableAfterLiveBusy: boolean;
   statusUncertain: boolean;
   cancellationRequested: boolean;
   terminalUnreconciled: boolean;
@@ -126,6 +127,7 @@ export class BackgroundJobBoard {
         objective: input.objective ?? existing.objective,
         state: 'running',
         timedOut: false,
+        recoverableAfterLiveBusy: false,
         statusUncertain: false,
         cancellationRequested: false,
         terminalUnreconciled: false,
@@ -152,6 +154,7 @@ export class BackgroundJobBoard {
       objective: input.objective,
       state: 'running',
       timedOut: false,
+      recoverableAfterLiveBusy: false,
       statusUncertain: false,
       cancellationRequested: false,
       terminalUnreconciled: false,
@@ -192,6 +195,12 @@ export class BackgroundJobBoard {
       ...existing,
       state: input.state,
       timedOut: input.timedOut ?? false,
+      recoverableAfterLiveBusy:
+        input.state !== 'running'
+          ? false
+          : input.timedOut === true
+            ? false
+            : existing.recoverableAfterLiveBusy,
       statusUncertain: input.statusUncertain ?? false,
       terminalUnreconciled: terminal ? true : existing.terminalUnreconciled,
       updatedAt: now,
@@ -254,6 +263,10 @@ export class BackgroundJobBoard {
       ...existing,
       updatedAt: now,
       lastLiveBusyAt: now,
+      timedOut: false,
+      recoverableAfterLiveBusy:
+        existing.recoverableAfterLiveBusy || existing.timedOut,
+      statusUncertain: false,
     };
 
     this.jobs.set(taskID, updated);
@@ -308,6 +321,7 @@ export class BackgroundJobBoard {
       ...existing,
       state: 'cancelled',
       timedOut: false,
+      recoverableAfterLiveBusy: false,
       statusUncertain: false,
       cancellationRequested: true,
       terminalUnreconciled: true,
@@ -381,6 +395,20 @@ export class BackgroundJobBoard {
     return job;
   }
 
+  resolveRecoverable(
+    parentSessionID: string,
+    taskIDOrAlias: string,
+    agent?: string,
+  ): BackgroundJobRecord | undefined {
+    const job = this.resolve(parentSessionID, taskIDOrAlias);
+    if (!job) return undefined;
+    if (agent && job.agent !== agent) return undefined;
+    if (job.state !== 'running' || !job.recoverableAfterLiveBusy) {
+      return undefined;
+    }
+    return job;
+  }
+
   markUsed(parentSessionID: string, key: string, now = Date.now()): void {
     const job = this.resolve(parentSessionID, key);
     if (!job) return;
@@ -459,7 +487,10 @@ export class BackgroundJobBoard {
     return [
       '### Background Job Board',
       'SENTINEL: background-job-board-v2',
-      'Do not poll running jobs. Wait for hook-driven completion, or use cancel_task only for explicit cancellation. Reconcile terminal jobs before final response. Reuse only completed sessions for the same specialist/context; never reuse cancelled or errored sessions.',
+      'Do not poll running jobs. Wait for hook-driven completion, or use cancel_task only for explicit cancellation. Reconcile terminal jobs before final response.',
+      'Completed or reconciled sessions are reusable by alias for the same specialist/context.',
+      'Timed-out running sessions are recoverable by alias for safe resume after a live busy signal.',
+      'Cancelled or errored sessions are not reusable.',
       '',
       '#### Active / Unreconciled',
       ...(active.length > 0