Explorar el Código

feat(council): support model fallback chain per councillor

Allow a councillor's model to be an ordered chain (array of model IDs
or { id, variant } entries) in addition to a single string. On any
failure or timeout the councillor advances to the next model in its
chain; the existing empty-response retry still applies per model.
Single-string configs are unchanged.
Jiajun0413 hace 1 mes
padre
commit
fc613dce5d

+ 39 - 3
docs/council.md

@@ -140,10 +140,38 @@ Each entry inside a preset is one councillor:
 
 | Field | Type | Required | Description |
 |-------|------|----------|-------------|
-| `model` | string | Yes | Model ID in `provider/model` format |
-| `variant` | string | No | Optional variant/reasoning setting |
+| `model` | string \| array | Yes | A `provider/model` string, or an ordered fallback chain tried until one responds |
+| `variant` | string | No | Optional variant/reasoning setting (applies to chain entries without their own) |
 | `prompt` | string | No | Optional role guidance prepended to the user prompt |
 
+#### Councillor model fallback
+
+`model` also accepts an ordered chain. When the primary model fails or times
+out, the councillor advances to the next model instead of dropping out of the
+council. Entries are `provider/model` strings or `{ "id", "variant" }` objects:
+
+```jsonc
+{
+  "council": {
+    "presets": {
+      "review": {
+        "reviewer": {
+          "model": [
+            "openai/gpt-5.5",
+            { "id": "google/gemini-3-pro", "variant": "high" },
+            "anthropic/claude-opus-4-6"
+          ],
+          "prompt": "Focus on bugs, edge cases, and failure modes."
+        }
+      }
+    }
+  }
+}
+```
+
+Empty-response retries (`councillor_retries`) apply per model before the chain
+advances. A single string keeps the previous single-model behavior.
+
 ### Council agent config
 
 The **synthesizer model** is **not** configured inside `council.presets`.
@@ -343,6 +371,13 @@ Council responses include a footer like:
 - timed-out councillors are marked `timed_out`
 - council still synthesizes from successful results
 
+### Model chain fallback
+
+When a councillor's `model` is an array, the councillors walks the chain in
+order. Empty-response retries apply per model; any other failure or timeout
+advances to the next model. The councillor only fails once every model in the
+chain is exhausted, and the reported model reflects the one that responded.
+
 ### Empty response retries
 
 Some providers silently return zero tokens. Council treats that as a retryable
@@ -350,7 +385,8 @@ failure.
 
 - `councillor_retries` defaults to `3`
 - retries only happen for **empty provider responses**
-- normal failures and timeouts are returned immediately
+- normal failures and timeouts are returned immediately (they advance to the
+  next model in the chain, if any)
 
 ### Failure behavior
 

+ 25 - 4
src/config/council-schema.test.ts

@@ -8,15 +8,36 @@ import {
 
 describe('CouncillorConfigSchema', () => {
   test('validates config with model and optional variant', () => {
-    const goodConfig: CouncillorConfig = {
+    const result = CouncillorConfigSchema.safeParse({
       model: 'openai/gpt-5.4-mini',
       variant: 'low',
-    };
+    });
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.model).toBe('openai/gpt-5.4-mini');
+      expect(result.data.variant).toBe('low');
+      // A single-model config normalizes to a one-entry chain.
+      expect(result.data.models).toEqual([
+        { id: 'openai/gpt-5.4-mini', variant: 'low' },
+      ]);
+    }
+  });
 
-    const result = CouncillorConfigSchema.safeParse(goodConfig);
+  test('accepts an ordered model fallback chain', () => {
+    const result = CouncillorConfigSchema.safeParse({
+      model: [
+        'openai/gpt-5.4-mini',
+        { id: 'google/gemini-3-pro', variant: 'high' },
+      ],
+    });
     expect(result.success).toBe(true);
     if (result.success) {
-      expect(result.data).toEqual(goodConfig);
+      // Primary model stays on `model` for backward compatibility.
+      expect(result.data.model).toBe('openai/gpt-5.4-mini');
+      expect(result.data.models).toEqual([
+        { id: 'openai/gpt-5.4-mini', variant: undefined },
+        { id: 'google/gemini-3-pro', variant: 'high' },
+      ]);
     }
   });
 

+ 61 - 12
src/config/council-schema.ts

@@ -11,6 +11,43 @@ const ModelIdSchema = z
     'Expected provider/model format (e.g. "openai/gpt-5.4-mini")',
   );
 
+/** A single model in a councillor fallback chain, with optional variant. */
+export type CouncillorModelEntry = { id: string; variant?: string };
+
+const CouncillorModelEntrySchema = z.object({
+  id: ModelIdSchema,
+  variant: z.string().optional(),
+});
+
+/**
+ * A councillor's model: either a single "provider/model" string, or an
+ * ordered fallback chain (array of strings and/or { id, variant } entries)
+ * tried in order until one responds.
+ */
+const CouncillorModelSchema = z
+  .union([
+    ModelIdSchema,
+    z.array(z.union([ModelIdSchema, CouncillorModelEntrySchema])).min(1),
+  ])
+  .describe(
+    'Model ID in provider/model format (e.g. "openai/gpt-5.4-mini"), or an ' +
+      'ordered fallback chain (array of model IDs or { id, variant } entries) ' +
+      'tried in order until one responds.',
+  );
+
+/** Flatten a councillor model config into an ordered list of model entries. */
+export function normalizeCouncillorModels(
+  model: string | Array<string | CouncillorModelEntry>,
+  fallbackVariant?: string,
+): CouncillorModelEntry[] {
+  const raw = Array.isArray(model) ? model : [model];
+  return raw.map((entry) =>
+    typeof entry === 'string'
+      ? { id: entry, variant: fallbackVariant }
+      : { id: entry.id, variant: entry.variant ?? fallbackVariant },
+  );
+}
+
 /**
  * Configuration for a single councillor within a preset.
  * Each councillor is an independent LLM that processes the same prompt.
@@ -18,19 +55,31 @@ const ModelIdSchema = z
  * Councillors run as agent sessions with read-only codebase access
  * (read, glob, grep, lsp, list). They can examine the codebase but
  * cannot modify files or spawn subagents.
+ *
+ * `model` accepts a single ID or an ordered fallback chain. The parsed config
+ * exposes `models` (the normalized chain) plus `model` (the primary, for
+ * backward compatibility).
  */
-export const CouncillorConfigSchema = z.object({
-  model: ModelIdSchema.describe(
-    'Model ID in provider/model format (e.g. "openai/gpt-5.4-mini")',
-  ),
-  variant: z.string().optional(),
-  prompt: z
-    .string()
-    .optional()
-    .describe(
-      'Optional role/guidance injected into the councillor user prompt',
-    ),
-});
+export const CouncillorConfigSchema = z
+  .object({
+    model: CouncillorModelSchema,
+    variant: z.string().optional(),
+    prompt: z
+      .string()
+      .optional()
+      .describe(
+        'Optional role/guidance injected into the councillor user prompt',
+      ),
+  })
+  .transform((c) => {
+    const models = normalizeCouncillorModels(c.model, c.variant);
+    return {
+      model: models[0].id,
+      variant: c.variant,
+      prompt: c.prompt,
+      models,
+    };
+  });
 
 export type CouncillorConfig = z.infer<typeof CouncillorConfigSchema>;
 

+ 51 - 0
src/council/council-manager.test.ts

@@ -848,6 +848,57 @@ describe('CouncilManager', () => {
       expect(result.councillorResults[0].error).toContain('timed out');
     });
 
+    test('falls back to next model in councillor chain on failure', async () => {
+      let sessionCount = 0;
+      const ctx = createMockContext({
+        sessionCreateResult: () => {
+          sessionCount++;
+          return { data: { id: `session-${sessionCount}` } };
+        },
+        promptImpl: async (args: any) => {
+          // First model (session-1) fails; second model (session-2) succeeds.
+          if (args.path?.id === 'session-1') {
+            throw new Error('Prompt timed out after 180000ms');
+          }
+          return {};
+        },
+        sessionMessagesResult: {
+          data: [
+            {
+              info: { role: 'assistant' },
+              parts: [{ type: 'text', text: 'Fallback success' }],
+            },
+          ],
+        },
+      });
+
+      const config: PluginConfig = {
+        council: {
+          presets: {
+            default: {
+              alpha: {
+                model: ['openai/gpt-5.4-mini', 'openai/gpt-5.3-codex'],
+              },
+            },
+          },
+        },
+      } as any;
+      const manager = new CouncilManager(ctx, config, undefined);
+
+      const result = await manager.runCouncil(
+        'test prompt',
+        undefined,
+        'parent-id',
+      );
+
+      expect(result.success).toBe(true);
+      expect(result.councillorResults).toHaveLength(1);
+      expect(result.councillorResults[0].status).toBe('completed');
+      expect(result.councillorResults[0].result).toBe('Fallback success');
+      // Reported model reflects the fallback that actually responded.
+      expect(result.councillorResults[0].model).toBe('openai/gpt-5.3-codex');
+    });
+
     test('exhausts councillor retries and returns failure', async () => {
       const ctx = createMockContext({
         promptImpl: async () => ({}),

+ 59 - 43
src/council/council-manager.ts

@@ -15,7 +15,11 @@ import {
   COUNCILLOR_STAGGER_MS,
   TMUX_SPAWN_DELAY_MS,
 } from '../config/constants';
-import type { CouncillorConfig, CouncilResult } from '../config/council-schema';
+import {
+  type CouncillorConfig,
+  type CouncilResult,
+  normalizeCouncillorModels,
+} from '../config/council-schema';
 import { log } from '../utils/logger';
 import {
   extractSessionResult,
@@ -405,9 +409,13 @@ export class CouncilManager {
   }
 
   /**
-   * Run a single councillor with retry logic for empty responses.
-   * Only retries on "Empty response from provider" errors — timeouts
-   * and other failures are returned immediately.
+   * Run a single councillor across its configured model chain.
+   *
+   * For each model in the chain, empty responses are retried up to
+   * `maxRetries` times (providers that silently rate-limit). Any other
+   * failure or timeout advances to the next model in the chain. The
+   * councillor only fails once every model has been exhausted; the reported
+   * `model` and `error` reflect the last model tried.
    */
   private async runCouncillorWithRetry(
     name: string,
@@ -423,60 +431,68 @@ export class CouncilManager {
     result?: string;
     error?: string;
   }> {
-    const modelLabel = shortModelLabel(config.model);
+    // Prefer the normalized chain from the schema transform. When configs are
+    // built without the transform (e.g. tests), derive it from the raw model.
+    const models =
+      config.models ?? normalizeCouncillorModels(config.model, config.variant);
     const totalAttempts = 1 + maxRetries;
 
-    for (let attempt = 1; attempt <= totalAttempts; attempt++) {
-      if (attempt > 1) {
-        log(
-          `[council-manager] Retrying councillor "${name}" (${modelLabel}), attempt ${attempt}/${totalAttempts}`,
-        );
-      }
+    let lastModel = models[0].id;
+    let lastStatus: 'failed' | 'timed_out' = 'failed';
+    let lastError = `Councillor "${name}": no model responded`;
 
-      try {
-        const result = await this.runAgentSession({
-          parentSessionId,
-          title: `Council ${name} (${modelLabel})`,
-          agent: 'councillor',
-          model: config.model,
-          promptText: formatCouncillorPrompt(prompt, config.prompt),
-          variant: config.variant,
-          timeout,
-          includeReasoning: false,
-        });
+    for (let modelIndex = 0; modelIndex < models.length; modelIndex++) {
+      const entry = models[modelIndex];
+      const modelLabel = shortModelLabel(entry.id);
+      lastModel = entry.id;
 
-        return {
-          name,
-          model: config.model,
-          status: 'completed' as const,
-          result,
-        };
-      } catch (error) {
-        const msg = error instanceof Error ? error.message : String(error);
+      for (let attempt = 1; attempt <= totalAttempts; attempt++) {
+        if (attempt > 1) {
+          log(
+            `[council-manager] Retrying councillor "${name}" (${modelLabel}), attempt ${attempt}/${totalAttempts}`,
+          );
+        } else if (modelIndex > 0) {
+          log(
+            `[council-manager] Councillor "${name}" falling back to ${modelLabel} (model ${modelIndex + 1}/${models.length})`,
+          );
+        }
 
-        // Only retry on empty responses (provider silently rate-limited)
-        const isEmptyResponse = msg.includes('Empty response from provider');
-        const canRetry = attempt < totalAttempts && isEmptyResponse;
+        try {
+          const result = await this.runAgentSession({
+            parentSessionId,
+            title: `Council ${name} (${modelLabel})`,
+            agent: 'councillor',
+            model: entry.id,
+            promptText: formatCouncillorPrompt(prompt, config.prompt),
+            variant: entry.variant,
+            timeout,
+            includeReasoning: false,
+          });
 
-        if (!canRetry) {
           return {
             name,
-            model: config.model,
-            status: msg.includes('timed out')
-              ? ('timed_out' as const)
-              : ('failed' as const),
-            error: `Councillor "${name}": ${msg}`,
+            model: entry.id,
+            status: 'completed' as const,
+            result,
           };
+        } catch (error) {
+          const msg = error instanceof Error ? error.message : String(error);
+          lastStatus = msg.includes('timed out') ? 'timed_out' : 'failed';
+          lastError = `Councillor "${name}": ${msg}`;
+
+          // Retry the same model only on empty responses (silent rate-limit);
+          // any other error moves on to the next model in the chain.
+          const isEmptyResponse = msg.includes('Empty response from provider');
+          if (!(attempt < totalAttempts && isEmptyResponse)) break;
         }
       }
     }
 
-    // Unreachable, but satisfies TypeScript
     return {
       name,
-      model: config.model,
-      status: 'failed' as const,
-      error: `Councillor "${name}": max retries exhausted`,
+      model: lastModel,
+      status: lastStatus,
+      error: lastError,
     };
   }
 }