Merge pull request #405 from alvinunreal/council-prompt-tune

docs/council.md

@@ -313,6 +313,17 @@ The orchestrator may delegate to `@council` for high-stakes or ambiguous
 decisions, but it does so sparingly because council is usually the most
 expensive path.
 
+### Output format
+
+Council responses include:
+
+1. `Council Response` — the synthesized final answer.
+2. `Councillor Details` — each responding councillor's individual response,
+   using the councillor names from the configured preset.
+3. `Council Summary` — agreement, disagreement resolution, remaining
+   uncertainty, and a consensus confidence rating of `unanimous`, `majority`,
+   or `split`.
+
 ### Output footer
 
 Council responses include a footer like:

src/agents/council.test.ts

@@ -1,224 +1,225 @@
-import { describe, expect, test } from "bun:test";
-import { formatCouncillorPrompt, formatCouncillorResults } from "./council";
+import { describe, expect, test } from 'bun:test';
+import { formatCouncillorPrompt, formatCouncillorResults } from './council';
 
-describe("formatCouncillorResults", () => {
+describe('formatCouncillorResults', () => {
   const originalPrompt =
-    "What is the best way to implement a REST API in TypeScript?";
+    'What is the best way to implement a REST API in TypeScript?';
 
-  test("formats completed councillor results correctly", () => {
+  test('formats completed councillor results correctly', () => {
     const councillorResults = [
       {
-        name: "alpha",
-        model: "anthropic/claude-opus-4-6",
-        status: "completed",
-        result: "Use Express.js with TypeScript interfaces for type safety.",
+        name: 'alpha',
+        model: 'anthropic/claude-opus-4-6',
+        status: 'completed',
+        result: 'Use Express.js with TypeScript interfaces for type safety.',
       },
       {
-        name: "beta",
-        model: "openai/gpt-5.5",
-        status: "completed",
+        name: 'beta',
+        model: 'openai/gpt-5.5',
+        status: 'completed',
         result:
-          "Consider Fastify for better performance and built-in type validation.",
+          'Consider Fastify for better performance and built-in type validation.',
       },
     ];
 
     const formatted = formatCouncillorResults(
       originalPrompt,
-      councillorResults
+      councillorResults,
     );
 
-    expect(formatted).toContain("**Original Prompt**:");
+    expect(formatted).toContain('**Original Prompt**:');
     expect(formatted).toContain(originalPrompt);
-    expect(formatted).toContain("**alpha** (claude-opus-4-6):");
-    expect(formatted).toContain("**beta** (gpt-5.5):");
+    expect(formatted).toContain('**alpha** (claude-opus-4-6):');
+    expect(formatted).toContain('**beta** (gpt-5.5):');
     expect(formatted).toContain(
-      "Use Express.js with TypeScript interfaces for type safety."
+      'Use Express.js with TypeScript interfaces for type safety.',
     );
     expect(formatted).toContain(
-      "Consider Fastify for better performance and built-in type validation."
+      'Consider Fastify for better performance and built-in type validation.',
     );
-    expect(formatted).toContain("**Councillor Responses**:");
+    expect(formatted).toContain('**Councillor Responses**:');
     expect(formatted).toContain(
-      "Synthesize the optimal response based on the above."
+      'You MUST follow the Synthesis Process steps before producing output',
     );
-    expect(formatted).not.toContain("**Failed/Timed-out Councillors**:");
+    expect(formatted).toContain(
+      'consensus confidence rating (unanimous, majority, or split)',
+    );
+    expect(formatted).not.toContain('**Failed/Timed-out Councillors**:');
   });
 
-  test("includes failed councillors section when some fail", () => {
+  test('includes failed councillors section when some fail', () => {
     const councillorResults = [
       {
-        name: "alpha",
-        model: "anthropic/claude-opus-4-6",
-        status: "completed",
-        result: "Use Express.js with TypeScript interfaces for type safety.",
+        name: 'alpha',
+        model: 'anthropic/claude-opus-4-6',
+        status: 'completed',
+        result: 'Use Express.js with TypeScript interfaces for type safety.',
       },
       {
-        name: "beta",
-        model: "openai/gpt-5.5",
-        status: "timed_out",
-        error: "Request timed out after 180000ms",
+        name: 'beta',
+        model: 'openai/gpt-5.5',
+        status: 'timed_out',
+        error: 'Request timed out after 180000ms',
       },
       {
-        name: "gamma",
-        model: "google/gemini-pro",
-        status: "failed",
-        error: "Provider returned empty response",
+        name: 'gamma',
+        model: 'google/gemini-pro',
+        status: 'failed',
+        error: 'Provider returned empty response',
       },
     ];
 
     const formatted = formatCouncillorResults(
       originalPrompt,
-      councillorResults
+      councillorResults,
     );
 
-    expect(formatted).toContain("**Councillor Responses**:");
-    expect(formatted).toContain("**alpha** (claude-opus-4-6):");
+    expect(formatted).toContain('**Councillor Responses**:');
+    expect(formatted).toContain('**alpha** (claude-opus-4-6):');
     expect(formatted).toContain(
-      "Use Express.js with TypeScript interfaces for type safety."
+      'Use Express.js with TypeScript interfaces for type safety.',
     );
-    expect(formatted).toContain("**Failed/Timed-out Councillors**:");
+    expect(formatted).toContain('**Failed/Timed-out Councillors**:');
     expect(formatted).toContain(
-      "**beta**: timed_out — Request timed out after 180000ms"
+      '**beta**: timed_out — Request timed out after 180000ms',
     );
     expect(formatted).toContain(
-      "**gamma**: failed — Provider returned empty response"
+      '**gamma**: failed — Provider returned empty response',
     );
-    expect(formatted).not.toContain("**beta** (gpt-5.5):");
-    expect(formatted).not.toContain("**gamma** (gemini-pro):");
+    expect(formatted).not.toContain('**beta** (gpt-5.5):');
+    expect(formatted).not.toContain('**gamma** (gemini-pro):');
   });
 
-  test("returns fallback message when all councillors fail", () => {
+  test('returns fallback message when all councillors fail', () => {
     const councillorResults = [
       {
-        name: "alpha",
-        model: "anthropic/claude-opus-4-6",
-        status: "timeout",
-        error: "Request timed out",
+        name: 'alpha',
+        model: 'anthropic/claude-opus-4-6',
+        status: 'timeout',
+        error: 'Request timed out',
       },
       {
-        name: "beta",
-        model: "openai/gpt-5.5",
-        status: "error",
-        error: "Provider error",
+        name: 'beta',
+        model: 'openai/gpt-5.5',
+        status: 'error',
+        error: 'Provider error',
       },
     ];
 
     const formatted = formatCouncillorResults(
       originalPrompt,
-      councillorResults
+      councillorResults,
     );
 
-    expect(formatted).toContain("**Original Prompt**:");
+    expect(formatted).toContain('**Original Prompt**:');
     expect(formatted).toContain(originalPrompt);
-    expect(formatted).toContain("**Councillor Responses**:");
-    expect(formatted).toContain("All councillors failed to produce output:");
-    expect(formatted).toContain("**alpha** (claude-opus-4-6):");
-    expect(formatted).toContain("**beta** (gpt-5.5):");
-    expect(formatted).toContain("Request timed out");
-    expect(formatted).toContain("Provider error");
+    expect(formatted).toContain('**Councillor Responses**:');
+    expect(formatted).toContain('All councillors failed to produce output:');
+    expect(formatted).toContain('**alpha** (claude-opus-4-6):');
+    expect(formatted).toContain('**beta** (gpt-5.5):');
+    expect(formatted).toContain('Request timed out');
+    expect(formatted).toContain('Provider error');
   });
 
-  test("handles councillors with result but completed status", () => {
+  test('handles councillors with result but completed status', () => {
     const councillorResults = [
       {
-        name: "alpha",
-        model: "anthropic/claude-opus-4-6",
-        status: "completed",
-        result: "Valid response",
+        name: 'alpha',
+        model: 'anthropic/claude-opus-4-6',
+        status: 'completed',
+        result: 'Valid response',
       },
       {
-        name: "beta",
-        model: "openai/gpt-5.5",
-        status: "completed",
-        result: "Another valid response",
+        name: 'beta',
+        model: 'openai/gpt-5.5',
+        status: 'completed',
+        result: 'Another valid response',
       },
     ];
 
     const formatted = formatCouncillorResults(
       originalPrompt,
-      councillorResults
+      councillorResults,
     );
 
-    expect(formatted).toContain("**alpha** (claude-opus-4-6):");
-    expect(formatted).toContain("Valid response");
-    expect(formatted).toContain("**beta** (gpt-5.5):");
-    expect(formatted).toContain("Another valid response");
-    expect(formatted).toContain(
-      "Synthesize the optimal response based on the above."
-    );
+    expect(formatted).toContain('**alpha** (claude-opus-4-6):');
+    expect(formatted).toContain('Valid response');
+    expect(formatted).toContain('**beta** (gpt-5.5):');
+    expect(formatted).toContain('Another valid response');
+    expect(formatted).toContain('review each councillor response individually');
   });
 });
 
-describe("formatCouncillorPrompt", () => {
-  const userPrompt = "How do I implement async/await in TypeScript?";
+describe('formatCouncillorPrompt', () => {
+  const userPrompt = 'How do I implement async/await in TypeScript?';
 
-  test("returns user prompt unchanged when no councillor prompt is provided", () => {
+  test('returns user prompt unchanged when no councillor prompt is provided', () => {
     const formatted = formatCouncillorPrompt(userPrompt);
     expect(formatted).toBe(userPrompt);
   });
 
-  test("prepends councillor prompt with separator when provided", () => {
+  test('prepends councillor prompt with separator when provided', () => {
     const councillorPrompt =
-      "You are a TypeScript expert. Focus on practical examples.";
+      'You are a TypeScript expert. Focus on practical examples.';
     const formatted = formatCouncillorPrompt(userPrompt, councillorPrompt);
 
     expect(formatted).toContain(councillorPrompt);
     expect(formatted).toContain(userPrompt);
-    expect(formatted).toContain("---");
+    expect(formatted).toContain('---');
     expect(formatted).toMatch(
       new RegExp(
         `^${councillorPrompt.replace(
           /[.*+?^${}()|[\]\\]/g,
-          "\\$&"
-        )}\\n\\n---\\n\\n${userPrompt.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`
-      )
+          '\\$&',
+        )}\\n\\n---\\n\\n${userPrompt.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`,
+      ),
     );
   });
 
-  test("handles multiline councillor prompt", () => {
+  test('handles multiline councillor prompt', () => {
     const councillorPrompt =
-      "You are an expert.\nFocus on clarity.\nProvide code examples.";
+      'You are an expert.\nFocus on clarity.\nProvide code examples.';
     const formatted = formatCouncillorPrompt(userPrompt, councillorPrompt);
 
     expect(formatted).toContain(councillorPrompt);
     expect(formatted).toContain(userPrompt);
-    expect(formatted).toContain("---");
+    expect(formatted).toContain('---');
     expect(formatted).toMatch(
       new RegExp(
         `^${councillorPrompt.replace(
           /[.*+?^${}()|[\]\\]/g,
-          "\\$&"
-        )}\\n\\n---\\n\\n${userPrompt.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`
-      )
+          '\\$&',
+        )}\\n\\n---\\n\\n${userPrompt.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`,
+      ),
     );
   });
 
-  test("handles empty councillor prompt", () => {
-    const formatted = formatCouncillorPrompt(userPrompt, "");
+  test('handles empty councillor prompt', () => {
+    const formatted = formatCouncillorPrompt(userPrompt, '');
     expect(formatted).toBe(userPrompt);
   });
 
-  test("handles multiline user prompt with councillor prompt", () => {
-    const councillorPrompt = "You are an expert.";
-    const multilineUserPrompt = "Line 1\nLine 2\nLine 3";
+  test('handles multiline user prompt with councillor prompt', () => {
+    const councillorPrompt = 'You are an expert.';
+    const multilineUserPrompt = 'Line 1\nLine 2\nLine 3';
     const formatted = formatCouncillorPrompt(
       multilineUserPrompt,
-      councillorPrompt
+      councillorPrompt,
     );
 
     expect(formatted).toContain(councillorPrompt);
     expect(formatted).toContain(multilineUserPrompt);
-    expect(formatted).toContain("---");
+    expect(formatted).toContain('---');
     expect(formatted).toMatch(
       new RegExp(
         `^${councillorPrompt.replace(
           /[.*+?^${}()|[\]\\]/g,
-          "\\$&"
+          '\\$&',
         )}\\n\\n---\\n\\n${multilineUserPrompt.replace(
           /[.*+?^${}()|[\]\\]/g,
-          "\\$&"
-        )}$`
-      )
+          '\\$&',
+        )}$`,
+      ),
     );
   });
 });

src/agents/council.ts

@@ -19,27 +19,52 @@ orchestration system that runs consensus across multiple models.
 1. Call the \`council_session\` tool with the user's prompt
 2. Optionally specify a preset (default: "default")
 3. Receive the councillor responses formatted for synthesis
-4. Synthesize the optimal final answer from the councillor responses
-5. Present the synthesized result to the user
-
-**Synthesis Guidelines**:
-When you receive councillor responses, synthesize them into the optimal final answer:
-- Review all councillor responses thoroughly and create the best possible answer
-- Credit specific insights from individual councillors by name (e.g., "alpha noted that...", "beta suggested...")
-- Clearly explain your reasoning for the chosen approach
-- Be transparent about trade-offs when different approaches have valid pros/cons
-- Note any remaining uncertainties or areas where further investigation is needed
-- If councillors disagree, explain the resolution and your reasoning
-- Acknowledge if consensus was impossible and explain why
-- Don't just average responses — choose the best approach and improve upon it
-- Present the synthesized solution with relevant code examples, concrete details, and clear explanations
+4. Follow the Synthesis Process below
+5. Present the result to the user
+
+**Synthesis Process** (MANDATORY — follow in order):
+1. Read the original user prompt
+2. Review each councillor's response individually — note each councillor's \
+key insight and unique contribution by name
+3. Identify agreements and contradictions between councillors
+4. Resolve contradictions with explicit reasoning
+5. Synthesize the optimal final answer
+6. Format output per the Required Output Format below
 
 **Behavior**:
 - Delegate requests directly to council_session
 - Don't pre-analyze or filter the prompt before calling council_session
-- Synthesize the councillor results into a comprehensive, coherent answer
-- Include attribution for valuable insights from specific councillors
-- If councillors disagree, explain why you chose one approach over another`;
+- Credit specific insights from individual councillors using their names
+- If councillors disagree, explain why you chose one approach over another
+- Do not omit per-councillor details from the final response
+- Do not collapse the output into only a final summary
+- Be transparent about trade-offs when different approaches have valid pros/cons
+- Don't just average responses — choose the best approach and improve upon it
+
+**Required Output Format**:
+Always include these sections in your final response:
+
+## Council Response
+Provide the best synthesized answer. Integrate the strongest points from the \
+councillors, resolve disagreements, and give the user a clear final \
+recommendation or answer. Include relevant code examples and concrete details.
+
+## Councillor Details
+Include each councillor's response separately.
+
+Use each councillor name exactly as provided in the tool result.
+
+Format each councillor like:
+
+### <councillor name>
+<that councillor's response>
+
+If a councillor failed or timed out, include that status briefly.
+
+## Council Summary
+Summarize where councillors agreed, where they disagreed, why you chose the \
+final answer, and any remaining uncertainty. Include a consensus confidence \
+rating: unanimous, majority, or split.`;
 
 export function createCouncilAgent(
   model: string,
@@ -128,7 +153,9 @@ export function formatCouncillorResults(
     const errorDetails = councillorResults
       .map(
         (cr) =>
-          `**${cr.name}** (${shortModelLabel(cr.model)}): ${cr.status} — ${cr.error ?? 'Unknown'}`,
+          `**${cr.name}** (${shortModelLabel(cr.model)}): ${cr.status} — ${
+            cr.error ?? 'Unknown'
+          }`,
       )
       .join('\n');
 
@@ -141,7 +168,8 @@ export function formatCouncillorResults(
     prompt += `\n\n---\n\n**Failed/Timed-out Councillors**:\n${failedSection}`;
   }
 
-  prompt += '\n\n---\n\nSynthesize the optimal response based on the above.';
+  prompt +=
+    '\n\n---\n\nYou MUST follow the Synthesis Process steps before producing output: review each councillor response individually, then produce the required output with a synthesized Council Response, per-councillor details using their exact names, and a Council Summary with consensus confidence rating (unanimous, majority, or split).';
 
   return prompt;
 }