Browse Source

fix(council): add mandatory numbered synthesis process, move output format to prompt end

Restore sequential per-councillor engagement enforcement that was lost
in PR #356 when council-master was merged into council agent.

Based on council research findings:
- Numbered process steps create distributional anchoring (Wei et al. 2022)
- Output format at prompt end leverages recency bias (Liu et al. 2023)
- MANDATORY + MUST instructions for strong enforcement

Co-authored-by: alvinunreal <alvinunreal@users.noreply.github.com>
ReqX 3 months ago
parent
commit
2ad1e207e8
2 changed files with 147 additions and 149 deletions
  1. 103 105
      src/agents/council.test.ts
  2. 44 44
      src/agents/council.ts

+ 103 - 105
src/agents/council.test.ts

@@ -1,227 +1,225 @@
-import { describe, expect, test } from "bun:test";
-import { formatCouncillorPrompt, formatCouncillorResults } from "./council";
+import { describe, expect, test } from 'bun:test';
+import { formatCouncillorPrompt, formatCouncillorResults } from './council';
 
-describe("formatCouncillorResults", () => {
+describe('formatCouncillorResults', () => {
   const originalPrompt =
-    "What is the best way to implement a REST API in TypeScript?";
+    'What is the best way to implement a REST API in TypeScript?';
 
-  test("formats completed councillor results correctly", () => {
+  test('formats completed councillor results correctly', () => {
     const councillorResults = [
       {
-        name: "alpha",
-        model: "anthropic/claude-opus-4-6",
-        status: "completed",
-        result: "Use Express.js with TypeScript interfaces for type safety.",
+        name: 'alpha',
+        model: 'anthropic/claude-opus-4-6',
+        status: 'completed',
+        result: 'Use Express.js with TypeScript interfaces for type safety.',
       },
       {
-        name: "beta",
-        model: "openai/gpt-5.5",
-        status: "completed",
+        name: 'beta',
+        model: 'openai/gpt-5.5',
+        status: 'completed',
         result:
-          "Consider Fastify for better performance and built-in type validation.",
+          'Consider Fastify for better performance and built-in type validation.',
       },
     ];
 
     const formatted = formatCouncillorResults(
       originalPrompt,
-      councillorResults
+      councillorResults,
     );
 
-    expect(formatted).toContain("**Original Prompt**:");
+    expect(formatted).toContain('**Original Prompt**:');
     expect(formatted).toContain(originalPrompt);
-    expect(formatted).toContain("**alpha** (claude-opus-4-6):");
-    expect(formatted).toContain("**beta** (gpt-5.5):");
+    expect(formatted).toContain('**alpha** (claude-opus-4-6):');
+    expect(formatted).toContain('**beta** (gpt-5.5):');
     expect(formatted).toContain(
-      "Use Express.js with TypeScript interfaces for type safety."
+      'Use Express.js with TypeScript interfaces for type safety.',
     );
     expect(formatted).toContain(
-      "Consider Fastify for better performance and built-in type validation."
+      'Consider Fastify for better performance and built-in type validation.',
     );
-    expect(formatted).toContain("**Councillor Responses**:");
+    expect(formatted).toContain('**Councillor Responses**:');
     expect(formatted).toContain(
-      "Produce the required final response: include a synthesized Council Response"
+      'You MUST follow the Synthesis Process steps before producing output',
     );
     expect(formatted).toContain(
-      "consensus confidence rating of unanimous, majority, or split."
+      'consensus confidence rating (unanimous, majority, or split)',
     );
-    expect(formatted).not.toContain("**Failed/Timed-out Councillors**:");
+    expect(formatted).not.toContain('**Failed/Timed-out Councillors**:');
   });
 
-  test("includes failed councillors section when some fail", () => {
+  test('includes failed councillors section when some fail', () => {
     const councillorResults = [
       {
-        name: "alpha",
-        model: "anthropic/claude-opus-4-6",
-        status: "completed",
-        result: "Use Express.js with TypeScript interfaces for type safety.",
+        name: 'alpha',
+        model: 'anthropic/claude-opus-4-6',
+        status: 'completed',
+        result: 'Use Express.js with TypeScript interfaces for type safety.',
       },
       {
-        name: "beta",
-        model: "openai/gpt-5.5",
-        status: "timed_out",
-        error: "Request timed out after 180000ms",
+        name: 'beta',
+        model: 'openai/gpt-5.5',
+        status: 'timed_out',
+        error: 'Request timed out after 180000ms',
       },
       {
-        name: "gamma",
-        model: "google/gemini-pro",
-        status: "failed",
-        error: "Provider returned empty response",
+        name: 'gamma',
+        model: 'google/gemini-pro',
+        status: 'failed',
+        error: 'Provider returned empty response',
       },
     ];
 
     const formatted = formatCouncillorResults(
       originalPrompt,
-      councillorResults
+      councillorResults,
     );
 
-    expect(formatted).toContain("**Councillor Responses**:");
-    expect(formatted).toContain("**alpha** (claude-opus-4-6):");
+    expect(formatted).toContain('**Councillor Responses**:');
+    expect(formatted).toContain('**alpha** (claude-opus-4-6):');
     expect(formatted).toContain(
-      "Use Express.js with TypeScript interfaces for type safety."
+      'Use Express.js with TypeScript interfaces for type safety.',
     );
-    expect(formatted).toContain("**Failed/Timed-out Councillors**:");
+    expect(formatted).toContain('**Failed/Timed-out Councillors**:');
     expect(formatted).toContain(
-      "**beta**: timed_out — Request timed out after 180000ms"
+      '**beta**: timed_out — Request timed out after 180000ms',
     );
     expect(formatted).toContain(
-      "**gamma**: failed — Provider returned empty response"
+      '**gamma**: failed — Provider returned empty response',
     );
-    expect(formatted).not.toContain("**beta** (gpt-5.5):");
-    expect(formatted).not.toContain("**gamma** (gemini-pro):");
+    expect(formatted).not.toContain('**beta** (gpt-5.5):');
+    expect(formatted).not.toContain('**gamma** (gemini-pro):');
   });
 
-  test("returns fallback message when all councillors fail", () => {
+  test('returns fallback message when all councillors fail', () => {
     const councillorResults = [
       {
-        name: "alpha",
-        model: "anthropic/claude-opus-4-6",
-        status: "timeout",
-        error: "Request timed out",
+        name: 'alpha',
+        model: 'anthropic/claude-opus-4-6',
+        status: 'timeout',
+        error: 'Request timed out',
       },
       {
-        name: "beta",
-        model: "openai/gpt-5.5",
-        status: "error",
-        error: "Provider error",
+        name: 'beta',
+        model: 'openai/gpt-5.5',
+        status: 'error',
+        error: 'Provider error',
       },
     ];
 
     const formatted = formatCouncillorResults(
       originalPrompt,
-      councillorResults
+      councillorResults,
     );
 
-    expect(formatted).toContain("**Original Prompt**:");
+    expect(formatted).toContain('**Original Prompt**:');
     expect(formatted).toContain(originalPrompt);
-    expect(formatted).toContain("**Councillor Responses**:");
-    expect(formatted).toContain("All councillors failed to produce output:");
-    expect(formatted).toContain("**alpha** (claude-opus-4-6):");
-    expect(formatted).toContain("**beta** (gpt-5.5):");
-    expect(formatted).toContain("Request timed out");
-    expect(formatted).toContain("Provider error");
+    expect(formatted).toContain('**Councillor Responses**:');
+    expect(formatted).toContain('All councillors failed to produce output:');
+    expect(formatted).toContain('**alpha** (claude-opus-4-6):');
+    expect(formatted).toContain('**beta** (gpt-5.5):');
+    expect(formatted).toContain('Request timed out');
+    expect(formatted).toContain('Provider error');
   });
 
-  test("handles councillors with result but completed status", () => {
+  test('handles councillors with result but completed status', () => {
     const councillorResults = [
       {
-        name: "alpha",
-        model: "anthropic/claude-opus-4-6",
-        status: "completed",
-        result: "Valid response",
+        name: 'alpha',
+        model: 'anthropic/claude-opus-4-6',
+        status: 'completed',
+        result: 'Valid response',
       },
       {
-        name: "beta",
-        model: "openai/gpt-5.5",
-        status: "completed",
-        result: "Another valid response",
+        name: 'beta',
+        model: 'openai/gpt-5.5',
+        status: 'completed',
+        result: 'Another valid response',
       },
     ];
 
     const formatted = formatCouncillorResults(
       originalPrompt,
-      councillorResults
+      councillorResults,
     );
 
-    expect(formatted).toContain("**alpha** (claude-opus-4-6):");
-    expect(formatted).toContain("Valid response");
-    expect(formatted).toContain("**beta** (gpt-5.5):");
-    expect(formatted).toContain("Another valid response");
-    expect(formatted).toContain(
-      "preserve each individual councillor response in Councillor Details"
-    );
+    expect(formatted).toContain('**alpha** (claude-opus-4-6):');
+    expect(formatted).toContain('Valid response');
+    expect(formatted).toContain('**beta** (gpt-5.5):');
+    expect(formatted).toContain('Another valid response');
+    expect(formatted).toContain('review each councillor response individually');
   });
 });
 
-describe("formatCouncillorPrompt", () => {
-  const userPrompt = "How do I implement async/await in TypeScript?";
+describe('formatCouncillorPrompt', () => {
+  const userPrompt = 'How do I implement async/await in TypeScript?';
 
-  test("returns user prompt unchanged when no councillor prompt is provided", () => {
+  test('returns user prompt unchanged when no councillor prompt is provided', () => {
     const formatted = formatCouncillorPrompt(userPrompt);
     expect(formatted).toBe(userPrompt);
   });
 
-  test("prepends councillor prompt with separator when provided", () => {
+  test('prepends councillor prompt with separator when provided', () => {
     const councillorPrompt =
-      "You are a TypeScript expert. Focus on practical examples.";
+      'You are a TypeScript expert. Focus on practical examples.';
     const formatted = formatCouncillorPrompt(userPrompt, councillorPrompt);
 
     expect(formatted).toContain(councillorPrompt);
     expect(formatted).toContain(userPrompt);
-    expect(formatted).toContain("---");
+    expect(formatted).toContain('---');
     expect(formatted).toMatch(
       new RegExp(
         `^${councillorPrompt.replace(
           /[.*+?^${}()|[\]\\]/g,
-          "\\$&"
-        )}\\n\\n---\\n\\n${userPrompt.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`
-      )
+          '\\$&',
+        )}\\n\\n---\\n\\n${userPrompt.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`,
+      ),
     );
   });
 
-  test("handles multiline councillor prompt", () => {
+  test('handles multiline councillor prompt', () => {
     const councillorPrompt =
-      "You are an expert.\nFocus on clarity.\nProvide code examples.";
+      'You are an expert.\nFocus on clarity.\nProvide code examples.';
     const formatted = formatCouncillorPrompt(userPrompt, councillorPrompt);
 
     expect(formatted).toContain(councillorPrompt);
     expect(formatted).toContain(userPrompt);
-    expect(formatted).toContain("---");
+    expect(formatted).toContain('---');
     expect(formatted).toMatch(
       new RegExp(
         `^${councillorPrompt.replace(
           /[.*+?^${}()|[\]\\]/g,
-          "\\$&"
-        )}\\n\\n---\\n\\n${userPrompt.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`
-      )
+          '\\$&',
+        )}\\n\\n---\\n\\n${userPrompt.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`,
+      ),
     );
   });
 
-  test("handles empty councillor prompt", () => {
-    const formatted = formatCouncillorPrompt(userPrompt, "");
+  test('handles empty councillor prompt', () => {
+    const formatted = formatCouncillorPrompt(userPrompt, '');
     expect(formatted).toBe(userPrompt);
   });
 
-  test("handles multiline user prompt with councillor prompt", () => {
-    const councillorPrompt = "You are an expert.";
-    const multilineUserPrompt = "Line 1\nLine 2\nLine 3";
+  test('handles multiline user prompt with councillor prompt', () => {
+    const councillorPrompt = 'You are an expert.';
+    const multilineUserPrompt = 'Line 1\nLine 2\nLine 3';
     const formatted = formatCouncillorPrompt(
       multilineUserPrompt,
-      councillorPrompt
+      councillorPrompt,
     );
 
     expect(formatted).toContain(councillorPrompt);
     expect(formatted).toContain(multilineUserPrompt);
-    expect(formatted).toContain("---");
+    expect(formatted).toContain('---');
     expect(formatted).toMatch(
       new RegExp(
         `^${councillorPrompt.replace(
           /[.*+?^${}()|[\]\\]/g,
-          "\\$&"
+          '\\$&',
         )}\\n\\n---\\n\\n${multilineUserPrompt.replace(
           /[.*+?^${}()|[\]\\]/g,
-          "\\$&"
-        )}$`
-      )
+          '\\$&',
+        )}$`,
+      ),
     );
   });
 });

+ 44 - 44
src/agents/council.ts

@@ -1,5 +1,5 @@
-import { shortModelLabel } from "../utils/session";
-import { type AgentDefinition, resolvePrompt } from "./orchestrator";
+import { shortModelLabel } from '../utils/session';
+import { type AgentDefinition, resolvePrompt } from './orchestrator';
 
 // NOTE: Councillor system prompts live in the councillor agent factory.
 // The format functions below only structure the USER message content — the
@@ -19,14 +19,35 @@ orchestration system that runs consensus across multiple models.
 1. Call the \`council_session\` tool with the user's prompt
 2. Optionally specify a preset (default: "default")
 3. Receive the councillor responses formatted for synthesis
-4. Synthesize the optimal final answer from the councillor responses
-5. Present both the synthesized answer and the per-councillor details to the user
+4. Follow the Synthesis Process below
+5. Present the result to the user
+
+**Synthesis Process** (MANDATORY — follow in order):
+1. Read the original user prompt
+2. Review each councillor's response individually — note each councillor's \
+key insight and unique contribution by name
+3. Identify agreements and contradictions between councillors
+4. Resolve contradictions with explicit reasoning
+5. Synthesize the optimal final answer
+6. Format output per the Required Output Format below
+
+**Behavior**:
+- Delegate requests directly to council_session
+- Don't pre-analyze or filter the prompt before calling council_session
+- Credit specific insights from individual councillors using their names
+- If councillors disagree, explain why you chose one approach over another
+- Do not omit per-councillor details from the final response
+- Do not collapse the output into only a final summary
+- Be transparent about trade-offs when different approaches have valid pros/cons
+- Don't just average responses — choose the best approach and improve upon it
 
 **Required Output Format**:
 Always include these sections in your final response:
 
 ## Council Response
-Provide the best synthesized answer. Integrate the strongest points from the councillors, resolve disagreements, and give the user a clear final recommendation or answer.
+Provide the best synthesized answer. Integrate the strongest points from the \
+councillors, resolve disagreements, and give the user a clear final \
+recommendation or answer. Include relevant code examples and concrete details.
 
 ## Councillor Details
 Include each councillor's response separately.
@@ -41,46 +62,25 @@ Format each councillor like:
 If a councillor failed or timed out, include that status briefly.
 
 ## Council Summary
-Summarize where councillors agreed, where they disagreed, why you chose the final answer, and any remaining uncertainty. Include a consensus confidence rating: unanimous, majority, or split.
-
-**Synthesis Guidelines**:
-When you receive councillor responses, synthesize them into the optimal final answer:
-- Review all councillor responses thoroughly and create the best possible answer
-- Preserve each councillor's individual response in the Councillor Details section
-- Credit specific insights from individual councillors using their provided names
-- Clearly explain your reasoning for the chosen approach
-- Be transparent about trade-offs when different approaches have valid pros/cons
-- Note any remaining uncertainties or areas where further investigation is needed
-- If councillors disagree, explain the resolution and your reasoning
-- Rate the consensus confidence as unanimous, majority, or split
-- Acknowledge if consensus was impossible and explain why
-- Don't just average responses — choose the best approach and improve upon it
-- Present the synthesized solution with relevant code examples, concrete details, and clear explanations
-
-**Behavior**:
-- Delegate requests directly to council_session
-- Don't pre-analyze or filter the prompt before calling council_session
-- Synthesize the councillor results into a comprehensive, coherent answer
-- Do not omit per-councillor details from the final response
-- Do not collapse the output into only a final summary
-- Include attribution for valuable insights from specific councillors
-- If councillors disagree, explain why you chose one approach over another`;
+Summarize where councillors agreed, where they disagreed, why you chose the \
+final answer, and any remaining uncertainty. Include a consensus confidence \
+rating: unanimous, majority, or split.`;
 
 export function createCouncilAgent(
   model: string,
   customPrompt?: string,
-  customAppendPrompt?: string
+  customAppendPrompt?: string,
 ): AgentDefinition {
   const prompt = resolvePrompt(
     COUNCIL_AGENT_PROMPT,
     customPrompt,
-    customAppendPrompt
+    customAppendPrompt,
   );
 
   const definition: AgentDefinition = {
-    name: "council",
+    name: 'council',
     description:
-      "Multi-LLM council agent that synthesizes responses from multiple models for higher-quality outputs",
+      'Multi-LLM council agent that synthesizes responses from multiple models for higher-quality outputs',
     config: {
       temperature: 0.1,
       prompt,
@@ -107,7 +107,7 @@ export function createCouncilAgent(
  */
 export function formatCouncillorPrompt(
   userPrompt: string,
-  councillorPrompt?: string
+  councillorPrompt?: string,
 ): string {
   if (!councillorPrompt) return userPrompt;
   return `${councillorPrompt}\n\n---\n\n${userPrompt}`;
@@ -129,10 +129,10 @@ export function formatCouncillorResults(
     status: string;
     result?: string;
     error?: string;
-  }>
+  }>,
 ): string {
   const completedWithResults = councillorResults.filter(
-    (cr) => cr.status === "completed" && cr.result
+    (cr) => cr.status === 'completed' && cr.result,
   );
 
   const councillorSection = completedWithResults
@@ -140,12 +140,12 @@ export function formatCouncillorResults(
       const shortModel = shortModelLabel(cr.model);
       return `**${cr.name}** (${shortModel}):\n${cr.result}`;
     })
-    .join("\n\n");
+    .join('\n\n');
 
   const failedSection = councillorResults
-    .filter((cr) => cr.status !== "completed")
-    .map((cr) => `**${cr.name}**: ${cr.status} — ${cr.error ?? "Unknown"}`)
-    .join("\n");
+    .filter((cr) => cr.status !== 'completed')
+    .map((cr) => `**${cr.name}**: ${cr.status} — ${cr.error ?? 'Unknown'}`)
+    .join('\n');
 
   // Defensive guard: caller (runCouncil) short-circuits when all fail,
   // but this function may be reused in other contexts.
@@ -154,10 +154,10 @@ export function formatCouncillorResults(
       .map(
         (cr) =>
           `**${cr.name}** (${shortModelLabel(cr.model)}): ${cr.status} — ${
-            cr.error ?? "Unknown"
-          }`
+            cr.error ?? 'Unknown'
+          }`,
       )
-      .join("\n");
+      .join('\n');
 
     return `---\n\n**Original Prompt**:\n${originalPrompt}\n\n---\n\n**Councillor Responses**:\nAll councillors failed to produce output:\n${errorDetails}\n\nPlease generate a response based on the original prompt alone.`;
   }
@@ -169,7 +169,7 @@ export function formatCouncillorResults(
   }
 
   prompt +=
-    "\n\n---\n\nProduce the required final response: include a synthesized Council Response, preserve each individual councillor response in Councillor Details using the provided councillor names exactly as given, and finish with a Council Summary that includes a consensus confidence rating of unanimous, majority, or split.";
+    '\n\n---\n\nYou MUST follow the Synthesis Process steps before producing output: review each councillor response individually, then produce the required output with a synthesized Council Response, per-councillor details using their exact names, and a Council Summary with consensus confidence rating (unanimous, majority, or split).';
 
   return prompt;
 }