council.test.ts 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558
  1. import { describe, expect, mock, test } from 'bun:test';
  2. import type { CouncilResult } from '../config/council-schema';
  3. import type { CouncilManager } from '../council/council-manager';
  4. import { createCouncilTool } from './council';
  5. function createMockPluginContext() {
  6. return {
  7. client: {
  8. session: {
  9. create: mock(async () => ({})),
  10. messages: mock(async () => ({})),
  11. prompt: mock(async () => ({})),
  12. abort: mock(async () => ({})),
  13. },
  14. },
  15. directory: '/tmp/test',
  16. } as any;
  17. }
  18. // Test mocks can omit 'model' field — it's filled by the manager, not the test
  19. type TestCouncillorResult = {
  20. name: string;
  21. model?: string;
  22. status: 'completed' | 'failed' | 'timed_out';
  23. result?: string;
  24. error?: string;
  25. };
  26. function createMockCouncilManager(
  27. results: {
  28. success?: boolean;
  29. result?: string;
  30. error?: string;
  31. councillorResults?: TestCouncillorResult[];
  32. } = {},
  33. ) {
  34. const councillorResults: CouncilResult['councillorResults'] = (
  35. results.councillorResults ?? [
  36. { name: 'alpha', status: 'completed', result: 'Alpha response' },
  37. { name: 'beta', status: 'completed', result: 'Beta response' },
  38. ]
  39. ).map((cr) => ({
  40. model: 'test/model',
  41. ...cr,
  42. }));
  43. const mockManager = {
  44. runCouncil: mock(async (): Promise<CouncilResult> => {
  45. return {
  46. success: results.success ?? true,
  47. result: 'result' in results ? results.result : 'Synthesized response',
  48. error: results.error,
  49. councillorResults,
  50. };
  51. }),
  52. getDeprecatedFields: mock(() => undefined),
  53. } as unknown as CouncilManager;
  54. return mockManager;
  55. }
  56. describe('council_session tool', () => {
  57. describe('tool definition', () => {
  58. test('creates council_session tool', () => {
  59. const ctx = createMockPluginContext();
  60. const councilManager = createMockCouncilManager();
  61. const tools = createCouncilTool(ctx, councilManager);
  62. expect(tools).toBeDefined();
  63. expect(tools.council_session).toBeDefined();
  64. expect(tools.council_session.description).toBeDefined();
  65. expect(tools.council_session.args).toBeDefined();
  66. });
  67. test('has correct tool description', () => {
  68. const ctx = createMockPluginContext();
  69. const councilManager = createMockCouncilManager();
  70. const tools = createCouncilTool(ctx, councilManager);
  71. expect(tools.council_session.description).toContain('multi-LLM');
  72. expect(tools.council_session.description).toContain('consensus');
  73. expect(tools.council_session.description).toContain('councillors');
  74. });
  75. test('defines required prompt argument', () => {
  76. const ctx = createMockPluginContext();
  77. const councilManager = createMockCouncilManager();
  78. const tools = createCouncilTool(ctx, councilManager);
  79. expect(tools.council_session.args.prompt).toBeDefined();
  80. expect(tools.council_session.args).toHaveProperty('prompt');
  81. });
  82. test('defines optional preset argument', () => {
  83. const ctx = createMockPluginContext();
  84. const councilManager = createMockCouncilManager();
  85. const tools = createCouncilTool(ctx, councilManager);
  86. expect(tools.council_session.args.preset).toBeDefined();
  87. expect(tools.council_session.args).toHaveProperty('preset');
  88. });
  89. });
  90. describe('execute', () => {
  91. test('calls councilManager.runCouncil with correct arguments', async () => {
  92. const ctx = createMockPluginContext();
  93. const councilManager = createMockCouncilManager();
  94. const tools = createCouncilTool(ctx, councilManager);
  95. const _result = await tools.council_session.execute(
  96. {
  97. prompt: 'Test prompt',
  98. preset: 'custom',
  99. },
  100. { sessionID: 'test-session-123' } as any,
  101. );
  102. expect(councilManager.runCouncil).toHaveBeenCalledTimes(1);
  103. expect(councilManager.runCouncil).toHaveBeenCalledWith(
  104. 'Test prompt',
  105. 'custom',
  106. 'test-session-123',
  107. );
  108. });
  109. test('uses default preset when not specified', async () => {
  110. const ctx = createMockPluginContext();
  111. const councilManager = createMockCouncilManager();
  112. const tools = createCouncilTool(ctx, councilManager);
  113. await tools.council_session.execute({ prompt: 'Test prompt' }, {
  114. sessionID: 'test-session-123',
  115. } as any);
  116. expect(councilManager.runCouncil).toHaveBeenCalledWith(
  117. 'Test prompt',
  118. undefined,
  119. 'test-session-123',
  120. );
  121. });
  122. test('returns successful council result with output', async () => {
  123. const ctx = createMockPluginContext();
  124. const councilManager = createMockCouncilManager({
  125. success: true,
  126. result: 'Synthesized answer from council',
  127. councillorResults: [
  128. {
  129. name: 'alpha',
  130. model: 'openai/gpt-5.4-mini',
  131. status: 'completed',
  132. result: 'Alpha says yes',
  133. },
  134. {
  135. name: 'beta',
  136. model: 'google/gemini-3-pro',
  137. status: 'completed',
  138. result: 'Beta says no',
  139. },
  140. ],
  141. });
  142. const tools = createCouncilTool(ctx, councilManager);
  143. const result = await tools.council_session.execute(
  144. { prompt: 'Test prompt' },
  145. { sessionID: 'test-session' } as any,
  146. );
  147. expect(result).toContain('Synthesized answer from council');
  148. expect(result).toContain('Council: 2/2 councillors responded');
  149. });
  150. test('appends councillor summary to successful result', async () => {
  151. const ctx = createMockPluginContext();
  152. const councilManager = createMockCouncilManager({
  153. success: true,
  154. result: 'Main answer',
  155. councillorResults: [
  156. { name: 'alpha', status: 'completed', result: 'A' },
  157. { name: 'beta', status: 'completed', result: 'B' },
  158. { name: 'gamma', status: 'completed', result: 'G' },
  159. ],
  160. });
  161. const tools = createCouncilTool(ctx, councilManager);
  162. const result = await tools.council_session.execute({ prompt: 'Test' }, {
  163. sessionID: 'test',
  164. } as any);
  165. expect(result).toContain('Main answer');
  166. expect(result).toContain('Council: 3/3 councillors responded');
  167. expect(result).toMatch(/---\s*\*Council:/);
  168. });
  169. test('handles mixed councillor success/failure in summary', async () => {
  170. const ctx = createMockPluginContext();
  171. const councilManager = createMockCouncilManager({
  172. success: true,
  173. result: 'Answer',
  174. councillorResults: [
  175. { name: 'alpha', status: 'completed', result: 'A' },
  176. { name: 'beta', status: 'failed', error: 'Error' },
  177. { name: 'gamma', status: 'completed', result: 'G' },
  178. ],
  179. });
  180. const tools = createCouncilTool(ctx, councilManager);
  181. const result = await tools.council_session.execute({ prompt: 'Test' }, {
  182. sessionID: 'test',
  183. } as any);
  184. // Summary should only count completed councillors
  185. expect(result).toContain('Council: 2/3 councillors responded');
  186. });
  187. test('handles all councillors failing', async () => {
  188. const ctx = createMockPluginContext();
  189. const councilManager = createMockCouncilManager({
  190. success: false,
  191. error: 'All councillors failed',
  192. result: undefined,
  193. councillorResults: [
  194. { name: 'alpha', status: 'failed', error: 'Failed' },
  195. { name: 'beta', status: 'timed_out', error: 'Timeout' },
  196. ],
  197. });
  198. const tools = createCouncilTool(ctx, councilManager);
  199. const result = await tools.council_session.execute({ prompt: 'Test' }, {
  200. sessionID: 'test',
  201. } as any);
  202. expect(result).toContain('Council session failed');
  203. expect(result).toContain('All councillors failed');
  204. });
  205. test('handles case when result is undefined', async () => {
  206. const ctx = createMockPluginContext();
  207. const councilManager = createMockCouncilManager({
  208. success: true,
  209. result: undefined,
  210. councillorResults: [
  211. { name: 'alpha', status: 'completed', result: 'A' },
  212. ],
  213. });
  214. const tools = createCouncilTool(ctx, councilManager);
  215. const result = await tools.council_session.execute({ prompt: 'Test' }, {
  216. sessionID: 'test',
  217. } as any);
  218. // Tool uses result ?? '(No output)', so it should show (No output)
  219. // But the mock manager is returning undefined in the outer object
  220. // The tool actually gets the result from the returned object
  221. expect(result).toContain('Council: 1/1 councillors responded');
  222. });
  223. test('converts prompt to string', async () => {
  224. const ctx = createMockPluginContext();
  225. const councilManager = createMockCouncilManager();
  226. const tools = createCouncilTool(ctx, councilManager);
  227. await tools.council_session.execute({ prompt: 12345 as any }, {
  228. sessionID: 'test',
  229. } as any);
  230. expect(councilManager.runCouncil).toHaveBeenCalledWith(
  231. '12345',
  232. undefined,
  233. 'test',
  234. );
  235. });
  236. test('handles preset as non-string (falls back to undefined)', async () => {
  237. const ctx = createMockPluginContext();
  238. const councilManager = createMockCouncilManager();
  239. const tools = createCouncilTool(ctx, councilManager);
  240. await tools.council_session.execute(
  241. { preset: 123 as any, prompt: 'Test' },
  242. { sessionID: 'test' } as any,
  243. );
  244. expect(councilManager.runCouncil).toHaveBeenCalledWith(
  245. 'Test',
  246. undefined,
  247. 'test',
  248. );
  249. });
  250. });
  251. describe('error handling', () => {
  252. test('throws error when toolContext is missing', async () => {
  253. const ctx = createMockPluginContext();
  254. const councilManager = createMockCouncilManager();
  255. const tools = createCouncilTool(ctx, councilManager);
  256. await expect(
  257. tools.council_session.execute({ prompt: 'Test' }, undefined as any),
  258. ).rejects.toThrow('Invalid toolContext');
  259. });
  260. test('throws error when toolContext is not object', async () => {
  261. const ctx = createMockPluginContext();
  262. const councilManager = createMockCouncilManager();
  263. const tools = createCouncilTool(ctx, councilManager);
  264. await expect(
  265. tools.council_session.execute({ prompt: 'Test' }, 'invalid' as any),
  266. ).rejects.toThrow('Invalid toolContext');
  267. });
  268. test('throws error when toolContext is missing sessionID', async () => {
  269. const ctx = createMockPluginContext();
  270. const councilManager = createMockCouncilManager();
  271. const tools = createCouncilTool(ctx, councilManager);
  272. await expect(
  273. tools.council_session.execute({ prompt: 'Test' }, {} as any),
  274. ).rejects.toThrow('Invalid toolContext');
  275. });
  276. test('handles CouncilManager throwing exception', async () => {
  277. const ctx = createMockPluginContext();
  278. const councilManager = {
  279. runCouncil: mock(async () => {
  280. throw new Error('Council manager crashed');
  281. }),
  282. getDeprecatedFields: mock(() => undefined),
  283. } as unknown as CouncilManager;
  284. const tools = createCouncilTool(ctx, councilManager);
  285. await expect(
  286. tools.council_session.execute({ prompt: 'Test' }, {
  287. sessionID: 'test',
  288. } as any),
  289. ).rejects.toThrow('Council manager crashed');
  290. });
  291. });
  292. describe('agent guard', () => {
  293. test('allows council agent to invoke council session', async () => {
  294. const ctx = createMockPluginContext();
  295. const councilManager = createMockCouncilManager({
  296. success: true,
  297. result: 'Synthesised answer',
  298. councillorResults: [
  299. { name: 'alpha', status: 'completed', result: 'A' },
  300. ],
  301. });
  302. const tools = createCouncilTool(ctx, councilManager);
  303. const result = await tools.council_session.execute({ prompt: 'Test' }, {
  304. sessionID: 'test',
  305. agent: 'council',
  306. } as any);
  307. expect(result).toContain('Synthesised answer');
  308. expect(councilManager.runCouncil).toHaveBeenCalledTimes(1);
  309. });
  310. test('blocks orchestrator agent from invoking council session', async () => {
  311. const ctx = createMockPluginContext();
  312. const councilManager = createMockCouncilManager();
  313. const tools = createCouncilTool(ctx, councilManager);
  314. expect(
  315. tools.council_session.execute({ prompt: 'Test' }, {
  316. sessionID: 'test',
  317. agent: 'orchestrator',
  318. } as any),
  319. ).rejects.toThrow(
  320. 'Council sessions can only be invoked by the council agent',
  321. );
  322. expect(councilManager.runCouncil).not.toHaveBeenCalled();
  323. });
  324. test('blocks disallowed agents from invoking council session', async () => {
  325. const ctx = createMockPluginContext();
  326. const councilManager = createMockCouncilManager();
  327. const tools = createCouncilTool(ctx, councilManager);
  328. expect(
  329. tools.council_session.execute({ prompt: 'Test' }, {
  330. sessionID: 'test',
  331. agent: 'explorer',
  332. } as any),
  333. ).rejects.toThrow(
  334. 'Council sessions can only be invoked by the council agent',
  335. );
  336. expect(councilManager.runCouncil).not.toHaveBeenCalled();
  337. });
  338. test('allows undefined agent (backward compatible)', async () => {
  339. const ctx = createMockPluginContext();
  340. const councilManager = createMockCouncilManager({
  341. success: true,
  342. result: 'Synthesised answer',
  343. councillorResults: [
  344. { name: 'alpha', status: 'completed', result: 'A' },
  345. ],
  346. });
  347. const tools = createCouncilTool(ctx, councilManager);
  348. const result = await tools.council_session.execute({ prompt: 'Test' }, {
  349. sessionID: 'test',
  350. } as any);
  351. expect(result).toContain('Synthesised answer');
  352. expect(councilManager.runCouncil).toHaveBeenCalledTimes(1);
  353. });
  354. });
  355. describe('edge cases', () => {
  356. test('handles empty councillor results', async () => {
  357. const ctx = createMockPluginContext();
  358. const councilManager = createMockCouncilManager({
  359. success: false,
  360. error: 'No councillors',
  361. result: undefined,
  362. councillorResults: [],
  363. });
  364. const tools = createCouncilTool(ctx, councilManager);
  365. const result = await tools.council_session.execute({ prompt: 'Test' }, {
  366. sessionID: 'test',
  367. } as any);
  368. // When success is false, tool returns error message without summary
  369. expect(result).toContain('Council session failed');
  370. expect(result).toContain('No councillors');
  371. });
  372. test('handles all councillors timed out', async () => {
  373. const ctx = createMockPluginContext();
  374. const councilManager = createMockCouncilManager({
  375. success: false,
  376. error: 'All timed out',
  377. result: undefined,
  378. councillorResults: [
  379. { name: 'alpha', status: 'timed_out', error: 'Timeout' },
  380. { name: 'beta', status: 'timed_out', error: 'Timeout' },
  381. ],
  382. });
  383. const tools = createCouncilTool(ctx, councilManager);
  384. const result = await tools.council_session.execute({ prompt: 'Test' }, {
  385. sessionID: 'test',
  386. } as any);
  387. // When success is false, tool returns error message without summary
  388. expect(result).toContain('Council session failed');
  389. expect(result).toContain('All timed out');
  390. });
  391. test('handles single successful councillor', async () => {
  392. const ctx = createMockPluginContext();
  393. const councilManager = createMockCouncilManager({
  394. success: true,
  395. result: 'Single result',
  396. councillorResults: [
  397. { name: 'solo', status: 'completed', result: 'Solo answer' },
  398. ],
  399. });
  400. const tools = createCouncilTool(ctx, councilManager);
  401. const result = await tools.council_session.execute({ prompt: 'Test' }, {
  402. sessionID: 'test',
  403. } as any);
  404. expect(result).toContain('Single result');
  405. expect(result).toContain('Council: 1/1 councillors responded');
  406. });
  407. test('handles many councillors', async () => {
  408. const ctx = createMockPluginContext();
  409. const councilManager = createMockCouncilManager({
  410. success: true,
  411. result: 'Multi result',
  412. councillorResults: Array.from({ length: 10 }, (_, i) => ({
  413. name: `councillor${i}`,
  414. status: 'completed',
  415. result: `Response ${i}`,
  416. })),
  417. });
  418. const tools = createCouncilTool(ctx, councilManager);
  419. const result = await tools.council_session.execute({ prompt: 'Test' }, {
  420. sessionID: 'test',
  421. } as any);
  422. expect(result).toContain('Council: 10/10 councillors responded');
  423. });
  424. test('includes deprecation warning when deprecated config fields detected', async () => {
  425. const ctx = createMockPluginContext();
  426. const councilManager = {
  427. runCouncil: mock(async () => ({
  428. success: true,
  429. result: 'Synthesized response',
  430. councillorResults: [
  431. {
  432. name: 'alpha',
  433. model: 'test/model',
  434. status: 'completed',
  435. result: 'Response',
  436. },
  437. ],
  438. })),
  439. getDeprecatedFields: mock(() => ['master', 'master_timeout']),
  440. getLegacyMasterModel: mock(() => undefined),
  441. } as unknown as CouncilManager;
  442. const tools = createCouncilTool(ctx, councilManager);
  443. const result = await tools.council_session.execute({ prompt: 'Test' }, {
  444. sessionID: 'test',
  445. } as any);
  446. expect(result).toContain('Config warning');
  447. expect(result).toContain('`council.master`');
  448. expect(result).toContain('`council.master_timeout`');
  449. // master with no legacy model → both treated as ignored
  450. expect(result).toContain('deprecated and ignored');
  451. });
  452. test('includes fallback warning when legacy master.model is used', async () => {
  453. const ctx = createMockPluginContext();
  454. const councilManager = {
  455. runCouncil: mock(async () => ({
  456. success: true,
  457. result: 'Synthesized response',
  458. councillorResults: [
  459. {
  460. name: 'alpha',
  461. model: 'test/model',
  462. status: 'completed',
  463. result: 'Response',
  464. },
  465. ],
  466. })),
  467. getDeprecatedFields: mock(() => ['master', 'master_timeout']),
  468. getLegacyMasterModel: mock(() => 'anthropic/claude-opus-4-6'),
  469. } as unknown as CouncilManager;
  470. const tools = createCouncilTool(ctx, councilManager);
  471. const result = await tools.council_session.execute({ prompt: 'Test' }, {
  472. sessionID: 'test',
  473. } as any);
  474. expect(result).toContain('Config warning');
  475. expect(result).toContain('`council.master`');
  476. // master with legacy model → fallback warning
  477. expect(result).toContain('fallback for the council agent');
  478. // master_timeout is still "ignored"
  479. expect(result).toContain('deprecated and ignored');
  480. });
  481. });
  482. });