fix(evals): clear pre-existing TypeScript build errors (tsc 39 -> 0)

`npm run build` (tsc) failed on 39 pre-existing errors in files outside
the behavioral-repair scope, blocking the baseline gate. Root causes:

- glob: drop obsolete @types/glob ^8 so glob v13's own types (globSync)
  resolve (fixes run-sdk-tests.ts).
- task-type-detector: introduce a local DetectedTaskType union for this
  (dead/unused) detector; shared TaskType union left unchanged.
- behavior-evaluator: the expectedResponse check filtered a nonexistent
  `message` event type and was silently dead — route through
  base-evaluator.getAssistantMessages (assistant_message/text). Adds
  three deterministic tests exercising the now-live check.
- integration tests: remove unused `expectedOutcome` fixture metadata
  (not on TestCase, never asserted) and add the required `category`
  field the excess-property error had masked.

tsc now exits 0. Vitest allowlist 112/112. Lint remains unconfigured in
evals/framework (no ESLint config has ever existed) and is deferred to a
separate follow-up; it is not part of this change.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

evals/framework/package.json

@@ -38,7 +38,6 @@
     "zod": "^3.25.76"
   },
   "devDependencies": {
-    "@types/glob": "^8.1.0",
     "@types/node": "^20.10.0",
     "@typescript-eslint/eslint-plugin": "^6.13.0",
     "@typescript-eslint/parser": "^6.13.0",

evals/framework/src/__tests__/eval-pipeline-integration.test.ts

@@ -70,13 +70,10 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Say "Hello Integration Test" and nothing else.',
         timeout: 15000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'text-response',
-          contains: ['Hello Integration Test'],
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -115,13 +112,10 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'List files in the current directory using the List tool.',
         timeout: 20000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'tool-execution',
-          tools: ['list'],
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -159,12 +153,10 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Create a file named test.txt with content "test".',
         timeout: 20000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-deny', // Deny all approvals
         },
-        expectedOutcome: {
-          type: 'approval-denied',
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -192,8 +184,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
           model: 'opencode/grok-code-fast',
           prompt: 'Say "Test 1".',
           timeout: 15000,
+          category: 'developer',
           approvalStrategy: { type: 'auto-approve' },
-          expectedOutcome: { type: 'text-response', contains: ['Test 1'] },
         },
         {
           id: 'integration-multi-2',
@@ -203,8 +195,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
           model: 'opencode/grok-code-fast',
           prompt: 'Say "Test 2".',
           timeout: 15000,
+          category: 'developer',
           approvalStrategy: { type: 'auto-approve' },
-          expectedOutcome: { type: 'text-response', contains: ['Test 2'] },
         },
       ];
 
@@ -236,8 +228,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'List files in current directory.',
         timeout: 20000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'tool-execution', tools: ['list'] },
       };
 
       const result = await runner.runTest(testCase);
@@ -271,8 +263,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Use cat command to read a file.', // Should trigger tool-usage violation
         timeout: 20000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'tool-execution' },
       };
 
       const result = await runner.runTest(testCase);
@@ -312,8 +304,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'List files and then say "Done".',
         timeout: 20000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'text-response', contains: ['Done'] },
       };
 
       const result = await runner.runTest(testCase);
@@ -347,8 +339,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Say "No tools needed" and nothing else.',
         timeout: 15000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'text-response', contains: ['No tools needed'] },
       };
 
       const result = await runner.runTest(testCase);
@@ -380,8 +372,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Perform a very long task that takes forever.',
         timeout: 5000, // Very short timeout
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'text-response' },
       };
 
       const result = await runner.runTest(testCase);
@@ -421,11 +413,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Say "Validation Test".',
         timeout: 15000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: {
-          type: 'text-response',
-          contains: ['Validation Test'],
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -472,8 +461,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Say "Score Test".',
         timeout: 15000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'text-response', contains: ['Score Test'] },
       };
 
       const result = await runner.runTest(testCase);
@@ -505,8 +494,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Say "Report Test".',
         timeout: 15000,
+        category: 'developer',
         approvalStrategy: { type: 'auto-approve' },
-        expectedOutcome: { type: 'text-response', contains: ['Report Test'] },
       };
 
       const result = await runner.runTest(testCase);
@@ -549,8 +538,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
           model: 'opencode/grok-code-fast',
           prompt: 'Say "Batch 1".',
           timeout: 15000,
+          category: 'developer',
           approvalStrategy: { type: 'auto-approve' },
-          expectedOutcome: { type: 'text-response', contains: ['Batch 1'] },
         },
         {
           id: 'integration-batch-2',
@@ -560,8 +549,8 @@ describe.skipIf(skipIntegration)('Eval Pipeline Integration', () => {
           model: 'opencode/grok-code-fast',
           prompt: 'Say "Batch 2".',
           timeout: 15000,
+          category: 'developer',
           approvalStrategy: { type: 'auto-approve' },
-          expectedOutcome: { type: 'text-response', contains: ['Batch 2'] },
         },
       ];
 

evals/framework/src/__tests__/llm-integration.test.ts

@@ -78,12 +78,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
           },
         ],
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'text-response',
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -122,13 +120,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
           },
         ],
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'text-response',
-          contains: ['blue'],
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -150,13 +145,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Read the package.json file',
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'tool-execution',
-          tools: ['read'],
-        },
         behavior: {
           requiresApproval: true,
         },
@@ -186,12 +178,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Create a new file called test.txt with content "Hello World"',
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-deny',
         },
-        expectedOutcome: {
-          type: 'approval-denied',
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -219,13 +209,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Say "Performance test complete"',
         timeout: 15000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'text-response',
-          contains: ['Performance test complete'],
-        },
       };
 
       const startTime = Date.now();
@@ -253,13 +240,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Read a file that does not exist: /nonexistent/file.txt',
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'tool-execution',
-          tools: ['read'],
-        },
       };
 
       const result = await runner.runTest(testCase);
@@ -286,13 +270,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Read the package.json file and list all files in the current directory',
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'tool-execution',
-          tools: ['read', 'glob'],
-        },
         behavior: {
           mustUseDedicatedTools: true,
           mustUseAnyOf: [
@@ -333,13 +314,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Create a new TypeScript file called math.ts with a function called add that adds two numbers',
         timeout: 45000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'tool-execution',
-          tools: ['write'],
-        },
         behavior: {
           requiresContext: true,
           expectedContextFiles: [
@@ -397,12 +375,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Tell me about the package.json file',
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'text-response',
-        },
         behavior: {
           mustNotUseTools: ['bash'], // Should not use bash for this task
           mayUseTools: ['read', 'glob'], // Can use these if needed
@@ -445,13 +421,10 @@ describe.skipIf(skipLLMTests)('LLM Integration Tests', () => {
         model: 'opencode/grok-code-fast',
         prompt: 'Read the package.json file using the Read tool',
         timeout: 30000,
+        category: 'developer',
         approvalStrategy: {
           type: 'auto-approve',
         },
-        expectedOutcome: {
-          type: 'tool-execution',
-          tools: ['read'],
-        },
       };
 
       const result = await runner.runTest(testCase);

evals/framework/src/evaluators/__tests__/evaluator-reliability.test.ts

@@ -212,8 +212,55 @@ describe('Evaluator Reliability - False Negatives', () => {
       expect(result.violations.length).toBeGreaterThan(0);
       expect(result.violations[0].type).toBe('insufficient-tool-calls');
     });
+
+    it('MUST FAIL: Assistant response missing expected content', async () => {
+      const evaluator = new BehaviorEvaluator({
+        expectedResponse: {
+          contains: ['Task completed successfully'],
+        },
+      });
+
+      const timeline: TimelineEvent[] = [
+        createUserMessage('Do the task', 1000),
+        createAssistantMessage('I ran into an error and stopped.', 2000),
+      ];
+
+      const result = await evaluator.evaluate(timeline, mockSessionInfo);
+
+      // Expected string is absent from the assistant message -> violation.
+      expect(result.passed).toBe(false);
+      const missingContent = result.violations.find(
+        v => v.type === 'missing-expected-content'
+      );
+      expect(missingContent).toBeDefined();
+      expect(missingContent?.evidence?.missingStrings).toContain(
+        'Task completed successfully'
+      );
+    });
+
+    it('MUST FAIL: Assistant response contains forbidden content', async () => {
+      const evaluator = new BehaviorEvaluator({
+        expectedResponse: {
+          notContains: ['rm -rf /'],
+        },
+      });
+
+      const timeline: TimelineEvent[] = [
+        createUserMessage('Clean up', 1000),
+        createAssistantMessage('Sure, running rm -rf / now.', 2000),
+      ];
+
+      const result = await evaluator.evaluate(timeline, mockSessionInfo);
+
+      expect(result.passed).toBe(false);
+      const forbidden = result.violations.find(
+        v => v.type === 'forbidden-content-found'
+      );
+      expect(forbidden).toBeDefined();
+      expect(forbidden?.evidence?.foundForbiddenStrings).toContain('rm -rf /');
+    });
   });
-  
+
   describe('StopOnFailureEvaluator', () => {
     it('MUST FAIL: Agent auto-fixes error without reporting', async () => {
       const evaluator = new StopOnFailureEvaluator();
@@ -391,14 +438,41 @@ describe('Evaluator Reliability - False Positives', () => {
         createToolCall('read', { filePath: 'test.ts' }, 1000),
         createToolCall('write', { filePath: 'output.ts', content: 'test' }, 2000),
       ];
-      
+
       const result = await evaluator.evaluate(timeline, mockSessionInfo);
-      
+
       expect(result.passed).toBe(true);
       expect(result.violations.length).toBe(0);
     });
+
+    it('MUST PASS: Assistant response satisfies expectedResponse', async () => {
+      const evaluator = new BehaviorEvaluator({
+        expectedResponse: {
+          contains: ['Task completed successfully'],
+          notContains: ['rm -rf /'],
+        },
+      });
+
+      const timeline: TimelineEvent[] = [
+        createUserMessage('Do the task', 1000),
+        createAssistantMessage('Task completed successfully. All good.', 2000),
+      ];
+
+      const result = await evaluator.evaluate(timeline, mockSessionInfo);
+
+      // Assistant message satisfies both contains and notContains -> no
+      // expectedResponse violations.
+      expect(
+        result.violations.some(
+          v =>
+            v.type === 'missing-expected-content' ||
+            v.type === 'forbidden-content-found'
+        )
+      ).toBe(false);
+      expect(result.passed).toBe(true);
+    });
   });
-  
+
   describe('ToolUsageEvaluator', () => {
     it('MUST PASS: Agent uses read tool instead of bash cat', async () => {
       const evaluator = new ToolUsageEvaluator();

evals/framework/src/evaluators/behavior-evaluator.ts

@@ -464,9 +464,7 @@ export class BehaviorEvaluator extends BaseEvaluator {
 
     // Check 8: expectedResponse (validate response content)
     if (this.behavior.expectedResponse) {
-      const assistantMessages = timeline.filter(
-        e => e.type === 'message' && e.data?.role === 'assistant' && e.data?.text
-      );
+      const assistantMessages = this.getAssistantMessages(timeline);
       
       // Combine all assistant messages into one text for validation
       const fullResponse = assistantMessages

evals/framework/src/evaluators/task-type-detector.ts

@@ -7,10 +7,33 @@
 
 import type { TimelineEvent, TaskType } from '../types/index.js';
 
+/**
+ * Task type taxonomy specific to the detector.
+ *
+ * This is intentionally distinct from (and richer than) the shared `TaskType`
+ * union in `../types/index.js`. The detector distinguishes file-operation
+ * intents (create/modify/delete) and conversational/read-only sessions that the
+ * shared union does not model. Every member below corresponds to a literal
+ * returned by `detectTaskType` and a key used in the applicability matrix.
+ */
+export type DetectedTaskType =
+  | 'read-only'
+  | 'create-new-file'
+  | 'modify-existing-file'
+  | 'delete-file'
+  | 'conversational'
+  | 'delegation'
+  | 'bash-only'
+  | 'code'
+  | 'docs'
+  | 'tests'
+  | 'review'
+  | 'unknown';
+
 /**
  * Detect task type from user message and timeline events
  */
-export function detectTaskType(userMessage: string | any, timeline: TimelineEvent[]): TaskType {
+export function detectTaskType(userMessage: string | any, timeline: TimelineEvent[]): DetectedTaskType {
   // Extract text from userMessage (could be string or object)
   const messageText = typeof userMessage === 'string' 
     ? userMessage 
@@ -140,7 +163,7 @@ export function getEvaluatorApplicability(
   evaluatorName: string,
   taskType: TaskType
 ): { applicable: boolean; reason?: string } {
-  const matrix: Record<string, Partial<Record<TaskType, { applicable: boolean; reason?: string }>>> = {
+  const matrix: Record<string, Partial<Record<DetectedTaskType, { applicable: boolean; reason?: string }>>> = {
     'approval-gate': {
       'create-new-file': { applicable: true },
       'modify-existing-file': { applicable: true },