Browse Source

Reduce Copilot mis-attribution for internal prompts (#168)

* feat: mark internal Copilot notifications as agent

* feat: move phase reminders to system transform

* fix: use stable text marker for internal prompts

* fix: avoid caching negative initiator lookups

* fix: skip phase reminders for internal turns

* fix: guard provider info lookup in chat headers

* fix: preserve positive initiator cache entries

* test: reset chat header cache between cases
NocturnesLK 5 months ago
parent
commit
0f4b783da7

+ 1 - 3
src/agents/orchestrator.ts

@@ -143,9 +143,7 @@ When user's approach seems problematic:
 `;
 
 export function createOrchestratorAgent(
-  model?:
-    | string
-    | Array<string | { id: string; variant?: string }>,
+  model?: string | Array<string | { id: string; variant?: string }>,
   customPrompt?: string,
   customAppendPrompt?: string,
 ): AgentDefinition {

+ 11 - 0
src/background/background-manager.test.ts

@@ -1,4 +1,5 @@
 import { describe, expect, mock, test } from 'bun:test';
+import { SLIM_INTERNAL_INITIATOR_MARKER } from '../utils';
 import { BackgroundTaskManager } from './background-manager';
 
 // Mock the plugin context
@@ -702,6 +703,16 @@ describe('BackgroundTaskManager', () => {
 
       // Should have called prompt.append for notification
       expect(ctx.client.session.prompt).toHaveBeenCalled();
+
+      const promptCalls = ctx.client.session.prompt.mock.calls as Array<
+        [{ body?: { parts?: Array<{ text?: string }> } }]
+      >;
+      const notificationCall = promptCalls[promptCalls.length - 1];
+      expect(
+        notificationCall[0].body?.parts?.[0]?.text?.includes(
+          SLIM_INTERNAL_INITIATOR_MARKER,
+        ),
+      ).toBe(true);
     });
   });
 

+ 7 - 5
src/background/background-manager.ts

@@ -20,7 +20,11 @@ import {
   SUBAGENT_DELEGATION_RULES,
 } from '../config';
 import type { TmuxConfig } from '../config/schema';
-import { applyAgentVariant, resolveAgentVariant } from '../utils';
+import {
+  applyAgentVariant,
+  createInternalAgentTextPart,
+  resolveAgentVariant,
+} from '../utils';
 import { log } from '../utils/logger';
 
 type PromptBody = {
@@ -244,9 +248,7 @@ export class BackgroundTaskManager {
     // primary may be a string, an array of string|{id,variant?}, or undefined
     let primaryIds: string[];
     if (Array.isArray(primary)) {
-      primaryIds = primary.map((m) =>
-        typeof m === 'string' ? m : m.id,
-      );
+      primaryIds = primary.map((m) => (typeof m === 'string' ? m : m.id));
     } else if (typeof primary === 'string') {
       primaryIds = [primary];
     } else {
@@ -611,7 +613,7 @@ export class BackgroundTaskManager {
     await this.client.session.prompt({
       path: { id: task.parentSessionId },
       body: {
-        parts: [{ type: 'text' as const, text: message }],
+        parts: [createInternalAgentTextPart(message)],
       },
     });
   }

+ 155 - 0
src/hooks/chat-headers.test.ts

@@ -0,0 +1,155 @@
+import { beforeEach, describe, expect, mock, test } from 'bun:test';
+import type { PluginInput } from '@opencode-ai/plugin';
+import { createInternalAgentTextPart } from '../utils';
+import {
+  __resetInternalMarkerCacheForTesting,
+  createChatHeadersHook,
+} from './chat-headers';
+
+function createMockContext(parts: unknown[] = []) {
+  return {
+    client: {
+      session: {
+        message: mock(async () => ({
+          data: {
+            info: { role: 'user' },
+            parts,
+          },
+        })),
+      },
+    },
+  } as unknown as PluginInput;
+}
+
+function createInput(
+  overrides?: Partial<{
+    providerID: string;
+    npm: string;
+    messageID: string;
+  }>,
+) {
+  return {
+    sessionID: 'session-1',
+    agent: 'orchestrator',
+    model: {
+      id: 'github-copilot/claude',
+      providerID: overrides?.providerID ?? 'github-copilot',
+      api: {
+        id: 'copilot',
+        url: 'https://example.com',
+        npm: overrides?.npm ?? '@custom/copilot',
+      },
+      name: 'Claude',
+      capabilities: {
+        temperature: true,
+        reasoning: true,
+        attachment: true,
+        toolcall: true,
+        input: {
+          text: true,
+          audio: false,
+          image: false,
+          video: false,
+          pdf: false,
+        },
+        output: {
+          text: true,
+          audio: false,
+          image: false,
+          video: false,
+          pdf: false,
+        },
+      },
+      cost: {
+        input: 0,
+        output: 0,
+        cache: { read: 0, write: 0 },
+      },
+      limit: { context: 0, output: 0 },
+      status: 'active' as const,
+      options: {},
+      headers: {},
+    },
+    provider: {
+      id: overrides?.providerID ?? 'github-copilot',
+      source: 'config' as const,
+      info: {
+        id: overrides?.providerID ?? 'github-copilot',
+      } as never,
+      options: {},
+    },
+    message: {
+      id: overrides?.messageID ?? 'message-1',
+      sessionID: 'session-1',
+      role: 'user' as const,
+      time: { created: Date.now() },
+      agent: 'orchestrator',
+      model: {
+        providerID: 'github-copilot',
+        modelID: 'claude',
+      },
+      tools: {},
+    },
+  };
+}
+
+describe('createChatHeadersHook', () => {
+  beforeEach(() => {
+    __resetInternalMarkerCacheForTesting();
+  });
+
+  test('sets x-initiator for marked Copilot messages', async () => {
+    const ctx = createMockContext([
+      createInternalAgentTextPart('internal notification'),
+    ]);
+    const hook = createChatHeadersHook(ctx);
+    const output = { headers: {} };
+
+    await hook['chat.headers'](createInput(), output);
+
+    expect(output.headers['x-initiator']).toBe('agent');
+  });
+
+  test('skips non-Copilot providers', async () => {
+    const ctx = createMockContext([
+      createInternalAgentTextPart('internal notification'),
+    ]);
+    const hook = createChatHeadersHook(ctx);
+    const output = { headers: {} };
+
+    await hook['chat.headers'](
+      createInput({ providerID: 'anthropic' }),
+      output,
+    );
+
+    expect(output.headers['x-initiator']).toBeUndefined();
+  });
+
+  test('skips requests handled by @ai-sdk/github-copilot', async () => {
+    const ctx = createMockContext([
+      createInternalAgentTextPart('internal notification'),
+    ]);
+    const hook = createChatHeadersHook(ctx);
+    const output = { headers: {} };
+
+    await hook['chat.headers'](
+      createInput({ npm: '@ai-sdk/github-copilot' }),
+      output,
+    );
+
+    expect(output.headers['x-initiator']).toBeUndefined();
+  });
+
+  test('skips normal user messages', async () => {
+    const ctx = createMockContext([{ type: 'text', text: 'normal prompt' }]);
+    const hook = createChatHeadersHook(ctx);
+    const output = { headers: {} };
+
+    await hook['chat.headers'](
+      createInput({ messageID: 'message-normal' }),
+      output,
+    );
+
+    expect(output.headers['x-initiator']).toBeUndefined();
+  });
+});

+ 97 - 0
src/hooks/chat-headers.ts

@@ -0,0 +1,97 @@
+import type { PluginInput, ProviderContext } from '@opencode-ai/plugin';
+import type { Model, UserMessage } from '@opencode-ai/sdk';
+import { hasInternalInitiatorMarker } from '../utils';
+
+interface ChatHeadersInput {
+  sessionID: string;
+  model: Model;
+  provider: ProviderContext;
+  message: UserMessage;
+}
+
+interface ChatHeadersOutput {
+  headers: Record<string, string>;
+}
+
+const INTERNAL_MARKER_CACHE_LIMIT = 1000;
+const internalMarkerCache = new Map<string, boolean>();
+
+export function __resetInternalMarkerCacheForTesting(): void {
+  internalMarkerCache.clear();
+}
+
+function getProviderID(input: ChatHeadersInput): string {
+  return input.provider.info?.id || input.model.providerID;
+}
+
+function isCopilotProvider(providerID: string): boolean {
+  return (
+    providerID === 'github-copilot' ||
+    providerID === 'github-copilot-enterprise'
+  );
+}
+
+async function hasInternalMarker(
+  client: PluginInput['client'],
+  sessionID: string,
+  messageID: string,
+): Promise<boolean> {
+  const cacheKey = `${sessionID}:${messageID}`;
+  const cached = internalMarkerCache.get(cacheKey);
+  if (cached !== undefined) {
+    return cached;
+  }
+
+  try {
+    const response = await client.session.message({
+      path: { id: sessionID, messageID },
+    });
+    const hasMarker = (response.data?.parts ?? []).some(
+      hasInternalInitiatorMarker,
+    );
+
+    if (hasMarker) {
+      if (internalMarkerCache.size >= INTERNAL_MARKER_CACHE_LIMIT) {
+        internalMarkerCache.clear();
+      }
+      internalMarkerCache.set(cacheKey, true);
+    }
+
+    return hasMarker;
+  } catch {
+    return false;
+  }
+}
+
+export function createChatHeadersHook(ctx: PluginInput) {
+  return {
+    'chat.headers': async (
+      input: ChatHeadersInput,
+      output: ChatHeadersOutput,
+    ): Promise<void> => {
+      if (!isCopilotProvider(getProviderID(input))) {
+        return;
+      }
+
+      if (input.model.api.npm === '@ai-sdk/github-copilot') {
+        return;
+      }
+
+      if (!input.message.id || input.message.role !== 'user') {
+        return;
+      }
+
+      if (
+        !(await hasInternalMarker(
+          ctx.client,
+          input.sessionID,
+          input.message.id,
+        ))
+      ) {
+        return;
+      }
+
+      output.headers['x-initiator'] = 'agent';
+    },
+  };
+}

+ 1 - 0
src/hooks/index.ts

@@ -1,5 +1,6 @@
 export type { AutoUpdateCheckerOptions } from './auto-update-checker';
 export { createAutoUpdateCheckerHook } from './auto-update-checker';
+export { createChatHeadersHook } from './chat-headers';
 export { createDelegateTaskRetryHook } from './delegate-task-retry';
 export { createJsonErrorRecoveryHook } from './json-error-recovery';
 export { createPhaseReminderHook } from './phase-reminder';

+ 63 - 0
src/hooks/phase-reminder/index.test.ts

@@ -0,0 +1,63 @@
+import { describe, expect, test } from 'bun:test';
+import { SLIM_INTERNAL_INITIATOR_MARKER } from '../../utils';
+import { createPhaseReminderHook, PHASE_REMINDER } from './index';
+
+describe('createPhaseReminderHook', () => {
+  test('prepends reminder for orchestrator sessions', async () => {
+    const hook = createPhaseReminderHook();
+    const output = {
+      messages: [
+        {
+          info: { role: 'user', agent: 'orchestrator' },
+          parts: [{ type: 'text', text: 'hello' }],
+        },
+      ],
+    };
+
+    await hook['experimental.chat.messages.transform']({}, output);
+
+    expect(output.messages[0].parts[0].text).toBe(
+      `${PHASE_REMINDER}\n\n---\n\nhello`,
+    );
+  });
+
+  test('skips non-orchestrator sessions', async () => {
+    const hook = createPhaseReminderHook();
+    const output = {
+      messages: [
+        {
+          info: { role: 'user', agent: 'explorer' },
+          parts: [{ type: 'text', text: 'hello' }],
+        },
+      ],
+    };
+
+    await hook['experimental.chat.messages.transform']({}, output);
+
+    expect(output.messages[0].parts[0].text).toBe('hello');
+  });
+
+  test('skips internal notification turns', async () => {
+    const hook = createPhaseReminderHook();
+    const output = {
+      messages: [
+        {
+          info: { role: 'user' },
+          parts: [
+            {
+              type: 'text',
+              text: `[Background task "x" completed]\n${SLIM_INTERNAL_INITIATOR_MARKER}`,
+            },
+          ],
+        },
+      ],
+    };
+
+    await hook['experimental.chat.messages.transform']({}, output);
+
+    expect(output.messages[0].parts[0].text).toContain(
+      SLIM_INTERNAL_INITIATOR_MARKER,
+    );
+    expect(output.messages[0].parts[0].text).not.toContain(PHASE_REMINDER);
+  });
+});

+ 8 - 2
src/hooks/phase-reminder/index.ts

@@ -8,7 +8,9 @@
  *
  * Uses experimental.chat.messages.transform so it doesn't show in UI.
  */
-const PHASE_REMINDER = `<reminder>Recall Workflow Rules:
+import { SLIM_INTERNAL_INITIATOR_MARKER } from '../../utils';
+
+export const PHASE_REMINDER = `<reminder>Recall Workflow Rules:
 Understand → find the best path (delegate based on rules and parallelize independent work) → execute → verify.
 If delegating, launch the specialist in the same turn you mention it.</reminder>`;
 
@@ -76,8 +78,12 @@ export function createPhaseReminderHook() {
         return;
       }
 
-      // Prepend the reminder to the existing text
       const originalText = lastUserMessage.parts[textPartIndex].text ?? '';
+      if (originalText.includes(SLIM_INTERNAL_INITIATOR_MARKER)) {
+        return;
+      }
+
+      // Prepend the reminder to the existing text
       lastUserMessage.parts[textPartIndex].text =
         `${PHASE_REMINDER}\n\n---\n\n${originalText}`;
     },

+ 6 - 3
src/index.ts

@@ -5,6 +5,7 @@ import { loadPluginConfig, type TmuxConfig } from './config';
 import { parseList } from './config/agent-mcps';
 import {
   createAutoUpdateCheckerHook,
+  createChatHeadersHook,
   createDelegateTaskRetryHook,
   createJsonErrorRecoveryHook,
   createPhaseReminderHook,
@@ -82,6 +83,8 @@ const OhMyOpenCodeLite: Plugin = async (ctx) => {
   // Initialize post-read nudge hook
   const postReadNudgeHook = createPostReadNudgeHook();
 
+  const chatHeadersHook = createChatHeadersHook(ctx);
+
   // Initialize delegate-task retry guidance hook
   const delegateTaskRetryHook = createDelegateTaskRetryHook(ctx);
 
@@ -128,9 +131,7 @@ const OhMyOpenCodeLite: Plugin = async (ctx) => {
           (opencodeConfig.provider as Record<string, unknown>) ?? {};
         const configuredProviders = Object.keys(providerConfig);
 
-        for (const [agentName, modelArray] of Object.entries(
-          modelArrayMap,
-        )) {
+        for (const [agentName, modelArray] of Object.entries(modelArrayMap)) {
           let resolved = false;
           for (const modelEntry of modelArray) {
             const slashIdx = modelEntry.id.indexOf('/');
@@ -264,6 +265,8 @@ const OhMyOpenCodeLite: Plugin = async (ctx) => {
       );
     },
 
+    'chat.headers': chatHeadersHook['chat.headers'],
+
     // Inject phase reminder before sending to API (doesn't show in UI)
     'experimental.chat.messages.transform':
       phaseReminderHook['experimental.chat.messages.transform'],

+ 1 - 0
src/utils/index.ts

@@ -1,5 +1,6 @@
 export * from './agent-variant';
 export * from './env';
+export * from './internal-initiator';
 export { log } from './logger';
 export * from './polling';
 export * from './tmux';

+ 28 - 0
src/utils/internal-initiator.ts

@@ -0,0 +1,28 @@
+export const SLIM_INTERNAL_INITIATOR_MARKER =
+  '<!-- SLIM_INTERNAL_INITIATOR -->';
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null;
+}
+
+export function createInternalAgentTextPart(text: string): {
+  type: 'text';
+  text: string;
+} {
+  return {
+    type: 'text',
+    text: `${text}\n${SLIM_INTERNAL_INITIATOR_MARKER}`,
+  };
+}
+
+export function hasInternalInitiatorMarker(part: unknown): boolean {
+  if (!isRecord(part) || part.type !== 'text') {
+    return false;
+  }
+
+  if (typeof part.text !== 'string') {
+    return false;
+  }
+
+  return part.text.includes(SLIM_INTERNAL_INITIATOR_MARKER);
+}