feat: add OpenCode cache verification tooling

README.md

@@ -125,6 +125,32 @@ bun install
 bun run build
 ```
 
+#### Cache verification
+
+Run `bun install` and build first, then point `OPENCODE_BIN` at a
+pre-provisioned `opencode-ai@1.18.2` binary. The verification uses the local
+plugin dependency tree, disables Bun auto-install, and uses a local capture
+server; it makes no external provider or dependency-resolution network calls:
+
+```bash
+bun run build
+OPENCODE_BIN=/absolute/path/to/opencode bun run verify:cache-stability
+```
+
+The live benchmark is manual and opt-in, not a CI check. It targets an already
+configured OpenCode server and writes redacted JSON telemetry:
+
+```bash
+bun scripts/benchmark-opencode-cache.ts \
+  --server http://127.0.0.1:4096 \
+  --provider anthropic \
+  --model claude-sonnet-4-5 \
+  --runs 10 \
+  --arm plugin-on \
+  --plugin-build "$(git rev-parse HEAD)" \
+  --output /tmp/opencode-cache-benchmark.json
+```
+
 ### Getting Started
 
 The installer generates both OpenAI and OpenCode Go presets, with OpenAI active by default.

package.json

@@ -61,6 +61,7 @@
     "generate-schema": "bun run scripts/generate-schema.ts",
     "verify:release": "bun run scripts/verify-release-artifact.ts",
     "verify:host-smoke": "bun run scripts/verify-opencode-host-smoke.ts",
+    "verify:cache-stability": "bun run scripts/verify-opencode-cache-stability.ts",
     "typecheck": "tsc --noEmit",
     "test": "bun test",
     "lint": "biome lint .",

scripts/benchmark-opencode-cache.ts

@@ -0,0 +1,764 @@
+#!/usr/bin/env bun
+export {};
+
+/**
+ * Example: bun scripts/benchmark-opencode-cache.ts --server http://127.0.0.1:4096 --provider anthropic --model claude-sonnet-4-5 --runs 10 --output /tmp/cache.json --arm plugin-on --plugin-build $(git rev-parse HEAD)
+ * Direct HTTP only: POST /session, POST/GET/DELETE /session/:id/message, GET /event, GET /global/health.
+ */
+
+type Json = Record<string, unknown>;
+type Event = {
+  at: number;
+  type: string;
+  sessionID?: string;
+  messageID?: string;
+  partID?: string;
+  field?: string;
+  partType?: string;
+  status?: string;
+  retryAttempt?: number;
+};
+type RequestObservation = {
+  position: number;
+  warmup: boolean;
+  input: number | null;
+  output: number | null;
+  cacheRead: number | null;
+  cacheWrite: number | null;
+  totalInput: number | null;
+};
+type PromptLoopObservation = {
+  turn: number;
+  warmup: boolean;
+  promptLoopTtftMs: number | null;
+  promptLoopLatencyMs: number | null;
+};
+
+const HELP = `Usage: bun scripts/benchmark-opencode-cache.ts [options]
+
+Required:
+  --server URL --provider ID --model ID --runs N --output PATH
+  --arm ID              Comparison arm identifier (for example plugin-on)
+  --plugin-build ID     Plugin build/commit identifier
+
+Options:
+  --timeout-ms N        Per-request timeout (default: 120000)
+  --retries N           Retry transient GET/create/probe failures (default: 2)
+  --retry-delay-ms N    Initial exponential-backoff delay (default: 500)
+  --header NAME:VALUE   Extra HTTP header; repeatable, never persisted
+  --overwrite           Replace an existing output file
+  --help                Show this help
+
+Each independent session explicitly targets agent "orchestrator" and the supplied
+provider/model. It requests Read(package.json), TodoWrite, a final answer, then a
+second user follow-up. Incomplete, retrying, compacted, or misrouted sessions are
+recorded as discards, never resampled. Output contains only normalized/redacted
+telemetry: no prompts, model text, tool I/O, workspace paths, headers, or event
+properties. Prompt-loop TTFT/latency are one observation per submitted user
+prompt, never copied to provider steps. SSE coverage must be established before
+prompts; an unavailable, malformed, or closed stream discards the session. A
+unique per-session nonce is kept only in live prompts.
+Sessions are deleted after collection.
+`;
+
+function fail(message: string): never {
+  console.error(`Error: ${message}\n\n${HELP}`);
+  process.exit(2);
+}
+function isRecord(value: unknown): value is Json {
+  return !!value && typeof value === 'object' && !Array.isArray(value);
+}
+function string(value: unknown) {
+  return typeof value === 'string' ? value : undefined;
+}
+function number(value: unknown) {
+  return typeof value === 'number' && Number.isFinite(value)
+    ? value
+    : undefined;
+}
+function urlPath(base: string, route: string) {
+  return `${base}${route.startsWith('/') ? route : `/${route}`}`;
+}
+const now = () => performance.timeOrigin + performance.now();
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+function parseArgs(argv: string[]) {
+  const values = new Map<string, string[]>();
+  const flags = new Set<string>();
+  const names = [
+    '--server',
+    '--provider',
+    '--model',
+    '--runs',
+    '--output',
+    '--arm',
+    '--plugin-build',
+    '--timeout-ms',
+    '--retries',
+    '--retry-delay-ms',
+    '--header',
+  ];
+  for (let i = 0; i < argv.length; i += 1) {
+    if (argv[i] === '--help') {
+      console.log(HELP);
+      process.exit(0);
+    }
+    if (argv[i] === '--overwrite') {
+      flags.add(argv[i]);
+      continue;
+    }
+    if (!names.includes(argv[i])) fail(`unknown argument ${argv[i]}`);
+    const value = argv[i + 1];
+    if (!value || value.startsWith('--')) fail(`missing value for ${argv[i]}`);
+    values.set(argv[i], [...(values.get(argv[i]) ?? []), value]);
+    i += 1;
+  }
+  const required = (name: string) => {
+    const value = values.get(name)?.at(-1);
+    if (!value) fail(`missing required ${name}`);
+    return value;
+  };
+  const integer = (name: string, fallback?: number) => {
+    const value = Number(values.get(name)?.at(-1) ?? fallback);
+    if (
+      !Number.isSafeInteger(value) ||
+      value < 0 ||
+      (name === '--runs' && value === 0)
+    )
+      fail(
+        `${name} must be a ${name === '--runs' ? 'positive' : 'non-negative'} integer`,
+      );
+    return value;
+  };
+  let server: URL;
+  try {
+    server = new URL(required('--server'));
+  } catch {
+    fail('--server must be an absolute http(s) URL');
+  }
+  if (!['http:', 'https:'].includes(server.protocol))
+    fail('--server must be an http(s) URL');
+  const headers = new Headers();
+  for (const value of values.get('--header') ?? []) {
+    const split = value.indexOf(':');
+    if (split < 1) fail('--header must be NAME:VALUE');
+    headers.append(value.slice(0, split), value.slice(split + 1).trim());
+  }
+  return {
+    server: server.toString().replace(/\/$/, ''),
+    provider: required('--provider'),
+    model: required('--model'),
+    runs: integer('--runs'),
+    output: required('--output'),
+    arm: required('--arm'),
+    pluginBuild: required('--plugin-build'),
+    timeoutMs: integer('--timeout-ms', 120_000),
+    retries: integer('--retries', 2),
+    retryDelayMs: integer('--retry-delay-ms', 500),
+    headers,
+    overwrite: flags.has('--overwrite'),
+  };
+}
+
+const args = parseArgs(process.argv.slice(2));
+if ((await Bun.file(args.output).exists()) && !args.overwrite)
+  fail(`output already exists: ${args.output} (use --overwrite)`);
+
+async function request(
+  method: string,
+  route: string,
+  body?: Json,
+  retry = false,
+) {
+  let last = 'request failed';
+  for (let attempt = 0; attempt <= args.retries; attempt += 1) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), args.timeoutMs);
+    try {
+      const headers = new Headers(args.headers);
+      if (body) headers.set('content-type', 'application/json');
+      const response = await fetch(urlPath(args.server, route), {
+        method,
+        headers,
+        body: body ? JSON.stringify(body) : undefined,
+        signal: controller.signal,
+      });
+      const text = await response.text();
+      if (response.ok)
+        return text.trim() ? (JSON.parse(text) as unknown) : undefined;
+      last = `${method} ${route} returned HTTP ${response.status}`;
+      if (!retry || response.status < 500 || attempt === args.retries)
+        throw new Error(last);
+    } catch (error) {
+      last = error instanceof Error ? error.message : String(error);
+      if (!retry || attempt === args.retries) throw new Error(last);
+    } finally {
+      clearTimeout(timeout);
+    }
+    await sleep(args.retryDelayMs * 2 ** attempt);
+  }
+  throw new Error(last);
+}
+
+function normalizeEvent(value: unknown): Event | undefined {
+  if (!isRecord(value) || !isRecord(value.properties)) return undefined;
+  const properties = value.properties;
+  const type = string(value.type);
+  const sessionID = string(properties.sessionID);
+  if (!type || !sessionID) return undefined;
+  if (type === 'message.part.delta')
+    return {
+      at: now(),
+      type,
+      sessionID,
+      messageID: string(properties.messageID),
+      partID: string(properties.partID),
+      field: string(properties.field),
+    };
+  if (type === 'message.part.updated' && isRecord(properties.part))
+    return {
+      at: now(),
+      type,
+      sessionID,
+      messageID: string(properties.part.messageID),
+      partID: string(properties.part.id),
+      partType: string(properties.part.type),
+      status: isRecord(properties.part.state)
+        ? string(properties.part.state.status)
+        : undefined,
+    };
+  if (type === 'message.updated' && isRecord(properties.info))
+    return {
+      at: now(),
+      type,
+      sessionID,
+      messageID: string(properties.info.id),
+    };
+  if (type === 'session.status' && isRecord(properties.status))
+    return {
+      at: now(),
+      type,
+      sessionID,
+      status: string(properties.status.type),
+      retryAttempt: number(properties.status.attempt),
+    };
+  if (type.toLowerCase().includes('compaction'))
+    return { at: now(), type, sessionID };
+  return undefined;
+}
+
+type StreamCoverage = {
+  ready: Promise<void>;
+  readyResolve: () => void;
+  issue: string | null;
+};
+
+function streamCoverage(): StreamCoverage {
+  let readyResolve = () => {};
+  const ready = new Promise<void>((resolve) => {
+    readyResolve = resolve;
+  });
+  return { ready, readyResolve, issue: null };
+}
+
+async function awaitSseReadiness(coverage: StreamCoverage) {
+  let timeout: ReturnType<typeof setTimeout> | undefined;
+  try {
+    await Promise.race([
+      coverage.ready,
+      new Promise<void>((_, reject) => {
+        timeout = setTimeout(
+          () => reject(new Error('SSE subscription readiness timed out')),
+          Math.min(args.timeoutMs, 10_000),
+        );
+      }),
+    ]);
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+  if (coverage.issue) throw new Error(coverage.issue);
+}
+
+async function eventStream(
+  events: Event[],
+  controller: AbortController,
+  coverage: StreamCoverage,
+) {
+  try {
+    const response = await fetch(urlPath(args.server, '/event'), {
+      headers: args.headers,
+      signal: controller.signal,
+    });
+    if (!response.ok || !response.body)
+      throw new Error(`GET /event returned HTTP ${response.status}`);
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = '';
+    while (!controller.signal.aborted) {
+      const chunk = await reader.read();
+      if (chunk.done) break;
+      buffer += decoder.decode(chunk.value, { stream: true });
+      const frames = buffer.split(/\r?\n\r?\n/);
+      buffer = frames.pop() ?? '';
+      for (const frame of frames) {
+        const data = frame
+          .split(/\r?\n/)
+          .find((line) => line.startsWith('data:'))
+          ?.slice(5)
+          .trim();
+        if (!data) continue;
+        try {
+          const parsed = JSON.parse(data) as unknown;
+          if (isRecord(parsed) && parsed.type === 'server.connected') {
+            coverage.readyResolve();
+            continue;
+          }
+          const event = normalizeEvent(parsed);
+          if (event) events.push(event);
+        } catch {
+          coverage.issue ??= 'SSE delivered a malformed frame';
+          coverage.readyResolve();
+        }
+      }
+    }
+    if (!controller.signal.aborted) {
+      coverage.issue ??= 'SSE stream closed before collection completed';
+      coverage.readyResolve();
+    }
+  } catch {
+    if (!controller.signal.aborted) {
+      coverage.issue ??= 'SSE stream connection failed';
+      coverage.readyResolve();
+    }
+  }
+}
+
+function normalizeMessage(value: unknown) {
+  if (!isRecord(value) || !isRecord(value.info)) return undefined;
+  const info = value.info;
+  const parts = Array.isArray(value.parts) ? value.parts : [];
+  return {
+    id: string(info.id),
+    parentID: string(info.parentID),
+    role: string(info.role),
+    agent: string(info.agent),
+    providerID: string(info.providerID),
+    modelID: string(info.modelID),
+    tokens: isRecord(info.tokens) ? info.tokens : undefined,
+    parts: parts.filter(isRecord).map((part) => ({
+      type: string(part.type),
+      tool: string(part.tool),
+      status: isRecord(part.state) ? string(part.state.status) : undefined,
+      tokens: isRecord(part.tokens) ? part.tokens : undefined,
+    })),
+  };
+}
+
+function tokenObservation(
+  tokens: unknown,
+  position: number,
+  warmup: boolean,
+): RequestObservation {
+  const tokenRecord = isRecord(tokens) ? tokens : {};
+  const cache = isRecord(tokenRecord.cache) ? tokenRecord.cache : undefined;
+  const input = number(tokenRecord.input) ?? null;
+  const cacheRead = cache ? (number(cache.read) ?? null) : null;
+  const cacheWrite = cache ? (number(cache.write) ?? null) : null;
+  const totalInput =
+    input === null || cacheRead === null || cacheWrite === null
+      ? null
+      : input + cacheRead + cacheWrite;
+  return {
+    position,
+    warmup,
+    input,
+    output: number(tokenRecord.output) ?? null,
+    cacheRead,
+    cacheWrite,
+    totalInput,
+  };
+}
+
+function percentile(values: number[], p: number) {
+  if (!values.length) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  return sorted[
+    Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)
+  ];
+}
+function boundedCoverage(observations: RequestObservation[]) {
+  const measured = observations.filter(
+    (item) => item.cacheRead !== null && item.totalInput !== null,
+  );
+  const total = measured.reduce((sum, item) => sum + (item.totalInput ?? 0), 0);
+  return total > 0
+    ? Math.min(
+        1,
+        Math.max(
+          0,
+          measured.reduce((sum, item) => sum + (item.cacheRead ?? 0), 0) /
+            total,
+        ),
+      )
+    : null;
+}
+function bootstrapCI<T>(
+  sessions: T[][],
+  metric: (items: T[]) => number | null,
+) {
+  const observed = metric(sessions.flat());
+  if (!sessions.length || observed === null)
+    return { estimate: observed, lower: null, upper: null, samples: 0 };
+  let state = 0x9e3779b9;
+  const random = () => {
+    state = (state * 1664525 + 1013904223) >>> 0;
+    return state / 2 ** 32;
+  };
+  const values: number[] = [];
+  for (let i = 0; i < 2000; i += 1) {
+    const sampled = Array.from(
+      { length: sessions.length },
+      () => sessions[Math.floor(random() * sessions.length)],
+    ).flat();
+    const value = metric(sampled);
+    if (value !== null) values.push(value);
+  }
+  return {
+    estimate: observed,
+    lower: percentile(values, 2.5),
+    upper: percentile(values, 97.5),
+    samples: values.length,
+  };
+}
+
+function firstPrompt(nonce: string) {
+  return `Session marker: ${nonce}. Use the Read tool to inspect package.json. Then use the TodoWrite tool to create one completed, low-priority todo named "cache benchmark trace". Do not modify files. After both tools finish, reply with exactly: benchmark complete.`;
+}
+
+function followUp(nonce: string) {
+  return `Session marker: ${nonce}. Without using tools, reply with exactly: follow-up complete.`;
+}
+
+async function run(index: number) {
+  const events: Event[] = [];
+  const controller = new AbortController();
+  const coverage = streamCoverage();
+  const stream = eventStream(events, controller, coverage);
+  let sessionID: string | undefined;
+  const record: Json = {
+    runIndex: index,
+    runOrder: index + 1,
+    requestedRouting: {
+      agent: 'orchestrator',
+      providerID: args.provider,
+      modelID: args.model,
+    },
+  };
+  try {
+    await awaitSseReadiness(coverage);
+    const created = await request('POST', '/session', {}, true);
+    sessionID = isRecord(created) ? string(created.id) : undefined;
+    if (!sessionID) throw new Error('POST /session returned no session id');
+    record.sessionID = sessionID;
+    const nonce = crypto.randomUUID();
+    const prompts: {
+      parentMessageID?: string;
+      startedAt: number;
+      completedAt: number;
+    }[] = [];
+    for (const text of [firstPrompt(nonce), followUp(nonce)]) {
+      const startedAt = now();
+      const response = await request(
+        'POST',
+        `/session/${encodeURIComponent(sessionID)}/message`,
+        {
+          agent: 'orchestrator',
+          model: { providerID: args.provider, modelID: args.model },
+          parts: [{ type: 'text', text }],
+        },
+      );
+      const normalized = normalizeMessage(response);
+      prompts.push({
+        // POST /message returns the assistant message; its parent is this turn's user message.
+        parentMessageID: normalized?.parentID,
+        startedAt,
+        completedAt: now(),
+      });
+    }
+    const [rawMessages, rawSession] = await Promise.all([
+      request(
+        'GET',
+        `/session/${encodeURIComponent(sessionID)}/message`,
+        undefined,
+        true,
+      ),
+      request(
+        'GET',
+        `/session/${encodeURIComponent(sessionID)}`,
+        undefined,
+        true,
+      ),
+    ]);
+    const messages = Array.isArray(rawMessages)
+      ? rawMessages
+          .map(normalizeMessage)
+          .filter((item): item is NonNullable<typeof item> => !!item)
+      : [];
+    const session = isRecord(rawSession) ? rawSession : {};
+    const sessionModel = isRecord(session.model) ? session.model : {};
+    const sessionEvents = events.filter(
+      (event) => event.sessionID === sessionID,
+    );
+    const assistants = messages.filter(
+      (message) => message.role === 'assistant',
+    );
+    const tools = assistants.flatMap((message) =>
+      message.parts
+        .filter((part) => part.type === 'tool')
+        .map((part) => ({ tool: part.tool, status: part.status })),
+    );
+    const observations: RequestObservation[] = [];
+    const promptLoops: PromptLoopObservation[] = [];
+    let position = 0;
+    for (const prompt of prompts) {
+      const assistantIDs = new Set(
+        assistants
+          .filter((message) => message.parentID === prompt.parentMessageID)
+          .map((message) => message.id)
+          .filter((id): id is string => !!id),
+      );
+      promptLoops.push({
+        turn: promptLoops.length + 1,
+        warmup: promptLoops.length === 0,
+        promptLoopTtftMs: (() => {
+          const event = sessionEvents.find(
+            (item) =>
+              item.at >= prompt.startedAt &&
+              item.at <= prompt.completedAt &&
+              item.messageID &&
+              assistantIDs.has(item.messageID) &&
+              item.type === 'message.part.delta' &&
+              ['text', 'reasoning'].includes(item.field ?? ''),
+          );
+          return event ? event.at - prompt.startedAt : null;
+        })(),
+        promptLoopLatencyMs: prompt.completedAt - prompt.startedAt,
+      });
+      const perPrompt = assistants
+        .filter((message) => message.parentID === prompt.parentMessageID)
+        .flatMap((message) =>
+          message.parts
+            .filter((part) => part.type === 'step-finish')
+            .map((part) => part.tokens),
+        );
+      const tokenSets = perPrompt.length
+        ? perPrompt
+        : assistants
+            .filter((message) => message.parentID === prompt.parentMessageID)
+            .map((message) => message.tokens);
+      for (const tokens of tokenSets) {
+        position += 1;
+        observations.push(tokenObservation(tokens, position, position === 1));
+      }
+    }
+    const retries = sessionEvents.filter(
+      (event) => event.type === 'session.status' && event.status === 'retry',
+    ).length;
+    const historyRetry = messages.some((message) =>
+      message.parts.some((part) => part.type === 'retry'),
+    );
+    const historyCompaction =
+      string(session.agent) === 'compaction' ||
+      messages.some(
+        (message) =>
+          message.agent === 'compaction' ||
+          message.parts.some((part) => part.type === 'compaction'),
+      ) ||
+      (isRecord(session.time) && number(session.time.compacting) !== undefined);
+    const compaction =
+      historyCompaction ||
+      sessionEvents.some((event) =>
+        event.type.toLowerCase().includes('compaction'),
+      );
+    const routingMatches =
+      string(session.agent) === 'orchestrator' &&
+      string(sessionModel.providerID) === args.provider &&
+      string(sessionModel.id) === args.model &&
+      assistants.length > 0 &&
+      assistants.every(
+        (message) =>
+          message.agent === 'orchestrator' &&
+          message.providerID === args.provider &&
+          message.modelID === args.model,
+      );
+    const reasons = [
+      ...(tools.some(
+        (tool) =>
+          tool.tool?.toLowerCase() === 'read' && tool.status === 'completed',
+      )
+        ? []
+        : ['missing completed Read tool']),
+      ...(tools.some(
+        (tool) =>
+          tool.tool?.toLowerCase() === 'todowrite' &&
+          tool.status === 'completed',
+      )
+        ? []
+        : ['missing completed harmless TodoWrite tool']),
+      ...(assistants.some((message) =>
+        message.parts.some((part) => part.type === 'text'),
+      )
+        ? []
+        : ['missing assistant final response']),
+      ...(messages.filter((message) => message.role === 'user').length >= 2
+        ? []
+        : ['missing second user follow-up']),
+      ...(routingMatches
+        ? []
+        : [
+            'observed session/assistant routing does not match orchestrator/provider/model',
+          ]),
+      ...(coverage.issue ? [coverage.issue] : []),
+      ...(retries || historyRetry ? ['session entered retry state'] : []),
+      ...(compaction ? ['session compaction observed'] : []),
+    ];
+    record.observation = {
+      routing: {
+        sessionAgent: string(session.agent) ?? null,
+        sessionProviderID: string(sessionModel.providerID) ?? null,
+        sessionModelID: string(sessionModel.id) ?? null,
+        assistantCount: assistants.length,
+      },
+      events: sessionEvents,
+      tools,
+      requests: observations,
+      promptLoops,
+      retries,
+      compaction,
+      historyRetry,
+      historyCompaction,
+    };
+    record.discarded = reasons.length > 0;
+    if (reasons.length) record.discardReasons = reasons;
+  } catch (error) {
+    record.discarded = true;
+    record.discardReasons = [
+      error instanceof Error ? error.message : String(error),
+    ];
+    record.observation = { events };
+  } finally {
+    if (sessionID) {
+      try {
+        await request('DELETE', `/session/${encodeURIComponent(sessionID)}`);
+      } catch {
+        record.cleanupError = 'session deletion failed';
+      }
+    }
+    controller.abort();
+    await stream;
+  }
+  return record;
+}
+
+let hostVersion: string | null = null;
+try {
+  const health = await request('GET', '/global/health', undefined, true);
+  hostVersion = isRecord(health) ? (string(health.version) ?? null) : null;
+} catch {
+  /* probe is optional */
+}
+const runs: Json[] = [];
+for (let index = 0; index < args.runs; index += 1) {
+  console.error(`Running session ${index + 1}/${args.runs}`);
+  runs.push(await run(index));
+}
+const eligible = runs.filter((run) => !run.discarded);
+const aggregateSessions = eligible.map((run) =>
+  isRecord(run.observation) && Array.isArray(run.observation.requests)
+    ? run.observation.requests
+        .filter(isRecord)
+        .map((item) => item as unknown as RequestObservation)
+        .filter((item) => !item.warmup)
+    : [],
+);
+const aggregate = aggregateSessions.flat();
+const promptLoopSessions = eligible.map((run) =>
+  isRecord(run.observation) && Array.isArray(run.observation.promptLoops)
+    ? run.observation.promptLoops
+        .filter(isRecord)
+        .map((item) => item as unknown as PromptLoopObservation)
+        .filter((item) => !item.warmup)
+    : [],
+);
+const promptLoops = promptLoopSessions.flat();
+const cacheMiss = (items: RequestObservation[]) => {
+  const measured = items.filter((item) => item.cacheRead !== null);
+  return measured.length
+    ? measured.filter((item) => item.cacheRead === 0).length / measured.length
+    : null;
+};
+const ttftP50 = (items: PromptLoopObservation[]) =>
+  percentile(
+    items
+      .map((item) => item.promptLoopTtftMs)
+      .filter((value): value is number => value !== null),
+    50,
+  );
+const latencyP50 = (items: PromptLoopObservation[]) =>
+  percentile(
+    items
+      .map((item) => item.promptLoopLatencyMs)
+      .filter((value): value is number => value !== null),
+    50,
+  );
+const report = {
+  schemaVersion: 2,
+  generatedAt: new Date().toISOString(),
+  comparison: {
+    arm: args.arm,
+    pluginBuild: args.pluginBuild,
+    hostOpenCodeVersion: hostVersion,
+    runOrder: 'sequential independent sessions',
+    routingFingerprint: {
+      agent: 'orchestrator',
+      providerID: args.provider,
+      modelID: args.model,
+    },
+  },
+  controls: {
+    runs: args.runs,
+    timeoutMs: args.timeoutMs,
+    retries: args.retries,
+    retryDelayMs: args.retryDelayMs,
+    warmupRule:
+      'exclude request position 1 in each eligible session from aggregates',
+  },
+  metrics: {
+    sessionsRequested: args.runs,
+    sessionsEligible: eligible.length,
+    sessionsDiscarded: runs.length - eligible.length,
+    requestObservationsRaw: eligible.flatMap((run) =>
+      isRecord(run.observation) && Array.isArray(run.observation.requests)
+        ? run.observation.requests
+        : [],
+    ).length,
+    requestObservationsAggregated: aggregate.length,
+    promptLoopObservationsRaw: eligible.flatMap((run) =>
+      isRecord(run.observation) && Array.isArray(run.observation.promptLoops)
+        ? run.observation.promptLoops
+        : [],
+    ).length,
+    promptLoopObservationsAggregated: promptLoops.length,
+    primaryMetricsSessionBootstrap95CI: {
+      zeroCacheMissRate: bootstrapCI(aggregateSessions, cacheMiss),
+      cacheCoverage: bootstrapCI(aggregateSessions, boundedCoverage),
+      promptLoopTtftMsP50: bootstrapCI(promptLoopSessions, ttftP50),
+      promptLoopLatencyMsP50: bootstrapCI(promptLoopSessions, latencyP50),
+    },
+  },
+  runs,
+};
+await Bun.write(args.output, `${JSON.stringify(report, null, 2)}\n`);
+console.error(
+  `Wrote ${args.output}: ${eligible.length}/${args.runs} eligible sessions`,
+);

scripts/verify-opencode-cache-stability.ts

@@ -0,0 +1,637 @@
+import { spawn, spawnSync } from 'node:child_process';
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { createServer } from 'node:http';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { PHASE_REMINDER, PHASE_REMINDER_TEXT } from '../src/config/constants';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = path.resolve(__dirname, '..');
+const distEntry = path.join(repoRoot, 'dist', 'index.js');
+const MODEL_ID = 'capture-model';
+const PROVIDER_ID = 'capture';
+const TIMEOUT_MS = process.platform === 'darwin' ? 60_000 : 30_000;
+
+type Capture = {
+  body: Record<string, unknown>;
+  headers: Record<string, string | string[] | undefined>;
+  url: string;
+};
+
+function fail(message: string): never {
+  throw new Error(message);
+}
+
+function run(command: string, args: string[], cwd = repoRoot): string {
+  const result = spawnSync(command, args, {
+    cwd,
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+  if (result.status === 0) return result.stdout.trim();
+  const detail = [result.stdout, result.stderr].filter(Boolean).join('\n');
+  fail(`Command failed: ${command} ${args.join(' ')}\n${detail}`);
+}
+
+function requireOpenCodeBinary(): string {
+  const opencode = process.env.OPENCODE_BIN;
+  if (!opencode) {
+    fail(
+      'OPENCODE_BIN is required and must point to a pre-provisioned opencode-ai@1.18.2 binary. Example: OPENCODE_BIN=/path/to/node_modules/.bin/opencode bun run verify:cache-stability',
+    );
+  }
+  if (!path.isAbsolute(opencode) || !existsSync(opencode)) {
+    fail(`OPENCODE_BIN must be an existing absolute path, got: ${opencode}`);
+  }
+  const version = run(opencode, ['--version']);
+  if (!/(^|\s)1\.18\.2(\s|$)/.test(version)) {
+    fail(`OPENCODE_BIN must be opencode-ai@1.18.2, got: ${version}`);
+  }
+  return opencode;
+}
+
+function requireLocalPluginTree() {
+  const nodeModules = path.join(repoRoot, 'node_modules');
+  if (!existsSync(nodeModules)) {
+    fail(
+      'Local plugin dependency tree is missing. Run `bun install` before verify:cache-stability; the harness never installs dependencies.',
+    );
+  }
+  for (const required of [
+    path.join(repoRoot, 'package.json'),
+    path.join(repoRoot, 'dist', 'index.js'),
+    path.join(nodeModules, '@opencode-ai', 'plugin'),
+    path.join(nodeModules, 'zod'),
+  ]) {
+    if (!existsSync(required)) {
+      fail(
+        `Local plugin dependency tree is incomplete at ${required}. Run \`bun install\` and \`bun run build\` before verify:cache-stability.`,
+      );
+    }
+  }
+  return { nodeModules, plugin: repoRoot };
+}
+
+function getFreePort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const server = createServer();
+    server.once('error', reject);
+    server.listen(0, '127.0.0.1', () => {
+      const address = server.address();
+      if (!address || typeof address === 'string') {
+        server.close();
+        reject(new Error('Failed to allocate a port'));
+        return;
+      }
+      server.close((error) => (error ? reject(error) : resolve(address.port)));
+    });
+  });
+}
+
+async function waitFor(url: string, predicate: () => boolean, label: string) {
+  const deadline = Date.now() + TIMEOUT_MS;
+  let lastError = '';
+  while (Date.now() < deadline) {
+    try {
+      const response = await fetch(url, { signal: AbortSignal.timeout(2_000) });
+      if (response.ok && predicate()) return;
+      lastError = `status=${response.status}`;
+    } catch (error) {
+      lastError = error instanceof Error ? error.message : String(error);
+    }
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+  fail(`Timed out waiting for ${label}: ${lastError}`);
+}
+
+async function stop(child: ReturnType<typeof spawn>) {
+  if (child.exitCode !== null) return;
+  child.kill('SIGTERM');
+  const exited = await Promise.race([
+    new Promise<boolean>((resolve) => child.once('exit', () => resolve(true))),
+    new Promise<boolean>((resolve) => setTimeout(() => resolve(false), 5_000)),
+  ]);
+  if (!exited && child.exitCode === null) {
+    child.kill('SIGKILL');
+    await new Promise((resolve) => child.once('exit', resolve));
+  }
+}
+
+function redact(value: unknown): string {
+  const text = JSON.stringify(value, null, 2)
+    .replaceAll(/Bearer\s+[^\s"']+/gi, 'Bearer [REDACTED]')
+    .replaceAll(
+      /"(api[_-]?key|authorization)"\s*:\s*"[^"]*"/gi,
+      '"$1":"[REDACTED]"',
+    );
+  return text.length > 12_000 ? `${text.slice(0, 12_000)}\n...[capped]` : text;
+}
+
+function describeDifference(left: string, right: string): string {
+  const offset = [...left].findIndex(
+    (character, index) => character !== right[index],
+  );
+  const index = offset === -1 ? Math.min(left.length, right.length) : offset;
+  return JSON.stringify({
+    leftLength: left.length,
+    rightLength: right.length,
+    offset: index,
+    left: left.slice(Math.max(0, index - 200), index + 400),
+    right: right.slice(Math.max(0, index - 200), index + 400),
+  });
+}
+
+function sse(payload: Record<string, unknown>): string {
+  return `data: ${JSON.stringify(payload)}\n\ndata: [DONE]\n\n`;
+}
+
+function toolResponse(id: string, name: string, argumentsJson: string): string {
+  return sse({
+    id: `capture-${id}`,
+    object: 'chat.completion.chunk',
+    choices: [
+      {
+        index: 0,
+        delta: {
+          tool_calls: [
+            {
+              index: 0,
+              id,
+              type: 'function',
+              function: { name, arguments: argumentsJson },
+            },
+          ],
+        },
+        finish_reason: 'tool_calls',
+      },
+    ],
+  });
+}
+
+function textResponse(text: string): string {
+  return sse({
+    id: 'capture-final',
+    object: 'chat.completion.chunk',
+    choices: [{ index: 0, delta: { content: text }, finish_reason: 'stop' }],
+  });
+}
+
+function isOrchestratorPayload(body: Record<string, unknown>): boolean {
+  return (
+    body.model === MODEL_ID &&
+    Array.isArray(body.tools) &&
+    body.tools.some(
+      (tool) =>
+        typeof tool === 'object' &&
+        tool !== null &&
+        'function' in tool &&
+        typeof tool.function === 'object' &&
+        tool.function !== null &&
+        'name' in tool.function &&
+        tool.function.name === 'read',
+    )
+  );
+}
+
+async function createCaptureServer(readPath: string) {
+  const requests: Capture[] = [];
+  const server = createServer(async (request, response) => {
+    const chunks: Buffer[] = [];
+    for await (const chunk of request) chunks.push(Buffer.from(chunk));
+    const raw = Buffer.concat(chunks).toString('utf8');
+    if (request.method !== 'POST' || request.url !== '/v1/chat/completions') {
+      response.writeHead(404).end('unexpected local capture request');
+      return;
+    }
+    let body: Record<string, unknown>;
+    try {
+      body = JSON.parse(raw) as Record<string, unknown>;
+    } catch {
+      response.writeHead(400).end('invalid JSON');
+      return;
+    }
+    requests.push({ body, headers: request.headers, url: request.url });
+    const primary = isOrchestratorPayload(body);
+    const index = requests.filter((item) =>
+      isOrchestratorPayload(item.body),
+    ).length;
+    const payload = !primary
+      ? textResponse('title')
+      : index === 1
+        ? toolResponse(
+            'call_read',
+            'read',
+            JSON.stringify({ filePath: readPath }),
+          )
+        : index === 2
+          ? toolResponse(
+              'call_todo',
+              'todowrite',
+              JSON.stringify({
+                todos: [
+                  {
+                    content: 'cache stability checked',
+                    status: 'completed',
+                    priority: 'low',
+                  },
+                ],
+              }),
+            )
+          : textResponse(
+              index === 3 ? 'tools completed' : 'second turn completed',
+            );
+    response.writeHead(200, {
+      'content-type': 'text/event-stream',
+      'cache-control': 'no-cache',
+      connection: 'keep-alive',
+    });
+    response.end(payload);
+  });
+  const port = await new Promise<number>((resolve, reject) => {
+    server.once('error', reject);
+    server.listen(0, '127.0.0.1', () => {
+      const address = server.address();
+      if (!address || typeof address === 'string')
+        return reject(new Error('Invalid capture address'));
+      resolve(address.port);
+    });
+  });
+  return { requests, server, url: `http://127.0.0.1:${port}/v1` };
+}
+
+function messages(
+  body: Record<string, unknown>,
+): Array<Record<string, unknown>> {
+  if (!Array.isArray(body.messages))
+    fail(`Provider payload omitted messages:\n${redact(body)}`);
+  return body.messages.filter(
+    (message): message is Record<string, unknown> =>
+      typeof message === 'object' && message !== null,
+  );
+}
+
+function contentText(message: Record<string, unknown>): string {
+  if (typeof message.content === 'string') return message.content;
+  if (!Array.isArray(message.content)) return '';
+  return message.content
+    .map((part) =>
+      typeof part === 'object' &&
+      part !== null &&
+      'text' in part &&
+      typeof part.text === 'string'
+        ? part.text
+        : '',
+    )
+    .join('');
+}
+
+function userProjection(body: Record<string, unknown>): string[] {
+  return messages(body)
+    .filter((message) => message.role === 'user')
+    .map(contentText)
+    .filter((content) => content.length > 0);
+}
+
+function promptProjection(body: Record<string, unknown>) {
+  const allMessages = messages(body);
+  const system = allMessages.filter(
+    (message) =>
+      typeof message === 'object' &&
+      message !== null &&
+      'role' in message &&
+      message.role === 'system',
+  );
+  const input = allMessages.filter(
+    (message) =>
+      typeof message === 'object' &&
+      message !== null &&
+      'role' in message &&
+      message.role !== 'system',
+  );
+  return {
+    system,
+    input,
+    tools: Array.isArray(body.tools) ? body.tools : [],
+    options: Object.fromEntries(
+      Object.entries(body).filter(([key]) =>
+        [
+          'model',
+          'temperature',
+          'top_p',
+          'max_tokens',
+          'max_completion_tokens',
+          'tool_choice',
+          'parallel_tool_calls',
+          'response_format',
+        ].includes(key),
+      ),
+    ),
+  };
+}
+
+function assertPromptPrefix(
+  previous: Record<string, unknown>,
+  next: Record<string, unknown>,
+  index: number,
+) {
+  const before = promptProjection(previous);
+  const after = promptProjection(next);
+  for (const field of ['system', 'tools', 'options'] as const) {
+    const left = JSON.stringify(before[field]);
+    const right = JSON.stringify(after[field]);
+    if (left !== right) {
+      fail(
+        `Prompt ${field} projection changed between requests ${index} and ${index + 1}:\n${describeDifference(left, right)}`,
+      );
+    }
+  }
+  if (after.input.length < before.input.length) {
+    fail(`Prompt input shrank between requests ${index} and ${index + 1}`);
+  }
+  for (const [partIndex, part] of before.input.entries()) {
+    if (JSON.stringify(part) !== JSON.stringify(after.input[partIndex])) {
+      fail(
+        `Prompt input is not prefix-stable between requests ${index} and ${index + 1}:\n${redact({ before, after })}`,
+      );
+    }
+  }
+}
+
+function assertStability(requests: Capture[]) {
+  const modelRequests = requests.filter((request) =>
+    isOrchestratorPayload(request.body),
+  );
+  if (modelRequests.length !== 4) {
+    fail(
+      `Expected exactly four primary-model requests, got ${modelRequests.length}:\n${redact(requests)}`,
+    );
+  }
+  const payloads = modelRequests.map((request) => request.body);
+  const systems = payloads.map((body) =>
+    messages(body)
+      .filter((message) => message.role === 'system')
+      .map((message) => JSON.stringify(message))
+      .join('\n'),
+  );
+  if (systems.some((system) => system.includes(PHASE_REMINDER))) {
+    fail(`PHASE_REMINDER leaked into system/instructions:\n${redact(systems)}`);
+  }
+  for (const [index, system] of systems.entries()) {
+    if (system !== systems[0]) {
+      const initial = systems[0];
+      if (initial === undefined) fail('Missing initial system projection');
+      fail(
+        `System/instruction projection changed at request ${index}:\n${describeDifference(initial, system)}`,
+      );
+    }
+  }
+  for (const [index, payload] of payloads.entries()) {
+    const next = payloads[index + 1];
+    if (next) assertPromptPrefix(payload, next, index);
+  }
+  const firstTurn = userProjection(payloads[0]);
+  const secondTurn = userProjection(payloads[3]);
+  if (!firstTurn.length || secondTurn.length < firstTurn.length) {
+    fail(`Missing transformed user prompt projection:\n${redact(payloads)}`);
+  }
+  for (const [index, projection] of firstTurn.entries()) {
+    if (secondTurn[index] !== projection) {
+      fail(
+        `Historical transformed prompt is not prefix-stable:\n${redact({ firstTurn, secondTurn })}`,
+      );
+    }
+  }
+  const reminderCount = (value: string) =>
+    value.split(PHASE_REMINDER_TEXT).length - 1;
+  const eligible = [...firstTurn, ...secondTurn];
+  if (eligible.some((projection) => reminderCount(projection) !== 1)) {
+    fail(
+      `Expected exactly one reminder per eligible user message:\n${redact(eligible)}`,
+    );
+  }
+  const toolNames = payloads
+    .flatMap((body) => (Array.isArray(body.tools) ? body.tools : []))
+    .map((tool) =>
+      typeof tool === 'object' &&
+      tool !== null &&
+      'function' in tool &&
+      typeof tool.function === 'object' &&
+      tool.function !== null &&
+      'name' in tool.function
+        ? String(tool.function.name)
+        : '',
+    );
+  if (!toolNames.includes('read') || !toolNames.includes('todowrite')) {
+    fail(`Expected read and todowrite to be available:\n${redact(toolNames)}`);
+  }
+  const transcript = payloads
+    .flatMap(messages)
+    .map((message) => JSON.stringify(message))
+    .join('\n');
+  if (!transcript.includes('call_read') || !transcript.includes('call_todo')) {
+    fail(
+      `Expected read and todowrite execution results in transcript:\n${redact(payloads)}`,
+    );
+  }
+}
+
+async function prompt(hostUrl: string, sessionID: string, text: string) {
+  const response = await fetch(`${hostUrl}/session/${sessionID}/message`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify({
+      agent: 'orchestrator',
+      model: { providerID: PROVIDER_ID, modelID: MODEL_ID },
+      parts: [{ type: 'text', text }],
+    }),
+    signal: AbortSignal.timeout(TIMEOUT_MS),
+  });
+  if (!response.ok)
+    fail(`Prompt failed (${response.status}): ${await response.text()}`);
+}
+
+async function main() {
+  if (!existsSync(distEntry)) {
+    fail(
+      'dist/index.js is missing. Run `bun run build` before verify:cache-stability.',
+    );
+  }
+  const opencode = requireOpenCodeBinary();
+  const localPlugin = requireLocalPluginTree();
+  const tempRoot = mkdtempSync(path.join(tmpdir(), 'omos-cache-stability-'));
+  let host: ReturnType<typeof spawn> | undefined;
+  let capture: Awaited<ReturnType<typeof createCaptureServer>> | undefined;
+  try {
+    const home = path.join(tempRoot, 'home');
+    const config = path.join(tempRoot, 'config');
+    const cache = path.join(tempRoot, 'cache');
+    const data = path.join(tempRoot, 'data');
+    const state = path.join(tempRoot, 'state');
+    const workspace = path.join(tempRoot, 'workspace');
+    for (const directory of [home, config, cache, data, state, workspace]) {
+      mkdirSync(directory, { recursive: true });
+    }
+    const readPath = path.join(workspace, 'fixture.txt');
+    writeFileSync(readPath, 'cache stability fixture\n');
+    const plugins = path.join(config, 'plugins');
+    mkdirSync(plugins, { recursive: true });
+    const configNodeModules = path.join(config, 'node_modules');
+    mkdirSync(configNodeModules, { recursive: true });
+    symlinkSync(
+      localPlugin.plugin,
+      path.join(configNodeModules, 'oh-my-opencode-slim'),
+    );
+    writeFileSync(
+      path.join(config, 'package.json'),
+      JSON.stringify({
+        type: 'module',
+        dependencies: { 'oh-my-opencode-slim': `file:${localPlugin.plugin}` },
+      }),
+    );
+    writeFileSync(
+      path.join(plugins, 'load-plugin.js'),
+      "export { default } from 'oh-my-opencode-slim';\n",
+    );
+    capture = await createCaptureServer(readPath);
+    writeFileSync(
+      path.join(workspace, 'opencode.json'),
+      JSON.stringify(
+        {
+          $schema: 'https://opencode.ai/config.json',
+          autoupdate: false,
+          share: 'disabled',
+          snapshot: false,
+          model: `${PROVIDER_ID}/${MODEL_ID}`,
+          small_model: `${PROVIDER_ID}/${MODEL_ID}`,
+          default_agent: 'orchestrator',
+          enabled_providers: [PROVIDER_ID],
+          provider: {
+            [PROVIDER_ID]: {
+              npm: '@ai-sdk/openai-compatible',
+              options: { apiKey: 'local-capture-key', baseURL: capture.url },
+              models: {
+                [MODEL_ID]: {
+                  name: MODEL_ID,
+                  tool_call: true,
+                  limit: { context: 32_000, output: 4_000 },
+                },
+              },
+            },
+          },
+        },
+        null,
+        2,
+      ),
+    );
+    const port = await getFreePort();
+    let logs = '';
+    host = spawn(
+      opencode,
+      [
+        'serve',
+        '--print-logs',
+        '--log-level',
+        'DEBUG',
+        '--hostname',
+        '127.0.0.1',
+        '--port',
+        String(port),
+      ],
+      {
+        cwd: workspace,
+        env: {
+          HOME: home,
+          XDG_CONFIG_HOME: config,
+          XDG_CACHE_HOME: cache,
+          XDG_DATA_HOME: data,
+          XDG_STATE_HOME: state,
+          OPENCODE_CONFIG_DIR: config,
+          OPENCODE_DISABLE_AUTOUPDATE: 'true',
+          OPENCODE_DISABLE_MODELS_FETCH: 'true',
+          OPENCODE_DISABLE_DEFAULT_PLUGINS: 'true',
+          BUN_AUTO_INSTALL: 'disable',
+          BUN_INSTALL_CACHE_DIR: path.join(tempRoot, 'empty-bun-cache'),
+          BUN_INSTALL_REGISTRY: 'http://127.0.0.1:9',
+          npm_config_offline: 'true',
+          NO_PROXY: '*',
+          no_proxy: '*',
+          PATH: process.env.PATH ?? '',
+        },
+        stdio: ['ignore', 'pipe', 'pipe'],
+      },
+    );
+    host.stdout?.on('data', (chunk) => (logs += String(chunk)));
+    host.stderr?.on('data', (chunk) => (logs += String(chunk)));
+    const hostUrl = `http://127.0.0.1:${port}`;
+    await waitFor(`${hostUrl}/global/health`, () => true, 'OpenCode health');
+    const created = await fetch(`${hostUrl}/session`, {
+      method: 'POST',
+      signal: AbortSignal.timeout(TIMEOUT_MS),
+    });
+    if (!created.ok)
+      fail(
+        `Session creation failed: ${await created.text()}\n${logs.slice(-12_000)}`,
+      );
+    const session = (await created.json()) as { id?: string };
+    if (!session.id) fail(`Session creation omitted id: ${redact(session)}`);
+    try {
+      await prompt(
+        hostUrl,
+        session.id,
+        'First cache-stability turn: use the requested tools.',
+      );
+    } catch (error) {
+      fail(
+        `${error instanceof Error ? error.message : String(error)}\nOpenCode logs:\n${logs.slice(-12_000)}`,
+      );
+    }
+    try {
+      await waitFor(
+        `${hostUrl}/global/health`,
+        () =>
+          capture?.requests.filter((request) =>
+            isOrchestratorPayload(request.body),
+          ).length === 3,
+        'tool execution',
+      );
+    } catch (error) {
+      fail(
+        `${error instanceof Error ? error.message : String(error)}\nCaptured requests:\n${redact(capture?.requests)}\nOpenCode logs:\n${logs.slice(-12_000)}`,
+      );
+    }
+    await prompt(
+      hostUrl,
+      session.id,
+      'Second cache-stability turn: finish without tools.',
+    );
+    await waitFor(
+      `${hostUrl}/global/health`,
+      () =>
+        capture?.requests.filter((request) =>
+          isOrchestratorPayload(request.body),
+        ).length === 4,
+      'second turn',
+    );
+    assertStability(capture.requests);
+    console.log('OpenCode cache-stability verification passed.');
+  } finally {
+    if (host) await stop(host);
+    const runningCapture = capture;
+    if (runningCapture) {
+      await new Promise<void>((resolve) =>
+        runningCapture.server.close(() => resolve()),
+      );
+    }
+    rmSync(tempRoot, { recursive: true, force: true });
+  }
+}
+
+await main();