oh-my-opencode-slim/scripts/cache-smoke.ts

/**
 * Cache smoke — live end-to-end probe answering "is provider prompt caching
 * working in my setup right now?"
 *
 * Starts a real `opencode serve` using your normal global config, auth, and
 * plugin, runs scripted conversations against your default (or given)
 * provider/model, and reads the provider-reported cache telemetry that
 * OpenCode stores on every assistant message (`tokens.cache.read/write`,
 * normalized from Anthropic's cache_control usage fields and OpenAI's
 * `prompt_tokens_details.cached_tokens`).
 *
 * The scenarios are designed to trigger this plugin's payload-touching
 * machinery on purpose — phase reminders, the post-file-tool nudge, todo
 * churn, background job board injection/reconciliation, repeated specialist
 * delegation with session reuse — so each injection path is validated
 * against a real provider, including the subagent child sessions it spawns.
 *
 * Usage:
 *   bun run cache:smoke [-- options]
 *
 * Options:
 *   --server URL        Use an already-running OpenCode server instead of
 *                       starting one (skips spawn/cleanup of the server).
 *   --provider ID       Route turns to this provider (requires --model).
 *   --model ID          Route turns to this model (requires --provider).
 *   --agent NAME        Agent for each turn (default: server default).
 *   --scenario LIST     Comma list of scenario names, or "extensive"/"all"
 *                       (default: plain,tools — the cheap probe).
 *   --turn-timeout-ms N Per-turn timeout (default 300000).
 *   --keep-sessions     Don't delete the probe sessions afterwards.
 *
 * Exit codes: 0 caching works · 1 bust detected · 2 inconclusive (provider
 * reported no cache telemetry) · 3 setup/runtime error.
 *
 * Each run costs real requests against your provider; the extensive set also
 * spawns background specialist sessions and takes several minutes. This is a
 * manual/operational probe, not a CI test — the CI-side guarantees live in
 * the cache-safety suites (see docs/cache-verification.md).
 */

import { spawn } from 'node:child_process';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { createServer } from 'node:http';
import { tmpdir } from 'node:os';
import path from 'node:path';

interface Args {
  server?: string;
  provider?: string;
  model?: string;
  agent?: string;
  scenarios: string[];
  turnTimeoutMs: number;
  keepSessions: boolean;
}

interface Turn {
  text: string;
  /** Wait after the turn completes (lets background work land). */
  pauseAfterMs?: number;
}

interface Scenario {
  name: string;
  description: string;
  /** Plugin machinery this scenario is designed to fire. */
  triggers: string;
  turns: (nonce: string) => Turn[];
}

interface RequestRow {
  messageID: string;
  input: number;
  output: number;
  cacheRead: number;
  cacheWrite: number;
}

interface SessionReport {
  label: string;
  rows: RequestRow[];
}

type Verdict = 'ok' | 'bust' | 'inconclusive';

/**
 * Requests at or above this input size with zero cache reads (after the
 * first request of their session) count as suspect — comfortably above
 * every provider's minimum cacheable prefix (OpenAI 1024, Anthropic ≤4096).
 */
const SUSPECT_INPUT_THRESHOLD = 4096;

const NO_TOOLS = 'Do not use any tools.';

const SCENARIOS: Scenario[] = [
  {
    name: 'plain',
    description: 'multi-turn conversation, no tools',
    triggers: 'phase reminder, skills filter, system transform',
    turns: (nonce) => [
      {
        text: `Cache smoke probe ${nonce}. Reply with exactly "ack 1" and nothing else. ${NO_TOOLS}`,
      },
      { text: `Reply with exactly "ack 2" and nothing else. ${NO_TOOLS}` },
      { text: `Reply with exactly "ack 3" and nothing else. ${NO_TOOLS}` },
    ],
  },
  {
    name: 'tools',
    description: 'tool loop within and across turns',
    triggers: 'tool-result growth across steps',
    turns: (nonce) => [
      {
        text: `Cache smoke probe ${nonce}. Read the file package.json in the current directory and reply with only the value of its "name" field.`,
      },
      { text: `Reply with exactly "ack done" and nothing else. ${NO_TOOLS}` },
    ],
  },
  {
    name: 'nudge',
    description: 'direct file read arms the post-file-tool nudge',
    triggers: 'post-file-tool-nudge injection, then phase-reminder equilibrium',
    turns: (nonce) => [
      {
        text: `Cache smoke probe ${nonce}. Do not delegate: use your read tool yourself on package.json and reply with only its "name" value.`,
      },
      { text: `Reply with exactly "ack nudged" and nothing else. ${NO_TOOLS}` },
      {
        text: `Reply with exactly "ack settled" and nothing else. ${NO_TOOLS}`,
      },
    ],
  },
  {
    name: 'todos',
    description: 'todo list created, updated, and completed across turns',
    triggers: 'todowrite churn, todo hygiene',
    turns: (nonce) => [
      {
        text: `Cache smoke probe ${nonce}. Use the todowrite tool to create exactly three todos named alpha, beta, gamma. Then reply with exactly "ack todos".`,
      },
      {
        text: 'Mark the todo alpha as completed and add a new todo named delta. Then reply with exactly "ack updated".',
      },
      {
        text: 'Mark every remaining todo as completed. Then reply with exactly "ack cleared".',
      },
      { text: `Reply with exactly "ack final" and nothing else. ${NO_TOOLS}` },
    ],
  },
  {
    name: 'long',
    description: 'six-turn conversation, growing history',
    triggers: 'sliding cache breakpoints over a long same-session history',
    turns: (nonce) => [
      {
        text: `Cache smoke probe ${nonce}. Reply with exactly "ack 1" and nothing else. ${NO_TOOLS}`,
      },
      { text: `Name one prime number below 10. One word only. ${NO_TOOLS}` },
      { text: `Name one planet. One word only. ${NO_TOOLS}` },
      { text: `Name one color. One word only. ${NO_TOOLS}` },
      { text: `Name one weekday. One word only. ${NO_TOOLS}` },
      {
        text: `Reply with exactly "ack long done" and nothing else. ${NO_TOOLS}`,
      },
    ],
  },
  {
    name: 'board',
    description:
      'background task launched without waiting; board appears, completion lands, reconcile',
    triggers:
      'background job board trailing injection, injected completion message, reconciliation',
    turns: (nonce) => [
      {
        text: `Cache smoke probe ${nonce}. Launch exactly one background @explorer task that lists the files in the current directory. Do not wait for it — reply immediately with exactly "ack launched".`,
        pauseAfterMs: 30_000,
      },
      {
        text: 'Reconcile any completed background tasks now, then reply with exactly "ack reconciled".',
      },
      {
        text: `Reply with exactly "ack board done" and nothing else. ${NO_TOOLS}`,
      },
    ],
  },
  {
    name: 'running-lane',
    description:
      'parent keeps talking while a background lane is still running (PR #871 window)',
    triggers:
      'running task tool_result sits mid-history across consecutive requests; byte churn there busts the cache tail',
    turns: (nonce) => [
      {
        text: `Cache smoke probe ${nonce}. Launch exactly one background @explorer task with this prompt: "Produce a very thorough report of at least 600 words describing every file in the current directory, its likely purpose, and recommendations." Do not wait for it — reply immediately with exactly "ack lane started".`,
      },
      {
        text: `Reply with exactly "ack while running" and nothing else. ${NO_TOOLS}`,
      },
      {
        text: `Reply with exactly "ack still running" and nothing else. ${NO_TOOLS}`,
        pauseAfterMs: 45_000,
      },
      {
        text: 'Reconcile any completed background tasks, then reply with exactly "ack lane done".',
      },
    ],
  },
  {
    name: 'agents',
    description:
      'repeated delegation: same specialist twice (session reuse), then a second specialist',
    triggers:
      'board churn across multiple tasks, task session reuse by alias, @mention rewriting, subagent session caching',
    turns: (nonce) => [
      {
        text: `Cache smoke probe ${nonce}. Launch a background @explorer task to list the files in the current directory. Wait for it to complete, then reply with exactly "ack explorer 1".`,
      },
      {
        text: 'Give @explorer one more task: report how many lines package.json has. Wait for completion, then reply with exactly "ack explorer 2".',
      },
      {
        text: 'Now launch a background @fixer task to create a file named hello.txt containing the single word "hi". Wait for completion, then reply with exactly "ack fixer".',
      },
      {
        text: `Reply with exactly "ack agents done" and nothing else. ${NO_TOOLS}`,
      },
    ],
  },
];

const CHEAP_SET = ['plain', 'tools'];
const EXTENSIVE_SET = SCENARIOS.map((scenario) => scenario.name);

function fail(message: string): never {
  console.error(`\ncache-smoke: ${message}`);
  process.exit(3);
}

function parseArgs(argv: string[]): Args {
  const args: Args = {
    scenarios: CHEAP_SET,
    turnTimeoutMs: 300_000,
    keepSessions: false,
  };
  for (let i = 0; i < argv.length; i += 1) {
    const flag = argv[i];
    const value = () => {
      const next = argv[i + 1];
      if (next === undefined) fail(`missing value for ${flag}`);
      i += 1;
      return next;
    };
    switch (flag) {
      case '--server':
        args.server = value().replace(/\/$/, '');
        break;
      case '--provider':
        args.provider = value();
        break;
      case '--model':
        args.model = value();
        break;
      case '--agent':
        args.agent = value();
        break;
      case '--scenario': {
        const requested = value();
        if (requested === 'all' || requested === 'extensive') {
          args.scenarios = EXTENSIVE_SET;
          break;
        }
        const names = requested.split(',');
        for (const name of names) {
          if (!SCENARIOS.some((scenario) => scenario.name === name)) {
            fail(
              `unknown scenario "${name}" (${EXTENSIVE_SET.join(' | ')} | extensive | all)`,
            );
          }
        }
        args.scenarios = names;
        break;
      }
      case '--turn-timeout-ms':
        args.turnTimeoutMs = Number(value());
        break;
      case '--keep-sessions':
        args.keepSessions = true;
        break;
      default:
        fail(`unknown flag ${flag}`);
    }
  }
  if (!!args.provider !== !!args.model) {
    fail('--provider and --model must be given together');
  }
  if (!Number.isFinite(args.turnTimeoutMs) || args.turnTimeoutMs <= 0) {
    fail('--turn-timeout-ms must be a positive number');
  }
  return args;
}

function getFreePort(): Promise<number> {
  return new Promise((resolve, reject) => {
    const server = createServer();
    server.once('error', reject);
    server.listen(0, '127.0.0.1', () => {
      const address = server.address();
      if (!address || typeof address === 'string') {
        server.close();
        reject(new Error('failed to allocate a port'));
        return;
      }
      server.close((error) => (error ? reject(error) : resolve(address.port)));
    });
  });
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return !!value && typeof value === 'object' && !Array.isArray(value);
}

function finiteNumber(value: unknown): number {
  return typeof value === 'number' && Number.isFinite(value) ? value : 0;
}

async function request(
  base: string,
  method: string,
  route: string,
  body?: unknown,
  timeoutMs = 30_000,
): Promise<unknown> {
  const response = await fetch(`${base}${route}`, {
    method,
    headers: body === undefined ? {} : { 'content-type': 'application/json' },
    body: body === undefined ? undefined : JSON.stringify(body),
    signal: AbortSignal.timeout(timeoutMs),
  });
  if (!response.ok) {
    throw new Error(`${method} ${route} returned HTTP ${response.status}`);
  }
  const text = await response.text();
  if (!text) return undefined;
  try {
    return JSON.parse(text);
  } catch {
    return undefined;
  }
}

async function waitForHealth(base: string): Promise<void> {
  const deadline = Date.now() + 40_000;
  while (Date.now() < deadline) {
    try {
      const response = await fetch(`${base}/global/health`, {
        signal: AbortSignal.timeout(2_000),
      });
      if (response.ok) return;
    } catch {
      // keep polling
    }
    await new Promise((resolve) => setTimeout(resolve, 400));
  }
  throw new Error('server did not become healthy within 40s');
}

function extractAssistantRows(rawMessages: unknown): RequestRow[] {
  if (!Array.isArray(rawMessages)) return [];
  const rows: RequestRow[] = [];
  for (const raw of rawMessages) {
    const info = isRecord(raw) && isRecord(raw.info) ? raw.info : undefined;
    if (info?.role !== 'assistant') continue;
    const tokens = isRecord(info.tokens) ? info.tokens : undefined;
    if (!tokens) continue;
    const cache = isRecord(tokens.cache) ? tokens.cache : undefined;
    rows.push({
      messageID: typeof info.id === 'string' ? info.id : '?',
      input: finiteNumber(tokens.input),
      output: finiteNumber(tokens.output),
      cacheRead: finiteNumber(cache?.read),
      cacheWrite: finiteNumber(cache?.write),
    });
  }
  return rows;
}

async function fetchSessionRows(
  base: string,
  sessionID: string,
): Promise<RequestRow[]> {
  const rawMessages = await request(
    base,
    'GET',
    `/session/${encodeURIComponent(sessionID)}/message`,
  );
  return extractAssistantRows(rawMessages);
}

async function listChildSessions(
  base: string,
  parentID: string,
): Promise<Array<{ id: string; title: string }>> {
  const raw = await request(base, 'GET', '/session').catch(() => undefined);
  if (!Array.isArray(raw)) return [];
  const children: Array<{ id: string; title: string }> = [];
  for (const item of raw) {
    if (!isRecord(item)) continue;
    if (item.parentID !== parentID || typeof item.id !== 'string') continue;
    children.push({
      id: item.id,
      title: typeof item.title === 'string' ? item.title : item.id,
    });
  }
  return children;
}

/** Suspect = non-first request with a sizeable prompt and zero cache reads. */
function suspectRows(rows: RequestRow[]): RequestRow[] {
  return rows
    .slice(1)
    .filter(
      (row) => row.cacheRead === 0 && row.input >= SUSPECT_INPUT_THRESHOLD,
    );
}

function judge(reports: SessionReport[]): Verdict {
  const allRows = reports.flatMap((report) => report.rows);
  if (allRows.length < 2) return 'inconclusive';
  const anyTelemetry = allRows.some(
    (row) => row.cacheRead > 0 || row.cacheWrite > 0,
  );
  if (!anyTelemetry) return 'inconclusive';
  const suspects = reports.flatMap((report) => suspectRows(report.rows));
  return suspects.length > 0 ? 'bust' : 'ok';
}

function coverage(rows: RequestRow[]): string {
  const later = rows.slice(1);
  const read = later.reduce((sum, row) => sum + row.cacheRead, 0);
  const input = later.reduce((sum, row) => sum + row.input, 0);
  const denominator = read + input;
  return denominator > 0
    ? `${((read / denominator) * 100).toFixed(1)}%`
    : 'n/a';
}

function printSessionTable(report: SessionReport): void {
  console.log(`  ${report.label}`);
  if (report.rows.length === 0) {
    console.log('    no assistant requests with token telemetry recorded');
    return;
  }
  console.log(
    '    req  input      output   cache-read  cache-write  read-coverage',
  );
  const suspects = new Set(suspectRows(report.rows));
  report.rows.forEach((row, index) => {
    const denominator = row.input + row.cacheRead;
    const rowCoverage =
      denominator > 0
        ? `${((row.cacheRead / denominator) * 100).toFixed(1)}%`
        : 'n/a';
    const marker = suspects.has(row) ? '  ← SUSPECT' : '';
    console.log(
      `    #${String(index + 1).padEnd(3)}${String(row.input).padEnd(11)}${String(row.output).padEnd(9)}${String(row.cacheRead).padEnd(12)}${String(row.cacheWrite).padEnd(13)}${rowCoverage}${marker}`,
    );
  });
  console.log(
    `    cache-read coverage after first request: ${coverage(report.rows)}`,
  );
}

function printScenarioReport(
  scenario: Scenario,
  reports: SessionReport[],
  verdict: Verdict,
): void {
  console.log(`\n━━ scenario: ${scenario.name} (${scenario.description})`);
  console.log(`  triggers: ${scenario.triggers}`);
  for (const report of reports) {
    printSessionTable(report);
  }
  const labels: Record<Verdict, string> = {
    ok: '✅ every sizeable follow-up request read the provider cache',
    bust: '❌ SUSPECT requests above read 0 cached tokens — the prompt prefix changed between requests',
    inconclusive:
      '⚠️ provider reported no cache telemetry — cannot verify (provider may not support or report caching)',
  };
  console.log(`  verdict: ${labels[verdict]}`);
}

async function runScenario(
  base: string,
  args: Args,
  scenario: Scenario,
): Promise<{ reports: SessionReport[]; verdict: Verdict }> {
  const created = await request(base, 'POST', '/session', {});
  const sessionID = isRecord(created) ? String(created.id ?? '') : '';
  if (!sessionID) throw new Error('POST /session returned no session id');
  const cleanupIDs = [sessionID];

  try {
    const nonce = crypto.randomUUID();
    for (const turn of scenario.turns(nonce)) {
      await request(
        base,
        'POST',
        `/session/${encodeURIComponent(sessionID)}/message`,
        {
          ...(args.agent ? { agent: args.agent } : {}),
          ...(args.provider && args.model
            ? { model: { providerID: args.provider, modelID: args.model } }
            : {}),
          parts: [{ type: 'text', text: turn.text }],
        },
        args.turnTimeoutMs,
      );
      if (turn.pauseAfterMs) {
        console.log(
          `  (waiting ${Math.round(turn.pauseAfterMs / 1000)}s for background work…)`,
        );
        await new Promise((resolve) => setTimeout(resolve, turn.pauseAfterMs));
      }
    }

    const reports: SessionReport[] = [
      {
        label: `session ${sessionID} (main)`,
        rows: await fetchSessionRows(base, sessionID),
      },
    ];
    for (const child of await listChildSessions(base, sessionID)) {
      cleanupIDs.push(child.id);
      reports.push({
        label: `session ${child.id} (subagent: ${child.title})`,
        rows: await fetchSessionRows(base, child.id),
      });
    }
    return { reports, verdict: judge(reports) };
  } finally {
    if (!args.keepSessions) {
      for (const id of cleanupIDs.reverse()) {
        await request(
          base,
          'DELETE',
          `/session/${encodeURIComponent(id)}`,
        ).catch(() => {});
      }
    } else {
      console.log(`  sessions kept: ${cleanupIDs.join(', ')}`);
    }
  }
}

async function main(): Promise<void> {
  const args = parseArgs(process.argv.slice(2));

  let base = args.server;
  let child: ReturnType<typeof spawn> | undefined;
  let scratch: string | undefined;

  if (!base) {
    const binary = process.env.OPENCODE_BIN ?? Bun.which('opencode');
    if (!binary) {
      fail(
        'opencode binary not found — install opencode or set OPENCODE_BIN, or pass --server URL',
      );
    }
    scratch = mkdtempSync(path.join(tmpdir(), 'cache-smoke-'));
    writeFileSync(
      path.join(scratch, 'package.json'),
      `${JSON.stringify({ name: 'cache-smoke-fixture', version: '0.0.0' }, null, 2)}\n`,
    );
    const port = await getFreePort();
    base = `http://127.0.0.1:${port}`;
    console.log(`starting opencode serve on ${base} (cwd: ${scratch})`);
    child = spawn(
      binary,
      ['serve', '--hostname', '127.0.0.1', '--port', String(port)],
      {
        cwd: scratch,
        stdio: ['ignore', 'pipe', 'pipe'],
      },
    );
    const stderrChunks: string[] = [];
    child.stderr?.on('data', (chunk: Buffer) => {
      stderrChunks.push(String(chunk));
    });
    child.once('exit', (code) => {
      if (code !== null && code !== 0) {
        console.error(stderrChunks.join('').slice(-2000));
        fail(`opencode serve exited early with code ${code}`);
      }
    });
    await waitForHealth(base);
  }

  const cleanup = () => {
    child?.kill('SIGTERM');
    if (scratch) rmSync(scratch, { recursive: true, force: true });
  };

  try {
    const scenarios = SCENARIOS.filter((scenario) =>
      args.scenarios.includes(scenario.name),
    );
    const verdicts: Verdict[] = [];
    for (const scenario of scenarios) {
      console.log(`\nrunning scenario: ${scenario.name}…`);
      const { reports, verdict } = await runScenario(base, args, scenario);
      printScenarioReport(scenario, reports, verdict);
      verdicts.push(verdict);
    }

    console.log('');
    if (verdicts.includes('bust')) {
      console.log(
        'RESULT: ❌ cache bust detected. Cross-check the plugin build (bun run build), then use docs/cache-verification.md to localize the changing prefix byte.',
      );
      process.exitCode = 1;
    } else if (verdicts.every((verdict) => verdict === 'inconclusive')) {
      console.log(
        'RESULT: ⚠️ inconclusive — the provider reported no cache telemetry for any request.',
      );
      process.exitCode = 2;
    } else {
      console.log(
        'RESULT: ✅ provider prompt caching is working across the tested scenarios.',
      );
    }
  } finally {
    cleanup();
  }
}

main().catch((error) => {
  fail(error instanceof Error ? error.message : String(error));
});