Browse Source

Add live cache smoke probe with provider telemetry verification

Alvin Unreal 1 week ago
parent
commit
36d3cbb715
3 changed files with 657 additions and 0 deletions
  1. 40 0
      docs/cache-verification.md
  2. 1 0
      package.json
  3. 616 0
      scripts/cache-smoke.ts

+ 40 - 0
docs/cache-verification.md

@@ -15,6 +15,10 @@ and should not be conflated.
   cache guarantee.
 - **Runtime cache monitoring** watches provider-reported cache telemetry
   during real sessions and logs a warning when a cache bust signature appears.
+- **Live cache smoke** (`bun run cache:smoke`) is a one-command operational
+  probe: it starts a real `opencode serve` with your normal config/auth/plugin,
+  runs short scripted conversations, and reports per-request provider cache
+  telemetry with a verdict.
 
 ## Continuous cache-safety tests (CI)
 
@@ -42,6 +46,42 @@ mistakes that have not been made before:
 All hook injections must go through `src/hooks/cache-safe-injection.ts`; see
 the Prompt Cache Safety section in `AGENTS.md` for the authoring rules.
 
+## Live cache smoke (`bun run cache:smoke`)
+
+Answers "is provider prompt caching working in my setup right now?" at the
+cost of a handful of real requests. It starts an isolated `opencode serve`
+(your global config, auth, and plugin apply), runs scripted scenarios in
+fresh sessions, then reads `tokens.cache.read/write` from the stored
+assistant messages — including the subagent child sessions that delegation
+scenarios spawn.
+
+Each scenario is designed to fire specific plugin payload machinery:
+
+| Scenario | Triggers | Default |
+|---|---|---|
+| `plain` | phase reminder, skills filter, system transform | ✅ |
+| `tools` | tool-result growth across steps | ✅ |
+| `nudge` | post-file-tool nudge injection, phase-reminder equilibrium | extensive |
+| `todos` | todowrite churn (create/update/complete across turns) | extensive |
+| `long` | sliding cache breakpoints over a six-turn history | extensive |
+| `board` | job-board trailing injection, injected completion, reconcile | extensive |
+| `agents` | repeated delegation, task-session reuse, subagent caching | extensive |
+
+```bash
+bun run cache:smoke                          # plain + tools (fast, cheap)
+bun run cache:smoke -- --scenario extensive  # full matrix, several minutes
+bun run cache:smoke -- --scenario board,agents
+bun run cache:smoke -- --provider anthropic --model claude-sonnet-5
+bun run cache:smoke -- --server http://127.0.0.1:4096   # reuse a running server
+```
+
+A request is flagged **SUSPECT** when it is not the first request of its
+session, its input is ≥4096 tokens (above every provider's minimum cacheable
+prefix), and it read zero cached tokens. Exit codes: `0` caching works, `1`
+bust detected (any suspect request), `2` inconclusive (provider reported no
+cache telemetry), `3` setup error. Providers with read-only telemetry
+(OpenAI-style `cached_tokens`) show `cache-write 0` — normal, not a failure.
+
 ## Runtime cache monitoring
 
 `src/hooks/cache-monitor/` observes `message.updated` events and the

+ 1 - 0
package.json

@@ -62,6 +62,7 @@
     "verify:release": "bun run scripts/verify-release-artifact.ts",
     "verify:host-smoke": "bun run scripts/verify-opencode-host-smoke.ts",
     "verify:cache-stability": "bun run scripts/verify-opencode-cache-stability.ts",
+    "cache:smoke": "bun run scripts/cache-smoke.ts",
     "typecheck": "tsc --noEmit",
     "test": "bun test",
     "lint": "biome lint .",

+ 616 - 0
scripts/cache-smoke.ts

@@ -0,0 +1,616 @@
+/**
+ * Cache smoke — live end-to-end probe answering "is provider prompt caching
+ * working in my setup right now?"
+ *
+ * Starts a real `opencode serve` using your normal global config, auth, and
+ * plugin, runs scripted conversations against your default (or given)
+ * provider/model, and reads the provider-reported cache telemetry that
+ * OpenCode stores on every assistant message (`tokens.cache.read/write`,
+ * normalized from Anthropic's cache_control usage fields and OpenAI's
+ * `prompt_tokens_details.cached_tokens`).
+ *
+ * The scenarios are designed to trigger this plugin's payload-touching
+ * machinery on purpose — phase reminders, the post-file-tool nudge, todo
+ * churn, background job board injection/reconciliation, repeated specialist
+ * delegation with session reuse — so each injection path is validated
+ * against a real provider, including the subagent child sessions it spawns.
+ *
+ * Usage:
+ *   bun run cache:smoke [-- options]
+ *
+ * Options:
+ *   --server URL        Use an already-running OpenCode server instead of
+ *                       starting one (skips spawn/cleanup of the server).
+ *   --provider ID       Route turns to this provider (requires --model).
+ *   --model ID          Route turns to this model (requires --provider).
+ *   --agent NAME        Agent for each turn (default: server default).
+ *   --scenario LIST     Comma list of scenario names, or "extensive"/"all"
+ *                       (default: plain,tools — the cheap probe).
+ *   --turn-timeout-ms N Per-turn timeout (default 300000).
+ *   --keep-sessions     Don't delete the probe sessions afterwards.
+ *
+ * Exit codes: 0 caching works · 1 bust detected · 2 inconclusive (provider
+ * reported no cache telemetry) · 3 setup/runtime error.
+ *
+ * Each run costs real requests against your provider; the extensive set also
+ * spawns background specialist sessions and takes several minutes. This is a
+ * manual/operational probe, not a CI test — the CI-side guarantees live in
+ * the cache-safety suites (see docs/cache-verification.md).
+ */
+
+import { spawn } from 'node:child_process';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { createServer } from 'node:http';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+interface Args {
+  server?: string;
+  provider?: string;
+  model?: string;
+  agent?: string;
+  scenarios: string[];
+  turnTimeoutMs: number;
+  keepSessions: boolean;
+}
+
+interface Turn {
+  text: string;
+  /** Wait after the turn completes (lets background work land). */
+  pauseAfterMs?: number;
+}
+
+interface Scenario {
+  name: string;
+  description: string;
+  /** Plugin machinery this scenario is designed to fire. */
+  triggers: string;
+  turns: (nonce: string) => Turn[];
+}
+
+interface RequestRow {
+  messageID: string;
+  input: number;
+  output: number;
+  cacheRead: number;
+  cacheWrite: number;
+}
+
+interface SessionReport {
+  label: string;
+  rows: RequestRow[];
+}
+
+type Verdict = 'ok' | 'bust' | 'inconclusive';
+
+/**
+ * Requests at or above this input size with zero cache reads (after the
+ * first request of their session) count as suspect — comfortably above
+ * every provider's minimum cacheable prefix (OpenAI 1024, Anthropic ≤4096).
+ */
+const SUSPECT_INPUT_THRESHOLD = 4096;
+
+const NO_TOOLS = 'Do not use any tools.';
+
+const SCENARIOS: Scenario[] = [
+  {
+    name: 'plain',
+    description: 'multi-turn conversation, no tools',
+    triggers: 'phase reminder, skills filter, system transform',
+    turns: (nonce) => [
+      {
+        text: `Cache smoke probe ${nonce}. Reply with exactly "ack 1" and nothing else. ${NO_TOOLS}`,
+      },
+      { text: `Reply with exactly "ack 2" and nothing else. ${NO_TOOLS}` },
+      { text: `Reply with exactly "ack 3" and nothing else. ${NO_TOOLS}` },
+    ],
+  },
+  {
+    name: 'tools',
+    description: 'tool loop within and across turns',
+    triggers: 'tool-result growth across steps',
+    turns: (nonce) => [
+      {
+        text: `Cache smoke probe ${nonce}. Read the file package.json in the current directory and reply with only the value of its "name" field.`,
+      },
+      { text: `Reply with exactly "ack done" and nothing else. ${NO_TOOLS}` },
+    ],
+  },
+  {
+    name: 'nudge',
+    description: 'direct file read arms the post-file-tool nudge',
+    triggers: 'post-file-tool-nudge injection, then phase-reminder equilibrium',
+    turns: (nonce) => [
+      {
+        text: `Cache smoke probe ${nonce}. Do not delegate: use your read tool yourself on package.json and reply with only its "name" value.`,
+      },
+      { text: `Reply with exactly "ack nudged" and nothing else. ${NO_TOOLS}` },
+      {
+        text: `Reply with exactly "ack settled" and nothing else. ${NO_TOOLS}`,
+      },
+    ],
+  },
+  {
+    name: 'todos',
+    description: 'todo list created, updated, and completed across turns',
+    triggers: 'todowrite churn, todo hygiene',
+    turns: (nonce) => [
+      {
+        text: `Cache smoke probe ${nonce}. Use the todowrite tool to create exactly three todos named alpha, beta, gamma. Then reply with exactly "ack todos".`,
+      },
+      {
+        text: 'Mark the todo alpha as completed and add a new todo named delta. Then reply with exactly "ack updated".',
+      },
+      {
+        text: 'Mark every remaining todo as completed. Then reply with exactly "ack cleared".',
+      },
+      { text: `Reply with exactly "ack final" and nothing else. ${NO_TOOLS}` },
+    ],
+  },
+  {
+    name: 'long',
+    description: 'six-turn conversation, growing history',
+    triggers: 'sliding cache breakpoints over a long same-session history',
+    turns: (nonce) => [
+      {
+        text: `Cache smoke probe ${nonce}. Reply with exactly "ack 1" and nothing else. ${NO_TOOLS}`,
+      },
+      { text: `Name one prime number below 10. One word only. ${NO_TOOLS}` },
+      { text: `Name one planet. One word only. ${NO_TOOLS}` },
+      { text: `Name one color. One word only. ${NO_TOOLS}` },
+      { text: `Name one weekday. One word only. ${NO_TOOLS}` },
+      {
+        text: `Reply with exactly "ack long done" and nothing else. ${NO_TOOLS}`,
+      },
+    ],
+  },
+  {
+    name: 'board',
+    description:
+      'background task launched without waiting; board appears, completion lands, reconcile',
+    triggers:
+      'background job board trailing injection, injected completion message, reconciliation',
+    turns: (nonce) => [
+      {
+        text: `Cache smoke probe ${nonce}. Launch exactly one background @explorer task that lists the files in the current directory. Do not wait for it — reply immediately with exactly "ack launched".`,
+        pauseAfterMs: 30_000,
+      },
+      {
+        text: 'Reconcile any completed background tasks now, then reply with exactly "ack reconciled".',
+      },
+      {
+        text: `Reply with exactly "ack board done" and nothing else. ${NO_TOOLS}`,
+      },
+    ],
+  },
+  {
+    name: 'agents',
+    description:
+      'repeated delegation: same specialist twice (session reuse), then a second specialist',
+    triggers:
+      'board churn across multiple tasks, task session reuse by alias, @mention rewriting, subagent session caching',
+    turns: (nonce) => [
+      {
+        text: `Cache smoke probe ${nonce}. Launch a background @explorer task to list the files in the current directory. Wait for it to complete, then reply with exactly "ack explorer 1".`,
+      },
+      {
+        text: 'Give @explorer one more task: report how many lines package.json has. Wait for completion, then reply with exactly "ack explorer 2".',
+      },
+      {
+        text: 'Now launch a background @fixer task to create a file named hello.txt containing the single word "hi". Wait for completion, then reply with exactly "ack fixer".',
+      },
+      {
+        text: `Reply with exactly "ack agents done" and nothing else. ${NO_TOOLS}`,
+      },
+    ],
+  },
+];
+
+const CHEAP_SET = ['plain', 'tools'];
+const EXTENSIVE_SET = SCENARIOS.map((scenario) => scenario.name);
+
+function fail(message: string): never {
+  console.error(`\ncache-smoke: ${message}`);
+  process.exit(3);
+}
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = {
+    scenarios: CHEAP_SET,
+    turnTimeoutMs: 300_000,
+    keepSessions: false,
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const flag = argv[i];
+    const value = () => {
+      const next = argv[i + 1];
+      if (next === undefined) fail(`missing value for ${flag}`);
+      i += 1;
+      return next;
+    };
+    switch (flag) {
+      case '--server':
+        args.server = value().replace(/\/$/, '');
+        break;
+      case '--provider':
+        args.provider = value();
+        break;
+      case '--model':
+        args.model = value();
+        break;
+      case '--agent':
+        args.agent = value();
+        break;
+      case '--scenario': {
+        const requested = value();
+        if (requested === 'all' || requested === 'extensive') {
+          args.scenarios = EXTENSIVE_SET;
+          break;
+        }
+        const names = requested.split(',');
+        for (const name of names) {
+          if (!SCENARIOS.some((scenario) => scenario.name === name)) {
+            fail(
+              `unknown scenario "${name}" (${EXTENSIVE_SET.join(' | ')} | extensive | all)`,
+            );
+          }
+        }
+        args.scenarios = names;
+        break;
+      }
+      case '--turn-timeout-ms':
+        args.turnTimeoutMs = Number(value());
+        break;
+      case '--keep-sessions':
+        args.keepSessions = true;
+        break;
+      default:
+        fail(`unknown flag ${flag}`);
+    }
+  }
+  if (!!args.provider !== !!args.model) {
+    fail('--provider and --model must be given together');
+  }
+  if (!Number.isFinite(args.turnTimeoutMs) || args.turnTimeoutMs <= 0) {
+    fail('--turn-timeout-ms must be a positive number');
+  }
+  return args;
+}
+
+function getFreePort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const server = createServer();
+    server.once('error', reject);
+    server.listen(0, '127.0.0.1', () => {
+      const address = server.address();
+      if (!address || typeof address === 'string') {
+        server.close();
+        reject(new Error('failed to allocate a port'));
+        return;
+      }
+      server.close((error) => (error ? reject(error) : resolve(address.port)));
+    });
+  });
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === 'object' && !Array.isArray(value);
+}
+
+function finiteNumber(value: unknown): number {
+  return typeof value === 'number' && Number.isFinite(value) ? value : 0;
+}
+
+async function request(
+  base: string,
+  method: string,
+  route: string,
+  body?: unknown,
+  timeoutMs = 30_000,
+): Promise<unknown> {
+  const response = await fetch(`${base}${route}`, {
+    method,
+    headers: body === undefined ? {} : { 'content-type': 'application/json' },
+    body: body === undefined ? undefined : JSON.stringify(body),
+    signal: AbortSignal.timeout(timeoutMs),
+  });
+  if (!response.ok) {
+    throw new Error(`${method} ${route} returned HTTP ${response.status}`);
+  }
+  const text = await response.text();
+  if (!text) return undefined;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+}
+
+async function waitForHealth(base: string): Promise<void> {
+  const deadline = Date.now() + 40_000;
+  while (Date.now() < deadline) {
+    try {
+      const response = await fetch(`${base}/global/health`, {
+        signal: AbortSignal.timeout(2_000),
+      });
+      if (response.ok) return;
+    } catch {
+      // keep polling
+    }
+    await new Promise((resolve) => setTimeout(resolve, 400));
+  }
+  throw new Error('server did not become healthy within 40s');
+}
+
+function extractAssistantRows(rawMessages: unknown): RequestRow[] {
+  if (!Array.isArray(rawMessages)) return [];
+  const rows: RequestRow[] = [];
+  for (const raw of rawMessages) {
+    const info = isRecord(raw) && isRecord(raw.info) ? raw.info : undefined;
+    if (info?.role !== 'assistant') continue;
+    const tokens = isRecord(info.tokens) ? info.tokens : undefined;
+    if (!tokens) continue;
+    const cache = isRecord(tokens.cache) ? tokens.cache : undefined;
+    rows.push({
+      messageID: typeof info.id === 'string' ? info.id : '?',
+      input: finiteNumber(tokens.input),
+      output: finiteNumber(tokens.output),
+      cacheRead: finiteNumber(cache?.read),
+      cacheWrite: finiteNumber(cache?.write),
+    });
+  }
+  return rows;
+}
+
+async function fetchSessionRows(
+  base: string,
+  sessionID: string,
+): Promise<RequestRow[]> {
+  const rawMessages = await request(
+    base,
+    'GET',
+    `/session/${encodeURIComponent(sessionID)}/message`,
+  );
+  return extractAssistantRows(rawMessages);
+}
+
+async function listChildSessions(
+  base: string,
+  parentID: string,
+): Promise<Array<{ id: string; title: string }>> {
+  const raw = await request(base, 'GET', '/session').catch(() => undefined);
+  if (!Array.isArray(raw)) return [];
+  const children: Array<{ id: string; title: string }> = [];
+  for (const item of raw) {
+    if (!isRecord(item)) continue;
+    if (item.parentID !== parentID || typeof item.id !== 'string') continue;
+    children.push({
+      id: item.id,
+      title: typeof item.title === 'string' ? item.title : item.id,
+    });
+  }
+  return children;
+}
+
+/** Suspect = non-first request with a sizeable prompt and zero cache reads. */
+function suspectRows(rows: RequestRow[]): RequestRow[] {
+  return rows
+    .slice(1)
+    .filter(
+      (row) => row.cacheRead === 0 && row.input >= SUSPECT_INPUT_THRESHOLD,
+    );
+}
+
+function judge(reports: SessionReport[]): Verdict {
+  const allRows = reports.flatMap((report) => report.rows);
+  if (allRows.length < 2) return 'inconclusive';
+  const anyTelemetry = allRows.some(
+    (row) => row.cacheRead > 0 || row.cacheWrite > 0,
+  );
+  if (!anyTelemetry) return 'inconclusive';
+  const suspects = reports.flatMap((report) => suspectRows(report.rows));
+  return suspects.length > 0 ? 'bust' : 'ok';
+}
+
+function coverage(rows: RequestRow[]): string {
+  const later = rows.slice(1);
+  const read = later.reduce((sum, row) => sum + row.cacheRead, 0);
+  const input = later.reduce((sum, row) => sum + row.input, 0);
+  const denominator = read + input;
+  return denominator > 0
+    ? `${((read / denominator) * 100).toFixed(1)}%`
+    : 'n/a';
+}
+
+function printSessionTable(report: SessionReport): void {
+  console.log(`  ${report.label}`);
+  if (report.rows.length === 0) {
+    console.log('    no assistant requests with token telemetry recorded');
+    return;
+  }
+  console.log(
+    '    req  input      output   cache-read  cache-write  read-coverage',
+  );
+  const suspects = new Set(suspectRows(report.rows));
+  report.rows.forEach((row, index) => {
+    const denominator = row.input + row.cacheRead;
+    const rowCoverage =
+      denominator > 0
+        ? `${((row.cacheRead / denominator) * 100).toFixed(1)}%`
+        : 'n/a';
+    const marker = suspects.has(row) ? '  ← SUSPECT' : '';
+    console.log(
+      `    #${String(index + 1).padEnd(3)}${String(row.input).padEnd(11)}${String(row.output).padEnd(9)}${String(row.cacheRead).padEnd(12)}${String(row.cacheWrite).padEnd(13)}${rowCoverage}${marker}`,
+    );
+  });
+  console.log(
+    `    cache-read coverage after first request: ${coverage(report.rows)}`,
+  );
+}
+
+function printScenarioReport(
+  scenario: Scenario,
+  reports: SessionReport[],
+  verdict: Verdict,
+): void {
+  console.log(`\n━━ scenario: ${scenario.name} (${scenario.description})`);
+  console.log(`  triggers: ${scenario.triggers}`);
+  for (const report of reports) {
+    printSessionTable(report);
+  }
+  const labels: Record<Verdict, string> = {
+    ok: '✅ every sizeable follow-up request read the provider cache',
+    bust: '❌ SUSPECT requests above read 0 cached tokens — the prompt prefix changed between requests',
+    inconclusive:
+      '⚠️ provider reported no cache telemetry — cannot verify (provider may not support or report caching)',
+  };
+  console.log(`  verdict: ${labels[verdict]}`);
+}
+
+async function runScenario(
+  base: string,
+  args: Args,
+  scenario: Scenario,
+): Promise<{ reports: SessionReport[]; verdict: Verdict }> {
+  const created = await request(base, 'POST', '/session', {});
+  const sessionID = isRecord(created) ? String(created.id ?? '') : '';
+  if (!sessionID) throw new Error('POST /session returned no session id');
+  const cleanupIDs = [sessionID];
+
+  try {
+    const nonce = crypto.randomUUID();
+    for (const turn of scenario.turns(nonce)) {
+      await request(
+        base,
+        'POST',
+        `/session/${encodeURIComponent(sessionID)}/message`,
+        {
+          ...(args.agent ? { agent: args.agent } : {}),
+          ...(args.provider && args.model
+            ? { model: { providerID: args.provider, modelID: args.model } }
+            : {}),
+          parts: [{ type: 'text', text: turn.text }],
+        },
+        args.turnTimeoutMs,
+      );
+      if (turn.pauseAfterMs) {
+        console.log(
+          `  (waiting ${Math.round(turn.pauseAfterMs / 1000)}s for background work…)`,
+        );
+        await new Promise((resolve) => setTimeout(resolve, turn.pauseAfterMs));
+      }
+    }
+
+    const reports: SessionReport[] = [
+      {
+        label: `session ${sessionID} (main)`,
+        rows: await fetchSessionRows(base, sessionID),
+      },
+    ];
+    for (const child of await listChildSessions(base, sessionID)) {
+      cleanupIDs.push(child.id);
+      reports.push({
+        label: `session ${child.id} (subagent: ${child.title})`,
+        rows: await fetchSessionRows(base, child.id),
+      });
+    }
+    return { reports, verdict: judge(reports) };
+  } finally {
+    if (!args.keepSessions) {
+      for (const id of cleanupIDs.reverse()) {
+        await request(
+          base,
+          'DELETE',
+          `/session/${encodeURIComponent(id)}`,
+        ).catch(() => {});
+      }
+    } else {
+      console.log(`  sessions kept: ${cleanupIDs.join(', ')}`);
+    }
+  }
+}
+
+async function main(): Promise<void> {
+  const args = parseArgs(process.argv.slice(2));
+
+  let base = args.server;
+  let child: ReturnType<typeof spawn> | undefined;
+  let scratch: string | undefined;
+
+  if (!base) {
+    const binary = process.env.OPENCODE_BIN ?? Bun.which('opencode');
+    if (!binary) {
+      fail(
+        'opencode binary not found — install opencode or set OPENCODE_BIN, or pass --server URL',
+      );
+    }
+    scratch = mkdtempSync(path.join(tmpdir(), 'cache-smoke-'));
+    writeFileSync(
+      path.join(scratch, 'package.json'),
+      `${JSON.stringify({ name: 'cache-smoke-fixture', version: '0.0.0' }, null, 2)}\n`,
+    );
+    const port = await getFreePort();
+    base = `http://127.0.0.1:${port}`;
+    console.log(`starting opencode serve on ${base} (cwd: ${scratch})`);
+    child = spawn(
+      binary,
+      ['serve', '--hostname', '127.0.0.1', '--port', String(port)],
+      {
+        cwd: scratch,
+        stdio: ['ignore', 'pipe', 'pipe'],
+      },
+    );
+    const stderrChunks: string[] = [];
+    child.stderr?.on('data', (chunk: Buffer) => {
+      stderrChunks.push(String(chunk));
+    });
+    child.once('exit', (code) => {
+      if (code !== null && code !== 0) {
+        console.error(stderrChunks.join('').slice(-2000));
+        fail(`opencode serve exited early with code ${code}`);
+      }
+    });
+    await waitForHealth(base);
+  }
+
+  const cleanup = () => {
+    child?.kill('SIGTERM');
+    if (scratch) rmSync(scratch, { recursive: true, force: true });
+  };
+
+  try {
+    const scenarios = SCENARIOS.filter((scenario) =>
+      args.scenarios.includes(scenario.name),
+    );
+    const verdicts: Verdict[] = [];
+    for (const scenario of scenarios) {
+      console.log(`\nrunning scenario: ${scenario.name}…`);
+      const { reports, verdict } = await runScenario(base, args, scenario);
+      printScenarioReport(scenario, reports, verdict);
+      verdicts.push(verdict);
+    }
+
+    console.log('');
+    if (verdicts.includes('bust')) {
+      console.log(
+        'RESULT: ❌ cache bust detected. Cross-check the plugin build (bun run build), then use docs/cache-verification.md to localize the changing prefix byte.',
+      );
+      process.exitCode = 1;
+    } else if (verdicts.every((verdict) => verdict === 'inconclusive')) {
+      console.log(
+        'RESULT: ⚠️ inconclusive — the provider reported no cache telemetry for any request.',
+      );
+      process.exitCode = 2;
+    } else {
+      console.log(
+        'RESULT: ✅ provider prompt caching is working across the tested scenarios.',
+      );
+    }
+  } finally {
+    cleanup();
+  }
+}
+
+main().catch((error) => {
+  fail(error instanceof Error ? error.message : String(error));
+});