|
|
@@ -0,0 +1,616 @@
|
|
|
+/**
|
|
|
+ * Cache smoke — live end-to-end probe answering "is provider prompt caching
|
|
|
+ * working in my setup right now?"
|
|
|
+ *
|
|
|
+ * Starts a real `opencode serve` using your normal global config, auth, and
|
|
|
+ * plugin, runs scripted conversations against your default (or given)
|
|
|
+ * provider/model, and reads the provider-reported cache telemetry that
|
|
|
+ * OpenCode stores on every assistant message (`tokens.cache.read/write`,
|
|
|
+ * normalized from Anthropic's cache_control usage fields and OpenAI's
|
|
|
+ * `prompt_tokens_details.cached_tokens`).
|
|
|
+ *
|
|
|
+ * The scenarios are designed to trigger this plugin's payload-touching
|
|
|
+ * machinery on purpose — phase reminders, the post-file-tool nudge, todo
|
|
|
+ * churn, background job board injection/reconciliation, repeated specialist
|
|
|
+ * delegation with session reuse — so each injection path is validated
|
|
|
+ * against a real provider, including the subagent child sessions it spawns.
|
|
|
+ *
|
|
|
+ * Usage:
|
|
|
+ * bun run cache:smoke [-- options]
|
|
|
+ *
|
|
|
+ * Options:
|
|
|
+ * --server URL Use an already-running OpenCode server instead of
|
|
|
+ * starting one (skips spawn/cleanup of the server).
|
|
|
+ * --provider ID Route turns to this provider (requires --model).
|
|
|
+ * --model ID Route turns to this model (requires --provider).
|
|
|
+ * --agent NAME Agent for each turn (default: server default).
|
|
|
+ * --scenario LIST Comma list of scenario names, or "extensive"/"all"
|
|
|
+ * (default: plain,tools — the cheap probe).
|
|
|
+ * --turn-timeout-ms N Per-turn timeout (default 300000).
|
|
|
+ * --keep-sessions Don't delete the probe sessions afterwards.
|
|
|
+ *
|
|
|
+ * Exit codes: 0 caching works · 1 bust detected · 2 inconclusive (provider
|
|
|
+ * reported no cache telemetry) · 3 setup/runtime error.
|
|
|
+ *
|
|
|
+ * Each run costs real requests against your provider; the extensive set also
|
|
|
+ * spawns background specialist sessions and takes several minutes. This is a
|
|
|
+ * manual/operational probe, not a CI test — the CI-side guarantees live in
|
|
|
+ * the cache-safety suites (see docs/cache-verification.md).
|
|
|
+ */
|
|
|
+
|
|
|
+import { spawn } from 'node:child_process';
|
|
|
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
|
+import { createServer } from 'node:http';
|
|
|
+import { tmpdir } from 'node:os';
|
|
|
+import path from 'node:path';
|
|
|
+
|
|
|
+interface Args {
|
|
|
+ server?: string;
|
|
|
+ provider?: string;
|
|
|
+ model?: string;
|
|
|
+ agent?: string;
|
|
|
+ scenarios: string[];
|
|
|
+ turnTimeoutMs: number;
|
|
|
+ keepSessions: boolean;
|
|
|
+}
|
|
|
+
|
|
|
+interface Turn {
|
|
|
+ text: string;
|
|
|
+ /** Wait after the turn completes (lets background work land). */
|
|
|
+ pauseAfterMs?: number;
|
|
|
+}
|
|
|
+
|
|
|
+interface Scenario {
|
|
|
+ name: string;
|
|
|
+ description: string;
|
|
|
+ /** Plugin machinery this scenario is designed to fire. */
|
|
|
+ triggers: string;
|
|
|
+ turns: (nonce: string) => Turn[];
|
|
|
+}
|
|
|
+
|
|
|
+interface RequestRow {
|
|
|
+ messageID: string;
|
|
|
+ input: number;
|
|
|
+ output: number;
|
|
|
+ cacheRead: number;
|
|
|
+ cacheWrite: number;
|
|
|
+}
|
|
|
+
|
|
|
+interface SessionReport {
|
|
|
+ label: string;
|
|
|
+ rows: RequestRow[];
|
|
|
+}
|
|
|
+
|
|
|
+type Verdict = 'ok' | 'bust' | 'inconclusive';
|
|
|
+
|
|
|
+/**
|
|
|
+ * Requests at or above this input size with zero cache reads (after the
|
|
|
+ * first request of their session) count as suspect — comfortably above
|
|
|
+ * every provider's minimum cacheable prefix (OpenAI 1024, Anthropic ≤4096).
|
|
|
+ */
|
|
|
+const SUSPECT_INPUT_THRESHOLD = 4096;
|
|
|
+
|
|
|
+const NO_TOOLS = 'Do not use any tools.';
|
|
|
+
|
|
|
+const SCENARIOS: Scenario[] = [
|
|
|
+ {
|
|
|
+ name: 'plain',
|
|
|
+ description: 'multi-turn conversation, no tools',
|
|
|
+ triggers: 'phase reminder, skills filter, system transform',
|
|
|
+ turns: (nonce) => [
|
|
|
+ {
|
|
|
+ text: `Cache smoke probe ${nonce}. Reply with exactly "ack 1" and nothing else. ${NO_TOOLS}`,
|
|
|
+ },
|
|
|
+ { text: `Reply with exactly "ack 2" and nothing else. ${NO_TOOLS}` },
|
|
|
+ { text: `Reply with exactly "ack 3" and nothing else. ${NO_TOOLS}` },
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ {
|
|
|
+ name: 'tools',
|
|
|
+ description: 'tool loop within and across turns',
|
|
|
+ triggers: 'tool-result growth across steps',
|
|
|
+ turns: (nonce) => [
|
|
|
+ {
|
|
|
+ text: `Cache smoke probe ${nonce}. Read the file package.json in the current directory and reply with only the value of its "name" field.`,
|
|
|
+ },
|
|
|
+ { text: `Reply with exactly "ack done" and nothing else. ${NO_TOOLS}` },
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ {
|
|
|
+ name: 'nudge',
|
|
|
+ description: 'direct file read arms the post-file-tool nudge',
|
|
|
+ triggers: 'post-file-tool-nudge injection, then phase-reminder equilibrium',
|
|
|
+ turns: (nonce) => [
|
|
|
+ {
|
|
|
+ text: `Cache smoke probe ${nonce}. Do not delegate: use your read tool yourself on package.json and reply with only its "name" value.`,
|
|
|
+ },
|
|
|
+ { text: `Reply with exactly "ack nudged" and nothing else. ${NO_TOOLS}` },
|
|
|
+ {
|
|
|
+ text: `Reply with exactly "ack settled" and nothing else. ${NO_TOOLS}`,
|
|
|
+ },
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ {
|
|
|
+ name: 'todos',
|
|
|
+ description: 'todo list created, updated, and completed across turns',
|
|
|
+ triggers: 'todowrite churn, todo hygiene',
|
|
|
+ turns: (nonce) => [
|
|
|
+ {
|
|
|
+ text: `Cache smoke probe ${nonce}. Use the todowrite tool to create exactly three todos named alpha, beta, gamma. Then reply with exactly "ack todos".`,
|
|
|
+ },
|
|
|
+ {
|
|
|
+ text: 'Mark the todo alpha as completed and add a new todo named delta. Then reply with exactly "ack updated".',
|
|
|
+ },
|
|
|
+ {
|
|
|
+ text: 'Mark every remaining todo as completed. Then reply with exactly "ack cleared".',
|
|
|
+ },
|
|
|
+ { text: `Reply with exactly "ack final" and nothing else. ${NO_TOOLS}` },
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ {
|
|
|
+ name: 'long',
|
|
|
+ description: 'six-turn conversation, growing history',
|
|
|
+ triggers: 'sliding cache breakpoints over a long same-session history',
|
|
|
+ turns: (nonce) => [
|
|
|
+ {
|
|
|
+ text: `Cache smoke probe ${nonce}. Reply with exactly "ack 1" and nothing else. ${NO_TOOLS}`,
|
|
|
+ },
|
|
|
+ { text: `Name one prime number below 10. One word only. ${NO_TOOLS}` },
|
|
|
+ { text: `Name one planet. One word only. ${NO_TOOLS}` },
|
|
|
+ { text: `Name one color. One word only. ${NO_TOOLS}` },
|
|
|
+ { text: `Name one weekday. One word only. ${NO_TOOLS}` },
|
|
|
+ {
|
|
|
+ text: `Reply with exactly "ack long done" and nothing else. ${NO_TOOLS}`,
|
|
|
+ },
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ {
|
|
|
+ name: 'board',
|
|
|
+ description:
|
|
|
+ 'background task launched without waiting; board appears, completion lands, reconcile',
|
|
|
+ triggers:
|
|
|
+ 'background job board trailing injection, injected completion message, reconciliation',
|
|
|
+ turns: (nonce) => [
|
|
|
+ {
|
|
|
+ text: `Cache smoke probe ${nonce}. Launch exactly one background @explorer task that lists the files in the current directory. Do not wait for it — reply immediately with exactly "ack launched".`,
|
|
|
+ pauseAfterMs: 30_000,
|
|
|
+ },
|
|
|
+ {
|
|
|
+ text: 'Reconcile any completed background tasks now, then reply with exactly "ack reconciled".',
|
|
|
+ },
|
|
|
+ {
|
|
|
+ text: `Reply with exactly "ack board done" and nothing else. ${NO_TOOLS}`,
|
|
|
+ },
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ {
|
|
|
+ name: 'agents',
|
|
|
+ description:
|
|
|
+ 'repeated delegation: same specialist twice (session reuse), then a second specialist',
|
|
|
+ triggers:
|
|
|
+ 'board churn across multiple tasks, task session reuse by alias, @mention rewriting, subagent session caching',
|
|
|
+ turns: (nonce) => [
|
|
|
+ {
|
|
|
+ text: `Cache smoke probe ${nonce}. Launch a background @explorer task to list the files in the current directory. Wait for it to complete, then reply with exactly "ack explorer 1".`,
|
|
|
+ },
|
|
|
+ {
|
|
|
+ text: 'Give @explorer one more task: report how many lines package.json has. Wait for completion, then reply with exactly "ack explorer 2".',
|
|
|
+ },
|
|
|
+ {
|
|
|
+ text: 'Now launch a background @fixer task to create a file named hello.txt containing the single word "hi". Wait for completion, then reply with exactly "ack fixer".',
|
|
|
+ },
|
|
|
+ {
|
|
|
+ text: `Reply with exactly "ack agents done" and nothing else. ${NO_TOOLS}`,
|
|
|
+ },
|
|
|
+ ],
|
|
|
+ },
|
|
|
+];
|
|
|
+
|
|
|
+const CHEAP_SET = ['plain', 'tools'];
|
|
|
+const EXTENSIVE_SET = SCENARIOS.map((scenario) => scenario.name);
|
|
|
+
|
|
|
+function fail(message: string): never {
|
|
|
+ console.error(`\ncache-smoke: ${message}`);
|
|
|
+ process.exit(3);
|
|
|
+}
|
|
|
+
|
|
|
+function parseArgs(argv: string[]): Args {
|
|
|
+ const args: Args = {
|
|
|
+ scenarios: CHEAP_SET,
|
|
|
+ turnTimeoutMs: 300_000,
|
|
|
+ keepSessions: false,
|
|
|
+ };
|
|
|
+ for (let i = 0; i < argv.length; i += 1) {
|
|
|
+ const flag = argv[i];
|
|
|
+ const value = () => {
|
|
|
+ const next = argv[i + 1];
|
|
|
+ if (next === undefined) fail(`missing value for ${flag}`);
|
|
|
+ i += 1;
|
|
|
+ return next;
|
|
|
+ };
|
|
|
+ switch (flag) {
|
|
|
+ case '--server':
|
|
|
+ args.server = value().replace(/\/$/, '');
|
|
|
+ break;
|
|
|
+ case '--provider':
|
|
|
+ args.provider = value();
|
|
|
+ break;
|
|
|
+ case '--model':
|
|
|
+ args.model = value();
|
|
|
+ break;
|
|
|
+ case '--agent':
|
|
|
+ args.agent = value();
|
|
|
+ break;
|
|
|
+ case '--scenario': {
|
|
|
+ const requested = value();
|
|
|
+ if (requested === 'all' || requested === 'extensive') {
|
|
|
+ args.scenarios = EXTENSIVE_SET;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ const names = requested.split(',');
|
|
|
+ for (const name of names) {
|
|
|
+ if (!SCENARIOS.some((scenario) => scenario.name === name)) {
|
|
|
+ fail(
|
|
|
+ `unknown scenario "${name}" (${EXTENSIVE_SET.join(' | ')} | extensive | all)`,
|
|
|
+ );
|
|
|
+ }
|
|
|
+ }
|
|
|
+ args.scenarios = names;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ case '--turn-timeout-ms':
|
|
|
+ args.turnTimeoutMs = Number(value());
|
|
|
+ break;
|
|
|
+ case '--keep-sessions':
|
|
|
+ args.keepSessions = true;
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ fail(`unknown flag ${flag}`);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (!!args.provider !== !!args.model) {
|
|
|
+ fail('--provider and --model must be given together');
|
|
|
+ }
|
|
|
+ if (!Number.isFinite(args.turnTimeoutMs) || args.turnTimeoutMs <= 0) {
|
|
|
+ fail('--turn-timeout-ms must be a positive number');
|
|
|
+ }
|
|
|
+ return args;
|
|
|
+}
|
|
|
+
|
|
|
+function getFreePort(): Promise<number> {
|
|
|
+ return new Promise((resolve, reject) => {
|
|
|
+ const server = createServer();
|
|
|
+ server.once('error', reject);
|
|
|
+ server.listen(0, '127.0.0.1', () => {
|
|
|
+ const address = server.address();
|
|
|
+ if (!address || typeof address === 'string') {
|
|
|
+ server.close();
|
|
|
+ reject(new Error('failed to allocate a port'));
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ server.close((error) => (error ? reject(error) : resolve(address.port)));
|
|
|
+ });
|
|
|
+ });
|
|
|
+}
|
|
|
+
|
|
|
+function isRecord(value: unknown): value is Record<string, unknown> {
|
|
|
+ return !!value && typeof value === 'object' && !Array.isArray(value);
|
|
|
+}
|
|
|
+
|
|
|
+function finiteNumber(value: unknown): number {
|
|
|
+ return typeof value === 'number' && Number.isFinite(value) ? value : 0;
|
|
|
+}
|
|
|
+
|
|
|
+async function request(
|
|
|
+ base: string,
|
|
|
+ method: string,
|
|
|
+ route: string,
|
|
|
+ body?: unknown,
|
|
|
+ timeoutMs = 30_000,
|
|
|
+): Promise<unknown> {
|
|
|
+ const response = await fetch(`${base}${route}`, {
|
|
|
+ method,
|
|
|
+ headers: body === undefined ? {} : { 'content-type': 'application/json' },
|
|
|
+ body: body === undefined ? undefined : JSON.stringify(body),
|
|
|
+ signal: AbortSignal.timeout(timeoutMs),
|
|
|
+ });
|
|
|
+ if (!response.ok) {
|
|
|
+ throw new Error(`${method} ${route} returned HTTP ${response.status}`);
|
|
|
+ }
|
|
|
+ const text = await response.text();
|
|
|
+ if (!text) return undefined;
|
|
|
+ try {
|
|
|
+ return JSON.parse(text);
|
|
|
+ } catch {
|
|
|
+ return undefined;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+async function waitForHealth(base: string): Promise<void> {
|
|
|
+ const deadline = Date.now() + 40_000;
|
|
|
+ while (Date.now() < deadline) {
|
|
|
+ try {
|
|
|
+ const response = await fetch(`${base}/global/health`, {
|
|
|
+ signal: AbortSignal.timeout(2_000),
|
|
|
+ });
|
|
|
+ if (response.ok) return;
|
|
|
+ } catch {
|
|
|
+ // keep polling
|
|
|
+ }
|
|
|
+ await new Promise((resolve) => setTimeout(resolve, 400));
|
|
|
+ }
|
|
|
+ throw new Error('server did not become healthy within 40s');
|
|
|
+}
|
|
|
+
|
|
|
+function extractAssistantRows(rawMessages: unknown): RequestRow[] {
|
|
|
+ if (!Array.isArray(rawMessages)) return [];
|
|
|
+ const rows: RequestRow[] = [];
|
|
|
+ for (const raw of rawMessages) {
|
|
|
+ const info = isRecord(raw) && isRecord(raw.info) ? raw.info : undefined;
|
|
|
+ if (info?.role !== 'assistant') continue;
|
|
|
+ const tokens = isRecord(info.tokens) ? info.tokens : undefined;
|
|
|
+ if (!tokens) continue;
|
|
|
+ const cache = isRecord(tokens.cache) ? tokens.cache : undefined;
|
|
|
+ rows.push({
|
|
|
+ messageID: typeof info.id === 'string' ? info.id : '?',
|
|
|
+ input: finiteNumber(tokens.input),
|
|
|
+ output: finiteNumber(tokens.output),
|
|
|
+ cacheRead: finiteNumber(cache?.read),
|
|
|
+ cacheWrite: finiteNumber(cache?.write),
|
|
|
+ });
|
|
|
+ }
|
|
|
+ return rows;
|
|
|
+}
|
|
|
+
|
|
|
+async function fetchSessionRows(
|
|
|
+ base: string,
|
|
|
+ sessionID: string,
|
|
|
+): Promise<RequestRow[]> {
|
|
|
+ const rawMessages = await request(
|
|
|
+ base,
|
|
|
+ 'GET',
|
|
|
+ `/session/${encodeURIComponent(sessionID)}/message`,
|
|
|
+ );
|
|
|
+ return extractAssistantRows(rawMessages);
|
|
|
+}
|
|
|
+
|
|
|
+async function listChildSessions(
|
|
|
+ base: string,
|
|
|
+ parentID: string,
|
|
|
+): Promise<Array<{ id: string; title: string }>> {
|
|
|
+ const raw = await request(base, 'GET', '/session').catch(() => undefined);
|
|
|
+ if (!Array.isArray(raw)) return [];
|
|
|
+ const children: Array<{ id: string; title: string }> = [];
|
|
|
+ for (const item of raw) {
|
|
|
+ if (!isRecord(item)) continue;
|
|
|
+ if (item.parentID !== parentID || typeof item.id !== 'string') continue;
|
|
|
+ children.push({
|
|
|
+ id: item.id,
|
|
|
+ title: typeof item.title === 'string' ? item.title : item.id,
|
|
|
+ });
|
|
|
+ }
|
|
|
+ return children;
|
|
|
+}
|
|
|
+
|
|
|
+/** Suspect = non-first request with a sizeable prompt and zero cache reads. */
|
|
|
+function suspectRows(rows: RequestRow[]): RequestRow[] {
|
|
|
+ return rows
|
|
|
+ .slice(1)
|
|
|
+ .filter(
|
|
|
+ (row) => row.cacheRead === 0 && row.input >= SUSPECT_INPUT_THRESHOLD,
|
|
|
+ );
|
|
|
+}
|
|
|
+
|
|
|
+function judge(reports: SessionReport[]): Verdict {
|
|
|
+ const allRows = reports.flatMap((report) => report.rows);
|
|
|
+ if (allRows.length < 2) return 'inconclusive';
|
|
|
+ const anyTelemetry = allRows.some(
|
|
|
+ (row) => row.cacheRead > 0 || row.cacheWrite > 0,
|
|
|
+ );
|
|
|
+ if (!anyTelemetry) return 'inconclusive';
|
|
|
+ const suspects = reports.flatMap((report) => suspectRows(report.rows));
|
|
|
+ return suspects.length > 0 ? 'bust' : 'ok';
|
|
|
+}
|
|
|
+
|
|
|
+function coverage(rows: RequestRow[]): string {
|
|
|
+ const later = rows.slice(1);
|
|
|
+ const read = later.reduce((sum, row) => sum + row.cacheRead, 0);
|
|
|
+ const input = later.reduce((sum, row) => sum + row.input, 0);
|
|
|
+ const denominator = read + input;
|
|
|
+ return denominator > 0
|
|
|
+ ? `${((read / denominator) * 100).toFixed(1)}%`
|
|
|
+ : 'n/a';
|
|
|
+}
|
|
|
+
|
|
|
+function printSessionTable(report: SessionReport): void {
|
|
|
+ console.log(` ${report.label}`);
|
|
|
+ if (report.rows.length === 0) {
|
|
|
+ console.log(' no assistant requests with token telemetry recorded');
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ console.log(
|
|
|
+ ' req input output cache-read cache-write read-coverage',
|
|
|
+ );
|
|
|
+ const suspects = new Set(suspectRows(report.rows));
|
|
|
+ report.rows.forEach((row, index) => {
|
|
|
+ const denominator = row.input + row.cacheRead;
|
|
|
+ const rowCoverage =
|
|
|
+ denominator > 0
|
|
|
+ ? `${((row.cacheRead / denominator) * 100).toFixed(1)}%`
|
|
|
+ : 'n/a';
|
|
|
+ const marker = suspects.has(row) ? ' ← SUSPECT' : '';
|
|
|
+ console.log(
|
|
|
+ ` #${String(index + 1).padEnd(3)}${String(row.input).padEnd(11)}${String(row.output).padEnd(9)}${String(row.cacheRead).padEnd(12)}${String(row.cacheWrite).padEnd(13)}${rowCoverage}${marker}`,
|
|
|
+ );
|
|
|
+ });
|
|
|
+ console.log(
|
|
|
+ ` cache-read coverage after first request: ${coverage(report.rows)}`,
|
|
|
+ );
|
|
|
+}
|
|
|
+
|
|
|
+function printScenarioReport(
|
|
|
+ scenario: Scenario,
|
|
|
+ reports: SessionReport[],
|
|
|
+ verdict: Verdict,
|
|
|
+): void {
|
|
|
+ console.log(`\n━━ scenario: ${scenario.name} (${scenario.description})`);
|
|
|
+ console.log(` triggers: ${scenario.triggers}`);
|
|
|
+ for (const report of reports) {
|
|
|
+ printSessionTable(report);
|
|
|
+ }
|
|
|
+ const labels: Record<Verdict, string> = {
|
|
|
+ ok: '✅ every sizeable follow-up request read the provider cache',
|
|
|
+ bust: '❌ SUSPECT requests above read 0 cached tokens — the prompt prefix changed between requests',
|
|
|
+ inconclusive:
|
|
|
+ '⚠️ provider reported no cache telemetry — cannot verify (provider may not support or report caching)',
|
|
|
+ };
|
|
|
+ console.log(` verdict: ${labels[verdict]}`);
|
|
|
+}
|
|
|
+
|
|
|
+async function runScenario(
|
|
|
+ base: string,
|
|
|
+ args: Args,
|
|
|
+ scenario: Scenario,
|
|
|
+): Promise<{ reports: SessionReport[]; verdict: Verdict }> {
|
|
|
+ const created = await request(base, 'POST', '/session', {});
|
|
|
+ const sessionID = isRecord(created) ? String(created.id ?? '') : '';
|
|
|
+ if (!sessionID) throw new Error('POST /session returned no session id');
|
|
|
+ const cleanupIDs = [sessionID];
|
|
|
+
|
|
|
+ try {
|
|
|
+ const nonce = crypto.randomUUID();
|
|
|
+ for (const turn of scenario.turns(nonce)) {
|
|
|
+ await request(
|
|
|
+ base,
|
|
|
+ 'POST',
|
|
|
+ `/session/${encodeURIComponent(sessionID)}/message`,
|
|
|
+ {
|
|
|
+ ...(args.agent ? { agent: args.agent } : {}),
|
|
|
+ ...(args.provider && args.model
|
|
|
+ ? { model: { providerID: args.provider, modelID: args.model } }
|
|
|
+ : {}),
|
|
|
+ parts: [{ type: 'text', text: turn.text }],
|
|
|
+ },
|
|
|
+ args.turnTimeoutMs,
|
|
|
+ );
|
|
|
+ if (turn.pauseAfterMs) {
|
|
|
+ console.log(
|
|
|
+ ` (waiting ${Math.round(turn.pauseAfterMs / 1000)}s for background work…)`,
|
|
|
+ );
|
|
|
+ await new Promise((resolve) => setTimeout(resolve, turn.pauseAfterMs));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ const reports: SessionReport[] = [
|
|
|
+ {
|
|
|
+ label: `session ${sessionID} (main)`,
|
|
|
+ rows: await fetchSessionRows(base, sessionID),
|
|
|
+ },
|
|
|
+ ];
|
|
|
+ for (const child of await listChildSessions(base, sessionID)) {
|
|
|
+ cleanupIDs.push(child.id);
|
|
|
+ reports.push({
|
|
|
+ label: `session ${child.id} (subagent: ${child.title})`,
|
|
|
+ rows: await fetchSessionRows(base, child.id),
|
|
|
+ });
|
|
|
+ }
|
|
|
+ return { reports, verdict: judge(reports) };
|
|
|
+ } finally {
|
|
|
+ if (!args.keepSessions) {
|
|
|
+ for (const id of cleanupIDs.reverse()) {
|
|
|
+ await request(
|
|
|
+ base,
|
|
|
+ 'DELETE',
|
|
|
+ `/session/${encodeURIComponent(id)}`,
|
|
|
+ ).catch(() => {});
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ console.log(` sessions kept: ${cleanupIDs.join(', ')}`);
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+async function main(): Promise<void> {
|
|
|
+ const args = parseArgs(process.argv.slice(2));
|
|
|
+
|
|
|
+ let base = args.server;
|
|
|
+ let child: ReturnType<typeof spawn> | undefined;
|
|
|
+ let scratch: string | undefined;
|
|
|
+
|
|
|
+ if (!base) {
|
|
|
+ const binary = process.env.OPENCODE_BIN ?? Bun.which('opencode');
|
|
|
+ if (!binary) {
|
|
|
+ fail(
|
|
|
+ 'opencode binary not found — install opencode or set OPENCODE_BIN, or pass --server URL',
|
|
|
+ );
|
|
|
+ }
|
|
|
+ scratch = mkdtempSync(path.join(tmpdir(), 'cache-smoke-'));
|
|
|
+ writeFileSync(
|
|
|
+ path.join(scratch, 'package.json'),
|
|
|
+ `${JSON.stringify({ name: 'cache-smoke-fixture', version: '0.0.0' }, null, 2)}\n`,
|
|
|
+ );
|
|
|
+ const port = await getFreePort();
|
|
|
+ base = `http://127.0.0.1:${port}`;
|
|
|
+ console.log(`starting opencode serve on ${base} (cwd: ${scratch})`);
|
|
|
+ child = spawn(
|
|
|
+ binary,
|
|
|
+ ['serve', '--hostname', '127.0.0.1', '--port', String(port)],
|
|
|
+ {
|
|
|
+ cwd: scratch,
|
|
|
+ stdio: ['ignore', 'pipe', 'pipe'],
|
|
|
+ },
|
|
|
+ );
|
|
|
+ const stderrChunks: string[] = [];
|
|
|
+ child.stderr?.on('data', (chunk: Buffer) => {
|
|
|
+ stderrChunks.push(String(chunk));
|
|
|
+ });
|
|
|
+ child.once('exit', (code) => {
|
|
|
+ if (code !== null && code !== 0) {
|
|
|
+ console.error(stderrChunks.join('').slice(-2000));
|
|
|
+ fail(`opencode serve exited early with code ${code}`);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ await waitForHealth(base);
|
|
|
+ }
|
|
|
+
|
|
|
+ const cleanup = () => {
|
|
|
+ child?.kill('SIGTERM');
|
|
|
+ if (scratch) rmSync(scratch, { recursive: true, force: true });
|
|
|
+ };
|
|
|
+
|
|
|
+ try {
|
|
|
+ const scenarios = SCENARIOS.filter((scenario) =>
|
|
|
+ args.scenarios.includes(scenario.name),
|
|
|
+ );
|
|
|
+ const verdicts: Verdict[] = [];
|
|
|
+ for (const scenario of scenarios) {
|
|
|
+ console.log(`\nrunning scenario: ${scenario.name}…`);
|
|
|
+ const { reports, verdict } = await runScenario(base, args, scenario);
|
|
|
+ printScenarioReport(scenario, reports, verdict);
|
|
|
+ verdicts.push(verdict);
|
|
|
+ }
|
|
|
+
|
|
|
+ console.log('');
|
|
|
+ if (verdicts.includes('bust')) {
|
|
|
+ console.log(
|
|
|
+ 'RESULT: ❌ cache bust detected. Cross-check the plugin build (bun run build), then use docs/cache-verification.md to localize the changing prefix byte.',
|
|
|
+ );
|
|
|
+ process.exitCode = 1;
|
|
|
+ } else if (verdicts.every((verdict) => verdict === 'inconclusive')) {
|
|
|
+ console.log(
|
|
|
+ 'RESULT: ⚠️ inconclusive — the provider reported no cache telemetry for any request.',
|
|
|
+ );
|
|
|
+ process.exitCode = 2;
|
|
|
+ } else {
|
|
|
+ console.log(
|
|
|
+ 'RESULT: ✅ provider prompt caching is working across the tested scenarios.',
|
|
|
+ );
|
|
|
+ }
|
|
|
+ } finally {
|
|
|
+ cleanup();
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+main().catch((error) => {
|
|
|
+ fail(error instanceof Error ? error.message : String(error));
|
|
|
+});
|