Browse Source

test(skills): wrap claude-api-ops/claude-code-ops/terraform-ops/process-compose-ops verifiers in CI suites

Add offline, deterministic, Linux-safe behavioural suites (tests/run.sh) that wrap
each skill's existing staleness/validation verifier, so a bug in a checker can no
longer fail silently in CI. Each suite: contract (py_compile/bash -n + --help),
offline happy path against the shipped skill, the §7 --json envelope, and one
negative assertion proving the verifier actually checks something (date-suffixed
model id -> 4; unknown hook event -> 10; malformed uses ref -> 4; mismatched
EXE_HASH -> nonzero). process-compose's behavioural layer is pwsh-gated
(structural checks always run) since verify-binary.ps1 is Windows/PowerShell-
locked; no suite ever touches the network.

Co-Authored-By: Claude <noreply@anthropic.com>
0xDarkMatter 6 days ago
parent
commit
1bdcc47c15

+ 85 - 0
skills/claude-api-ops/tests/run.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# Self-test for claude-api-ops — fully offline: no network, no Anthropic API.
+#
+# Wraps the skill's §7 staleness verifier (scripts/check-model-table.py), which
+# guards the two fast-moving fact tables (SKILL.md "Current Models" and
+# references/caching-and-cost.md cache-minimums) against silent drift. Contract
+# (py_compile + --help), offline happy path against the shipped skill, the
+# --json §7 envelope, and a NEGATIVE proving the verifier actually rejects a bad
+# model id (a date-suffixed alias — exactly what SKILL.md forbids). --live is
+# NEVER invoked: it hits the Models API, and a network blip must never fail a PR.
+#
+# Usage:   bash tests/run.sh
+# Exit:    0 all pass, 1 one or more failures
+
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL="$(dirname "$HERE")"
+V="$SKILL/scripts/check-model-table.py"
+
+# Pick a python that actually executes (the Windows Store python3 stub exists
+# on PATH but exits non-zero; probe by running it).
+PYTHON=""
+for c in python python3 py; do
+  if command -v "$c" >/dev/null 2>&1 && "$c" -c "" >/dev/null 2>&1; then PYTHON="$c"; break; fi
+done
+
+SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT
+PASS=0; FAIL=0
+ok() { PASS=$((PASS+1)); printf '  PASS  %s\n' "$1"; }
+no() { FAIL=$((FAIL+1)); printf '  FAIL  %s\n' "$1"; }
+expect_exit() { [[ "$2" == "$3" ]] && ok "$1 (exit $3)" || no "$1 (want $2 got $3)"; }
+expect_has()  { case "$3" in *"$2"*) ok "$1";; *) no "$1 (missing '$2')";; esac; }
+
+echo "=== claude-api-ops self-test ==="
+
+if [[ -z "$PYTHON" ]]; then
+  echo "  SKIP  no working python (verifier is python) — cannot test"
+  [[ "$FAIL" -eq 0 ]] || exit 1
+  exit 0
+fi
+
+# ── contract ──────────────────────────────────────────────────────────────────
+echo "-- contract --"
+"$PYTHON" -m py_compile "$V" 2>/dev/null && ok "py_compile check-model-table.py" || no "py_compile check-model-table.py"
+"$PYTHON" "$V" --help >/dev/null 2>&1; expect_exit "--help exits 0" 0 $?
+out="$("$PYTHON" "$V" --help 2>&1)"
+expect_has "--help has EXAMPLES" "EXAMPLES" "$out"
+"$PYTHON" "$V" --bogus >/dev/null 2>&1; expect_exit "unknown flag -> 2" 2 $?
+
+# ── offline structural mode (§7 seam: --offline default, --live advisory) ─────
+echo "-- offline structural --"
+"$PYTHON" "$V" --offline >/dev/null 2>&1; expect_exit "--offline clean on shipped skill" 0 $?
+out="$("$PYTHON" "$V" --offline --json 2>/dev/null)"
+expect_has "--offline --json envelope schema" '"schema": "claude-mods.claude-api-ops.model-table/v1"' "$out"
+expect_has "--offline --json consistent" '"consistent": true' "$out"
+
+# ── negative: a date-suffixed alias must be rejected (exit 4 VALIDATION) ──────
+# Run the verifier from a doctored copy; never mutate the shipped skill.
+echo "-- negative --"
+cp -r "$SKILL" "$SB/copy"
+# Append the date suffix SKILL.md explicitly forbids ("Never append date
+# suffixes"). The verifier's DATE_SUFFIX_RE must flag it as VALIDATION drift.
+# Target the table cell uniquely (the prose never pairs "Opus 4.8 |" with the
+# backticked id) so the edit is surgical.
+"$PYTHON" - "$SB/copy/SKILL.md" <<'PY'
+import pathlib, sys
+p = pathlib.Path(sys.argv[1])
+t = p.read_text(encoding="utf-8")
+t = t.replace("Opus 4.8 | `claude-opus-4-8`", "Opus 4.8 | `claude-opus-4-8-20251114`")
+p.write_text(t, encoding="utf-8")
+PY
+"$PYTHON" "$SB/copy/scripts/check-model-table.py" --offline >"$SB/neg.out" 2>&1
+expect_exit "--offline flags date-suffixed id -> 4" 4 $?
+expect_has "finding names the date suffix" "date suffix" "$(cat "$SB/neg.out")"
+
+# ── SKILL.md sanity ───────────────────────────────────────────────────────────
+echo "-- SKILL.md --"
+grep -q '^name: claude-api-ops$' "$SKILL/SKILL.md" && ok "frontmatter name" || no "frontmatter name"
+grep -q 'check-model-table.py' "$SKILL/SKILL.md" && ok "verifier cited from SKILL.md" || no "verifier cited from SKILL.md"
+
+echo ""
+echo "=== $PASS passed, $FAIL failed ==="
+[[ "$FAIL" -eq 0 ]] || exit 1
+exit 0

+ 89 - 0
skills/claude-code-ops/tests/run.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+# Self-test for claude-code-ops — fully offline: no network.
+#
+# Wraps the skill's offline hooks-contract verifier (scripts/validate-hooks-json.py),
+# which lints a hooks.json / settings.json hooks block against the 30-event Claude
+# Code hook catalog. Contract (py_compile + --help), offline happy path against
+# the shipped skill (assets/hooks.json.template, stripped of its JSONC comments
+# exactly as the template's own header instructs — "STRIP EVERY COMMENT"), the
+# --json §7 envelope, and a NEGATIVE proving the verifier rejects an unknown
+# event (exit 10, names the event). The verifier is structural-only by design;
+# there is no --live mode to avoid.
+#
+# Usage:   bash tests/run.sh
+# Exit:    0 all pass, 1 one or more failures
+
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL="$(dirname "$HERE")"
+V="$SKILL/scripts/validate-hooks-json.py"
+TPL="$SKILL/assets/hooks.json.template"
+
+# Pick a python that actually executes (the Windows Store python3 stub exists on
+# PATH but exits non-zero; probe by running it).
+PYTHON=""
+for c in python python3 py; do
+  if command -v "$c" >/dev/null 2>&1 && "$c" -c "" >/dev/null 2>&1; then PYTHON="$c"; break; fi
+done
+
+SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT
+PASS=0; FAIL=0
+ok() { PASS=$((PASS+1)); printf '  PASS  %s\n' "$1"; }
+no() { FAIL=$((FAIL+1)); printf '  FAIL  %s\n' "$1"; }
+expect_exit() { [[ "$2" == "$3" ]] && ok "$1 (exit $3)" || no "$1 (want $2 got $3)"; }
+expect_has()  { case "$3" in *"$2"*) ok "$1";; *) no "$1 (missing '$2')";; esac; }
+
+echo "=== claude-code-ops self-test ==="
+
+if [[ -z "$PYTHON" ]]; then
+  echo "  SKIP  no working python (verifier is python) — cannot test"
+  [[ "$FAIL" -eq 0 ]] || exit 1
+  exit 0
+fi
+
+# ── contract ──────────────────────────────────────────────────────────────────
+echo "-- contract --"
+"$PYTHON" -m py_compile "$V" 2>/dev/null && ok "py_compile validate-hooks-json.py" || no "py_compile validate-hooks-json.py"
+"$PYTHON" "$V" --help >/dev/null 2>&1; expect_exit "--help exits 0" 0 $?
+out="$("$PYTHON" "$V" --help 2>&1)"
+expect_has "--help has EXAMPLES" "EXAMPLES" "$out"
+"$PYTHON" "$V" --bogus >/dev/null 2>&1; expect_exit "unknown flag -> 2" 2 $?
+
+# ── offline happy path: shipped template, comments stripped (its own rule) ─────
+echo "-- offline structural --"
+# The shipped assets/hooks.json.template is intentionally JSONC. Stripping // lines
+# yields strict JSON that must lint clean against the catalog. The path is passed
+# as an argv (MSYS translates it for Windows Python); never embed it in -c.
+grep -vE '^[[:space:]]*//' "$TPL" > "$SB/hooks.json"
+"$PYTHON" -c "import json,sys; json.load(open(sys.argv[1],encoding='utf-8'))" "$SB/hooks.json" \
+  >/dev/null 2>&1 && ok "stripped template is strict JSON" || no "stripped template is strict JSON"
+"$PYTHON" "$V" "$SB/hooks.json" >/dev/null 2>&1; expect_exit "lint clean on shipped skill" 0 $?
+out="$("$PYTHON" "$V" --json "$SB/hooks.json" 2>/dev/null)"
+expect_has "--json envelope schema" '"schema": "claude-mods.claude-code-ops.hooks-lint/v1"' "$out"
+expect_has "--json zero errors" '"errors": 0' "$out"
+
+# ── negative: an unknown event must be flagged (exit 10, names the event) ─────
+echo "-- negative --"
+cat > "$SB/bad.json" <<'EOF'
+{
+  "hooks": {
+    "PreToolUs": [
+      { "matcher": "Bash", "hooks": [ { "type": "command", "command": "echo hi" } ] }
+    ]
+  }
+}
+EOF
+"$PYTHON" "$V" "$SB/bad.json" >"$SB/neg.out" 2>&1
+expect_exit "unknown event -> 10" 10 $?
+expect_has "finding names the bad event" "PreToolUs" "$(cat "$SB/neg.out")"
+
+# ── SKILL.md sanity ───────────────────────────────────────────────────────────
+echo "-- SKILL.md --"
+grep -q '^name: claude-code-ops$' "$SKILL/SKILL.md" && ok "frontmatter name" || no "frontmatter name"
+grep -q 'validate-hooks-json.py' "$SKILL/SKILL.md" && ok "verifier cited from SKILL.md" || no "verifier cited from SKILL.md"
+
+echo ""
+echo "=== $PASS passed, $FAIL failed ==="
+[[ "$FAIL" -eq 0 ]] || exit 1
+exit 0

+ 86 - 0
skills/process-compose-ops/tests/run.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Self-test for process-compose-ops — fully offline: no network, no download.
+#
+# Wraps the skill's binary-integrity verifier (scripts/verify-binary.ps1), which
+# re-checks the committed process-compose.exe SHA-256 against the recorded
+# EXE_HASH. Unlike the three §7 staleness scripts this verifier is PowerShell and
+# platform-locked to a Windows binary, so the suite has two layers:
+#   * contract / structural (run anywhere, no PowerShell host required): the .ps1
+#     exists and encodes the Get-FileHash/EXE_HASH compare plus the mismatch
+#     throw — proving it is a real check, not a vacuous rubber stamp;
+#   * behavioural happy + negative against a temp bin/ — exercised only when a
+#     PowerShell host (pwsh | powershell) is present. A dummy .exe and a
+#     hand-written EXE_HASH stand in for the real binary, so no download and no
+#     network ever occur. SKIP'd (suite stays green) on a PowerShell-less Linux
+#     runner — the structural layer still runs.
+#
+# Usage:   bash tests/run.sh
+# Exit:    0 all pass, 1 one or more failures
+
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL="$(dirname "$HERE")"
+V="$SKILL/scripts/verify-binary.ps1"
+
+# A PowerShell host, if any (pwsh = PowerShell 7 cross-platform; powershell =
+# Windows PowerShell 5.1). Absent on most Linux CI runners.
+PSHOST=""
+for c in pwsh powershell; do
+  if command -v "$c" >/dev/null 2>&1 && "$c" -NoProfile -Command 'exit 0' >/dev/null 2>&1; then PSHOST="$c"; break; fi
+done
+
+SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT
+PASS=0; FAIL=0
+ok() { PASS=$((PASS+1)); printf '  PASS  %s\n' "$1"; }
+no() { FAIL=$((FAIL+1)); printf '  FAIL  %s\n' "$1"; }
+expect_exit() { [[ "$2" == "$3" ]] && ok "$1 (exit $3)" || no "$1 (want $2 got $3)"; }
+expect_has()  { case "$3" in *"$2"*) ok "$1";; *) no "$1 (missing '$2')";; esac; }
+
+echo "=== process-compose-ops self-test ==="
+
+# ── contract / structural (runs anywhere, no PowerShell host needed) ──────────
+echo "-- contract --"
+[[ -f "$V" ]] && ok "verify-binary.ps1 present" || no "verify-binary.ps1 present"
+[[ -s "$V" ]] && ok "verify-binary.ps1 non-empty" || no "verify-binary.ps1 non-empty"
+# The verifier must encode the integrity check it claims to run: a SHA-256
+# compare of the binary against EXE_HASH that throws on mismatch. If these
+# markers were gutted the verifier would be a vacuous rubber stamp.
+vtxt="$(cat "$V")"
+expect_has "computes SHA-256 of the binary" "Get-FileHash" "$vtxt"
+expect_has "reads recorded EXE_HASH" "EXE_HASH" "$vtxt"
+expect_has "fails loud on mismatch" "MISMATCH" "$vtxt"
+
+# ── behavioural (happy + negative) under a PowerShell host, if present ─────────
+echo "-- behavioural (${PSHOST:-no powershell host}) --"
+if [[ -z "$PSHOST" ]]; then
+  echo "  SKIP  happy/negative (no pwsh/powershell on this runner — structural layer above still ran)"
+else
+  # happy: a dummy .exe whose recorded EXE_HASH matches its actual SHA-256.
+  mkdir -p "$SB/ok/bin"
+  printf 'process-compose-fake-bytes' > "$SB/ok/bin/process-compose.exe"
+  # sha256sum and Get-FileHash compute the identical digest; the verifier
+  # compares lowercased + trimmed, so this matches what it recomputes.
+  sha256sum "$SB/ok/bin/process-compose.exe" | awk '{print $1}' > "$SB/ok/bin/EXE_HASH"
+  "$PSHOST" -NoProfile -File "$V" -BinDir "$SB/ok/bin" >/dev/null 2>&1
+  expect_exit "matching EXE_HASH -> 0" 0 $?
+
+  # negative: a recorded EXE_HASH that disagrees with the actual SHA-256.
+  mkdir -p "$SB/bad/bin"
+  printf 'process-compose-fake-bytes' > "$SB/bad/bin/process-compose.exe"
+  printf '0000000000000000000000000000000000000000000000000000000000000000' > "$SB/bad/bin/EXE_HASH"
+  "$PSHOST" -NoProfile -File "$V" -BinDir "$SB/bad/bin" >"$SB/neg.out" 2>&1
+  rc=$?
+  [[ "$rc" -ne 0 ]] && ok "mismatched EXE_HASH -> nonzero (exit $rc)" || no "mismatched EXE_HASH -> nonzero (got 0)"
+  expect_has "verifier reports MISMATCH" "MISMATCH" "$(cat "$SB/neg.out")"
+fi
+
+# ── SKILL.md sanity ───────────────────────────────────────────────────────────
+echo "-- SKILL.md --"
+grep -q '^name: process-compose-ops$' "$SKILL/SKILL.md" && ok "frontmatter name" || no "frontmatter name"
+grep -q 'verify-binary.ps1' "$SKILL/SKILL.md" && ok "verifier cited from SKILL.md" || no "verifier cited from SKILL.md"
+
+echo ""
+echo "=== $PASS passed, $FAIL failed ==="
+[[ "$FAIL" -eq 0 ]] || exit 1
+exit 0

+ 75 - 0
skills/terraform-ops/tests/run.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Self-test for terraform-ops — fully offline: no network, no GitHub API.
+#
+# Wraps the skill's §7 staleness verifier (scripts/check-action-refs.sh), which
+# lints `uses: owner/repo@ref` lines in GitHub Actions workflow YAML. Contract
+# (bash -n + --help), offline happy path against the shipped
+# assets/github-actions-terraform.yml, the --json §7 envelope (jq-guarded), and a
+# NEGATIVE proving the verifier rejects a malformed uses (a ref with no @ ->
+# exit 4 MALFORMED). --live is NEVER invoked (it resolves refs against the GitHub
+# API); a network blip must never fail a PR.
+#
+# Usage:   bash tests/run.sh
+# Exit:    0 all pass, 1 one or more failures
+
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL="$(dirname "$HERE")"
+V="$SKILL/scripts/check-action-refs.sh"
+DEFAULT="$SKILL/assets/github-actions-terraform.yml"
+
+SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT
+PASS=0; FAIL=0
+ok() { PASS=$((PASS+1)); printf '  PASS  %s\n' "$1"; }
+no() { FAIL=$((FAIL+1)); printf '  FAIL  %s\n' "$1"; }
+expect_exit() { [[ "$2" == "$3" ]] && ok "$1 (exit $3)" || no "$1 (want $2 got $3)"; }
+expect_has()  { case "$3" in *"$2"*) ok "$1";; *) no "$1 (missing '$2')";; esac; }
+
+echo "=== terraform-ops self-test ==="
+
+# ── contract ──────────────────────────────────────────────────────────────────
+echo "-- contract --"
+bash -n "$V" && ok "bash -n check-action-refs.sh" || no "bash -n check-action-refs.sh"
+bash "$V" --help >/dev/null 2>&1; expect_exit "--help exits 0" 0 $?
+out="$(bash "$V" --help 2>&1)"
+expect_has "--help has Examples" "xamples" "$out"
+bash "$V" --bogus >/dev/null 2>&1; expect_exit "unknown flag -> 2" 2 $?
+
+# ── offline structural mode (§7 seam: --offline default, --live advisory) ─────
+echo "-- offline structural --"
+[[ -f "$DEFAULT" ]] && ok "shipped workflow present" || no "shipped workflow present"
+bash "$V" --offline >/dev/null 2>&1; expect_exit "--offline clean on shipped skill" 0 $?
+# --json needs jq; SKIP the envelope checks where jq is absent (a no-jq runner is
+# legitimate, and the verifier itself exits 5 for --json without jq).
+if command -v jq >/dev/null 2>&1; then
+  out="$(bash "$V" --offline --json 2>/dev/null)"
+  expect_has "--offline --json envelope schema" '"schema": "claude-mods.terraform-ops.action-refs/v1"' "$out"
+  expect_has "--offline --json mode" '"mode": "offline"' "$out"
+else
+  echo "  SKIP  --json envelope (no jq on this runner)"
+fi
+
+# ── negative: a uses with no @ref must be flagged malformed (exit 4) ──────────
+echo "-- negative --"
+cat > "$SB/bad.yml" <<'EOF'
+name: bad
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: hashicorp/setup-terraform
+EOF
+bash "$V" --offline "$SB/bad.yml" >"$SB/neg.out" 2>&1
+expect_exit "--offline flags missing @ref -> 4" 4 $?
+expect_has "finding names the malformed ref" "malformed" "$(cat "$SB/neg.out")"
+
+# ── SKILL.md sanity ───────────────────────────────────────────────────────────
+echo "-- SKILL.md --"
+grep -q '^name: terraform-ops$' "$SKILL/SKILL.md" && ok "frontmatter name" || no "frontmatter name"
+grep -q 'check-action-refs.sh' "$SKILL/SKILL.md" && ok "verifier cited from SKILL.md" || no "verifier cited from SKILL.md"
+
+echo ""
+echo "=== $PASS passed, $FAIL failed ==="
+[[ "$FAIL" -eq 0 ]] || exit 1
+exit 0