Browse Source

test: Add doc-drift gate + skill behavioural suites to CI

doc-drift.sh fails CI when README/AGENTS.md/PLAN.md counts diverge from
disk, when a skill lacks a README table row, or when a doc links a
non-existent path. run-skill-tests.sh discovers and runs every
skills/*/tests/run.sh (4 suites today; OS-gated suites self-skip).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
0xDarkMatter 1 month ago
parent
commit
dc1101fd81
3 changed files with 141 additions and 0 deletions
  1. 6 0
      .github/workflows/validate.yml
  2. 102 0
      tests/doc-drift.sh
  3. 33 0
      tests/run-skill-tests.sh

+ 6 - 0
.github/workflows/validate.yml

@@ -28,3 +28,9 @@ jobs:
 
       - name: Validate extensions (frontmatter, naming, structure)
         run: bash tests/validate.sh
+
+      - name: Doc-drift gate (docs must match disk)
+        run: bash tests/doc-drift.sh
+
+      - name: Run skill behavioural test suites
+        run: bash tests/run-skill-tests.sh

+ 102 - 0
tests/doc-drift.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# Doc-drift gate: documentation must describe what is actually on disk.
+#
+# Checks:
+#   1. Component counts on disk vs claims in README.md header, AGENTS.md
+#      overview bullets, and docs/PLAN.md inventory table
+#   2. Every skill directory has a row in a README skill table
+#   3. Every repo-relative markdown link in README.md / AGENTS.md resolves
+#      to an existing file or directory (no ghost references)
+#
+# Exit 0 = clean, exit 1 = drift detected.
+set -u
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT" || exit 1
+
+errors=0
+err() { echo "DRIFT: $*"; errors=$((errors + 1)); }
+
+# --- 1. Counts on disk ------------------------------------------------------
+skills_disk=0
+for d in skills/*/; do
+    [ -f "$d/SKILL.md" ] && skills_disk=$((skills_disk + 1))
+done
+agents_disk=$(find agents -maxdepth 1 -name '*.md' | wc -l)
+hooks_disk=$(find hooks -maxdepth 1 -name '*.sh' | wc -l)
+rules_disk=$(find rules -maxdepth 1 -name '*.md' | wc -l)
+styles_disk=$(find output-styles -maxdepth 1 -name '*.md' | wc -l)
+commands_disk=$(find commands -maxdepth 1 -name '*.md' | wc -l)
+
+echo "Disk: agents=$agents_disk skills=$skills_disk styles=$styles_disk hooks=$hooks_disk rules=$rules_disk commands=$commands_disk"
+
+# --- README header claim: "**N agents. N skills. N styles. N hooks. N rules. ...**"
+header="$(grep -oE '\*\*[0-9]+ agents\. [0-9]+ skills\. [0-9]+ styles\. [0-9]+ hooks\. [0-9]+ rules\.' README.md | head -1)"
+if [ -z "$header" ]; then
+    err "README.md: count header line not found (expected '**N agents. N skills. ...**')"
+else
+    read -r r_agents r_skills r_styles r_hooks r_rules <<< \
+        "$(echo "$header" | grep -oE '[0-9]+' | tr '\n' ' ')"
+    [ "$r_agents" = "$agents_disk" ] || err "README header: $r_agents agents claimed, $agents_disk on disk"
+    [ "$r_skills" = "$skills_disk" ] || err "README header: $r_skills skills claimed, $skills_disk on disk"
+    [ "$r_styles" = "$styles_disk" ] || err "README header: $r_styles styles claimed, $styles_disk on disk"
+    [ "$r_hooks"  = "$hooks_disk"  ] || err "README header: $r_hooks hooks claimed, $hooks_disk on disk"
+    [ "$r_rules"  = "$rules_disk"  ] || err "README header: $r_rules rules claimed, $rules_disk on disk"
+fi
+
+# --- AGENTS.md overview bullets ---------------------------------------------
+check_agents_md() { # $1=regex $2=disk-count $3=label
+    local claim
+    claim="$(grep -oE "$1" AGENTS.md | head -1 | grep -oE '[0-9]+')"
+    if [ -z "$claim" ]; then
+        err "AGENTS.md: no '$3' count bullet found"
+    elif [ "$claim" != "$2" ]; then
+        err "AGENTS.md: $claim $3 claimed, $2 on disk"
+    fi
+}
+check_agents_md '\*\*[0-9]+ expert agents\*\*' "$agents_disk" "agents"
+check_agents_md '\*\*[0-9]+ skills\*\*' "$skills_disk" "skills"
+check_agents_md '\*\*[0-9]+ output styles\*\*' "$styles_disk" "output styles"
+check_agents_md '\*\*[0-9]+ hooks\*\*' "$hooks_disk" "hooks"
+check_agents_md '\*\*[0-9]+ commands\*\*' "$commands_disk" "commands"
+
+# --- docs/PLAN.md inventory table -------------------------------------------
+check_plan() { # $1=row-label $2=disk-count
+    local claim
+    claim="$(grep -E "^\| $1 \|" docs/PLAN.md | head -1 | awk -F'|' '{gsub(/ /,"",$3); print $3}')"
+    if [ -n "$claim" ] && [ "$claim" != "$2" ]; then
+        err "docs/PLAN.md: $1 = $claim claimed, $2 on disk"
+    fi
+}
+check_plan "Agents" "$agents_disk"
+check_plan "Skills" "$skills_disk"
+check_plan "Commands" "$commands_disk"
+check_plan "Rules" "$rules_disk"
+check_plan "Output Styles" "$styles_disk"
+check_plan "Hooks" "$hooks_disk"
+
+# --- 2. Every skill has a README row ----------------------------------------
+for d in skills/*/; do
+    n="$(basename "$d")"
+    [ -f "$d/SKILL.md" ] || continue
+    grep -q "skills/$n/" README.md || err "README.md: skill '$n' has no table row"
+done
+
+# --- 3. Ghost-link check (README.md + AGENTS.md) ----------------------------
+for doc in README.md AGENTS.md; do
+    while IFS= read -r path; do
+        path="${path%%#*}"   # strip anchors
+        [ -z "$path" ] && continue
+        [ -e "$path" ] || err "$doc: link target does not exist: $path"
+    done < <(grep -oE '\]\((skills|agents|hooks|rules|output-styles|commands|docs|tools|tests|scripts)/[^)]*\)' "$doc" \
+             | sed -E 's/^\]\(//; s/\)$//')
+done
+
+echo
+if [ "$errors" -eq 0 ]; then
+    echo "doc-drift: clean"
+    exit 0
+else
+    echo "doc-drift: $errors issue(s) found"
+    exit 1
+fi

+ 33 - 0
tests/run-skill-tests.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Run every skill's behavioural test suite (skills/*/tests/run.sh).
+# Suites are responsible for their own OS gating (exit 0 with a skip
+# message on unsupported platforms). Any nonzero exit fails this runner.
+set -u
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT" || exit 1
+
+suites=(skills/*/tests/run.sh)
+if [ ! -e "${suites[0]}" ]; then
+    echo "No skill test suites found (skills/*/tests/run.sh)"
+    exit 0
+fi
+
+failed=0
+total=0
+for suite in "${suites[@]}"; do
+    total=$((total + 1))
+    name="$(basename "$(dirname "$(dirname "$suite")")")"
+    echo "=== $name"
+    if bash "$suite"; then
+        echo "--- $name: PASS"
+    else
+        rc=$?
+        echo "--- $name: FAIL (exit $rc)"
+        failed=$((failed + 1))
+    fi
+    echo
+done
+
+echo "Skill test suites: $((total - failed))/$total passed"
+[ "$failed" -eq 0 ] || exit 1