Bläddra i källkod

test(validate): add skill description budget gate (700 hard / 35k soft, warn mode)

Every skill's description+when_to_use is loaded into every session -
an unbounded per-session token tax. Gate caps the COMBINED pair at 700
chars/skill (the measured knee: 674->780 gap in the distribution) with
a 35k-char catalog soft budget. Ships in DESC_BUDGET_MODE=warn - 26
skills currently exceed the cap (55,424 combined chars catalog-wide);
the trim wave lands next, then the mode flips to fail (one-word
change). Combined-field measurement is deliberate: moving bloat from
description into when_to_use is a field-shuffle, not a saving.
PyYAML extraction with sed fallback when python3 is absent.

Built by fleetflow lane gates/g1-desc-budget (codex); committed by the
orchestrator - codex sandbox cannot write worktree git metadata.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
0xDarkMatter 1 vecka sedan
förälder
incheckning
e14b0cc0e1
1 ändrade filer med 88 tillägg och 0 borttagningar
  1. 88 0
      tests/validate.sh

+ 88 - 0
tests/validate.sh

@@ -262,6 +262,93 @@ validate_skills() {
     done < <(find "$skills_dir" -mindepth 1 -maxdepth 1 -type d -print0)
     done < <(find "$skills_dir" -mindepth 1 -maxdepth 1 -type d -print0)
 }
 }
 
 
+# Validate the session-wide cost of skill descriptions
+validate_description_budget() {
+    echo ""
+    echo "=== Validating Skill Description Budget ==="
+
+    # Ship in warn mode while the 18 known offenders are trimmed; flipping to fail is a one-word change.
+    # The cap combines description + when_to_use because Claude Code loads both, so moving text between
+    # fields does not reduce the per-session cost.
+    DESC_BUDGET_MODE="warn" # warn|fail
+
+    local skills_dir="$PROJECT_DIR/skills"
+    local hard_cap=700
+    local soft_budget=35000
+    local catalog_total=0
+    local name
+    local combined_len
+    local budget_rows
+
+    if [[ ! -d "$skills_dir" ]]; then
+        log_warn "skills/ directory not found"
+        return
+    fi
+
+    local python_status=1
+    if command -v python3 >/dev/null 2>&1; then
+        set +e
+        budget_rows=$(python3 - "$skills_dir" 2>/dev/null <<'PY'
+import pathlib
+import sys
+
+import yaml
+
+skills_dir = pathlib.Path(sys.argv[1])
+for skill_dir in sorted(path for path in skills_dir.iterdir() if path.is_dir() and not path.name.startswith("_")):
+    skill_file = skill_dir / "SKILL.md"
+    if not skill_file.is_file():
+        continue
+    lines = skill_file.read_text(encoding="utf-8-sig").splitlines()
+    markers = [index for index, line in enumerate(lines) if line.strip() == "---"]
+    if len(markers) < 2:
+        continue
+    frontmatter = yaml.safe_load("\n".join(lines[markers[0] + 1:markers[1]])) or {}
+    description = str(frontmatter.get("description") or "")
+    when_to_use = str(frontmatter.get("when_to_use") or "")
+    print(f"{skill_dir.name}\t{len(description) + len(when_to_use)}")
+PY
+        )
+        python_status=$?
+        set -e
+    fi
+
+    if ((python_status != 0)); then
+        local skill_subdir
+        local skill_file
+        local description
+        local when_to_use
+        budget_rows=""
+        while IFS= read -r -d '' skill_subdir; do
+            [[ "$(basename "$skill_subdir")" == _* ]] && continue
+            skill_file="$skill_subdir/SKILL.md"
+            [[ -f "$skill_file" ]] || continue
+            description=$(get_yaml_field "$skill_file" "description" || true)
+            when_to_use=$(get_yaml_field "$skill_file" "when_to_use" || true)
+            budget_rows+="$(printf '%s\t%s' "$(basename "$skill_subdir")" "$((${#description} + ${#when_to_use}))")"$'\n'
+        done < <(find "$skills_dir" -mindepth 1 -maxdepth 1 -type d -print0)
+    fi
+
+    while IFS=$'\t' read -r name combined_len; do
+        [[ -z "$name" ]] && continue
+        catalog_total=$((catalog_total + combined_len))
+
+        if ((combined_len > hard_cap)); then
+            if [[ "$DESC_BUDGET_MODE" == "fail" ]]; then
+                log_fail "$name - description + when_to_use is $combined_len chars (hard cap: $hard_cap)"
+            else
+                log_warn "$name - description + when_to_use is $combined_len chars (hard cap: $hard_cap)"
+            fi
+        fi
+    done <<< "$budget_rows"
+
+    if ((catalog_total > soft_budget)); then
+        log_warn "Skill catalog descriptions total $catalog_total chars (soft budget: $soft_budget)"
+    else
+        log_pass "Skill catalog descriptions total $catalog_total chars (soft budget: $soft_budget)"
+    fi
+}
+
 # Validate rules (optional YAML frontmatter with optional paths field)
 # Validate rules (optional YAML frontmatter with optional paths field)
 validate_rules() {
 validate_rules() {
     echo ""
     echo ""
@@ -465,6 +552,7 @@ main() {
     validate_agents
     validate_agents
     validate_commands
     validate_commands
     validate_skills
     validate_skills
+    validate_description_budget
     validate_rules
     validate_rules
     validate_settings
     validate_settings
     validate_plugin
     validate_plugin