Browse Source

test(skills): Fix SIGPIPE grep -q flakes in five more suites

Same anti-pattern fixed in fleet-ops (9a64810): under 'set -o pipefail',
'producer | grep -q <pattern>' intermittently fails even when the pattern
is present — grep -q exits at the first match, the producer takes SIGPIPE
(exit 141) while still writing, and pipefail surfaces 141 as the pipeline
status. Swept all skills/*/tests/*.sh for pipefail + '| grep -q' combos
and converted each to capture-then-match (case / expect_has / grep -q on
a here-string, which cannot SIGPIPE), keeping each suite's idiom:

- github-ops: three --help asserts, five source-invariant greps (the
  inverted -v ones get an emptiness guard — an empty capture would feed
  grep -v a matching empty line via the here-string), one printf pipe
- pypi-ops: --help '--build' assert
- ffmpeg-ops: cube-header assert; the libvmaf gate, where a 141 silently
  took the no-vmaf branch and skipped the vmaf tests
- mapbox-ops, threejs-ops: 'head -N | grep -Eq Examples' asserts
  (missed by the original sweep; same class)

fleetflow/net-ops/mac-ops grep -q pipes are safe (no pipefail set).
fleet-ops itself left untouched — already fixed on its own branch.

Verified: each touched suite stress-run 20x, 0 failures; all 20
ffmpeg-ops runs took the libvmaf branch (no silent SKIPs).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
0xDarkMatter 4 days ago
parent
commit
dc46bfe697

+ 8 - 2
skills/ffmpeg-ops/tests/run.sh

@@ -95,7 +95,10 @@ echo "-- gen-luts --"
 out="$("$PYTHON" "$S/gen-luts.py" --variants warm_filmic --size 17 --out-dir "$SB/luts" 2>/dev/null)"; rc=$?
 expect_exit "gen-luts size 17 -> 0" 0 "$rc"
 [[ -f "$SB/luts/warm_filmic.cube" ]] && ok "cube file written" || no "cube file written"
-head -5 "$SB/luts/warm_filmic.cube" | grep -q "LUT_3D_SIZE 17" && ok "cube header size" || no "cube header size"
+# Captured, not `head | grep -q`: under `set -o pipefail` grep -q's early exit
+# SIGPIPEs the producer (141) and flakes the assert even on a match.
+cube_hdr="$(head -5 "$SB/luts/warm_filmic.cube")"
+expect_has "cube header size" "LUT_3D_SIZE 17" "$cube_hdr"
 rows="$(grep -cE '^[0-9]' "$SB/luts/warm_filmic.cube")"
 [[ "$rows" == "4913" ]] && ok "cube row count 17^3" || no "cube row count (want 4913 got $rows)"
 out="$("$PYTHON" "$S/gen-luts.py" --variants neutral709 --size 17 --out-dir "$SB/luts" --json 2>/dev/null)"
@@ -174,7 +177,10 @@ else
   expect_exit "quality self-compare -> 0" 0 $?
   out="$("$PYTHON" "$S/quality-compare.py" "$FIX" "$FIX" --metrics ssim --json 2>/dev/null)"
   expect_has "ssim of identical ~1" '"all": 1' "$out"
-  if ffmpeg -hide_banner -filters 2>/dev/null | grep -q libvmaf; then
+  # Captured, not `-filters | grep -q`: under pipefail a SIGPIPE'd ffmpeg (141)
+  # would silently take the no-vmaf branch even when libvmaf is present.
+  filter_list="$(ffmpeg -hide_banner -filters 2>/dev/null)"
+  if grep -q libvmaf <<<"$filter_list"; then
     "$PYTHON" "$S/quality-compare.py" "$FIX" "$FIX" --metrics vmaf --min-vmaf 95 >/dev/null 2>&1
     expect_exit "vmaf self-compare above threshold -> 0" 0 $?
   else

+ 22 - 10
skills/github-ops/tests/run.sh

@@ -43,7 +43,12 @@ bash -n "$SP" && ok "sp: bash -n clean" || no "sp: bash -n"
 
 bash "$SP" --help >/dev/null 2>&1; expect "sp: --help" 0 $?
 # --help must advertise EXAMPLES so the tool is discoverable.
-if bash "$SP" --help 2>&1 | grep -q "Examples:"; then ok "sp: --help has EXAMPLES"
+# Never assert via `producer | grep -q` in this suite: under `set -o pipefail`,
+# grep -q exits at the first match and the producer dies with SIGPIPE (141),
+# flaking the pipeline non-zero even when the pattern is present. Capture the
+# output once, then grep the variable (a here-string can't SIGPIPE).
+sp_help="$(bash "$SP" --help 2>&1)"
+if grep -q "Examples:" <<<"$sp_help"; then ok "sp: --help has EXAMPLES"
 else no "sp: --help missing EXAMPLES"; fi
 
 bash "$SP" --frobnicate >/dev/null 2>&1; expect "sp: unknown flag -> usage" 2 $?
@@ -76,12 +81,16 @@ else no "sp: SECURITY.md.template asset missing/empty"; fi
 # Read-only guarantee. The ONLY executor in this script is `runner gh api …`
 # (every -X PUT/PATCH lives inside an emitted *_cmd string, never executed). Assert
 # no `runner gh api` invocation carries a mutating verb.
-if grep -E 'runner gh api' "$SP" | grep -Eq '\-X (PUT|PATCH|POST|DELETE)'; then
+sp_api_calls="$(grep -E 'runner gh api' "$SP")"   # captured, not piped — see SIGPIPE note above
+if grep -Eq '\-X (PUT|PATCH|POST|DELETE)' <<<"$sp_api_calls"; then
   no "sp: found an executed mutating gh api call (must be read-only)"
 else ok "sp: no executed mutating gh api call (read-only)"; fi
 # And every mutating verb that DOES appear must be inside a quoted command string
 # (assigned to a *_cmd var), proving it is emitted-as-text only.
-if grep -nE '\-X (PUT|PATCH|POST|DELETE)' "$SP" | grep -vqE '_cmd='; then
+# Inverted greps (-v) need the emptiness guard: an empty capture would feed the
+# here-string's single empty line to grep -v, which would wrongly match.
+sp_mut="$(grep -nE '\-X (PUT|PATCH|POST|DELETE)' "$SP")"
+if [ -n "$sp_mut" ] && grep -vqE '_cmd=' <<<"$sp_mut"; then
   no "sp: a mutating verb appears outside an emitted *_cmd string"
 else ok "sp: all mutating verbs are emitted text only"; fi
 
@@ -93,10 +102,11 @@ RS="$SCRIPTS/repo-scorecard.sh"
 bash -n "$RS" && ok "rs: bash -n clean" || no "rs: bash -n"
 
 bash "$RS" --help >/dev/null 2>&1; expect "rs: --help" 0 $?
-if bash "$RS" --help 2>&1 | grep -q "Examples:"; then ok "rs: --help has EXAMPLES"
+rs_help="$(bash "$RS" --help 2>&1)"   # captured, not piped — see SIGPIPE note above
+if grep -q "Examples:" <<<"$rs_help"; then ok "rs: --help has EXAMPLES"
 else no "rs: --help missing EXAMPLES"; fi
 # The scoring rubric must be documented in the header (transparent, auditable).
-if bash "$RS" --help 2>&1 | grep -q "SCORING MODEL"; then ok "rs: --help documents SCORING MODEL"
+if grep -q "SCORING MODEL" <<<"$rs_help"; then ok "rs: --help documents SCORING MODEL"
 else no "rs: --help missing SCORING MODEL"; fi
 
 bash "$RS" --frobnicate >/dev/null 2>&1; expect "rs: unknown flag -> usage" 2 $?
@@ -121,18 +131,20 @@ else no "rs: does not reference check-issues.sh"; fi
 # Read-only guarantee: no executed mutating gh verb anywhere. Every gh call must
 # be a GET (the remediation pointers it prints are text, not executed). Assert no
 # `gh api -X PUT/PATCH/POST/DELETE` and no `gh repo edit`/`gh release create` etc.
-if grep -E '\bgh (api )?-X (PUT|PATCH|POST|DELETE)' "$RS" | grep -vqE '^\s*#'; then
+rs_mut="$(grep -E '\bgh (api )?-X (PUT|PATCH|POST|DELETE)' "$RS")"   # captured — SIGPIPE note above
+if [ -n "$rs_mut" ] && grep -vqE '^\s*#' <<<"$rs_mut"; then
   no "rs: found an executed mutating gh -X call (must be read-only)"
 else ok "rs: no executed mutating gh -X call (read-only)"; fi
 # Belt-and-braces: every `runner gh …` (the only network executor) is a read-only
 # subcommand — `gh api <GET path>` or `gh repo list`. No mutating subcommand runs.
-if grep -nE 'runner gh ' "$RS" | grep -Evq 'runner gh (api|repo list)'; then
+rs_runner="$(grep -nE 'runner gh ' "$RS")"
+if [ -n "$rs_runner" ] && grep -Evq 'runner gh (api|repo list)' <<<"$rs_runner"; then
   no "rs: a 'runner gh' call uses a non-read-only subcommand"
 else ok "rs: every executed 'runner gh' is read-only (api / repo list)"; fi
 # And mutating gh subcommands, where they appear, are inside printed fix strings only
 # (the remediation pointers), never executed. Verify they sit on addfix/echo lines.
-if grep -nE 'gh (release create|repo edit|release delete|secret set|pr merge)' "$RS" \
-     | grep -vqE 'addfix|→'; then
+rs_ghsub="$(grep -nE 'gh (release create|repo edit|release delete|secret set|pr merge)' "$RS")"
+if [ -n "$rs_ghsub" ] && grep -vqE 'addfix|→' <<<"$rs_ghsub"; then
   no "rs: a mutating gh subcommand appears outside a printed remediation string"
 else ok "rs: mutating gh subcommands only appear as printed remediation text"; fi
 
@@ -156,7 +168,7 @@ if [ -f "$LIBTERM" ]; then
     "$(term_mark unknown)" "$(term_header hdr)" "$TERM_ARROW" \
     "$(term_panel_open github-ops PANEL meta)" "$(term_panel_line body)" \
     "$(term_section "" sect 3)" "$(term_panel_close hk "$(term_health warning x)")"')"
-  if printf '%s' "$marks" | LC_ALL=C grep -q '[^[:print:][:cntrl:]]'; then
+  if LC_ALL=C grep -q '[^[:print:][:cntrl:]]' <<<"$marks"; then
     no "term.sh TERM_ASCII=1 still emits non-ASCII bytes"
   else ok "term.sh TERM_ASCII=1 primitives are pure ASCII"; fi
   # A fallback that silently drops the glyph (empty) is a bug, not a fallback.

+ 6 - 2
skills/mapbox-ops/tests/run.sh

@@ -62,7 +62,10 @@ done
 py="$here/scripts/screenshot_map.py"
 if [ -f "$py" ]; then
   "$PY" -m py_compile "$py" && ok "py_compile clean" || bad "py_compile failed"
-  head -25 "$py" | grep -Eq '^(# )?Examples:' && ok "has Examples block" || bad "no Examples block"
+  # Captured, not `head | grep -q`: under `set -o pipefail` grep -q's early exit
+  # SIGPIPEs the producer (141) and flakes the assert even on a match.
+  py_head="$(head -25 "$py")"
+  grep -Eq '^(# )?Examples:' <<<"$py_head" && ok "has Examples block" || bad "no Examples block"
   "$PY" "$py" --help >/dev/null 2>&1 && ok "--help exits 0" || bad "--help nonzero"
   # USAGE (exit 2) on a file:// URL — happens before the playwright import, so this is offline-safe
   "$PY" "$py" "file:///tmp/x.html" /tmp/o.png >/dev/null 2>&1
@@ -76,7 +79,8 @@ facts="$here/scripts/check-mapbox-facts.py"
 if [ -f "$facts" ]; then
   ok "resource present: scripts/check-mapbox-facts.py"
   "$PY" -m py_compile "$facts" && ok "facts: py_compile clean" || bad "facts: py_compile failed"
-  head -30 "$facts" | grep -Eq '^# +Examples:' && ok "facts: has Examples block" || bad "facts: no Examples block"
+  facts_head="$(head -30 "$facts")"   # captured, not piped — see SIGPIPE note above
+  grep -Eq '^# +Examples:' <<<"$facts_head" && ok "facts: has Examples block" || bad "facts: no Examples block"
   "$PY" "$facts" --help >/dev/null 2>&1 && ok "facts: --help exits 0" || bad "facts: --help nonzero"
   # Offline mode must pass on the skill's own content (internal consistency).
   "$PY" "$facts" --offline >/dev/null 2>&1 && ok "facts: --offline consistent (exit 0)" || bad "facts: --offline found drift"

+ 5 - 1
skills/pypi-ops/tests/run.sh

@@ -39,7 +39,11 @@ PF="$SCRIPTS/publish-preflight.sh"
 bash "$PF" --help >/dev/null 2>&1;  expect_exit "--help" 0 $?
 bash "$PF" --bogus >/dev/null 2>&1; expect_exit "bad flag -> 2" 2 $?
 # --build is wired (the actual build is network/backend-dependent, not exercised offline)
-bash "$PF" --help 2>&1 | grep -q -- '--build' && ok "--help documents --build" || no "--help missing --build"
+# Captured, not `--help | grep -q`: under `set -o pipefail` grep -q's early exit
+# SIGPIPEs the producer (141) and flakes the assert even on a match. See
+# skills/fleet-ops/tests/run.sh for the pattern.
+pf_help="$(bash "$PF" --help 2>&1)"
+expect_has "--help documents --build" "--build" "$pf_help"
 mkdir -p "$SB/empty"
 bash "$PF" "$SB/empty" >/dev/null 2>&1; expect_exit "no pyproject -> 3" 3 $?
 

+ 4 - 1
skills/threejs-ops/tests/run.sh

@@ -62,7 +62,10 @@ done
 facts="$here/scripts/check-three-facts.py"
 if [ -f "$facts" ]; then
   "$PY" -m py_compile "$facts" && ok "facts: py_compile clean" || bad "facts: py_compile failed"
-  head -35 "$facts" | grep -Eq '^# +Examples:' && ok "facts: has Examples block" || bad "facts: no Examples block"
+  # Captured, not `head | grep -q`: under `set -o pipefail` grep -q's early exit
+  # SIGPIPEs the producer (141) and flakes the assert even on a match.
+  facts_head="$(head -35 "$facts")"
+  grep -Eq '^# +Examples:' <<<"$facts_head" && ok "facts: has Examples block" || bad "facts: no Examples block"
   "$PY" "$facts" --help >/dev/null 2>&1 && ok "facts: --help exits 0" || bad "facts: --help nonzero"
   # Offline mode must pass on the skill's own content (internal consistency).
   "$PY" "$facts" --offline >/dev/null 2>&1 && ok "facts: --offline consistent (exit 0)" || bad "facts: --offline found inconsistency"