|
|
@@ -0,0 +1,337 @@
|
|
|
+#!/usr/bin/env python3
|
|
|
+# Staleness verifier for the fast-moving facts the mapbox-ops skill encodes.
|
|
|
+#
|
|
|
+# Two modes (SKILL-RESOURCE-PROTOCOL.md §7):
|
|
|
+# --offline (default): NO network. Asserts the skill is internally consistent —
|
|
|
+# style-catalog.json parses, the v3 Standard config enums
|
|
|
+# (lightPreset/theme) agree between catalog and references,
|
|
|
+# the terrain tileset IDs and version gates (weather >= 3.7,
|
|
|
+# camera roll >= 3.5) are stated consistently, every classic
|
|
|
+# style url matches its id, every third-party entry is
|
|
|
+# addressable. Runs in PR CI and MAY block.
|
|
|
+# --live: network. Resolves the concrete third-party style-JSON URLs
|
|
|
+# and probes whether Mapbox GL JS has shipped a major beyond
|
|
|
+# v3 (which would mean the whole skill needs a review pass).
|
|
|
+# Runs in the scheduled freshness workflow and NEVER blocks a
|
|
|
+# PR: a transient network failure is UNAVAILABLE (exit 7), only
|
|
|
+# a confirmed change is DRIFT (exit 10).
|
|
|
+#
|
|
|
+# Usage: check-mapbox-facts.py [--offline|--live] [--json] [-q] [--timeout SEC]
|
|
|
+# Input: none (reads the skill's own assets/ + references/ relative to this file)
|
|
|
+# Output: stdout = data only (text findings, or the --json envelope)
|
|
|
+# Stderr: headers, progress, warnings, errors
|
|
|
+# Exit: 0 ok, 2 usage, 3 not-found (skill files missing), 4 validation
|
|
|
+# (offline inconsistency), 5 missing-dep, 7 unavailable (live network),
|
|
|
+# 10 drift (live: a URL 404'd, or GL JS major bumped past v3)
|
|
|
+#
|
|
|
+# Examples:
|
|
|
+# check-mapbox-facts.py --offline
|
|
|
+# check-mapbox-facts.py --offline --json | jq '.data[] | select(.status!="ok")'
|
|
|
+# check-mapbox-facts.py --live --timeout 15
|
|
|
+"""Staleness verifier for mapbox-ops (see header comment)."""
|
|
|
+
|
|
|
+from __future__ import annotations
|
|
|
+
|
|
|
+import argparse
|
|
|
+import json
|
|
|
+import re
|
|
|
+import sys
|
|
|
+from pathlib import Path
|
|
|
+
|
|
|
+EXIT_OK = 0
|
|
|
+EXIT_USAGE = 2
|
|
|
+EXIT_NOT_FOUND = 3
|
|
|
+EXIT_VALIDATION = 4
|
|
|
+EXIT_MISSING_DEP = 5
|
|
|
+EXIT_UNAVAILABLE = 7
|
|
|
+EXIT_DRIFT = 10
|
|
|
+
|
|
|
+SCHEMA = "claude-mods.mapbox-ops.facts/v1"
|
|
|
+
|
|
|
+SKILL_ROOT = Path(__file__).resolve().parent.parent
|
|
|
+CATALOG = SKILL_ROOT / "assets" / "style-catalog.json"
|
|
|
+REFS = SKILL_ROOT / "references"
|
|
|
+SKILL_MD = SKILL_ROOT / "SKILL.md"
|
|
|
+
|
|
|
+# Facts the skill commits to. Changing these is a deliberate edit; the verifier
|
|
|
+# asserts the skill states them consistently across catalog + references.
|
|
|
+EXPECTED_LIGHT_PRESET = {"dawn", "day", "dusk", "night"}
|
|
|
+EXPECTED_THEME = {"default", "faded", "monochrome"}
|
|
|
+TERRAIN_DEM_ID = "mapbox.mapbox-terrain-dem-v1"
|
|
|
+TERRAIN_VECTOR_ID = "mapbox.mapbox-terrain-v2"
|
|
|
+GLJS_MAJOR = 3 # the skill is scoped to mapbox-gl-js v3.x
|
|
|
+
|
|
|
+
|
|
|
+class Finding:
|
|
|
+ __slots__ = ("check", "status", "detail")
|
|
|
+
|
|
|
+ def __init__(self, check: str, status: str, detail: str) -> None:
|
|
|
+ self.check = check
|
|
|
+ self.status = status # ok | fail | drift | unavailable
|
|
|
+ self.detail = detail
|
|
|
+
|
|
|
+ def as_dict(self) -> dict:
|
|
|
+ return {"check": self.check, "status": self.status, "detail": self.detail}
|
|
|
+
|
|
|
+
|
|
|
+def read_text(path: Path) -> str:
|
|
|
+ return path.read_text(encoding="utf-8", errors="replace")
|
|
|
+
|
|
|
+
|
|
|
+# --------------------------------------------------------------------------- #
|
|
|
+# Offline checks #
|
|
|
+# --------------------------------------------------------------------------- #
|
|
|
+def run_offline(findings: list[Finding]) -> None:
|
|
|
+ # Required files present (else NOT_FOUND, distinct from inconsistency).
|
|
|
+ missing = [p for p in (CATALOG, SKILL_MD, REFS) if not p.exists()]
|
|
|
+ if missing:
|
|
|
+ for p in missing:
|
|
|
+ findings.append(Finding("files-present", "fail", f"missing: {p}"))
|
|
|
+ raise _NotFound()
|
|
|
+
|
|
|
+ # O1 — catalog parses.
|
|
|
+ try:
|
|
|
+ catalog = json.loads(read_text(CATALOG))
|
|
|
+ findings.append(Finding("catalog-json", "ok", "style-catalog.json parses"))
|
|
|
+ except json.JSONDecodeError as exc:
|
|
|
+ findings.append(Finding("catalog-json", "fail", f"invalid JSON: {exc}"))
|
|
|
+ return # nothing else is checkable
|
|
|
+
|
|
|
+ presets = catalog.get("standard_presets", {})
|
|
|
+ v3_md = read_text(REFS / "v3-standard-style.md") if (REFS / "v3-standard-style.md").exists() else ""
|
|
|
+
|
|
|
+ # O2 — lightPreset enum: catalog matches the committed set AND each value is
|
|
|
+ # documented in v3-standard-style.md.
|
|
|
+ light = set(presets.get("lightPreset", []))
|
|
|
+ if light != EXPECTED_LIGHT_PRESET:
|
|
|
+ findings.append(Finding("lightPreset-enum", "fail",
|
|
|
+ f"catalog {sorted(light)} != expected {sorted(EXPECTED_LIGHT_PRESET)}"))
|
|
|
+ else:
|
|
|
+ undoc = [v for v in EXPECTED_LIGHT_PRESET if v not in v3_md]
|
|
|
+ if undoc:
|
|
|
+ findings.append(Finding("lightPreset-enum", "fail",
|
|
|
+ f"values not documented in v3-standard-style.md: {undoc}"))
|
|
|
+ else:
|
|
|
+ findings.append(Finding("lightPreset-enum", "ok", "dawn|day|dusk|night consistent"))
|
|
|
+
|
|
|
+ # O3 — theme enum.
|
|
|
+ theme = set(presets.get("theme", []))
|
|
|
+ if theme != EXPECTED_THEME:
|
|
|
+ findings.append(Finding("theme-enum", "fail",
|
|
|
+ f"catalog {sorted(theme)} != expected {sorted(EXPECTED_THEME)}"))
|
|
|
+ else:
|
|
|
+ findings.append(Finding("theme-enum", "ok", "default|faded|monochrome consistent"))
|
|
|
+
|
|
|
+ # O4 — terrain tileset IDs present in terrain.md.
|
|
|
+ terrain_md = read_text(REFS / "terrain.md") if (REFS / "terrain.md").exists() else ""
|
|
|
+ for tid in (TERRAIN_DEM_ID, TERRAIN_VECTOR_ID):
|
|
|
+ if tid in terrain_md:
|
|
|
+ findings.append(Finding(f"terrain-id:{tid}", "ok", "present in terrain.md"))
|
|
|
+ else:
|
|
|
+ findings.append(Finding(f"terrain-id:{tid}", "fail", "absent from terrain.md"))
|
|
|
+
|
|
|
+ # O5 — weather version gate agrees between catalog effects comment and dataviz ref.
|
|
|
+ effects_comment = catalog.get("effects", {}).get("_comment", "")
|
|
|
+ dataviz_md = read_text(REFS / "dataviz-and-3d.md") if (REFS / "dataviz-and-3d.md").exists() else ""
|
|
|
+ cat_ver = _first_gl_gate(effects_comment)
|
|
|
+ ref_ver = _first_gl_gate(dataviz_md, near="setRain") or _first_gl_gate(dataviz_md, near="Weather")
|
|
|
+ if cat_ver and ref_ver and cat_ver == ref_ver == "3.7":
|
|
|
+ findings.append(Finding("weather-gate", "ok", "GL JS >= 3.7 consistent (catalog + dataviz-and-3d.md)"))
|
|
|
+ else:
|
|
|
+ findings.append(Finding("weather-gate", "fail",
|
|
|
+ f"weather version gate mismatch (catalog={cat_ver!r}, ref={ref_ver!r}, want 3.7)"))
|
|
|
+
|
|
|
+ # O6 — camera roll gate >= 3.5 stated in camera-and-animation.md.
|
|
|
+ camera_md = read_text(REFS / "camera-and-animation.md") if (REFS / "camera-and-animation.md").exists() else ""
|
|
|
+ roll_ver = _first_gl_gate(camera_md, near="roll")
|
|
|
+ if roll_ver == "3.5":
|
|
|
+ findings.append(Finding("camera-roll-gate", "ok", "native roll GL JS >= 3.5 stated"))
|
|
|
+ else:
|
|
|
+ findings.append(Finding("camera-roll-gate", "fail",
|
|
|
+ f"camera roll gate = {roll_ver!r}, want 3.5"))
|
|
|
+
|
|
|
+ # O7 — GL JS major scope: SKILL.md says v3.
|
|
|
+ skill_md = read_text(SKILL_MD)
|
|
|
+ if re.search(rf"v{GLJS_MAJOR}\.x", skill_md) and re.search(rf"v{GLJS_MAJOR}\b", skill_md):
|
|
|
+ findings.append(Finding("gljs-major", "ok", f"skill scoped to v{GLJS_MAJOR}.x"))
|
|
|
+ else:
|
|
|
+ findings.append(Finding("gljs-major", "fail", f"SKILL.md no longer clearly scopes v{GLJS_MAJOR}.x"))
|
|
|
+
|
|
|
+ # O8 — every classic style url tail matches its id.
|
|
|
+ bad_urls = []
|
|
|
+ for s in catalog.get("styles", []):
|
|
|
+ sid, url = s.get("id", ""), s.get("url", "")
|
|
|
+ if not url.endswith("/" + sid) and not url.endswith(sid):
|
|
|
+ bad_urls.append(f"{sid} -> {url}")
|
|
|
+ if bad_urls:
|
|
|
+ findings.append(Finding("style-url-id", "fail", "url/id mismatch: " + "; ".join(bad_urls)))
|
|
|
+ else:
|
|
|
+ findings.append(Finding("style-url-id", "ok", f"{len(catalog.get('styles', []))} style urls match ids"))
|
|
|
+
|
|
|
+ # O9 — every third-party entry is addressable (has a url or an explanatory note).
|
|
|
+ unaddressable = [t.get("id", "?") for t in catalog.get("third_party", [])
|
|
|
+ if not t.get("url") and not t.get("note")]
|
|
|
+ if unaddressable:
|
|
|
+ findings.append(Finding("third-party-addressable", "fail",
|
|
|
+ "no url and no note: " + ", ".join(unaddressable)))
|
|
|
+ else:
|
|
|
+ findings.append(Finding("third-party-addressable", "ok",
|
|
|
+ f"{len(catalog.get('third_party', []))} third-party entries addressable"))
|
|
|
+
|
|
|
+
|
|
|
+def _first_gl_gate(text: str, near: str | None = None) -> str | None:
|
|
|
+ """Return the first 'GL JS >= 3.N' version found, optionally on a line mentioning `near`."""
|
|
|
+ pat = re.compile(r"(?:GL JS\s*)?[>≥]=?\s*(3\.\d+)")
|
|
|
+ if near:
|
|
|
+ for line in text.splitlines():
|
|
|
+ if near in line:
|
|
|
+ m = pat.search(line)
|
|
|
+ if m:
|
|
|
+ return m.group(1)
|
|
|
+ return None
|
|
|
+ m = pat.search(text)
|
|
|
+ return m.group(1) if m else None
|
|
|
+
|
|
|
+
|
|
|
+class _NotFound(Exception):
|
|
|
+ pass
|
|
|
+
|
|
|
+
|
|
|
+# --------------------------------------------------------------------------- #
|
|
|
+# Live checks #
|
|
|
+# --------------------------------------------------------------------------- #
|
|
|
+def run_live(findings: list[Finding], timeout: float) -> None:
|
|
|
+ import urllib.error
|
|
|
+ import urllib.request
|
|
|
+
|
|
|
+ try:
|
|
|
+ catalog = json.loads(read_text(CATALOG))
|
|
|
+ except (OSError, json.JSONDecodeError) as exc:
|
|
|
+ findings.append(Finding("catalog-json", "fail", f"cannot read catalog: {exc}"))
|
|
|
+ raise _NotFound()
|
|
|
+
|
|
|
+ def probe(url: str) -> str:
|
|
|
+ """Return resolved | notfound | unavailable for a URL (HEAD, GET fallback)."""
|
|
|
+ for method in ("HEAD", "GET"):
|
|
|
+ req = urllib.request.Request(url, method=method,
|
|
|
+ headers={"User-Agent": "mapbox-ops-staleness/1"})
|
|
|
+ try:
|
|
|
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
|
+ return "resolved" if resp.status < 400 else "unavailable"
|
|
|
+ except urllib.error.HTTPError as e:
|
|
|
+ if e.code in (404, 410):
|
|
|
+ return "notfound"
|
|
|
+ if e.code in (403, 405, 429):
|
|
|
+ # forbidden/method-not-allowed/rate-limited: exists or can't tell.
|
|
|
+ if method == "HEAD":
|
|
|
+ continue # retry with GET
|
|
|
+ return "unavailable" if e.code == 429 else "resolved"
|
|
|
+ return "unavailable"
|
|
|
+ except (urllib.error.URLError, TimeoutError, OSError):
|
|
|
+ return "unavailable"
|
|
|
+ return "unavailable"
|
|
|
+
|
|
|
+ # L1 — concrete third-party style URLs (skip templated/keyed ones).
|
|
|
+ for t in catalog.get("third_party", []):
|
|
|
+ url = t.get("url")
|
|
|
+ if not url or "<" in url or "key=" in url:
|
|
|
+ continue
|
|
|
+ res = probe(url)
|
|
|
+ status = {"resolved": "ok", "notfound": "drift", "unavailable": "unavailable"}[res]
|
|
|
+ findings.append(Finding(f"url:{t.get('id', url)}", status, url))
|
|
|
+
|
|
|
+ # L2 — has Mapbox GL JS shipped a major beyond v3? A live v4.0.0 on the CDN
|
|
|
+ # means the skill's scope assumption needs a human review pass (drift, not error).
|
|
|
+ cdn = "https://api.mapbox.com/mapbox-gl-js/v{}.0.0/mapbox-gl.js"
|
|
|
+ v3 = probe(cdn.format(GLJS_MAJOR))
|
|
|
+ if v3 == "unavailable":
|
|
|
+ findings.append(Finding("gljs-cdn", "unavailable", "Mapbox CDN unreachable"))
|
|
|
+ else:
|
|
|
+ nxt = probe(cdn.format(GLJS_MAJOR + 1))
|
|
|
+ if nxt == "resolved":
|
|
|
+ findings.append(Finding("gljs-major-bump", "drift",
|
|
|
+ f"mapbox-gl-js v{GLJS_MAJOR + 1}.0.0 is live — review skill scope"))
|
|
|
+ elif nxt == "unavailable":
|
|
|
+ findings.append(Finding("gljs-major-bump", "unavailable",
|
|
|
+ f"could not probe v{GLJS_MAJOR + 1} (network)"))
|
|
|
+ else:
|
|
|
+ findings.append(Finding("gljs-major-bump", "ok",
|
|
|
+ f"v{GLJS_MAJOR} current; no v{GLJS_MAJOR + 1} GA"))
|
|
|
+
|
|
|
+
|
|
|
+# --------------------------------------------------------------------------- #
|
|
|
+# Main #
|
|
|
+# --------------------------------------------------------------------------- #
|
|
|
+def main(argv: list[str]) -> int:
|
|
|
+ ap = argparse.ArgumentParser(add_help=True, description="mapbox-ops staleness verifier")
|
|
|
+ mode = ap.add_mutually_exclusive_group()
|
|
|
+ mode.add_argument("--offline", action="store_true", help="structural/internal-consistency only (default)")
|
|
|
+ mode.add_argument("--live", action="store_true", help="resolve URLs + probe GL JS major (network)")
|
|
|
+ ap.add_argument("--json", action="store_true", help="emit the JSON envelope on stdout")
|
|
|
+ ap.add_argument("-q", "--quiet", action="store_true", help="suppress stderr progress")
|
|
|
+ ap.add_argument("--timeout", type=float, default=10.0, help="per-request timeout for --live (seconds)")
|
|
|
+ try:
|
|
|
+ args = ap.parse_args(argv)
|
|
|
+ except SystemExit as e:
|
|
|
+ # argparse exits 2 on bad args (matches USAGE); 0 on --help.
|
|
|
+ return EXIT_USAGE if e.code not in (0, None) else EXIT_OK
|
|
|
+
|
|
|
+ live = args.live
|
|
|
+ mode_name = "live" if live else "offline"
|
|
|
+
|
|
|
+ def emit(msg: str) -> None:
|
|
|
+ if not args.quiet:
|
|
|
+ print(msg, file=sys.stderr)
|
|
|
+
|
|
|
+ findings: list[Finding] = []
|
|
|
+ emit(f"== check-mapbox-facts ({mode_name}) ==")
|
|
|
+ try:
|
|
|
+ if live:
|
|
|
+ run_live(findings, args.timeout)
|
|
|
+ else:
|
|
|
+ run_offline(findings)
|
|
|
+ except _NotFound:
|
|
|
+ if args.json:
|
|
|
+ print(json.dumps({"error": {"code": "NOT_FOUND",
|
|
|
+ "message": "skill files missing",
|
|
|
+ "details": [f.as_dict() for f in findings]}}))
|
|
|
+ for f in findings:
|
|
|
+ emit(f" [{f.status.upper()}] {f.check}: {f.detail}")
|
|
|
+ return EXIT_NOT_FOUND
|
|
|
+
|
|
|
+ n_fail = sum(1 for f in findings if f.status == "fail")
|
|
|
+ n_drift = sum(1 for f in findings if f.status == "drift")
|
|
|
+ n_unavail = sum(1 for f in findings if f.status == "unavailable")
|
|
|
+
|
|
|
+ # Output: stdout is data only.
|
|
|
+ if args.json:
|
|
|
+ print(json.dumps({
|
|
|
+ "data": [f.as_dict() for f in findings],
|
|
|
+ "meta": {"mode": mode_name, "count": len(findings),
|
|
|
+ "fail": n_fail, "drift": n_drift, "unavailable": n_unavail,
|
|
|
+ "schema": SCHEMA},
|
|
|
+ }, indent=2))
|
|
|
+ else:
|
|
|
+ for f in findings:
|
|
|
+ print(f"{f.check}\t{f.status}\t{f.detail}")
|
|
|
+
|
|
|
+ # Progress summary to stderr.
|
|
|
+ for f in findings:
|
|
|
+ if f.status != "ok":
|
|
|
+ emit(f" [{f.status.upper()}] {f.check}: {f.detail}")
|
|
|
+ emit(f"-- {len(findings)} checks: {n_fail} fail, {n_drift} drift, {n_unavail} unavailable")
|
|
|
+
|
|
|
+ # Exit precedence: a real inconsistency (offline) or 404 (live) is the loudest
|
|
|
+ # signal; an unavailable network is advisory and must never mask a clean run as
|
|
|
+ # failing — but if the ONLY non-ok results are unavailable, exit 7, never 0.
|
|
|
+ if n_fail:
|
|
|
+ return EXIT_VALIDATION
|
|
|
+ if n_drift:
|
|
|
+ return EXIT_DRIFT
|
|
|
+ if n_unavail:
|
|
|
+ return EXIT_UNAVAILABLE
|
|
|
+ return EXIT_OK
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ sys.exit(main(sys.argv[1:]))
|