OpenAgentsControl/.github/workflows/validate-test-suites.yml

name: Validate Test Suites

on:
  push:
    paths:
      - 'evals/agents/**/config/**/*.json'
      - 'evals/agents/**/tests/**/*.yaml'
      - 'scripts/validation/validate-test-suites.sh'
      - 'package.json'
      - 'pnpm-lock.yaml'
      - 'pnpm-workspace.yaml'
      - '.github/workflows/validate-test-suites.yml'
  pull_request:
    paths:
      - 'evals/agents/**/config/**/*.json'
      - 'evals/agents/**/tests/**/*.yaml'
      - 'scripts/validation/validate-test-suites.sh'
      - 'package.json'
      - 'pnpm-lock.yaml'
      - 'pnpm-workspace.yaml'
      - '.github/workflows/validate-test-suites.yml'
  workflow_dispatch:

permissions:
  contents: read

jobs:
  validate:
    name: Validate Test Suite Definitions
    runs-on: ubuntu-latest
    
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          persist-credentials: false
      
      - name: Setup pnpm
        uses: pnpm/action-setup@b0f76dfb45f55f8421693e4803ac7bb65143bd34 # v6

      - name: Setup Node.js
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: '20'
          cache: 'pnpm'
          cache-dependency-path: 'pnpm-lock.yaml'
      
      - name: Install workspace dependencies
        run: pnpm install --frozen-lockfile
      
      - name: Validate all test suites
        run: pnpm --dir evals/framework run validate:suites:all
      
      # NOTE: A "Comment on PR (if validation failed)" step was removed here.
      # It called github.rest.issues.createComment, which needs `issues: write`.
      # This workflow is fork-reachable (pull_request) and intentionally runs with
      # a read-only token, so the step would 403; and for fork PRs GitHub forces a
      # read-only token regardless, so it could never work there. A validation
      # failure is already surfaced as a failed check plus the uploaded report.

      - name: Upload validation report
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: suite-validation-report
          path: |
            evals/agents/**/config/**/*.json
            scripts/validation/validate-test-suites.sh
          retention-days: 7