OpenAgentsControl/.github/workflows/validate-test-suites.yml

name: Validate Test Suites

on:
  push:
    paths:
      - 'evals/agents/*/config/**/*.json'
      - 'evals/agents/*/tests/**/*.yaml'
      - 'scripts/validation/validate-test-suites.sh'
      - '.github/workflows/validate-test-suites.yml'
  pull_request:
    paths:
      - 'evals/agents/*/config/**/*.json'
      - 'evals/agents/*/tests/**/*.yaml'
      - 'scripts/validation/validate-test-suites.sh'
      - '.github/workflows/validate-test-suites.yml'
  workflow_dispatch:

permissions:
  contents: read

jobs:
  validate:
    name: Validate Test Suite Definitions
    runs-on: ubuntu-latest
    
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      
      - name: Setup Node.js
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: '20'
          cache: 'npm'
          cache-dependency-path: 'package-lock.json'
      
      - name: Install dependencies
        run: |
          npm ci
      
      - name: Validate all test suites
        run: |
          cd evals/framework
          npm run validate:suites:all
      
      - name: Comment on PR (if validation failed)
        if: failure() && github.event_name == 'pull_request'
        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
        with:
          script: |
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: '❌ **Test Suite Validation Failed**\n\nPlease check the test suite JSON files for errors. Run `npm run validate:suites` locally to see details.'
            })
      
      - name: Upload validation report
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: suite-validation-report
          path: |
            evals/agents/*/config/**/*.json
            scripts/validation/validate-test-suites.sh
          retention-days: 7