OpenAgentsControl/.github/workflows/validate-test-suites.yml

name: Validate Test Suites

on:
  push:
    paths:
      - 'evals/agents/*/config/**/*.json'
      - 'evals/agents/*/tests/**/*.yaml'
      - 'scripts/validation/validate-test-suites.sh'
      - '.github/workflows/validate-test-suites.yml'
  pull_request:
    paths:
      - 'evals/agents/*/config/**/*.json'
      - 'evals/agents/*/tests/**/*.yaml'
      - 'scripts/validation/validate-test-suites.sh'
      - '.github/workflows/validate-test-suites.yml'
  workflow_dispatch:

jobs:
  validate:
    name: Validate Test Suite Definitions
    runs-on: ubuntu-latest
    
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'
          cache-dependency-path: 'evals/framework/package-lock.json'
      
      - name: Install dependencies
        run: |
          cd evals/framework
          npm ci
      
      - name: Validate all test suites
        run: |
          cd evals/framework
          npm run validate:suites:all
      
      - name: Comment on PR (if validation failed)
        if: failure() && github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: '❌ **Test Suite Validation Failed**\n\nPlease check the test suite JSON files for errors. Run `npm run validate:suites` locally to see details.'
            })
      
      - name: Upload validation report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: suite-validation-report
          path: |
            evals/agents/*/config/**/*.json
            scripts/validation/validate-test-suites.sh
          retention-days: 7