wandb__skills/.github/workflows/eval-skills.yml

name: Evaluate Skills

# Evaluates skills from this repo using the WandBAgentFactory eval framework.
#
# Self-contained: copies skills into WBAF, writes a temp agent config
# referencing those skills, generates a temp eval YAML, and runs it.
# No changes needed in WandBAgentFactory to test new skills.
#
# Benchmarks against multiple agents via matrix strategy.

on:
  push:
    branches: [main]
    paths: ["skills/**"]
    
  schedule:
    # Weekly on Sundays at 11 PM PT (7 AM UTC Monday)
    - cron: "0 7 * * 1"

jobs:
  eval:
    name: "Eval skills (${{ matrix.base_agent }})"
    runs-on: ubuntu-latest
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        base_agent: ${{ fromJson(inputs.base_agents || '["codex", "claude-code"]') }}

    env:
      WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
      MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

    steps:
      - name: Checkout skills
        uses: actions/checkout@v4
        with:
          path: skills

      - name: Checkout WandBAgentFactory
        uses: actions/checkout@v4
        with:
          repository: wandb/WandBAgentFactory
          ref: main
          token: ${{ secrets.WBAF_PAT }}
          path: wbaf

      - name: Install uv
        uses: astral-sh/setup-uv@v4

      - name: Set up Python
        working-directory: wbaf
        run: uv python install

      - name: Install dependencies
        working-directory: wbaf
        run: uv sync

      - name: Configure Weave project
        working-directory: wbaf
        run: |
          printf 'entity: wandb\nproject: wb-agent-frozen\n' > wandb_settings.yaml

      - name: Regenerate eval task list and copy skills
        id: setup
        run: |
          INPUT_SKILLS="${{ inputs.skills }}"
          BASE_AGENT="${{ matrix.base_agent }}"

          # ── Regenerate wba-all.yaml to pick up any new tasks ──
          uv run --directory wbaf python developer/scripts/update_all_eval.py

          # ── Discover skills ──
          if [[ -n "$INPUT_SKILLS" ]]; then
            IFS=',' read -ra SKILL_NAMES <<< "$INPUT_SKILLS"
          else
            SKILL_NAMES=()
            for d in skills/skills/*/; do
              [[ -f "$d/SKILL.md" ]] && SKILL_NAMES+=("$(basename "$d")")
            done
          fi

          echo "Skills to test: ${SKILL_NAMES[*]}"
          echo "Base agent: $BASE_AGENT"

          # ── Copy skills into WBAF ──
          for name in "${SKILL_NAMES[@]}"; do
            name="$(echo "$name" | xargs)"
            src="skills/skills/$name"
            dst="wbaf/skills/$name"
            if [[ -d "$src" ]]; then
              cp -r "$src" "$dst"
              echo "  copied: $name"
            else
              echo "  WARNING: $name not found in skills/skills/, skipping"
            fi
          done

          # ── Write temp agent config ──
          # Paths: shell uses wbaf/ prefix (cwd=workspace root),
          # Python uses no prefix (uv --directory wbaf sets cwd=wbaf/).
          AGENT_NAME="ci-skill-eval-${BASE_AGENT}"
          mkdir -p "wbaf/agents/${AGENT_NAME}"

          uv run --directory wbaf python -c "
          import yaml
          from pathlib import Path

          skills = '${SKILL_NAMES[*]}'.split()
          config = {
              'extends': '$BASE_AGENT',
              'skills': skills,
          }
          out = Path('agents/$AGENT_NAME/config.yaml')
          out.parent.mkdir(parents=True, exist_ok=True)
          out.write_text(yaml.dump(config, default_flow_style=False))
          print(f'Agent: $AGENT_NAME (extends $BASE_AGENT, skills={skills})')
          "

          # ── Write temp eval config from wba-all.yaml ──
          uv run --directory wbaf python -c "
          import yaml
          from pathlib import Path

          eval_file = '${{ inputs.eval_tasks || 'wba-all' }}'
          src = yaml.safe_load(Path(f'evals/{eval_file}.yaml').read_text())
          tasks = src.get('tasks', [])

          eval_spec = {
              'agents': ['$AGENT_NAME'],
              'scorers': src.get('scorers', ['rubric']),
              'tasks': tasks,
              'trials': 1,
          }

          out = Path('evals/ci-skill-eval.yaml')
          out.write_text(yaml.dump(eval_spec, default_flow_style=False, sort_keys=False))
          print(f'Eval: {len(tasks)} tasks, scorers={eval_spec[\"scorers\"]}')
          "

          echo "agent_name=$AGENT_NAME" >> "$GITHUB_OUTPUT"
          echo "skills=${SKILL_NAMES[*]}" >> "$GITHUB_OUTPUT"

      - name: Run evaluation
        working-directory: wbaf
        run: |
          uv run python -W ignore -m core.run_eval \
            evals/ci-skill-eval.yaml \
            --weave-parallelism 15 \
            2>&1 | tee eval-output.log

      - name: Extract results summary
        if: always()
        id: results
        run: |
          # Extract only aggregate pass/fail counts — no task names or details.
          uv run --directory wbaf python -c "
          import re, json

          log = open('eval-output.log').read()

          # Find the last summary line (agent-name: {dict})
          summary = {}
          for line in log.splitlines():
              m = re.match(r'^[\w-]+:\s*(\{.+\})\s*$', line)
              if m:
                  try:
                      summary = eval(m.group(1))
                  except Exception:
                      pass

          # Extract only aggregate counts
          scorer = summary.get('scorer') or summary.get('rubric') or summary.get('correctness') or {}
          output = summary.get('output', {})

          result = {
              'passed': scorer.get('passed', {}).get('true_count', 0),
              'failed': scorer.get('passed', {}).get('true_count', 0),
              'total': output.get('succeeded', {}).get('true_count', 0)
                       + output.get('succeeded', {}).get('false_count', 0),
              'score_mean': scorer.get('score', {}).get('mean', 0),
          }

          # Correct failed count
          result['failed'] = result['total'] - result['passed']

          print(json.dumps(result))
          with open('results-summary.json', 'w') as f:
              json.dump(result, f)
          " || echo '{"passed":0,"failed":0,"total":0,"score_mean":0}' > wbaf/results-summary.json

      - name: Upload results summary
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: results-${{ matrix.base_agent }}
          path: wbaf/results-summary.json
          retention-days: 90

  # ── Update README and badges with aggregated results ──
  update-readme:
    name: Update README & Badges
    needs: eval
    if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
    runs-on: ubuntu-latest

    steps:
      - name: Checkout skills
        uses: actions/checkout@v4
        with:
          token: ${{ secrets.WBAF_PAT }}

      - name: Download all result summaries
        uses: actions/download-artifact@v4
        with:
          pattern: results-*
          path: results/

      - name: Update README and badges
        run: |
          python3 -c "
          import json, re, yaml
          from pathlib import Path

          # ── Parse results from all agents ──
          agents = {}
          for result_dir in sorted(Path('results').iterdir()):
              if not result_dir.is_dir():
                  continue
              agent = result_dir.name.replace('results-', '')
              summary_file = result_dir / 'results-summary.json'
              if summary_file.exists():
                  agents[agent] = json.loads(summary_file.read_text())

          # ── Generate per-agent badges ──
          badges_dir = Path('.badges')
          badges_dir.mkdir(exist_ok=True)

          for agent, data in agents.items():
              passed = data.get('passed', 0)
              total = data.get('total', 0)
              score = data.get('score_mean', 0)

              if total > 0:
                  pct = passed / total
                  label = f'{passed}/{total} ({pct:.0%})'
                  color = 'brightgreen' if pct >= 0.8 else 'yellow' if pct >= 0.5 else 'red'
              else:
                  label = 'no data'
                  color = 'lightgrey'

              badge = {
                  'schemaVersion': 1,
                  'label': agent,
                  'message': label,
                  'color': color,
              }
              (badges_dir / f'{agent}.json').write_text(
                  json.dumps(badge, indent=2) + '\n'
              )

          # ── Build status string for skill table ──
          parts = []
          for agent, data in sorted(agents.items()):
              passed = data.get('passed', 0)
              total = data.get('total', 0)
              if total > 0:
                  parts.append(f'{agent}: {passed}/{total} ({passed/total:.0%})')
          status = ' | '.join(parts) if parts else 'evaluated'

          # ── Build skill table ──
          skills_dir = Path('skills')
          readme = Path('README.md')

          rows = []
          for skill_dir in sorted(skills_dir.iterdir()):
              if not skill_dir.is_dir() or not (skill_dir / 'SKILL.md').is_file():
                  continue
              name = skill_dir.name

              text = (skill_dir / 'SKILL.md').read_text()
              desc = ''
              if text.startswith('---'):
                  end = text.find('\n---', 3)
                  if end != -1:
                      fm = yaml.safe_load(text[3:end])
                      desc = fm.get('description', '') if isinstance(fm, dict) else ''
              if len(desc) > 100:
                  desc = desc[:97] + '...'

              rows.append(f'| [\`{name}\`](skills/{name}/) | {desc} | {status} |')

          table = '\n'.join([
              '| Skill | Description | Status |',
              '|-------|-------------|--------|',
              *rows,
          ])

          content = readme.read_text()
          pattern = r'<!-- BEGIN SKILL TABLE -->.*?<!-- END SKILL TABLE -->'
          replacement = f'<!-- BEGIN SKILL TABLE -->\n{table}\n<!-- END SKILL TABLE -->'
          content = re.sub(pattern, replacement, content, flags=re.DOTALL)
          readme.write_text(content)
          print(f'Updated README - status: {status}')
          "

      - name: Commit results
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add README.md .badges/
          if git diff --cached --quiet; then
            echo "No changes"
          else
            git commit -m "update skill eval results and badges"
            git push
          fi