dotnet__skills/.github/workflows/evaluation.yml

# Unified evaluation workflow for all PRs (same-repo and fork) and scheduled runs.
#
# IMPORTANT: The /evaluate command uses the `issue_comment` trigger, which
# ALWAYS runs the workflow YAML from the default branch (main), NOT from the
# PR branch. Changes to this file in a PR will not take effect until merged.
# The skill-validator binary IS built from the PR branch for same-repo PRs,
# so eng/skill-validator/ changes are tested before merge.
#
# For PRs (same-repo and fork):
#   - On PR open/sync, the `pr-status` job posts an initial commit status:
#     - "success" if no skills changed (required check passes immediately)
#     - "pending" if skills changed (maintainer must post /evaluate to trigger)
#   - When a maintainer posts "/evaluate" on the PR, the `gate` job validates
#     permissions and triggers the full evaluation pipeline.
#
# For scheduled runs:
#   - Runs daily, evaluates all plugins with skills and tests.
#
# Security model for fork PRs:
#   - Workflow YAML: always from the default branch (enforced by issue_comment
#     and pull_request_target triggers)
#   - Validator binary: built from the default branch (fork PRs) or PR branch
#     (same-repo PRs, selected via needs.gate.outputs.is_fork in the ref: expression)
#   - Skill/test content: checked out from the fork PR (untrusted data, read-only)
#   - Secret access: only users with write+ permission can trigger evaluation
name: evaluation

# For a triage/manual single-PR dispatch, surface the PR + head in the run name.
# This run name also doubles as the idempotency key the PR-triage worker matches
# on (eval_run_exists_for_head in pr-triage-act.sh): a workflow_dispatch run's
# head_sha is the default branch — not the PR head — so the run name is how the
# worker recognises that an evaluation has already been dispatched for this head.
# For all other events the expression yields '' and GitHub uses the default name.
run-name: "${{ inputs.pr_number != '' && format('Evaluate PR #{0} @ {1}', inputs.pr_number, inputs.head_sha) || '' }}"

on:
  # Manual trigger for one-off deploys (e.g., AGENTVIZ SPA update) and
  # targeted re-runs of a single plugin without committing to main.
  # The pr_number/head_sha inputs are also the PR-triage worker's entry point:
  # the worker dispatches this workflow directly (workflow_dispatch is exempt
  # from GitHub's GITHUB_TOKEN recursion guard) instead of relying on the
  # `evaluate-now` label, which never fires when applied by the bot.
  workflow_dispatch:
    inputs:
      plugin:
        description: "Specific plugin to evaluate (leave blank for all)"
        type: string
        required: false
      pr_number:
        description: "PR number to evaluate. Routes the dispatch through the gate/PR pipeline instead of a full/plugin run."
        type: string
        required: false
      head_sha:
        description: "Short head SHA of the PR (informational; used for the run name and triage idempotency). The gate re-fetches the live head."
        type: string
        required: false

  # Same-repo PRs: post initial status
  pull_request:

  # Fork PRs: post initial status (runs from base branch for security).
  # Also receives `labeled` events: a human applying the `evaluate-now` label
  # is one entry point into this pipeline (alongside `/evaluate`). The triage
  # worker runs as github-actions[bot] and cannot use the label (label events
  # emitted by GITHUB_TOKEN do not start workflows), so it uses the
  # workflow_dispatch pr_number input above instead.
  pull_request_target:
    types: [opened, synchronize, reopened, labeled]

  # /evaluate command trigger
  issue_comment:
    types: [created]

  # Daily scheduled evaluation
  schedule:
    - cron: '0 0 * * *'  # Once daily at midnight UTC

concurrency:
  # /evaluate (issue_comment), the human-applied `evaluate-now` label
  # (pull_request_target labeled), and the triage worker's workflow_dispatch
  # (pr_number input) all share the same `eval-<pr_number>` group, so a race
  # between any of them collapses to a single run for the PR.
  group: ${{ github.workflow }}-${{ (github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '/evaluate')) && format('eval-{0}', github.event.issue.number) || (github.event_name == 'pull_request_target' && github.event.action == 'labeled' && github.event.label.name == 'evaluate-now') && format('eval-{0}', github.event.pull_request.number) || (github.event_name == 'workflow_dispatch' && inputs.pr_number != '') && format('eval-{0}', inputs.pr_number) || (github.event_name == 'issue_comment' && format('eval-noop-{0}-{1}', github.event.issue.number, github.event.comment.id)) || (github.event_name == 'pull_request' && format('eval-status-{0}', github.event.pull_request.number)) || (github.event_name == 'pull_request_target' && format('eval-fork-status-{0}', github.event.pull_request.number)) || github.run_id }}
  cancel-in-progress: true

env:
  DASHBOARD_RETENTION_DAYS: 14
  # Agent/judge models are resolved per-leg from eng/eval-models.json via
  # eng/resolve-judge.mjs (see the discover job's "Resolve model/judge matrix"
  # step). The judge is never the same model as the agent.

permissions:
  contents: write
  pull-requests: write
  statuses: write

jobs:
  # ==========================================================================
  # PR STATUS JOBS
  # Post initial commit status so the required check is never stuck as "Expected".
  # Posts success (no skills) or pending (needs /evaluate).
  # ==========================================================================

  # Same-repo PRs: use pull_request trigger (has direct access to PR content)
  pr-status:
    if: >-
      github.event_name == 'pull_request' &&
      github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    permissions:
      contents: read
      statuses: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Discover changes requiring evaluation
        id: discover
        shell: pwsh
        run: |
          $base = "${{ github.event.pull_request.base.sha }}"
          $head = "${{ github.event.pull_request.head.sha }}"
          $mergeBase = git merge-base $base $head
          $changedFiles = git diff --name-only --diff-filter=ACMR $mergeBase $head

          $hasSkillChanges = $changedFiles |
            Where-Object { $_ -match '^(plugins/[^/]+/skills|tests/[^/]+)/[^/]+/' } |
            Select-Object -First 1

          # Skill-validator and evaluation pipeline changes need evaluation.
          # Documentation files under src/ don't affect evaluation.
          $hasInfraChanges = $changedFiles |
            Where-Object {
              ($_ -match '^eng/skill-validator/' -and $_ -notmatch '^eng/skill-validator/src/(README\.md|docs/)') -or
              ($_ -match '^eng/vally-adapter/') -or
              $_ -match '^\.github/workflows/(evaluation|vally-evaluation)\.yml$'
            } |
            Select-Object -First 1

          if ($hasSkillChanges -or $hasInfraChanges) {
            echo "needs_eval=true" >> $env:GITHUB_OUTPUT
          } else {
            echo "needs_eval=false" >> $env:GITHUB_OUTPUT
          }

      - name: Post evaluation commit status
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          if [[ "${{ steps.discover.outputs.needs_eval }}" == "true" ]]; then
            STATE="pending"
            DESC="Post /evaluate to trigger evaluation"
          else
            STATE="success"
            DESC="No skills to evaluate"
          fi

          gh api "repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }}" \
            -f state="$STATE" \
            -f context="evaluation-status" \
            -f description="$DESC" \
            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  # Fork PRs: use pull_request_target (runs from base branch, fetches PR metadata safely)
  fork-pr-status:
    if: >-
      github.event_name == 'pull_request_target' &&
      github.event.action != 'labeled' &&
      github.event.action != 'unlabeled' &&
      github.event.pull_request.head.repo.full_name != github.repository
    runs-on: ubuntu-latest
    permissions:
      contents: read
      statuses: write
    steps:
      - name: Checkout base branch
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          fetch-depth: 0
          persist-credentials: false

      - name: Fetch PR head for diff
        run: git fetch origin +refs/pull/${{ github.event.pull_request.number }}/head

      - name: Discover changes requiring evaluation
        id: discover
        shell: pwsh
        run: |
          $base = "${{ github.event.pull_request.base.sha }}"
          $head = "FETCH_HEAD"
          $mergeBase = git merge-base $base $head
          $changedFiles = git diff --name-only --diff-filter=ACMR $mergeBase $head

          $hasSkillChanges = $changedFiles |
            Where-Object { $_ -match '^(plugins/[^/]+/skills|tests/[^/]+)/[^/]+/' } |
            Select-Object -First 1

          $hasInfraChanges = $changedFiles |
            Where-Object {
              ($_ -match '^eng/skill-validator/' -and $_ -notmatch '^eng/skill-validator/src/(README\.md|docs/)') -or
              ($_ -match '^eng/vally-adapter/') -or
              $_ -match '^\.github/workflows/(evaluation|vally-evaluation)\.yml$'
            } |
            Select-Object -First 1

          if ($hasSkillChanges -or $hasInfraChanges) {
            echo "needs_eval=true" >> $env:GITHUB_OUTPUT
          } else {
            echo "needs_eval=false" >> $env:GITHUB_OUTPUT
          }

      - name: Post evaluation commit status
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          if [[ "${{ steps.discover.outputs.needs_eval }}" == "true" ]]; then
            STATE="pending"
            DESC="Fork PR — post /evaluate to trigger evaluation"
          else
            STATE="success"
            DESC="No skills to evaluate"
          fi

          gh api "repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }}" \
            -f state="$STATE" \
            -f context="evaluation-status" \
            -f description="$DESC" \
            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  # ==========================================================================
  # GATE JOB
  # Validate evaluation trigger. Three entry points:
  #   1. /evaluate comment (issue_comment) — original human path
  #   2. evaluate-now label (pull_request_target labeled) — a human applying
  #      the label (the bot cannot: GITHUB_TOKEN label events don't start runs)
  #   3. workflow_dispatch with pr_number — the pr-triage worker's path. It runs
  #      as github-actions[bot] and dispatches this workflow directly, since
  #      workflow_dispatch is exempt from the GITHUB_TOKEN recursion guard.
  # All must come from a trusted actor (write+ on the repo, or the workflow's
  # own github-actions[bot] identity for paths 2 and 3).
  # ==========================================================================
  gate:
    if: >-
      (github.event.issue.pull_request &&
       startsWith(github.event.comment.body, '/evaluate'))
      ||
      (github.event_name == 'pull_request_target' &&
       github.event.action == 'labeled' &&
       github.event.label.name == 'evaluate-now')
      ||
      (github.event_name == 'workflow_dispatch' &&
       inputs.pr_number != '')
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write
      statuses: write
      issues: write
    outputs:
      head_sha: ${{ steps.pr.outputs.head_sha }}
      base_sha: ${{ steps.pr.outputs.base_sha }}
      pr_number: ${{ steps.pr.outputs.pr_number }}
      is_fork: ${{ steps.pr.outputs.is_fork }}
    steps:
      - name: Check actor permissions
        id: perms
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          if [[ "${{ github.event_name }}" == "issue_comment" ]]; then
            ACTOR='${{ github.event.comment.user.login }}'
          else
            ACTOR='${{ github.event.sender.login }}'
          fi
          # The triage worker dispatches this workflow as github-actions[bot]
          # (workflow_dispatch). A human may instead apply the evaluate-now label,
          # which arrives under that human's identity. The bot identity has no
          # entry in /collaborators/* but is implicitly trusted because only this
          # repository's own GITHUB_TOKEN can act under it.
          if [[ "$ACTOR" == "github-actions[bot]" ]]; then
            echo "Actor is github-actions[bot] — trusted by construction"
            exit 0
          fi
          PERMISSION=$(gh api "repos/${{ github.repository }}/collaborators/${ACTOR}/permission" --jq '.permission')
          echo "Actor ${ACTOR} has permission: $PERMISSION"
          if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "write" && "$PERMISSION" != "maintain" ]]; then
            echo "::error::Actor does not have write access"
            exit 1
          fi

      - name: Get PR details
        id: pr
        env:
          GH_TOKEN: ${{ github.token }}
          # Read the dispatch input via env (not direct interpolation) so the
          # numeric validation below runs before the value reaches any gh api
          # call — a crafted input cannot inject shell or path traversal.
          PR_NUMBER_INPUT: ${{ inputs.pr_number }}
        run: |
          if [[ "${{ github.event_name }}" == "issue_comment" ]]; then
            PR_NUMBER='${{ github.event.issue.number }}'
          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            PR_NUMBER="$PR_NUMBER_INPUT"
            if [[ ! "$PR_NUMBER" =~ ^[0-9]+$ ]]; then
              echo "::error::workflow_dispatch input pr_number='$PR_NUMBER' must be a positive integer"
              exit 1
            fi
          else
            PR_NUMBER='${{ github.event.pull_request.number }}'
          fi
          PR_DATA=$(gh api "repos/${{ github.repository }}/pulls/${PR_NUMBER}")
          HEAD_SHA=$(echo "$PR_DATA" | jq -r '.head.sha')
          HEAD_REPO=$(echo "$PR_DATA" | jq -r '.head.repo.full_name')
          BASE_REPO=$(echo "$PR_DATA" | jq -r '.base.repo.full_name')
          BASE_SHA=$(echo "$PR_DATA" | jq -r '.base.sha')

          if [[ "$HEAD_REPO" != "$BASE_REPO" ]]; then
            echo "is_fork=true" >> $GITHUB_OUTPUT
          else
            echo "is_fork=false" >> $GITHUB_OUTPUT
          fi

          echo "PR #${PR_NUMBER}: head=${HEAD_SHA} base=${BASE_SHA} fork=$([[ "$HEAD_REPO" != "$BASE_REPO" ]] && echo true || echo false)"
          echo "head_sha=${HEAD_SHA}" >> $GITHUB_OUTPUT
          echo "base_sha=${BASE_SHA}" >> $GITHUB_OUTPUT
          echo "pr_number=${PR_NUMBER}" >> $GITHUB_OUTPUT

      - name: Add reaction to comment
        if: github.event_name == 'issue_comment'
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          gh api "repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions" \
            -X POST -f content='eyes' || true

      - name: Remove evaluate-now label
        if: github.event_name == 'pull_request_target' || github.event_name == 'workflow_dispatch'
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          # For the label entry point, removing the label consumes the trigger so
          # re-applying re-fires. For the workflow_dispatch entry point there is no
          # label to consume, but we still strip any stale human-applied
          # evaluate-now label so it doesn't linger. Removal uses the workflow's
          # own GITHUB_TOKEN; per GitHub's recursion rules, label events emitted by
          # GITHUB_TOKEN do not start new workflow runs.
          gh pr edit "${{ steps.pr.outputs.pr_number }}" \
            --repo "${{ github.repository }}" \
            --remove-label "evaluate-now" || true

      - name: Set pending commit status
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          gh api "repos/${{ github.repository }}/statuses/${{ steps.pr.outputs.head_sha }}" \
            -f state=pending \
            -f context="evaluation-status" \
            -f description="Evaluation in progress..." \
            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  # ==========================================================================
  # DISCOVER JOB
  # Find skills to evaluate based on changed files.
  # ==========================================================================
  discover:
    needs: gate
    if: >-
      always() &&
      (needs.gate.result == 'success' || github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.pr_number == '')) &&
      (github.event_name != 'schedule' || github.repository == 'dotnet/skills')
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
    outputs:
      entries: ${{ steps.find.outputs.entries }}
      has_entries: ${{ steps.find.outputs.has_entries }}
      is_infra: ${{ steps.find.outputs.is_infra }}
      plugins: ${{ steps.find.outputs.plugins }}
      has_plugins: ${{ steps.find.outputs.has_plugins }}
      # Model/judge fan-out. Gating (PR) stays single-model (the gating model)
      # with a resolved judge != agent; nightly (schedule) fans out to the full
      # matrix. Both come from eng/eval-models.json via eng/resolve-judge.mjs.
      model_pairs: ${{ steps.models.outputs.pairs }}
      models: ${{ steps.models.outputs.models }}
      # Union of every agent model and its resolved judge for the active scope —
      # the exact set the ListModels preflight must find on the CI token.
      required_models: ${{ steps.models.outputs.required }}
    steps:
      - name: Check for new commits since last evaluation
        if: github.event_name == 'schedule'
        id: check-changes
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          # Bypass the skip guard on manual reruns so "Re-run jobs" always executes.
          if [ "${GITHUB_RUN_ATTEMPT}" != "1" ]; then
            echo "Manual rerun (attempt ${GITHUB_RUN_ATTEMPT}) — bypassing skip guard"
            echo "has_changes=true" >> $GITHUB_OUTPUT
            exit 0
          fi

          # Determine whether a new evaluation is needed by inspecting the most
          # recent completed scheduled run.
          LATEST=$(gh api "repos/${{ github.repository }}/actions/workflows/evaluation.yml/runs?event=schedule&status=completed&per_page=1" \
            --jq '(.workflow_runs[0] // empty) | "\(.head_sha) \(.conclusion)"' 2>/dev/null) || LATEST=""

          if [ -z "$LATEST" ]; then
            echo "No previous completed scheduled run found — proceeding"
            echo "has_changes=true" >> $GITHUB_OUTPUT
            exit 0
          fi

          LAST_SHA="${LATEST%% *}"
          LAST_CONCLUSION="${LATEST##* }"
          CURRENT_SHA="${{ github.sha }}"

          if [ "$LAST_SHA" = "$CURRENT_SHA" ] && [ "$LAST_CONCLUSION" = "success" ]; then
            echo "Last scheduled evaluation at $LAST_SHA succeeded — skipping"
            echo "has_changes=false" >> $GITHUB_OUTPUT
          else
            if [ "$LAST_SHA" != "$CURRENT_SHA" ]; then
              COUNT=$(gh api "repos/${{ github.repository }}/compare/${LAST_SHA}...${CURRENT_SHA}" --jq '.total_commits' 2>/dev/null) || COUNT="unknown"
              echo "$COUNT new commit(s) since last evaluation ($LAST_SHA)"
            else
              echo "Last scheduled evaluation at $LAST_SHA concluded with '$LAST_CONCLUSION' — retrying"
            fi
            echo "has_changes=true" >> $GITHUB_OUTPUT
          fi

      - name: Checkout repository
        if: github.event_name != 'schedule' || steps.check-changes.outputs.has_changes == 'true' || github.event_name == 'workflow_dispatch'
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Fetch PR head
        if: needs.gate.outputs.pr_number != ''
        run: git fetch origin +refs/pull/${{ needs.gate.outputs.pr_number }}/head:refs/remotes/origin/pr-head

      - name: Find skills to evaluate
        if: github.event_name != 'schedule' || steps.check-changes.outputs.has_changes == 'true' || github.event_name == 'workflow_dispatch'
        id: find
        run: |
          $entries = @()
          $plugins = @()

          # Build matrix entries for a full-plugin evaluation, sharding the
          # plugin's skills by the optional `executionShard:` top-level tag
          # in tests/<plugin>/<skill>/eval.yaml. Untagged skills fall into a
          # synthetic "default" bucket. Plugins with all skills in one bucket
          # produce a single entry (unchanged behavior); plugins with multiple
          # buckets fan out into one matrix entry per shard so each shard
          # runs on a fresh GitHub-hosted runner. This is the mitigation for
          # MCP-heavy plugins like dotnet-msbuild, where cumulative runner
          # resource usage across ~90 min of continuous MCP traffic
          # consistently triggered host-level termination.
          function Get-PluginShardEntries {
            param([string]$plugin, [string]$contentRoot = ".")
            $skillsDir = "plugins/$plugin/skills"
            $skillsRoot = Join-Path $contentRoot $skillsDir
            $singleEntry = @{ name = $plugin; plugin = $plugin; skills_path = $skillsDir }
            if (-not (Test-Path $skillsRoot)) { return @($singleEntry) }
            $skillDirs = @(Get-ChildItem -Path $skillsRoot -Directory -ErrorAction SilentlyContinue |
              Sort-Object Name | Select-Object -ExpandProperty Name)
            if ($skillDirs.Count -eq 0) { return @($singleEntry) }
            $shardGroups = @{}
            foreach ($skill in $skillDirs) {
              $evalPath = Join-Path $contentRoot "tests" $plugin $skill "eval.yaml"
              $shard = "default"
              if (Test-Path $evalPath) {
                # Allow optional leading whitespace so an accidentally indented
                # executionShard: key still groups correctly (rather than silently
                # collapsing back to the default bucket).
                $m = Select-String -Path $evalPath -Pattern '^\s*executionShard:\s*[\x27"]?([\w.\-]+)' -List
                if ($m) { $shard = $m.Matches[0].Groups[1].Value }
              }
              if (-not $shardGroups.ContainsKey($shard)) { $shardGroups[$shard] = @() }
              $shardGroups[$shard] += $skill
            }
            if ($shardGroups.Count -le 1) { return @($singleEntry) }
            return @($shardGroups.Keys | Sort-Object | ForEach-Object {
              $shardName = $_
              $paths = ($shardGroups[$shardName] | ForEach-Object { "$skillsDir/$_" }) -join ' '
              @{ name = "$plugin--shard-$shardName"; plugin = $plugin; skills_path = $paths }
            })
          }

          if ("${{ needs.gate.outputs.pr_number }}" -ne "") {
            # Single-PR run (either /evaluate or evaluate-now label):
            # detect individual changed skills using gate outputs
            $base = "${{ needs.gate.outputs.base_sha }}"
            $head = (git rev-parse origin/pr-head)

            # Use a worktree so Test-Path checks are against PR content
            git worktree add /tmp/pr-content origin/pr-head 2>$null
            $contentRoot = "/tmp/pr-content"

            $mergeBase = git merge-base $base $head
            $changedFiles = git diff --name-only --diff-filter=ACMR $mergeBase $head

            # Check if any changed files are in infrastructure paths
            # Documentation files under src/ don't affect evaluation.
            $hasInfraChanges = $changedFiles |
              Where-Object {
                ($_ -match '^eng/skill-validator/' -and $_ -notmatch '^eng/skill-validator/src/(README\.md|docs/)') -or
                ($_ -match '^eng/vally-adapter/') -or
                $_ -match '^\.github/workflows/(evaluation|vally-evaluation)\.yml$'
              } |
              Select-Object -First 1

            # Also check for skill/test changes so we don't lose them
            $hasSkillChanges = $changedFiles |
              Where-Object { $_ -match '^(?:plugins/([^/]+)/skills|tests/([^/]+))/([^/]+)/' } |
              Select-Object -First 1

            if ($hasInfraChanges -and -not $hasSkillChanges) {
              echo "is_infra=true" >> $env:GITHUB_OUTPUT
              # Infra-only: evaluate a small random subset of plugins to keep
              # the smoke-test fast while still catching regressions.
              $allPlugins = @(Get-ChildItem -Path (Join-Path $contentRoot "plugins") -Directory -ErrorAction SilentlyContinue |
                Where-Object {
                  (Test-Path (Join-Path $_.FullName "skills")) -and
                  (Test-Path (Join-Path $contentRoot "tests" $_.Name))
                } |
                Select-Object -ExpandProperty Name)
              $plugins = @($allPlugins | Get-Random -Count ([Math]::Min(2, $allPlugins.Count)))
              Write-Host "Infrastructure changes detected, evaluating random subset: $($plugins -join ', ')"
              $entries = @($plugins | ForEach-Object { Get-PluginShardEntries -plugin $_ -contentRoot $contentRoot })
            } else {
              # Extract unique plugin/skill pairs from changed files
              $changedPairs = @($changedFiles |
                Where-Object { $_ -match '^(?:plugins/([^/]+)/skills|tests/([^/]+))/([^/]+)/' } |
                ForEach-Object {
                  $p = if ($Matches[1]) { $Matches[1] } else { $Matches[2] }
                  "$p/$($Matches[3])"
                } |
                Sort-Object -Unique)

              # Filter to skills that have a SKILL.md and a tests directory.
              # Sharding does not apply to per-skill PR runs — each changed
              # skill gets its own matrix entry already.
              $entries = @($changedPairs | ForEach-Object {
                $parts = $_ -split '/'
                $plugin = $parts[0]
                $skill = $parts[1]
                $skillMd = Join-Path $contentRoot "plugins" $plugin "skills" $skill "SKILL.md"
                $testsDir = Join-Path $contentRoot "tests" $plugin
                if ((Test-Path $skillMd) -and (Test-Path $testsDir)) {
                  @{
                    name = "$plugin--$skill"
                    plugin = $plugin
                    skills_path = "plugins/$plugin/skills/$skill"
                  }
                }
              } | Where-Object { $_ })

              $plugins = @($entries | ForEach-Object { $_.plugin } | Sort-Object -Unique)
            }

            git worktree remove /tmp/pr-content --force 2>$null
          } else {
            # Schedule and workflow_dispatch: evaluate full plugins.
            # Schedule covers everything; workflow_dispatch can optionally
            # narrow to a single plugin via the `plugin` input for ad-hoc
            # smoke tests without committing to main.
            # Exclude experimental plugins from scheduled runs — they are
            # evaluated only on-demand via /evaluate on PRs.
            $excludeFromSchedule = @('dotnet-experimental')
            $dispatchPlugin = "${{ inputs.plugin }}"
            if ("${{ github.event_name }}" -eq "workflow_dispatch" -and $dispatchPlugin) {
              # Restrict to a simple directory-name shape before any path
              # construction, so values like '../foo' or 'foo/bar' cannot
              # traverse outside plugins/ or tests/.
              if ($dispatchPlugin -notmatch '^[a-zA-Z0-9._-]+$') {
                throw "workflow_dispatch input plugin='$dispatchPlugin' must match ^[a-zA-Z0-9._-]+$ (single directory name, no path separators)"
              }
              if (-not (Test-Path (Join-Path "plugins" $dispatchPlugin "skills")) -or
                  -not (Test-Path (Join-Path "tests" $dispatchPlugin))) {
                throw "workflow_dispatch input plugin='$dispatchPlugin' is not a valid plugin (must have plugins/<name>/skills and tests/<name>)"
              }
              $plugins = @($dispatchPlugin)
              Write-Host "workflow_dispatch: evaluating only $dispatchPlugin"
            } else {
              $plugins = @(Get-ChildItem -Path "plugins" -Directory |
                Where-Object { (Test-Path (Join-Path $_.FullName "skills")) -and (Test-Path (Join-Path "tests" $_.Name)) -and ($_.Name -notin $excludeFromSchedule) } |
                Select-Object -ExpandProperty Name)
            }
            $entries = @($plugins | ForEach-Object { Get-PluginShardEntries -plugin $_ })
          }

          # Output entries for evaluate matrix
          if (-not $entries -or $entries.Count -eq 0) {
            Write-Host "No entries to evaluate"
            echo "entries=[]" >> $env:GITHUB_OUTPUT
            echo "has_entries=false" >> $env:GITHUB_OUTPUT
          } else {
            $json = $entries | ConvertTo-Json -Compress -AsArray
            Write-Host "Entries to evaluate: $json"
            echo "entries=$json" >> $env:GITHUB_OUTPUT
            echo "has_entries=true" >> $env:GITHUB_OUTPUT
          }

          # Output plugins for publish jobs
          if (-not $plugins -or $plugins.Count -eq 0) {
            echo "plugins=[]" >> $env:GITHUB_OUTPUT
            echo "has_plugins=false" >> $env:GITHUB_OUTPUT
          } else {
            $cjson = $plugins | ConvertTo-Json -Compress -AsArray
            echo "plugins=$cjson" >> $env:GITHUB_OUTPUT
            echo "has_plugins=true" >> $env:GITHUB_OUTPUT
          }
        shell: pwsh

      - name: Resolve model/judge matrix
        if: github.event_name != 'schedule' || steps.check-changes.outputs.has_changes == 'true' || github.event_name == 'workflow_dispatch'
        id: models
        shell: bash
        env:
          PR_NUMBER: ${{ needs.gate.outputs.pr_number }}
        run: |
          # Fail loudly if eng/eval-models.json is broken (judge == agent, etc.)
          node eng/resolve-judge.mjs selftest
          # Gating (a PR is in context) stays single-model; nightly/main fans out.
          if [ -n "$PR_NUMBER" ]; then
            SCOPE="--gating"
          else
            SCOPE="--nightly"
          fi
          PAIRS=$(node eng/resolve-judge.mjs matrix $SCOPE)
          MODELS=$(node eng/resolve-judge.mjs models $SCOPE)
          REQUIRED=$(node eng/resolve-judge.mjs required $SCOPE)
          echo "pairs=$PAIRS" >> "$GITHUB_OUTPUT"
          echo "models=$MODELS" >> "$GITHUB_OUTPUT"
          echo "required=$REQUIRED" >> "$GITHUB_OUTPUT"
          echo "Model/judge pairs ($SCOPE): $PAIRS"
          echo "Required model ids ($SCOPE): $REQUIRED"

  # ==========================================================================
  # BUILD VALIDATOR
  # Build the skill-validator from the appropriate ref.
  # ==========================================================================
  build-validator:
    needs: [gate, discover]
    if: >-
      always() &&
      needs.discover.outputs.has_entries == 'true' &&
      needs.discover.result == 'success' &&
      (needs.gate.result == 'success' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      cache-key: ${{ steps.compute-key.outputs.cache-key }}
    steps:
      # For same-repo PRs (trusted), build from the PR branch so skill-validator
      # changes are tested. For fork PRs (untrusted), always build from the
      # base branch to prevent untrusted code from modifying tooling.
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ needs.gate.outputs.is_fork != 'true' && needs.gate.outputs.head_sha || '' }}
          persist-credentials: false

      - name: Compute cache key
        id: compute-key
        run: echo "cache-key=skill-validator-${{ runner.os }}-${{ hashFiles('eng/skill-validator/src/**', 'eng/skill-validator/Directory.Build.props', 'global.json') }}" >> "$GITHUB_OUTPUT"

      - name: Cache validator archive
        id: cache-validator
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
        with:
          path: skill-validator-dist.tar.gz
          key: ${{ steps.compute-key.outputs.cache-key }}

      - name: Setup .NET SDK
        if: steps.cache-validator.outputs.cache-hit != 'true'
        uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5
        with:
          global-json-file: global.json

      - name: Build skill-validator
        if: steps.cache-validator.outputs.cache-hit != 'true'
        run: dotnet publish eng/skill-validator/src/SkillValidator.csproj

      - name: Create validator archive
        if: steps.cache-validator.outputs.cache-hit != 'true'
        run: tar -czf skill-validator-dist.tar.gz -C artifacts/publish/SkillValidator/release .

      - name: Upload built validator
        if: steps.cache-validator.outputs.cache-hit != 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: skill-validator-dist.tar.gz
          path: skill-validator-dist.tar.gz
          archive: false
          retention-days: 1

  # ==========================================================================
  # PREFLIGHT MODELS
  # Fail fast if any model id the active scope depends on (agent ∪ judge, from
  # eng/eval-models.json) is unavailable to the CI Copilot token. Runs ONCE,
  # before the evaluate matrix fans out across every plugin × model, so a bad
  # or renamed model id surfaces in seconds instead of after N parallel jobs
  # each hit the same "Invalid model" error mid-run. Uses the same
  # skill-validator binary and the same ListModelsAsync() call the eval makes.
  # ==========================================================================
  preflight-models:
    needs: [gate, discover, build-validator]
    if: always() && !cancelled() && needs.build-validator.result == 'success'
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 10
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ needs.gate.outputs.is_fork != 'true' && needs.gate.outputs.head_sha || '' }}
          persist-credentials: false

      - name: Restore cached validator
        id: cache-validator
        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
        with:
          path: skill-validator-dist.tar.gz
          key: ${{ needs.build-validator.outputs.cache-key }}

      - name: Download built validator
        if: steps.cache-validator.outputs.cache-hit != 'true'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: skill-validator-dist.tar.gz

      - name: Extract validator
        run: |
          mkdir -p artifacts/publish/SkillValidator/release
          tar -xzf skill-validator-dist.tar.gz -C artifacts/publish/SkillValidator/release

      - name: Select random Copilot token
        id: select-token
        env:
          TOKEN_1: ${{ secrets.COPILOT_GITHUB_TOKEN }}
          TOKEN_2: ${{ secrets.COPILOT_GITHUB_TOKEN_2 }}
          TOKEN_3: ${{ secrets.COPILOT_GITHUB_TOKEN_3 }}
          TOKEN_4: ${{ secrets.COPILOT_GITHUB_TOKEN_4 }}
          TOKEN_5: ${{ secrets.COPILOT_GITHUB_TOKEN_5 }}
          TOKEN_6: ${{ secrets.COPILOT_GITHUB_TOKEN_6 }}
          TOKEN_7: ${{ secrets.COPILOT_GITHUB_TOKEN_7 }}
          TOKEN_8: ${{ secrets.COPILOT_GITHUB_TOKEN_8 }}
        run: |
          # Collect all non-empty token secrets
          TOKENS=()
          NAMES=()
          for i in 1 2 3 4 5 6 7 8; do
            var="TOKEN_$i"
            val="${!var}"
            if [ -n "$val" ]; then
              TOKENS+=("$val")
              if [ "$i" -eq 1 ]; then
                NAMES+=("COPILOT_GITHUB_TOKEN")
              else
                NAMES+=("COPILOT_GITHUB_TOKEN_$i")
              fi
            fi
          done

          if [ ${#TOKENS[@]} -eq 0 ]; then
            echo "::error::No COPILOT_GITHUB_TOKEN secrets are configured"
            exit 1
          fi

          IDX=$((RANDOM % ${#TOKENS[@]}))
          echo "Selected ${NAMES[$IDX]} (1 of ${#TOKENS[@]} available tokens)"
          echo "::add-mask::${TOKENS[$IDX]}"
          echo "token=${TOKENS[$IDX]}" >> $GITHUB_OUTPUT

      - name: Verify required models are available
        env:
          GITHUB_TOKEN: ${{ steps.select-token.outputs.token }}
          # JSON array of required ids from the resolver, passed via env so the
          # shell never parses the brackets/quotes; list-models tolerates JSON.
          REQUIRED_MODELS: ${{ needs.discover.outputs.required_models }}
        run: |
          if [ -z "$REQUIRED_MODELS" ]; then
            echo "::error::No required models were resolved; discover job did not run the model resolver."
            exit 1
          fi
          echo "Required models for this run: $REQUIRED_MODELS"
          artifacts/publish/SkillValidator/release/skill-validator list-models --require "$REQUIRED_MODELS"

  # ==========================================================================
  # EVALUATE
  # Run skill-validator across changed skills (matrix).
  # ==========================================================================
  evaluate:
    needs: [gate, discover, build-validator, preflight-models]
    if: always() && !cancelled() && needs.build-validator.result == 'success' && needs.preflight-models.result == 'success'
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 180
    name: evaluate (${{ matrix.entry.name }} · ${{ matrix.pair.model }})
    strategy:
      fail-fast: false
      matrix:
        entry: ${{ fromJson(needs.discover.outputs.entries || '[]') }}
        pair: ${{ fromJson(needs.discover.outputs.model_pairs || '[]') }}

    steps:
      - name: Checkout skills content
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ needs.gate.outputs.head_sha || '' }}
          persist-credentials: false

      - name: Setup .NET SDK
        uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5
        with:
          global-json-file: global.json

      - name: Restore cached validator
        id: cache-validator
        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
        with:
          path: skill-validator-dist.tar.gz
          key: ${{ needs.build-validator.outputs.cache-key }}

      - name: Download built validator
        if: steps.cache-validator.outputs.cache-hit != 'true'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: skill-validator-dist.tar.gz

      - name: Extract validator
        run: |
          mkdir -p artifacts/publish/SkillValidator/release
          tar -xzf skill-validator-dist.tar.gz -C artifacts/publish/SkillValidator/release

      - name: Select random Copilot token
        id: select-token
        env:
          TOKEN_1: ${{ secrets.COPILOT_GITHUB_TOKEN }}
          TOKEN_2: ${{ secrets.COPILOT_GITHUB_TOKEN_2 }}
          TOKEN_3: ${{ secrets.COPILOT_GITHUB_TOKEN_3 }}
          TOKEN_4: ${{ secrets.COPILOT_GITHUB_TOKEN_4 }}
          TOKEN_5: ${{ secrets.COPILOT_GITHUB_TOKEN_5 }}
          TOKEN_6: ${{ secrets.COPILOT_GITHUB_TOKEN_6 }}
          TOKEN_7: ${{ secrets.COPILOT_GITHUB_TOKEN_7 }}
          TOKEN_8: ${{ secrets.COPILOT_GITHUB_TOKEN_8 }}
        run: |
          # Collect all non-empty token secrets
          TOKENS=()
          NAMES=()
          for i in 1 2 3 4 5 6 7 8; do
            var="TOKEN_$i"
            val="${!var}"
            if [ -n "$val" ]; then
              TOKENS+=("$val")
              if [ "$i" -eq 1 ]; then
                NAMES+=("COPILOT_GITHUB_TOKEN")
              else
                NAMES+=("COPILOT_GITHUB_TOKEN_$i")
              fi
            fi
          done

          if [ ${#TOKENS[@]} -eq 0 ]; then
            echo "::error::No COPILOT_GITHUB_TOKEN secrets are configured"
            exit 1
          fi

          # Assign token deterministically by matrix job index to avoid collisions.
          # Falls back to RANDOM if strategy.job-index is unavailable.
          JOB_INDEX="${{ strategy.job-index }}"
          if [ -n "$JOB_INDEX" ]; then
            IDX=$(( JOB_INDEX % ${#TOKENS[@]} ))
          else
            IDX=$((RANDOM % ${#TOKENS[@]}))
          fi
          echo "Selected ${NAMES[$IDX]} (1 of ${#TOKENS[@]} available tokens, job-index=${JOB_INDEX:-random})"

          # Mask the value so it won't appear in logs, then export
          echo "::add-mask::${TOKENS[$IDX]}"
          echo "token=${TOKENS[$IDX]}" >> $GITHUB_OUTPUT

      - name: Detect plugin MCP usage
        id: detect-mcp
        env:
          # Pass matrix value through env so that hostile plugin directory
          # names (e.g. containing $()/backticks) cannot be evaluated by
          # bash at script-parse time.
          PLUGIN: ${{ matrix.entry.plugin }}
        run: |
          PLUGIN_JSON="plugins/${PLUGIN}/plugin.json"
          HAS_MCP=false
          if [ -f "$PLUGIN_JSON" ] && grep -q '"mcpServers"' "$PLUGIN_JSON"; then
            HAS_MCP=true
            echo "::notice::Plugin ${PLUGIN} declares MCP servers — forcing serial execution to avoid runner OOM."
          fi
          echo "has_mcp=$HAS_MCP" >> "$GITHUB_OUTPUT"

      - name: Run skill-validator
        env:
          GITHUB_TOKEN: ${{ steps.select-token.outputs.token }}
          RESULTS_PATH: artifacts/TestResults/skill-validator/${{ matrix.entry.name }}
          HAS_MCP: ${{ steps.detect-mcp.outputs.has_mcp }}
          # Matrix values are passed via env so they reach bash as data,
          # not as script text — preventing shell metacharacters in
          # user-controlled repo content (plugin / skill directory names)
          # from being interpreted at parse time.
          PLUGIN: ${{ matrix.entry.plugin }}
          SKILLS_PATH: ${{ matrix.entry.skills_path }}
          MODEL: ${{ matrix.pair.model }}
          JUDGE_MODEL: ${{ matrix.pair.judge }}
          # When a plugin declares MCP servers, every session spawns extra
          # `dotnet dnx <mcp-package>` child processes. Two failure modes
          # we mitigate independently:
          #   1. Acute OOM at default 18-way concurrency (~17-30 min) —
          #      capped here at 1 x 3 x 1 = 3 concurrent sessions.
          #   2. Cumulative runner-host exhaustion at low concurrency
          #      (~87 min) — addressed by sharding the plugin via the
          #      `executionShard:` tag in eval.yaml, which fans out into
          #      multiple matrix legs each on a fresh runner. See the
          #      Get-PluginShardEntries helper in the discover job.
          # Both mitigations are required; either alone reproduces a
          # different failure. RUNS stays unchanged so CV remains meaningful.
          RUNS: ${{ needs.discover.outputs.is_infra == 'true' && '1' || (github.event_name == 'schedule' && '5' || '3') }}
          PARALLEL_SKILLS: ${{ steps.detect-mcp.outputs.has_mcp == 'true' && '1' || ((needs.discover.outputs.is_infra == 'true' || github.event_name == 'schedule') && '2' || '5') }}
          PARALLEL_SCENARIOS: ${{ steps.detect-mcp.outputs.has_mcp == 'true' && '3' || ((needs.discover.outputs.is_infra == 'true' || github.event_name == 'schedule') && '3' || '5') }}
          PARALLEL_RUNS: ${{ steps.detect-mcp.outputs.has_mcp == 'true' && '1' || ((needs.discover.outputs.is_infra == 'true' || github.event_name == 'schedule') && '3' || '5') }}
        run: |
          ARGS="--verdict-warn-only --verbose"
          ARGS="$ARGS --results-dir $RESULTS_PATH --reporter console --reporter json --reporter markdown"
          ARGS="$ARGS --model $MODEL"
          ARGS="$ARGS --judge-model $JUDGE_MODEL"
          ARGS="$ARGS --runs $RUNS"
          ARGS="$ARGS --parallel-skills $PARALLEL_SKILLS"
          ARGS="$ARGS --parallel-scenarios $PARALLEL_SCENARIOS"
          ARGS="$ARGS --parallel-runs $PARALLEL_RUNS"
          ARGS="$ARGS --keep-sessions"
          # When MCP servers are in play, the runner is memory-pressured and
          # judge API calls can occasionally exceed the 300 s default before
          # responses are parsed. Bump to 600 s to avoid retry storms.
          if [ "$HAS_MCP" = "true" ]; then
            ARGS="$ARGS --judge-timeout 600"
          fi

          echo "Effective parallelism: runs=$RUNS skills=$PARALLEL_SKILLS scenarios=$PARALLEL_SCENARIOS runs-parallel=$PARALLEL_RUNS (has_mcp=$HAS_MCP)"

          # SKILLS_PATH may be a single path ("plugins/foo/skills") or a
          # space-separated list of paths ("plugins/foo/skills/a plugins/foo/skills/b")
          # when the discover job has sharded a plugin across matrix legs.
          # Intentionally unquoted so multiple paths expand as separate args.
          # PLUGIN and SKILLS_PATH come in via env (see above) so shell
          # metacharacters in user-controlled directory names cannot be
          # evaluated at script-parse time.
          artifacts/publish/SkillValidator/release/skill-validator evaluate $ARGS --tests-dir "./tests/${PLUGIN}" $SKILLS_PATH

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          # `@<model>` suffix keeps the three model legs from colliding on one
          # artifact name. `@` is a safe separator: plugin/skill/shard names
          # match ^[a-zA-Z0-9._-]+$ and can never contain it, so publish jobs
          # split on the LAST `@` to recover (entry, model) unambiguously.
          name: skill-validator-results-${{ matrix.entry.name }}@${{ matrix.pair.model }}
          path: artifacts/TestResults/skill-validator/${{ matrix.entry.name }}/
          include-hidden-files: true
          retention-days: 30

  # ==========================================================================
  # VALLY EVALUATION
  # Runs vally evaluations in parallel with skill-validator.
  # NOTE: These results do not gate PRs for now.
  # ==========================================================================
  vally-evaluate:
    needs: [gate, discover]
    if: >-
      always() && !cancelled() &&
      needs.discover.outputs.has_entries == 'true' &&
      needs.discover.result == 'success'
    uses: ./.github/workflows/vally-evaluation.yml
    with:
      entries: ${{ needs.discover.outputs.entries }}
      head_sha: ${{ needs.gate.outputs.head_sha }}
      pr_number: ${{ needs.gate.outputs.pr_number }}
    secrets: inherit

  # ==========================================================================
  # COMMENT ON PR
  # Post consolidated evaluation results as a PR comment.
  # ==========================================================================
  comment-on-pr:
    needs: [gate, discover, build-validator, evaluate, publish-session-data]
    if: >-
      always() &&
      needs.evaluate.result != 'cancelled' &&
      needs.gate.outputs.pr_number != '' &&
      needs.discover.outputs.has_entries == 'true'
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    steps:
      - name: Download all result artifacts
        if: needs.evaluate.result != 'skipped'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: skill-validator-results-*
          path: all-results/
          merge-multiple: false
        continue-on-error: ${{ needs.evaluate.result != 'success' }}

      - name: Restore cached validator
        if: needs.evaluate.result != 'skipped'
        id: cache-validator
        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
        with:
          path: skill-validator-dist.tar.gz
          key: ${{ needs.build-validator.outputs.cache-key }}
        continue-on-error: true

      - name: Download built validator
        if: needs.evaluate.result != 'skipped' && steps.cache-validator.outputs.cache-hit != 'true'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: skill-validator-dist.tar.gz
        continue-on-error: ${{ needs.evaluate.result != 'success' }}

      - name: Extract validator
        if: needs.evaluate.result != 'skipped' && hashFiles('skill-validator-dist.tar.gz') != ''
        run: |
          mkdir -p artifacts/publish/SkillValidator/release
          tar -xzf skill-validator-dist.tar.gz -C artifacts/publish/SkillValidator/release

      - name: Consolidate and post results
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          PR_NUMBER=${{ needs.gate.outputs.pr_number }}
          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

          # If results exist, consolidate them into a summary comment
          JSON_FILES=$(find all-results/ -name results.json 2>/dev/null || true)
          VALIDATOR="artifacts/publish/SkillValidator/release/skill-validator"
          if [ -n "$JSON_FILES" ] && [ -x "$VALIDATOR" ]; then
            "$VALIDATOR" evaluate consolidate \
              --output summary-body.md \
              $JSON_FILES

            # Create a simplified table for the PR comment by stripping
            # "Quality (Plugin)" and "Agents Invoked" columns (GH UI concern).
            # The full table is preserved in the workflow step summary.
            python3 - <<'PY'
          lines = open('summary-body.md').read().split('\n')
          result, remove, prev = [], set(), False
          for line in lines:
              is_tbl = line.startswith('|')
              if is_tbl and not prev:
                  parts = line.split('|')
                  remove = {i for i, p in enumerate(parts) if p.strip() in ('Quality (Plugin)', 'Agents Invoked')}
                  parts = [p.replace('Quality (Isolated)', 'Quality') for p in parts]
                  result.append('|'.join(p for i, p in enumerate(parts) if i not in remove))
              elif is_tbl:
                  parts = line.split('|')
                  result.append('|'.join(p for i, p in enumerate(parts) if i not in remove))
              else:
                  result.append(line)
              prev = is_tbl
          open('simplified-body.md', 'w').write('\n'.join(result))
          PY

            INVESTIGATE_PROMPT=""
            # If any skill failed, build a copy-paste prompt for AI-assisted investigation
            if jq -se '[.[].verdicts[] | select(.passed == false)] | length > 0' $JSON_FILES > /dev/null 2>&1; then
              RUN_ID="${{ github.run_id }}"
              INVESTIGATE_PROMPT=$(printf '\n> **To investigate failures**, paste this to your AI coding agent:\n>\n> _For PR %s in %s, download eval artifacts with `gh run download %s --repo %s --pattern "skill-validator-results-*" --dir ./eval-results`, then fetch https://raw.githubusercontent.com/%s/%s/eng/skill-validator/src/docs/InvestigatingResults.md and follow it to analyze the results.json files. Diagnose each failure, suggest fixes to the eval.yaml and skill content, and tell me what to fix first._' \
                "${PR_NUMBER}" "${{ github.repository }}" "${RUN_ID}" "${{ github.repository }}" "${{ github.repository }}" "${{ needs.gate.outputs.head_sha }}")
            fi

            # PR comment: simplified table + "Full results" link + investigation prompt
            {
              cat simplified-body.md
              echo ""
              echo "[🔍 Full Results - additional metrics and failure investigation steps]($RUN_URL)"
              if [ -n "$INVESTIGATE_PROMPT" ]; then
                echo "$INVESTIGATE_PROMPT"
              fi
            } > consolidated-comment.md

            # Append AGENTVIZ replay link if session data was published
            if [[ "${{ needs.publish-session-data.result }}" == "success" ]]; then
              # Session data lives in the standalone dotnet/skills-data repo.
              MANIFEST_URL="https://raw.githubusercontent.com/dotnet/skills-data/dashboard-session-data/data/manifest.json"
              # Derive the Pages base URL from the repository (org.github.io/repo)
              ORG=$(echo "${{ github.repository }}" | cut -d/ -f1)
              REPO=$(echo "${{ github.repository }}" | cut -d/ -f2)
              REPLAY_URL="https://${ORG}.github.io/${REPO}/replay/index.html"
              MANIFEST_ENCODED=$(printf '%s' "$MANIFEST_URL" | jq -sRr @uri)
              FULL_URL="${REPLAY_URL}?manifest=${MANIFEST_ENCODED}&tag=pr-${PR_NUMBER}"
              echo "" >> consolidated-comment.md
              echo "**[▶ Sessions Visualisation](${FULL_URL})** -- interactive replay of all evaluation sessions" >> consolidated-comment.md
              ANALYTICS_BASE_URL="https://server.mangowater-996ff7b2.swedencentral.azurecontainerapps.io/"
              FACET_ENCODED=$(printf '%s' "label:pr-${PR_NUMBER}" | jq -sRr @uri)
              ANALYTICS_URL="${ANALYTICS_BASE_URL}?facet=${FACET_ENCODED}"
              echo "**[📊 Session Analytics (preview)](${ANALYTICS_URL})** -- aggregated metrics across evaluation sessions" >> consolidated-comment.md
            fi

            # Action summary: full table (all columns) + investigation prompt
            {
              cat summary-body.md
              if [ -n "$INVESTIGATE_PROMPT" ]; then
                echo "$INVESTIGATE_PROMPT"
              fi
            } >> "$GITHUB_STEP_SUMMARY"

            gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
              -X POST -F "body=@consolidated-comment.md"
          else
            # No results — evaluate was skipped or failed before producing artifacts
            if [[ "${{ needs.evaluate.result }}" == "skipped" ]]; then
              BODY="❌ Evaluation did not complete (upstream job failed or was skipped). [View workflow run](${RUN_URL})"
            else
              BODY="❌ Evaluation failed. [View workflow run](${RUN_URL})"
            fi

            gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" -X POST -f body="$BODY"
          fi

  # ==========================================================================
  # REPORT STATUS
  # Post final evaluation status via commit status API.
  # ==========================================================================
  report-status:
    needs: [gate, discover, build-validator, evaluate]
    if: always() && needs.gate.result == 'success'
    runs-on: ubuntu-latest
    permissions:
      statuses: write
      pull-requests: write
      issues: write
    steps:
      - name: Remove eyes reaction from trigger comment
        if: github.event_name == 'issue_comment'
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          REACTION_ID=$(gh api "repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions" \
            --jq '.[] | select(.content == "eyes" and .user.login == "github-actions[bot]") | .id' | head -1 || echo "")
          if [[ -n "$REACTION_ID" && "$REACTION_ID" != "null" ]]; then
            gh api "repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions/${REACTION_ID}" \
              -X DELETE || true
          fi

      - name: Set final commit status
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          if [[ "${{ needs.evaluate.result }}" == "success" ]]; then
            STATE="success"
            DESC="Evaluation passed"
          elif [[ "${{ needs.evaluate.result }}" == "skipped" && "${{ needs.discover.result }}" == "success" && "${{ needs.discover.outputs.has_entries }}" != "true" ]]; then
            STATE="success"
            DESC="No skills to evaluate"
          elif [[ "${{ needs.build-validator.result }}" == "failure" ]]; then
            STATE="failure"
            DESC="Validator build failed"
          elif [[ "${{ needs.evaluate.result }}" == "failure" ]]; then
            STATE="failure"
            DESC="Evaluation failed"
          else
            STATE="error"
            DESC="Evaluation did not complete (build: ${{ needs.build-validator.result }}, evaluate: ${{ needs.evaluate.result }}, discover: ${{ needs.discover.result }})"
          fi
          gh api "repos/${{ github.repository }}/statuses/${{ needs.gate.outputs.head_sha }}" \
            -f state="$STATE" \
            -f context="evaluation-status" \
            -f description="$DESC" \
            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

      - name: Post completion comment for skipped evaluation
        if: needs.evaluate.result == 'skipped'
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          PR_NUMBER=${{ needs.gate.outputs.pr_number }}
          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

          if [[ "${{ needs.discover.result }}" == "success" && "${{ needs.discover.outputs.has_entries }}" != "true" ]]; then
            BODY="⏭️ No skills to evaluate — no changed skills with tests were found in this PR. [View workflow run](${RUN_URL})"
          else
            BODY="❌ Evaluation did not complete (upstream job failed or was skipped). [View workflow run](${RUN_URL})"
          fi
          gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" -X POST -f body="$BODY"

  # ==========================================================================
  # PUBLISH TOKEN DATA
  # Both scheduled and PR runs: generate token-usage data → dashboard-token-data
  # ==========================================================================
  publish-token-data:
    needs: [gate, discover, evaluate]
    if: >-
      always() &&
      !cancelled() &&
      needs.discover.result == 'success' &&
      needs.discover.outputs.has_plugins == 'true' &&
      (
        github.ref == 'refs/heads/main' ||
        needs.gate.result == 'success'
      )
    concurrency:
      group: publish-token-data
      cancel-in-progress: false
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Download evaluation artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: skill-validator-results-*
          path: all-results/
          merge-multiple: false
        continue-on-error: ${{ needs.evaluate.result != 'success' }}

      - name: Fetch existing token data from dashboard-token-data
        run: |
          git fetch origin dashboard-token-data:dashboard-token-data 2>/dev/null || true
          mkdir -p /tmp/token-data/data
          git checkout dashboard-token-data -- data/token-usage.json 2>/dev/null && \
            cp data/token-usage.json /tmp/token-data/data/ && \
            git checkout HEAD -- . || true

      - name: Get PR title
        if: needs.gate.outputs.pr_number != ''
        id: pr-info
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          PR_TITLE=$(gh api "repos/${{ github.repository }}/pulls/${{ needs.gate.outputs.pr_number }}" --jq '.title' 2>/dev/null) || PR_TITLE=""
          echo "pr_title=${PR_TITLE}" >> $GITHUB_OUTPUT

      - name: Generate token usage data
        env:
          PR_TITLE: ${{ steps.pr-info.outputs.pr_title }}
        run: |
          $source = if ("${{ needs.gate.outputs.pr_number }}" -ne "") { "pr" } else { "scheduled" }
          $plugins = '${{ needs.discover.outputs.plugins }}' | ConvertFrom-Json
          foreach ($plugin in $plugins) {
            # Artifact names are "skill-validator-results-<entry>@<model>", where
            # <entry> is "<plugin>" (scheduled / infra / full-plugin PR) or
            # "<plugin>--<skill>" (skill-change PR). We split on the LAST '@' to
            # recover <entry> (plugin/skill names cannot contain '@'), then match
            # the plugin. Every matching artifact is processed — each results.json
            # carries its own `model`, so token entries are model-tagged already
            # and the three model legs never overwrite each other.
            $artifactDirs = @(Get-ChildItem -Path "all-results" -Directory -ErrorAction SilentlyContinue |
              Where-Object {
                $at = $_.Name.LastIndexOf('@')
                if ($at -lt 0) { return $false }
                $entry = $_.Name.Substring(0, $at)
                $entry -eq "skill-validator-results-$plugin" -or $entry -like "skill-validator-results-$plugin--*"
              })
            if ($artifactDirs.Count -eq 0) {
              Write-Warning "No artifacts found for $plugin, skipping"
              continue
            }
            foreach ($artifactDir in $artifactDirs) {
              $runDir = Get-ChildItem -Path $artifactDir.FullName -Directory -ErrorAction SilentlyContinue |
                Where-Object { $_.Name -match '^\d{8}-\d{6}$' } |
                Sort-Object Name -Descending |
                Select-Object -First 1
              if (-not $runDir) {
                Write-Warning "No run results found in $($artifactDir.Name), skipping"
                continue
              }

              $resultsFile = Join-Path $runDir.FullName "results.json"
              Write-Host "`n=== Collecting token usage for: $plugin (from $($artifactDir.Name)) ==="
              $params = @{
                ResultsFile   = $resultsFile
                PluginName    = $plugin
                OutputDir     = "/tmp/token-data/data"
                Source        = $source
                RetentionDays = $env:DASHBOARD_RETENTION_DAYS
              }
              if ($source -eq "pr") {
                $params.PRNumber = ${{ needs.gate.outputs.pr_number }}
                $params.PRTitle = $env:PR_TITLE
              }
              $params.SkipBenchmarkData = $true
              & ./eng/dashboard/generate-benchmark-data.ps1 @params
            }
          }
        shell: pwsh

      - name: Push to dashboard-token-data branch
        run: |
          cd /tmp
          REPO_URL="https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git"

          if git ls-remote --exit-code --heads "$REPO_URL" dashboard-token-data > /dev/null 2>&1; then
            git clone --branch dashboard-token-data --single-branch "$REPO_URL" token-deploy
          else
            mkdir token-deploy && cd token-deploy && git init && git checkout -b dashboard-token-data
            git remote add origin "$REPO_URL"
            cd /tmp
          fi

          cd /tmp/token-deploy
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          # Remove any stale files so this branch stays token-only
          git rm -rf data/ --ignore-unmatch --quiet 2>/dev/null || true
          mkdir -p data
          # Ensure token-usage.json exists; create an empty fallback if missing
          if [ ! -f /tmp/token-data/data/token-usage.json ]; then
            mkdir -p /tmp/token-data/data
            printf '{ "entries": [] }\n' > /tmp/token-data/data/token-usage.json
          fi
          cp /tmp/token-data/data/token-usage.json data/

          git add data/token-usage.json
          git diff --cached --quiet && echo "No changes to deploy" && exit 0

          if [[ -n "${{ needs.gate.outputs.pr_number }}" ]]; then
            git commit -m "Update PR token usage data (PR #${{ needs.gate.outputs.pr_number }})"
          else
            git commit -m "Update scheduled token usage data"
          fi
          git push origin dashboard-token-data

  # ==========================================================================
  # PUBLISH SESSION DATA
  # Both scheduled and PR runs: flatten JSONL sessions → dashboard-session-data
  # branch on dotnet/skills-data (separate repo to keep this one small).
  # ==========================================================================
  publish-session-data:
    needs: [gate, discover, evaluate]
    if: >-
      always() && !cancelled() &&
      needs.discover.result == 'success' &&
      needs.discover.outputs.has_plugins == 'true' &&
      (
        github.ref == 'refs/heads/main' ||
        needs.gate.result == 'success'
      )
    runs-on: ubuntu-latest
    concurrency:
      group: publish-session-data
      cancel-in-progress: false
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Download evaluation artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: skill-validator-results-*
          path: all-results/
          merge-multiple: false
        continue-on-error: ${{ needs.evaluate.result != 'success' }}

      - name: Inspect downloaded artifacts
        if: always()
        run: |
          echo "=== all-results/ directory listing (3 levels) ==="
          if [ -d all-results ]; then
            find all-results -maxdepth 3 -ls 2>/dev/null | head -80
            echo "---"
            echo "sessions.db files:"
            find all-results -name 'sessions.db' -ls 2>/dev/null
            echo "events.jsonl files:"
            find all-results -name 'events.jsonl' 2>/dev/null | head -20
          else
            echo "all-results/ directory does not exist!"
          fi

      - name: Determine source metadata
        id: meta
        run: |
          if [ -n "${{ needs.gate.outputs.pr_number }}" ]; then
            echo "source=pr" >> "$GITHUB_OUTPUT"
            echo "pr_number=${{ needs.gate.outputs.pr_number }}" >> "$GITHUB_OUTPUT"
            echo "subdir=pr/${{ needs.gate.outputs.pr_number }}" >> "$GITHUB_OUTPUT"
          else
            echo "source=scheduled" >> "$GITHUB_OUTPUT"
            echo "pr_number=" >> "$GITHUB_OUTPUT"
            echo "subdir=scheduled/$(date -u +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
          fi

      - name: Build session manifest
        shell: pwsh
        run: |
          ./eng/dashboard/build-replay-sessions.ps1 `
            -ResultsDir all-results `
            -OutputDir staging `
            -Source ${{ steps.meta.outputs.source }} `
            -PrNumber "${{ steps.meta.outputs.pr_number }}"

      - name: Clone existing session data branch
        env:
          # Cross-repo PAT with contents:write on dotnet/skills-data.
          # GITHUB_TOKEN cannot push to a different repo, so a fine-grained PAT is required.
          SKILLS_DATA_TOKEN: ${{ secrets.SKILLS_DATA_TOKEN }}
        run: |
          cd /tmp
          REPO_URL="https://x-access-token:${SKILLS_DATA_TOKEN}@github.com/dotnet/skills-data.git"

          if git ls-remote --exit-code --heads "$REPO_URL" dashboard-session-data > /dev/null 2>&1; then
            git clone --branch dashboard-session-data --single-branch --depth 1 "$REPO_URL" session-deploy
          else
            mkdir session-deploy && cd session-deploy && git init && git checkout -b dashboard-session-data
            git remote add origin "$REPO_URL"
            cd /tmp
          fi

          cd /tmp/session-deploy
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

      - name: Merge and purge old sessions
        shell: pwsh
        run: |
          ./eng/dashboard/purge-replay-sessions.ps1 `
            -ExistingDir /tmp/session-deploy/data `
            -NewDir staging `
            -OutputDir /tmp/session-deploy/data `
            -RetentionDays 7

      - name: Push to dashboard-session-data branch (dotnet/skills-data)
        run: |
          cd /tmp/session-deploy
          git add data/
          git diff --cached --quiet && echo "No changes to deploy" && exit 0

          if [[ -n "${{ needs.gate.outputs.pr_number }}" ]]; then
            git commit -m "Update session data (PR #${{ needs.gate.outputs.pr_number }})"
          else
            git commit -m "Update scheduled session data"
          fi
          git push origin dashboard-session-data

  # ==========================================================================
  # PUBLISH EVAL DATA
  # Scheduled runs only: generate benchmark data → dashboard-eval-data
  # ==========================================================================
  publish-eval-data:
    needs: [gate, discover, evaluate]
    if: >-
      always() &&
      !cancelled() &&
      needs.discover.result == 'success' &&
      needs.discover.outputs.has_plugins == 'true' &&
      github.event_name == 'schedule'
    concurrency:
      group: publish-eval-data
      cancel-in-progress: false
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Download evaluation artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: skill-validator-results-*
          path: all-results/
          merge-multiple: false
        continue-on-error: ${{ needs.evaluate.result != 'success' }}

      - name: Fetch existing eval data from dashboard-eval-data
        run: |
          git fetch origin dashboard-eval-data:dashboard-eval-data 2>/dev/null || true
          mkdir -p /tmp/eval-data/data
          git checkout dashboard-eval-data -- data/ 2>/dev/null && \
            cp -r data/* /tmp/eval-data/data/ && \
            git checkout HEAD -- . || true

      - name: Generate benchmark data
        run: |
          $sha = "${{ github.sha }}"
          $commitMsg = git log -1 --format='%s' $sha
          $commitTimestamp = git log -1 --format='%aI' $sha
          $commitAuthor = git log -1 --format='%an' $sha

          $commitJson = @{
            id        = $sha
            message   = $commitMsg
            timestamp = $commitTimestamp
            url       = "https://github.com/${{ github.repository }}/commit/$sha"
            author    = @{ name = $commitAuthor; username = "${{ github.actor }}" }
          } | ConvertTo-Json -Compress

          $plugins = '${{ needs.discover.outputs.plugins }}' | ConvertFrom-Json
          $models = '${{ needs.discover.outputs.models }}' | ConvertFrom-Json
          foreach ($plugin in $plugins) {
            # Fan out one datapoint per (plugin, model). Artifacts are named
            # "skill-validator-results-<entry>@<model>" where <entry> is
            # "<plugin>" or "<plugin>--shard-<tag>". For each model we take only
            # that model's artifacts (matched by the "@<model>" suffix; '@' can
            # never appear in a plugin/shard name), shard-merge them into ONE
            # synthetic results.json, and append a single model-tagged datapoint
            # to <plugin>.json. Existing single-model history stays valid — new
            # entries are simply tagged with their model.
            foreach ($model in $models) {
              $suffix = "@$model"
              $artifactDirs = @(Get-ChildItem -Path "all-results" -Directory -ErrorAction SilentlyContinue |
                Where-Object {
                  if (-not $_.Name.EndsWith($suffix)) { return $false }
                  $entry = $_.Name.Substring(0, $_.Name.Length - $suffix.Length)
                  $entry -eq "skill-validator-results-$plugin" -or $entry -like "skill-validator-results-$plugin--shard-*"
                })

              if ($artifactDirs.Count -eq 0) {
                Write-Warning "No run results found for $plugin ($model), skipping"
                continue
              }

              $resultsFiles = @()
              foreach ($ad in $artifactDirs) {
                $runDir = Get-ChildItem -Path $ad.FullName -Directory -ErrorAction SilentlyContinue |
                  Where-Object { $_.Name -match '^\d{8}-\d{6}$' } |
                  Sort-Object Name -Descending |
                  Select-Object -First 1
                if ($runDir) {
                  $rf = Join-Path $runDir.FullName "results.json"
                  if (Test-Path $rf) { $resultsFiles += $rf }
                }
              }

              if ($resultsFiles.Count -eq 0) {
                Write-Warning "No results.json found in any artifact for $plugin ($model), skipping"
                continue
              }

              if ($resultsFiles.Count -eq 1) {
                $resultsFile = $resultsFiles[0]
              } else {
                # Merge shard verdicts (same model) into a single results.json.
                Write-Host "Merging $($resultsFiles.Count) shard results.json files for $plugin ($model)"
                $merged = $null
                $allVerdicts = [System.Collections.Generic.List[object]]::new()
                foreach ($rf in ($resultsFiles | Sort-Object)) {
                  $r = Get-Content $rf -Raw | ConvertFrom-Json
                  if ($null -eq $merged) { $merged = $r }
                  if ($r.verdicts) { foreach ($v in $r.verdicts) { $allVerdicts.Add($v) } }
                }
                $merged.verdicts = $allVerdicts.ToArray()
                $mergedDir = "all-results/_merged"
                New-Item -ItemType Directory -Force -Path $mergedDir | Out-Null
                $resultsFile = Join-Path $mergedDir "$plugin$suffix.results.json"
                $merged | ConvertTo-Json -Depth 100 | Out-File -FilePath $resultsFile -Encoding utf8
                Write-Host "  Merged verdict count: $($allVerdicts.Count) -> $resultsFile"
              }

              Write-Host "`n=== Generating benchmark data for: $plugin ($model) ==="
              # Re-check existence each model iteration: the previous model's
              # call has already written <plugin>.json, so this model appends to
              # the up-to-date file rather than clobbering the earlier entry.
              $existingFile = "/tmp/eval-data/data/$plugin.json"
              $params = @{
                ResultsFile    = $resultsFile
                PluginName     = $plugin
                OutputDir      = "/tmp/eval-data/data"
                CommitJson     = $commitJson
                RetentionDays  = $env:DASHBOARD_RETENTION_DAYS
                Source         = 'scheduled'
                SkipTokenUsage = $true
              }
              if ((Test-Path $existingFile) -and (Get-Content $existingFile -Raw -ErrorAction SilentlyContinue)) {
                $params.ExistingDataFile = $existingFile
              }
              & ./eng/dashboard/generate-benchmark-data.ps1 @params
            }
          }

          # Purge entries older than retention window
          & ./eng/dashboard/generate-benchmark-data.ps1 -PurgeStaleFiles -DataDir "/tmp/eval-data/data" -RetentionDays $env:DASHBOARD_RETENTION_DAYS

          # Generate components.json manifest (exclude token-usage.json if present)
          $plugins = Get-ChildItem -Path "/tmp/eval-data/data" -Filter "*.json" -File -ErrorAction SilentlyContinue |
            Where-Object { $_.Name -notin @("components.json", "token-usage.json") } |
            ForEach-Object { $_.BaseName }
          @($plugins) | ConvertTo-Json -AsArray | Out-File -FilePath "/tmp/eval-data/data/components.json" -Encoding utf8
        shell: pwsh

      - name: Push to dashboard-eval-data branch
        run: |
          cd /tmp
          REPO_URL="https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git"

          if git ls-remote --exit-code --heads "$REPO_URL" dashboard-eval-data > /dev/null 2>&1; then
            git clone --branch dashboard-eval-data --single-branch "$REPO_URL" eval-deploy
          else
            mkdir eval-deploy && cd eval-deploy && git init && git checkout -b dashboard-eval-data
            git remote add origin "$REPO_URL"
            cd /tmp
          fi

          cd /tmp/eval-deploy
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          # Remove stale files so this branch only contains current eval data
          git rm -rf data/ --ignore-unmatch --quiet 2>/dev/null || true
          mkdir -p data
          # Copy only eval data files (not token-usage.json)
          cp /tmp/eval-data/data/components.json data/
          for f in /tmp/eval-data/data/*.json; do
            fname=$(basename "$f")
            [ "$fname" = "token-usage.json" ] && continue
            cp "$f" "data/$fname"
          done

          git add data/
          git diff --cached --quiet && echo "No changes to deploy" && exit 0
          git commit -m "Update benchmark data"
          git push origin dashboard-eval-data

  # ==========================================================================
  # DEPLOY DASHBOARD
  # Scheduled runs + manual dispatch on main: assemble data from both
  # branches + UI → gh-pages. No data generation — pure copy and deploy.
  # ==========================================================================
  deploy-dashboard:
    needs: [discover, publish-token-data, publish-eval-data, publish-session-data]
    if: >-
      always() &&
      !cancelled() &&
      (
        (needs.discover.result == 'success' && needs.discover.outputs.has_plugins == 'true' && github.event_name == 'schedule') ||
        (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main')
      )
    concurrency:
      group: deploy-dashboard
      cancel-in-progress: false
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository (for dashboard UI files)
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 20

      - name: Fetch eval data from dashboard-eval-data branch
        run: |
          mkdir -p /tmp/gh-pages/data
          git fetch origin dashboard-eval-data:dashboard-eval-data 2>/dev/null || true
          git checkout dashboard-eval-data -- data/ 2>/dev/null && \
            cp -r data/* /tmp/gh-pages/data/ && \
            git checkout HEAD -- . || true

      - name: Fetch token data from dashboard-token-data branch
        run: |
          git fetch origin dashboard-token-data:dashboard-token-data 2>/dev/null || true
          git show dashboard-token-data:data/token-usage.json > /tmp/gh-pages/data/token-usage.json 2>/dev/null || \
            echo '{"entries":[]}' > /tmp/gh-pages/data/token-usage.json

      - name: Check if AGENTVIZ SPA needs update
        id: check-replay
        run: |
          AGENTVIZ_REPO="https://github.com/jayparikh/agentviz.git"
          AGENTVIZ_BRANCH="main"

          # Resolve the latest commit on the AGENTVIZ branch (no clone needed)
          TARGET_SHA=$(git ls-remote "$AGENTVIZ_REPO" "refs/heads/$AGENTVIZ_BRANCH" | cut -f1)
          if [ -z "$TARGET_SHA" ]; then
            echo "::error::Could not resolve AGENTVIZ branch $AGENTVIZ_BRANCH"
            exit 1
          fi
          echo "target_sha=$TARGET_SHA" >> "$GITHUB_OUTPUT"
          echo "AGENTVIZ target commit: $TARGET_SHA"

          # Read the currently deployed commit SHA from gh-pages (no clone needed)
          DEPLOYED_SHA=""
          DEPLOYED_SHA=$(curl -fsSL \
            "https://raw.githubusercontent.com/${{ github.repository }}/gh-pages/replay/.agentviz-commit" \
            2>/dev/null) || true
          echo "deployed_sha=$DEPLOYED_SHA" >> "$GITHUB_OUTPUT"

          if [ "$TARGET_SHA" = "$DEPLOYED_SHA" ]; then
            echo "skip=true" >> "$GITHUB_OUTPUT"
            echo "AGENTVIZ SPA is up-to-date (commit $TARGET_SHA), skipping build."
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
            echo "AGENTVIZ SPA needs update: deployed=$DEPLOYED_SHA target=$TARGET_SHA"
          fi

      - name: Build AGENTVIZ SPA
        if: steps.check-replay.outputs.skip != 'true'
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
        id: agentviz-cache
        with:
          path: /tmp/agentviz-dist
          key: agentviz-dist-${{ steps.check-replay.outputs.target_sha }}

      - name: Build AGENTVIZ SPA (on cache miss)
        if: steps.check-replay.outputs.skip != 'true' && steps.agentviz-cache.outputs.cache-hit != 'true'
        run: |
          TARGET_SHA="${{ steps.check-replay.outputs.target_sha }}"

          git clone https://github.com/jayparikh/agentviz.git /tmp/agentviz-src
          cd /tmp/agentviz-src

          # Check out the exact resolved commit for deterministic builds
          if ! git checkout "$TARGET_SHA"; then
            echo "::error::Failed to check out AGENTVIZ commit $TARGET_SHA"
            exit 1
          fi

          npm ci
          npm run build
          mkdir -p /tmp/agentviz-dist
          cp -r dist/* /tmp/agentviz-dist/

      - name: Deploy to GitHub Pages
        run: |
          cd /tmp
          REPO_URL="https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git"

          if git ls-remote --exit-code --heads "$REPO_URL" gh-pages > /dev/null 2>&1; then
            git clone --branch gh-pages --single-branch "$REPO_URL" deploy
          else
            mkdir deploy && cd deploy && git init && git checkout -b gh-pages
            git remote add origin "$REPO_URL"
            git config user.name "github-actions[bot]"
            git config user.email "github-actions[bot]@users.noreply.github.com"
            cd /tmp
          fi

          cd /tmp/deploy
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          # Copy data from both branches
          mkdir -p data
          cp /tmp/gh-pages/data/*.json data/

          # Copy dashboard UI from source tree
          cp ${{ github.workspace }}/eng/dashboard/dashboard.html index.html
          cp ${{ github.workspace }}/eng/dashboard/dashboard.js dashboard.js
          cp ${{ github.workspace }}/eng/dashboard/token-usage.js token-usage.js

          # Deploy AGENTVIZ SPA (skip if already present and unchanged)
          if [ "${{ steps.check-replay.outputs.skip }}" != "true" ]; then
            rm -rf replay
            mkdir -p replay
            if [ -d /tmp/agentviz-dist ]; then
              cp -r /tmp/agentviz-dist/* replay/
            else
              echo "::error::No AGENTVIZ build artifacts found"
              exit 1
            fi
            echo "${{ steps.check-replay.outputs.target_sha }}" > replay/.agentviz-commit
          fi

          git add .
          git diff --cached --quiet && echo "No changes to deploy" && exit 0
          git commit -m "Update dashboard"
          git push origin gh-pages