adobe__skills/.github/workflows/tessl-eval.yml

name: Tessl Skill Eval

on:
  pull_request:
    branches: [main]

jobs:
  tessl-eval:
    runs-on: ubuntu-latest
    # Uses the "eval" environment for protection rules (e.g. reviewer approval)
    # to guard the TESSL_TOKEN secret against exfiltration via same-repo branches.
    environment: eval
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Check eval trigger
        id: trigger
        run: |
          LAST_MSG=$(git log -1 --format=%s)
          echo "commit_message=$LAST_MSG"
          if [[ ! "$LAST_MSG" =~ ^eval: ]]; then
            echo "skip=true" >> "$GITHUB_OUTPUT"
            echo "Last commit does not start with 'eval:' — skipping eval."
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
            echo "Eval triggered by commit: $LAST_MSG"
          fi

      - name: Post skip summary (no eval trigger)
        if: steps.trigger.outputs.skip == 'true'
        run: |
          {
            echo "## Skill Eval"
            echo ""
            echo "Eval not requested — last commit message does not start with \`eval:\`."
            echo "To trigger evals, create a commit with a message like:"
            echo "\`\`\`"
            echo 'git commit --allow-empty -m "eval: test building-blocks changes"'
            echo "\`\`\`"
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Detect changed skills with tiles
        if: steps.trigger.outputs.skip != 'true'
        id: detect
        run: |
          CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)

          # Build list of all tile directories (parents of tile.json)
          ALL_TILE_DIRS=$(find skills -name tile.json -exec dirname {} \; 2>/dev/null | sort)

          if [ -z "$ALL_TILE_DIRS" ]; then
            echo "skip=true" >> "$GITHUB_OUTPUT"
            echo "No tile.json files found in repo."
          else
            # For each changed file, find which tile directory it belongs to
            DIRS=""
            for changed_file in $CHANGED_FILES; do
              for tile_dir in $ALL_TILE_DIRS; do
                case "$changed_file" in
                  "$tile_dir"/*)
                    # Check not already added
                    case " $DIRS " in
                      *" $tile_dir "*) ;;
                      *) DIRS="$DIRS $tile_dir" ;;
                    esac
                    break
                    ;;
                esac
              done
            done

            DIRS=$(echo "$DIRS" | xargs)  # trim whitespace

            # Check for unmatched changes under skills/
            UNMATCHED=false
            for changed_file in $CHANGED_FILES; do
              case "$changed_file" in
                skills/*)
                  MATCHED=false
                  for tile_dir in $ALL_TILE_DIRS; do
                    case "$changed_file" in
                      "$tile_dir"/*) MATCHED=true; break ;;
                    esac
                  done
                  if [ "$MATCHED" = "false" ]; then
                    UNMATCHED=true
                    break
                  fi
                  ;;
              esac
            done

            if [ "$UNMATCHED" = "true" ]; then
              echo "Unmatched changes under skills/ detected — evaluating all tiles"
              DIRS=$(find skills -name tile.json -exec dirname {} \; 2>/dev/null | tr "\n" " " | xargs)
            fi

            if [ -z "$DIRS" ]; then
              echo "skip=true" >> "$GITHUB_OUTPUT"
              echo "Changed files don't belong to any tile — nothing to eval."
            else
              echo "skip=false" >> "$GITHUB_OUTPUT"
              echo "dirs=$DIRS" >> "$GITHUB_OUTPUT"
              echo "Changed tiles: $DIRS"
            fi
          fi

      - name: Post skip summary (no qualifying tiles)
        if: steps.trigger.outputs.skip != 'true' && steps.detect.outputs.skip == 'true'
        run: |
          {
            echo "## Skill Eval"
            echo ""
            echo "No qualifying skills to evaluate — either no skill files were changed or changed skills have no \`tile.json\`."
          } >> "$GITHUB_STEP_SUMMARY"

      - uses: tesslio/setup-tessl@v2
        if: steps.trigger.outputs.skip != 'true' && steps.detect.outputs.skip != 'true'
        with:
          token: ${{ secrets.TESSL_TOKEN }}

      - name: Run evals
        if: steps.trigger.outputs.skip != 'true' && steps.detect.outputs.skip != 'true'
        run: |
          PASS=0
          FAIL=0
          SUMMARY_ROWS=""

          for tile_dir in ${{ steps.detect.outputs.dirs }}; do
            TILE_NAME=$(basename "$tile_dir")
            echo "::group::Evaluating $TILE_NAME ($tile_dir)"

            EXIT_CODE=0
            OUTPUT=$(tessl eval run "$tile_dir" 2>&1) || EXIT_CODE=$?
            echo "$OUTPUT"
            echo "::endgroup::"

            if [ "$EXIT_CODE" -ne 0 ]; then
              echo "::warning::tessl eval run failed for $TILE_NAME (exit code $EXIT_CODE)"
              FAIL=$((FAIL + 1))
              SUMMARY_ROWS="$SUMMARY_ROWS| $TILE_NAME | error | ❌ |\n"
            else
              PASS=$((PASS + 1))
              SUMMARY_ROWS="$SUMMARY_ROWS| $TILE_NAME | passed | ✅ |\n"
              # Show detailed results only for successful eval
              echo "::group::Eval results for $TILE_NAME"
              tessl eval view --last 2>&1 || true
              echo "::endgroup::"
            fi
          done

          TOTAL=$((PASS + FAIL))

          {
            echo "## Skill Eval"
            echo ""
            echo "| Tile | Result | Status |"
            echo "|------|--------|--------|"
            echo -e "$SUMMARY_ROWS"
            echo "| **Total** | **$PASS/$TOTAL passed** | $([ "$FAIL" -eq 0 ] && echo '✅' || echo '❌') |"
          } >> "$GITHUB_STEP_SUMMARY"

          echo ""
          echo "============================="
          echo "  Skill Eval Summary"
          echo "============================="
          echo "  Total:  $TOTAL"
          echo "  Passed: $PASS"
          echo "  Failed: $FAIL"
          echo "============================="

          if [ "$FAIL" -gt 0 ]; then
            echo "::error::$FAIL tile(s) failed evaluation"
            exit 1
          fi