vercel__workflow/.github/workflows/benchmark-community-world.yml

# Reusable workflow for running benchmarks against a community world
# Supports optional service containers (mongodb, redis) via service-type input
# Called by benchmarks.yml

name: Benchmark Community World

on:
  workflow_call:
    inputs:
      world-id:
        description: 'World identifier (turso, mongodb, etc.)'
        required: true
        type: string
      world-name:
        description: 'Display name for the world'
        required: true
        type: string
      world-package:
        description: 'NPM package name for the world'
        required: true
        type: string
      package-version:
        description: 'Optional npm version/dist-tag to pin the world package to (empty = latest)'
        required: false
        type: string
        default: ''
      app-name:
        description: 'App to test (default: nextjs-turbopack)'
        required: false
        type: string
        default: 'nextjs-turbopack'
      env-vars:
        description: 'JSON object of environment variables to set'
        required: false
        type: string
        default: '{}'
      service-type:
        description: 'Service container to run (none, mongodb, redis, docker)'
        required: false
        type: string
        default: 'none'
      services:
        description: 'JSON array of service definitions from the manifest (used when service-type is docker)'
        required: false
        type: string
        default: '[]'
      full-suite:
        description: 'Run full benchmark suite including long-running tests'
        required: false
        type: boolean
        default: false

jobs:
  benchmark:
    name: Benchmark ${{ inputs.world-name }}
    runs-on: ubuntu-latest
    timeout-minutes: 60

    env:
      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
      TURBO_TEAM: ${{ vars.TURBO_TEAM }}
      WORKFLOW_PUBLIC_MANIFEST: '1'

    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4

      - name: Start MongoDB
        if: ${{ inputs.service-type == 'mongodb' }}
        run: |
          docker run -d --name mongodb -p 27017:27017 mongo:7
          echo "Waiting for MongoDB to be ready..."
          for i in {1..30}; do
            if docker exec mongodb mongosh --eval 'db.runCommand({ ping: 1 })' &>/dev/null; then
              echo "MongoDB is ready"
              break
            fi
            sleep 2
          done

      - name: Start Redis
        if: ${{ inputs.service-type == 'redis' }}
        run: |
          docker run -d --name redis -p 6379:6379 redis:7-alpine
          echo "Waiting for Redis to be ready..."
          for i in {1..30}; do
            if docker exec redis redis-cli ping | grep -q PONG; then
              echo "Redis is ready"
              break
            fi
            sleep 2
          done

      - name: Start Docker services
        if: ${{ inputs.service-type == 'docker' }}
        # Services start serially in manifest order. Each service's health check
        # must pass before the next one starts, so any service that depends on
        # another (e.g. a workflow service that needs postgres) must be listed
        # AFTER its dependency in the manifest's `services` array.
        #
        # `healthCheck.cmd` is executed via `sh -c` below, which means the
        # manifest is a shell-execution surface. Safe here because
        # `worlds-manifest.json` is in-repo and PR-reviewed.
        run: |
          set -euo pipefail
          SERVICES='${{ inputs.services }}'
          count=$(echo "$SERVICES" | jq '. | length')
          for idx in $(seq 0 $((count - 1))); do
            svc=$(echo "$SERVICES" | jq -c ".[$idx]")
            name=$(echo "$svc" | jq -r '.name')
            image=$(echo "$svc" | jq -r '.image')
            health_cmd=$(echo "$svc" | jq -r '.healthCheck.cmd // empty')
            retries=$(echo "$svc" | jq -r '.healthCheck.retries // 10')

            # Build docker run as an array so env values with spaces/quotes
            # survive intact (no shell re-interpretation via eval).
            args=(docker run -d --name "$name" --network host)
            env_lines=$(echo "$svc" | jq -r '.env // {} | to_entries[] | "\(.key)=\(.value)"')
            if [ -n "$env_lines" ]; then
              while IFS= read -r line; do
                args+=(-e "$line")
              done <<< "$env_lines"
            fi
            args+=("$image")

            echo "Starting $name ($image)..."
            "${args[@]}"

            # Health check runs on the host (--network host shares the network).
            if [ -n "$health_cmd" ]; then
              echo "Waiting for $name to be ready..."
              for i in $(seq 1 "$retries"); do
                if sh -c "$health_cmd" &>/dev/null; then
                  echo "$name is ready"
                  break
                fi
                if [ "$i" -eq "$retries" ]; then
                  echo "ERROR: $name health check did not pass after $retries retries"
                  exit 1
                fi
                sleep 2
              done
            fi
          done

      - name: Setup environment
        uses: ./.github/actions/setup-workflow-dev
        with:
          build-packages: 'true'

      - name: Validate inputs
        env:
          APP_NAME: ${{ inputs.app-name }}
          WORLD_ID: ${{ inputs.world-id }}
          WORLD_PACKAGE: ${{ inputs.world-package }}
        run: |
          if [[ ! "$APP_NAME" =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]]; then
            echo "Invalid app-name: $APP_NAME" >&2
            exit 1
          fi
          if [[ ! "$WORLD_ID" =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]]; then
            echo "Invalid world-id: $WORLD_ID" >&2
            exit 1
          fi
          if [[ ! "$WORLD_PACKAGE" =~ ^(@[a-zA-Z0-9][a-zA-Z0-9_.-]*/)?[a-zA-Z0-9][a-zA-Z0-9_.-]*(@[a-zA-Z0-9][a-zA-Z0-9_.+-]*)?$ ]]; then
            echo "Invalid world-package: $WORLD_PACKAGE" >&2
            exit 1
          fi

      - name: Install ${{ inputs.world-name }} World
        env:
          APP_NAME: ${{ inputs.app-name }}
          WORLD_PACKAGE: ${{ inputs.world-package }}
          WORLD_PACKAGE_VERSION: ${{ inputs.package-version }}
        run: |
          SPEC="${WORLD_PACKAGE}${WORLD_PACKAGE_VERSION:+@$WORLD_PACKAGE_VERSION}"
          pnpm --filter "$APP_NAME" add "$SPEC"

      # Per-world setup. Hardcoded (not taken from the matrix) so a malicious
      # fork PR cannot smuggle arbitrary shell through matrix.world.setup-command.
      - name: Per-world setup
        env:
          APP_NAME: ${{ inputs.app-name }}
          WORLD_ID: ${{ inputs.world-id }}
        run: |
          cd "workbench/$APP_NAME"
          case "$WORLD_ID" in
            turso) pnpm exec workflow-turso-setup ;;
            *) echo "No setup required for $WORLD_ID" ;;
          esac

      - name: Resolve symlinks
        env:
          APP_NAME: ${{ inputs.app-name }}
        run: ./scripts/resolve-symlinks.sh "workbench/$APP_NAME"

      - name: Set environment variables
        env:
          ENV_VARS_JSON: ${{ inputs.env-vars }}
        run: |
          printf '%s' "$ENV_VARS_JSON" | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> "$GITHUB_ENV"

      - name: Build workbench
        env:
          APP_NAME: ${{ inputs.app-name }}
        run: pnpm turbo run build --filter="./workbench/$APP_NAME"

      - name: Run benchmarks
        env:
          DEPLOYMENT_URL: "http://localhost:3000"
          APP_NAME: ${{ inputs.app-name }}
          WORLD_ID: ${{ inputs.world-id }}
          WORKFLOW_BENCH_BACKEND: ${{ inputs.world-id }}
          BENCHMARK_FULL_SUITE: ${{ inputs.full-suite }}
        run: |
          cd "workbench/$APP_NAME"
          pnpm start &
          echo "Waiting for server to start..."
          sleep 15
          cd "$GITHUB_WORKSPACE"
          pnpm vitest bench packages/core/e2e/bench.bench.ts --run "--outputJson=bench-results-$APP_NAME-$WORLD_ID.json"

      - name: Render benchmark results
        uses: ./.github/actions/render-benchmarks
        with:
          benchmark-file: bench-results-${{ inputs.app-name }}-${{ inputs.world-id }}.json
          app-name: ${{ inputs.app-name }}
          backend: ${{ inputs.world-id }}

      - name: Upload benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: bench-results-${{ inputs.app-name }}-${{ inputs.world-id }}
          path: |
            bench-results-${{ inputs.app-name }}-${{ inputs.world-id }}.json
            bench-timings-${{ inputs.app-name }}-${{ inputs.world-id }}.json

      - name: Stop services
        if: always()
        run: |
          docker stop mongodb 2>/dev/null || true
          docker stop redis 2>/dev/null || true
          if [ '${{ inputs.service-type }}' = 'docker' ]; then
            echo '${{ inputs.services }}' | jq -r '.[].name' 2>/dev/null | while read -r name; do
              docker stop "$name" 2>/dev/null || true
            done
          fi