vercel__workflow/.github/workflows/event-log-race-repro.yml

name: Event Log Race Repro

on:
  pull_request:
    branches: [main, stable]
    types: [opened, reopened, synchronize, labeled]
  # Every input below is passed straight through to the harness, and a blank one
  # lands on the harness' own default in
  # `packages/core/e2e/event-log-race-repro.test.ts` — which is the only place
  # these numbers are defined. Do not reintroduce `default:` values here: a second
  # copy drifts, and the copy CI uses would stop matching the one a local run gets.
  #
  # The defaults are sized for a per-PR regression check. To soak for a *rate*
  # instead, dispatch with the historical scale: step_storm_attempts 600,
  # hook_storm_attempts 600, attempts 200, concurrency 40, budget_ms 4500000 —
  # and raise `timeout-minutes` below, which is sized for the default scale.
  workflow_dispatch:
    inputs:
      step_storm_attempts:
        description: 'Number of step-storm runs (racing steps vs. watchdog + poke-hook pressure)'
        required: false
      hook_storm_attempts:
        description: 'Number of hook-storm runs (racing hook deliveries vs. watchdog — the production shape)'
        required: false
      attempts:
        description: 'Number of hook-sleep control runs (calibration baseline, ~0.1% historical corruption rate)'
        required: false
      concurrency:
        description: 'Number of concurrent workflow runs'
        required: false
      rounds:
        description: 'Rounds of racing branches per storm run (longer log = more room for a divergence to surface)'
        required: false
      width:
        description: 'Racing branches per round (each one is an independent wake source, so this is the main concurrency dial)'
        required: false
      watchdog_ms:
        description: 'Per-branch watchdog timeout. Branches that time out emit an extra step, which is what shifts the correlation-ID sequence'
        required: false
      step_delay_ms:
        description: 'step-storm: raced step duration. Must straddle watchdog_ms once jitter is applied'
        required: false
      hook_resume_stagger_ms:
        description: 'hook-storm: per-index delay inside a round resume burst. width * this should exceed watchdog_ms'
        required: false
      blocked_branch_attempts:
        description: 'Number of blocked-branch runs (launch step before the hook/watchdog race; resume burst aimed at the launch-completion tail)'
        required: false
      launch_stagger_ms:
        description: 'blocked-branch: per-index spacing of launch step durations, so a round completions commit spread out'
        required: false
      resume_burst_offset_ms:
        description: 'blocked-branch: delay from a round hooks existing to the resume burst. Aim at step_delay_ms + (width-1) * launch_stagger_ms'
        required: false
      resume_burst_jitter_ms:
        description: 'blocked-branch: uniform jitter on the burst offset, so rounds sweep the window'
        required: false
      blocked_branch_watchdog_ms:
        description: 'blocked-branch: hook-race watchdog. Size it to normally lose to the burst; it exists to enter a wait into the race'
        required: false
      poke_interval_ms:
        description: 'step-storm: cadence of out-of-band resumes to the never-read poke hook'
        required: false
      poke_max:
        description: 'step-storm: ceiling on poke resumes per run, so a slow run cannot collect unbounded out-of-band writes. Raise it with budget_ms when soaking'
        required: false
      run_timeout_ms:
        description: 'Per-run timeout before classifying as stuck'
        required: false
      budget_ms:
        description: 'Wall-clock budget for launching attempts. Must stay far enough below the job timeout to leave room for rendering the summary'
        required: false

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/stable' }}

jobs:
  event-log-race-repro:
    name: Event Log Race Repro
    runs-on: ubuntu-latest
    # Sized for the harness' default scale: its own test timeout is
    # `budget_ms + run_timeout_ms + 60s` (~17 min at the defaults), and the rest is
    # checkout, build, the deployment wait, and rendering the summary. A soak
    # dispatch that raises `budget_ms` has to raise this too, or the runner kills
    # the job before the summary is written.
    timeout-minutes: 25
    if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'event-log-race-repro') }}
    # No comment-writing scopes: the PR comment for every lane is written by
    # `event-log-race-repro-comment` below.
    permissions:
      contents: read
      deployments: read
      statuses: read
      id-token: write
    env:
      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
      TURBO_TEAM: ${{ vars.TURBO_TEAM }}
      WORKFLOW_PUBLIC_MANIFEST: '1'

    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

      - name: Setup environment
        uses: ./.github/actions/setup-workflow-dev
        with:
          build-packages: 'false'

      - name: Build CLI
        run: pnpm turbo run build --filter='@workflow/cli'

      - name: Wait for Vercel deployment
        id: waitForDeployment
        uses: vercel/wait-for-deployment-action@0e2b0c5c5cce31f1648108aeec56467187aca037
        with:
          project-slug: example-nextjs-workflow-turbopack
          timeout: 1000
          check-interval: 15
          environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }}

      - name: Run event log race repro
        id: repro
        continue-on-error: true
        run: pnpm run test:e2e:event-log-race-repro --reporter=default
        env:
          NODE_OPTIONS: "--enable-source-maps"
          DEPLOYMENT_URL: ${{ steps.waitForDeployment.outputs.deployment-url }}
          VERCEL_DEPLOYMENT_ID: ${{ steps.waitForDeployment.outputs.deployment-id }}
          APP_NAME: nextjs-turbopack
          WORKFLOW_VERCEL_ENV: ${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }}
          WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }}
          WORKFLOW_VERCEL_TEAM: "team_nO2mCG4W8IxPIeKoSsqwAxxB"
          WORKFLOW_VERCEL_PROJECT: "prj_yjkM7UdHliv8bfxZ1sMJQf1pMpdi"
          WORKFLOW_VERCEL_PROJECT_SLUG: example-nextjs-workflow-turbopack
          VERCEL_WORKFLOW_SERVER_URL: ${{ github.ref != 'refs/heads/main' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }}
          # Deliberately unguarded pass-through: on a `pull_request` event every
          # `inputs.*` is empty, and the harness reads an empty variable as unset
          # and falls back to its own default. See the note on the inputs above.
          EVENT_LOG_RACE_REPRO_STEP_STORM_ATTEMPTS: ${{ inputs.step_storm_attempts }}
          EVENT_LOG_RACE_REPRO_HOOK_STORM_ATTEMPTS: ${{ inputs.hook_storm_attempts }}
          EVENT_LOG_RACE_REPRO_ATTEMPTS: ${{ inputs.attempts }}
          EVENT_LOG_RACE_REPRO_CONCURRENCY: ${{ inputs.concurrency }}
          EVENT_LOG_RACE_REPRO_ROUNDS: ${{ inputs.rounds }}
          EVENT_LOG_RACE_REPRO_WIDTH: ${{ inputs.width }}
          EVENT_LOG_RACE_REPRO_WATCHDOG_MS: ${{ inputs.watchdog_ms }}
          EVENT_LOG_RACE_REPRO_STEP_DELAY_MS: ${{ inputs.step_delay_ms }}
          EVENT_LOG_RACE_REPRO_HOOK_RESUME_STAGGER_MS: ${{ inputs.hook_resume_stagger_ms }}
          EVENT_LOG_RACE_REPRO_BLOCKED_BRANCH_ATTEMPTS: ${{ inputs.blocked_branch_attempts }}
          EVENT_LOG_RACE_REPRO_LAUNCH_STAGGER_MS: ${{ inputs.launch_stagger_ms }}
          EVENT_LOG_RACE_REPRO_RESUME_BURST_OFFSET_MS: ${{ inputs.resume_burst_offset_ms }}
          EVENT_LOG_RACE_REPRO_RESUME_BURST_JITTER_MS: ${{ inputs.resume_burst_jitter_ms }}
          EVENT_LOG_RACE_REPRO_BLOCKED_BRANCH_WATCHDOG_MS: ${{ inputs.blocked_branch_watchdog_ms }}
          EVENT_LOG_RACE_REPRO_POKE_INTERVAL_MS: ${{ inputs.poke_interval_ms }}
          EVENT_LOG_RACE_REPRO_POKE_MAX: ${{ inputs.poke_max }}
          EVENT_LOG_RACE_REPRO_RUN_TIMEOUT_MS: ${{ inputs.run_timeout_ms }}
          EVENT_LOG_RACE_REPRO_BUDGET_MS: ${{ inputs.budget_ms }}

      # This lane's own numbers, for the job summary. The PR comment covering
      # all three lanes is rendered once, from the artifacts, by
      # `event-log-race-repro-comment`.
      - name: Render repro summary
        if: always()
        run: |
          node .github/scripts/render-event-log-race-repro-results.js \
            event-log-race-repro-results.json \
            --label vercel \
            --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
            --run-attempt "${{ github.run_attempt }}" \
            --timestamp "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
            | tee event-log-race-repro-summary.md >> "$GITHUB_STEP_SUMMARY"

      - name: Upload repro results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: event-log-race-repro-results
          path: |
            event-log-race-repro-results.json
            event-log-race-repro-summary.md
          retention-days: 7
          if-no-files-found: ignore

      - name: Fail on corrupted, failed, or stuck runs
        if: always()
        run: node .github/scripts/render-event-log-race-repro-results.js event-log-race-repro-results.json --check

  # The same harness against the self-hosted Worlds, one parallel job per World.
  # `scripts/event-log-race-repro-local.sh` owns the whole lifecycle: it builds
  # the packages and the workbench app with WORKFLOW_TARGET_WORLD and
  # WORKFLOW_PUBLIC_MANIFEST=1 set at *build* time (both are build-time inputs),
  # brings up Postgres and applies migrations for `--world postgres`, starts the
  # app, runs the harness, and tears the server down.
  #
  # Unlike the Vercel lane above, these lanes publish numbers instead of a
  # verdict. The local storms bite much harder than the Vercel preview does —
  # the world-postgres step-storm has red baselines at the default scale (see
  # "Event Log Race Repro" in CLAUDE.md) — so a gate here would be red on PRs
  # that changed nothing, and a gate that is always red is a gate everyone
  # learns to ignore. Their numbers reach the PR through the shared comment
  # rendered by `event-log-race-repro-comment`. The only thing that fails these
  # jobs is plumbing: a harness that produced no result file at all.
  #
  # A soak dispatch that raises `budget_ms` has to raise `timeout-minutes` here
  # too, same as the Vercel lane.
  event-log-race-repro-local-worlds:
    name: Event Log Race Repro (world-${{ matrix.world }})
    runs-on: ubuntu-latest
    # The Vercel lane's 25 minutes, plus room for what it never pays for:
    # pnpm build, the app build, and (postgres) container startup + migrations.
    timeout-minutes: 45
    if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'event-log-race-repro') }}
    strategy:
      fail-fast: false
      matrix:
        world: [local, postgres]
    # No comment-writing scopes: see the Vercel lane's permissions.
    permissions:
      contents: read
    env:
      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
      TURBO_TEAM: ${{ vars.TURBO_TEAM }}

    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

      - name: Setup environment
        uses: ./.github/actions/setup-workflow-dev
        with:
          # The script runs its own `pnpm build` so the packages are built in
          # the same environment as the app.
          build-packages: 'false'

      - name: Run event log race repro (world-${{ matrix.world }})
        id: repro
        # The script exits with the harness' own status, which fails on any
        # regression outcome — expected on some baselines, see the job comment.
        continue-on-error: true
        run: ./scripts/event-log-race-repro-local.sh --world ${{ matrix.world }}
        env:
          # Deliberately unguarded pass-through: on a `pull_request` event every
          # `inputs.*` is empty, and the harness reads an empty variable as unset
          # and falls back to its own default. See the note on the inputs above.
          EVENT_LOG_RACE_REPRO_STEP_STORM_ATTEMPTS: ${{ inputs.step_storm_attempts }}
          EVENT_LOG_RACE_REPRO_HOOK_STORM_ATTEMPTS: ${{ inputs.hook_storm_attempts }}
          EVENT_LOG_RACE_REPRO_ATTEMPTS: ${{ inputs.attempts }}
          EVENT_LOG_RACE_REPRO_CONCURRENCY: ${{ inputs.concurrency }}
          EVENT_LOG_RACE_REPRO_ROUNDS: ${{ inputs.rounds }}
          EVENT_LOG_RACE_REPRO_WIDTH: ${{ inputs.width }}
          EVENT_LOG_RACE_REPRO_WATCHDOG_MS: ${{ inputs.watchdog_ms }}
          EVENT_LOG_RACE_REPRO_STEP_DELAY_MS: ${{ inputs.step_delay_ms }}
          EVENT_LOG_RACE_REPRO_HOOK_RESUME_STAGGER_MS: ${{ inputs.hook_resume_stagger_ms }}
          EVENT_LOG_RACE_REPRO_POKE_INTERVAL_MS: ${{ inputs.poke_interval_ms }}
          EVENT_LOG_RACE_REPRO_POKE_MAX: ${{ inputs.poke_max }}
          EVENT_LOG_RACE_REPRO_RUN_TIMEOUT_MS: ${{ inputs.run_timeout_ms }}
          EVENT_LOG_RACE_REPRO_BUDGET_MS: ${{ inputs.budget_ms }}

      # This lane's own numbers, for the job summary — see the Vercel lane.
      - name: Render repro summary
        if: always()
        run: |
          node .github/scripts/render-event-log-race-repro-results.js \
            event-log-race-repro-results.json \
            --label "world-${{ matrix.world }}" \
            --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
            --run-attempt "${{ github.run_attempt }}" \
            --timestamp "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
            | tee event-log-race-repro-summary.md >> "$GITHUB_STEP_SUMMARY"

      - name: Upload repro results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: event-log-race-repro-results-world-${{ matrix.world }}
          path: |
            event-log-race-repro-results.json
            event-log-race-repro-summary.md
            event-log-race-repro-server.log
          retention-days: 7
          if-no-files-found: ignore

      # Plumbing gate only: regressions are reported, not gated (see the job
      # comment), but a run that produced no result file at all is a broken
      # lane, and a broken lane that stays green never gets fixed.
      - name: Fail if the harness produced no results
        if: always()
        run: test -f event-log-race-repro-results.json

  # One PR comment for all three lanes. They run as parallel jobs and each
  # uploads its own results file, so the comment is rendered here, from the
  # artifacts, once every lane has finished. One comment means one stored
  # history, which is the point: the three worlds' numbers sit on the same rows
  # of one table, and a push adds three rows rather than editing three comments.
  event-log-race-repro-comment:
    name: Event Log Race Repro (comment)
    runs-on: ubuntu-latest
    needs: [event-log-race-repro, event-log-race-repro-local-worlds]
    # `always()`, because a lane that failed, timed out, or produced nothing is
    # exactly what the comment has to report. The label condition is repeated
    # because `always()` also fires when every dependency was skipped.
    if: ${{ always() && (github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'event-log-race-repro')) }}
    timeout-minutes: 10
    permissions:
      contents: read
      issues: write
      pull-requests: write

    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

      - name: Download lane results
        # A lane that died before its first checkpoint uploads nothing, and
        # `download-artifact` treats a pattern that matched nothing as an error.
        # The renderer reports a lane with no results file as exactly that, so
        # this step is allowed to come back empty.
        continue-on-error: true
        uses: actions/download-artifact@v4
        with:
          pattern: event-log-race-repro-results*
          path: lane-results

      - name: Fetch previous repro comment
        if: github.event_name == 'pull_request'
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          gh api \
            "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments?per_page=100" \
            --jq '[.[] | select(.body | contains("<!-- event-log-race-repro-results -->"))][-1].body // ""' \
            > event-log-race-repro-previous-comment.md

      - name: Render repro summary
        run: |
          previous_comment_args=()
          if [ -f event-log-race-repro-previous-comment.md ]; then
            previous_comment_args=(--previous-comment event-log-race-repro-previous-comment.md)
          fi

          # Lane order here is the row order in the comment: the gating lane
          # first, then the two report-only worlds.
          node .github/scripts/render-event-log-race-repro-results.js \
            --lane vercel=lane-results/event-log-race-repro-results/event-log-race-repro-results.json \
            --lane local=lane-results/event-log-race-repro-results-world-local/event-log-race-repro-results.json \
            --lane postgres=lane-results/event-log-race-repro-results-world-postgres/event-log-race-repro-results.json \
            --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
            --run-attempt "${{ github.run_attempt }}" \
            --timestamp "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
            "${previous_comment_args[@]}" \
            | tee event-log-race-repro-summary.md >> "$GITHUB_STEP_SUMMARY"

      - name: Update PR comment
        if: github.event_name == 'pull_request'
        uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
        with:
          header: event-log-race-repro-results
          path: event-log-race-repro-summary.md