copilotkit__copilotkit/.github/workflows/showcase_promote.yml

name: "Showcase: Promote (staging → prod)"

# Promotes the staging-tested digest of one or more services to prod.
# Workflow_dispatch only. Humans trigger. No automatic prod promotes.
#
# Order:
#   0. resolve-targets              → expand the workflow_dispatch `service`
#                                     input (SSOT key, dispatch_name, or
#                                     'all') into the canonical services_csv
#                                     consumed by every downstream job.
#   1. verify-staging-precondition  → live-probe staging for the target service(s).
#                                     Refuse on red (matches bin/railway
#                                     --require-staging-green default).
#   2. promote                      → bin/railway promote <service>; runs the
#                                     spec §7 hardening (P1..P6).
#   3. verify-prod                  → verify-deploy.ts --env prod for the
#                                     target service(s). Feature-level probes,
#                                     not naked 200.
#   4. notify                       → Slack #oss-alerts on any red. Never #engr.
#                                     success → #team-showcase.

on:
  workflow_dispatch:
    inputs:
      service:
        description: "Service to promote (dispatch_name or SSOT key). 'all' = whole fleet. Leave the placeholder to abort."
        required: true
        type: choice
        # >>> BEGIN GENERATED service options (showcase/scripts/sync-promote-service-options.ts) — DO NOT EDIT
        default: __select_a_service__
        options:
          - __select_a_service__
          - all
          - ag2
          - agno
          - built-in-agent
          - claude-sdk-python
          - claude-sdk-typescript
          - crewai-crews
          - google-adk
          - langgraph-fastapi
          - langgraph-python
          - langgraph-typescript
          - langroid
          - llamaindex
          - mastra
          - ms-agent-dotnet
          - ms-agent-harness-dotnet
          - ms-agent-python
          - pydantic-ai
          - shell
          - shell-dashboard
          - shell-docs
          - shell-dojo
          - showcase-aimock
          - showcase-harness
          - showcase-pocketbase
          - spring-ai
          - strands
          - webhooks
        # <<< END GENERATED service options
      digest:
        description: "Optional digest override (default: snapshot from staging)"
        required: false
        type: string

# Serialize ALL promotes on a single input-agnostic group so concurrent runs
# (e.g. `all` + a single-service promote) can't pin the same Railway service
# at once; `cancel-in-progress: false` ensures an in-flight prod promote is
# never cancelled by a newer run queued behind it.
concurrency:
  group: showcase-promote
  cancel-in-progress: false

permissions:
  contents: read

jobs:
  resolve-targets:
    runs-on: ubuntu-latest
    timeout-minutes: 3
    permissions:
      contents: read
    outputs:
      services_csv: ${{ steps.resolve.outputs.services_csv }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          persist-credentials: false
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 22.x
      - name: Generate SSOT artifact
        working-directory: showcase/scripts
        run: |
          npm ci
          npx tsx emit-railway-envs-json.ts
      - name: Resolve target service set
        id: resolve
        env:
          INPUT: ${{ inputs.service }}
          DIGEST: ${{ inputs.digest }}
        run: |
          set -euo pipefail
          GENERATED="showcase/scripts/railway-envs.generated.json"
          if [ "$INPUT" = "__select_a_service__" ]; then
            echo "::error::No service selected. Re-run and pick a service (or 'all') from the dropdown."
            exit 1
          fi
          if [ "$INPUT" = "all" ]; then
            # A single digest identifies at most one service's image, so a
            # fleet-wide promote pinned to one digest is always wrong. The
            # per-service promote loop would slip past bin/railway's own
            # --digest guard (each call passes a positional service), so reject
            # the combination loud and early here.
            if [ -n "${DIGEST:-}" ]; then
              echo "::error::--digest cannot be combined with 'all' (a single digest is meaningless across multiple services); pick one service."
              exit 1
            fi
            CSV=$(jq -r '.services[] | select(.probe.prod == true) | .name' "$GENERATED" | sort -u | tr '\n' ',' | sed 's/,$//')
            # Fail loud if 'all' resolved to nothing (e.g. an SSOT regression
            # dropped every probe.prod entry). An empty CSV would otherwise
            # propagate downstream with exit 0; mirror the single-service
            # branch's fail-loud style.
            if [ -z "$CSV" ]; then
              echo "::error::'all' resolved to zero prod-eligible services"
              exit 1
            fi
          else
            # Capture the FULL match set (no `head` — that would silently mask
            # an ambiguous match, and piping jq into head under pipefail can
            # SIGPIPE jq and abort with no ::error:: annotation). Then count
            # and branch fail-loud.
            # Independently enforce prod-eligibility here: the dropdown
            # advertises a prod-eligible set, but a stale/edited dropdown could
            # offer a probe.prod:false service. Filtering on probe.prod == true
            # ensures the single-service path never promotes a non-eligible
            # service to prod, matching the `all` branch's gate.
            MATCHES=$(jq -r --arg s "$INPUT" '
              .services[]
              | select(.name == $s or .dispatchName == $s)
              | select(.probe.prod == true)
              | .name
            ' "$GENERATED")
            # `grep -c` on empty input exits 1 under set -e; guard with || true.
            COUNT=$(printf '%s' "$MATCHES" | grep -c . || true)
            if [ "$COUNT" -eq 0 ]; then
              echo "::error::Unknown or not prod-eligible service '$INPUT' (not an SSOT key/dispatch_name, or probe.prod is not true)"
              exit 1
            elif [ "$COUNT" -gt 1 ]; then
              LIST=$(printf '%s' "$MATCHES" | tr '\n' ',' | sed 's/,$//')
              echo "::error::Ambiguous service '$INPUT' matches multiple SSOT entries: $LIST"
              exit 1
            fi
            CSV="$MATCHES"
          fi
          echo "services_csv=$CSV" >> "$GITHUB_OUTPUT"

  verify-staging-precondition:
    needs: [resolve-targets]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    environment: railway
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          persist-credentials: false
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 22.x
      - working-directory: showcase/scripts
        run: npm ci
      - name: Live-probe staging for promote precondition
        working-directory: showcase/scripts
        env:
          RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
          SERVICES_CSV: ${{ needs.resolve-targets.outputs.services_csv }}
        run: |
          if [ -z "$RAILWAY_TOKEN" ]; then
            echo "::error::RAILWAY_TOKEN is not set"
            exit 1
          fi
          # Spec §7.2 P3: the live staging probe at promote time is
          # authoritative. CI verify history is a leading indicator only.
          # Run from showcase/scripts (where `npm ci` installed tsx) so npx
          # uses the local install instead of network-fetching it.
          npx tsx verify-deploy.ts --env staging --services "$SERVICES_CSV"

  promote:
    needs: [resolve-targets, verify-staging-precondition]
    # Do NOT hard-depend on verify-staging-precondition's RESULT. That job
    # probes the FULL requested set all-or-nothing, so a single staging-red
    # service (e.g. a chronically-broken integration in `all`) fails it and
    # — under the default `if: success()` — would SKIP promote entirely,
    # re-blocking the whole fleet. bin/railway enforces staging-green
    # PER-SERVICE (spec §7 P2 = latest staging deploy must be SUCCESS, P3 =
    # live staging probe, default-on), so a red service is refused on its own
    # and lands in promote-fleet.sh's failed set (reds the run) without
    # taking the greens down with it. verify-staging-precondition therefore
    # stays as an advisory early signal surfaced in the notify payload, not a
    # hard blocker. `!cancelled()` keeps a human-cancelled run from promoting;
    # mirrors the verify-prod gate idiom below.
    if: ${{ !cancelled() && needs.resolve-targets.result == 'success' }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    environment: railway
    permissions:
      contents: read
      packages: read
    outputs:
      # CSV of the services that ACTUALLY promoted (best-effort: a partial
      # failure still exposes the succeeded subset). verify-prod scopes its
      # prod verification to exactly this set so a single failed service can't
      # red the verification of the services that did promote.
      succeeded_csv: ${{ steps.promote.outputs.succeeded_csv }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          persist-credentials: false
      - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
        with:
          ruby-version: "3.3"
          bundler-cache: false
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 22.x
      - working-directory: showcase/scripts
        run: |
          npm ci
          npx tsx emit-railway-envs-json.ts
      - name: bin/railway promote
        id: promote
        env:
          RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SERVICES_CSV: ${{ needs.resolve-targets.outputs.services_csv }}
          DIGEST: ${{ inputs.digest }}
        run: |
          set -euo pipefail
          if [ -z "$RAILWAY_TOKEN" ]; then
            echo "::error::RAILWAY_TOKEN is not set"
            exit 1
          fi
          # promote-fleet.sh runs each service in turn BEST-EFFORT: a single
          # red service (e.g. a chronically-broken integration in the `all`
          # set) must not abort promotion of the rest of the fleet. The script
          # attempts every service, aggregates the succeeded/failed sets, emits
          # a step summary, and exits non-zero iff ANY service failed (so the
          # notify job still fires) — but only AFTER attempting all of them.
          # bin/railway itself handles spec §7 preconditions (P1..P6);
          # --require-staging-green is default-on and the prior job already
          # established staging is green (defense in depth). We deliberately do
          # NOT pass --confirm-divergence: WARN-divergence refusals are a real
          # signal that must fail the run.
          RAILWAY_BIN="showcase/bin/railway" \
            showcase/scripts/promote-fleet.sh

  verify-prod:
    needs: [resolve-targets, promote]
    # Run whenever promote actually RAN — success OR partial failure — so the
    # services that DID promote still get prod verification. `!cancelled()`
    # excludes a human-cancelled run; `needs.promote.result != 'skipped'`
    # excludes the case where promote never ran (e.g. an upstream abort/skip).
    # Under the default `if: success()` this job was SKIPPED on any partial
    # promote failure, leaving the promoted services with zero prod
    # verification — that is the bug this gate fixes.
    if: ${{ !cancelled() && needs.promote.result != 'skipped' }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    environment: railway
    permissions:
      contents: read
    outputs:
      # Distinguishes a job that actually PROBED prod (`success`) from one that
      # SKIPPED probing because nothing promoted (`skipped`). The GitHub job
      # `result` is `success` in BOTH cases (the skip path exits 0), so the
      # notify job must read THIS output — not `needs.verify-prod.result` — to
      # avoid reporting a misleading `verify-prod=success` when prod was never
      # touched. A real probe failure / contract violation exits non-zero, so
      # the job `result` becomes `failure` and this output is never written
      # (notify falls back to the job result for that case).
      status: ${{ steps.verify.outputs.status }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          persist-credentials: false
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 22.x
      - working-directory: showcase/scripts
        run: npm ci
      - name: Run verify-deploy --env prod
        id: verify
        working-directory: showcase/scripts
        env:
          RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
          # Scope verification to the services that ACTUALLY promoted (from the
          # promote job's best-effort succeeded set), NOT the full requested set
          # (resolve-targets) — verifying a service that failed to promote would
          # guarantee a red verify and mask the health of the ones that did
          # promote.
          SERVICES_CSV: ${{ needs.promote.outputs.succeeded_csv }}
          # The promote job's result, so the empty-CSV branch can tell a genuine
          # all-failed run (empty succeeded set is expected) apart from a
          # contract violation (promote reported success yet emitted no CSV).
          PROMOTE_RESULT: ${{ needs.promote.result }}
        run: |
          set -euo pipefail
          if [ -z "$RAILWAY_TOKEN" ]; then
            echo "::error::RAILWAY_TOKEN is not set"
            exit 1
          fi
          # If NOTHING promoted (every requested service failed, so the
          # succeeded set is empty), there is nothing to verify — skip with a
          # clear log line rather than calling verify-deploy.ts with an empty
          # --services (which would either error or vacuously pass). The promote
          # job already exited non-zero in that case, so the overall run is red
          # via the notify state machine regardless.
          #
          # BUT: an empty succeeded set is only legitimate when promote did NOT
          # succeed. If promote reported success and STILL emitted no CSV, the
          # "promote already failed" assumption that justifies the vacuous skip
          # is violated — fail loud instead of silently exiting 0.
          if [ -z "$SERVICES_CSV" ]; then
            if [ "$PROMOTE_RESULT" = "success" ]; then
              echo "::error::promote reported success but succeeded_csv is empty — contract violation"
              exit 1
            fi
            echo "::notice::succeeded_csv is empty (no services promoted, or promote did not run); skipping prod verification. The run is red via the promote job result if anything failed."
            # Record that prod was SKIPPED, not verified. The job still exits 0
            # (its `result` is `success`), so the notify job reads this `status`
            # output to render `verify-prod=skipped` instead of a misleading
            # `verify-prod=success`.
            echo "status=skipped" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          # Run from showcase/scripts (where `npm ci` installed tsx) so npx uses
          # the local install instead of network-fetching it. A non-zero exit
          # here fails the step (job `result` = failure) and `status` is never
          # written, so notify falls back to the job result.
          npx tsx verify-deploy.ts --env prod --services "$SERVICES_CSV"
          # Prod was actually probed and passed.
          echo "status=success" >> "$GITHUB_OUTPUT"

  notify:
    # Slack #oss-alerts only. Never #engr (engr is sacred — release alerts only).
    needs: [resolve-targets, verify-staging-precondition, promote, verify-prod]
    if: always()
    runs-on: ubuntu-latest
    timeout-minutes: 3
    permissions:
      contents: read
      actions: read
    env:
      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
      SLACK_WEBHOOK_TS: ${{ secrets.SLACK_WEBHOOK_TEAM_SHOWCASE }}
    steps:
      # Best-effort promote invariant (do NOT reintroduce a PRE gate below):
      #   resolve-targets              = HARD precondition (must succeed).
      #   verify-staging-precondition  = ADVISORY only — surfaced in the Slack
      #                                  payload (`pre-staging`), never gates
      #                                  promote or run success. bin/railway
      #                                  enforces staging-green per-service, so
      #                                  PRE is a leading indicator, not a gate.
      #   promote                      = best-effort (exits non-zero iff a
      #                                  service failed).
      #   verify-prod                  = verifies the succeeded subset.
      #   run success                  = PROMOTE == success AND PROD == success.
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          persist-credentials: false
      - name: Compute state
        id: state
        env:
          RESOLVE: ${{ needs.resolve-targets.result }}
          PRE: ${{ needs.verify-staging-precondition.result }}
          PROMOTE: ${{ needs.promote.result }}
          PROD: ${{ needs.verify-prod.result }}
          # The verify-prod job's own status output: `success` (prod actually
          # probed + passed) or `skipped` (nothing promoted, so prod was never
          # touched). Empty when the job failed/cancelled or never wrote it.
          PROD_STATUS: ${{ needs.verify-prod.outputs.status }}
          CSV: ${{ needs.resolve-targets.outputs.services_csv }}
          INPUT: ${{ inputs.service }}
        run: |
          set -euo pipefail
          # Displayed verify-prod value for the Slack line. The job `result` is
          # `success` for BOTH a real pass AND the empty-CSV skip (which exits
          # 0), so a bare `result` would read `verify-prod=success` even when
          # prod was never probed. The mapping (prefer the job's `status` output
          # when it ran cleanly, else fall back to the raw result) lives in the
          # bats-tested showcase/scripts/verify-prod-display.sh so it can't
          # drift from its test (see __tests__/verify-prod-display.bats).
          PROD_DISPLAY="$(PROD="$PROD" PROD_STATUS="$PROD_STATUS" showcase/scripts/verify-prod-display.sh)"
          if [ "$INPUT" = "__select_a_service__" ]; then
            # Deliberate no-op abort: a human clicked Run without picking a
            # service. resolve-targets exits 1 and the downstream jobs skip.
            # This is NOT a failed promote — emit a neutral state so no red
            # alert pages #oss-alerts.
            STATE="aborted"; ICON=":heavy_minus_sign:"
          elif [ "$PROMOTE" = "success" ] && [ "$PROD" = "success" ]; then
            # PRE (verify-staging-precondition) is ADVISORY and intentionally
            # NOT in this gate: bin/railway enforces staging-green per-service,
            # so a service that was staging-red at precondition time but still
            # promoted+verified cleanly must not red the run. PRE remains
            # surfaced in the Slack payload as the `pre-staging` field.
            STATE="success"; ICON=":white_check_mark:"
          elif [ "$RESOLVE" = "cancelled" ] || [ "$PRE" = "cancelled" ] || [ "$PROMOTE" = "cancelled" ] || [ "$PROD" = "cancelled" ]; then
            # A human cancelled the run (e.g. via the Actions UI). This is a
            # deliberate stop, not a failed promote — emit a neutral state so
            # no red alert pages #oss-alerts. Cancelling DURING resolve-targets
            # leaves the three downstream jobs `skipped` (not `cancelled`), so
            # we must consult RESOLVE here too or the cancel would fall through
            # to `failure` and fire a false red page.
            STATE="cancelled"; ICON=":heavy_minus_sign:"
          else
            STATE="failure"; ICON=":x:"
          fi
          {
            echo "state=$STATE"
            echo "icon=$ICON"
            echo "csv=$CSV"
            echo "prod_display=$PROD_DISPLAY"
          } >> "$GITHUB_OUTPUT"
      - name: Post to #oss-alerts
        if: steps.state.outputs.state == 'failure' && inputs.service != '__select_a_service__' && env.SLACK_WEBHOOK != ''
        uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0
        with:
          webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
          webhook-type: incoming-webhook
          # Newlines are injected via fromJSON('"\n"') (a real LF char), NOT a
          # literal '\n' in the template: GitHub Actions expression string
          # literals do not interpret backslash escapes, so a literal '\n' would
          # survive toJSON as the two chars \\n and Slack would render it
          # verbatim as "\n" instead of a line break.
          payload: |
            {
              "text": ${{ toJSON(format(
                '{0} *Showcase Promote Failed*{9}Services: `{1}`{9}resolve-targets={2} pre-staging={3} promote={4} verify-prod={5}{9}<{6}/{7}/actions/runs/{8}|View run>',
                steps.state.outputs.icon,
                steps.state.outputs.csv,
                needs.resolve-targets.result,
                needs.verify-staging-precondition.result,
                needs.promote.result,
                steps.state.outputs.prod_display,
                github.server_url,
                github.repository,
                github.run_id,
                fromJSON('"\n"')
              )) }}
            }
      - name: Post to #team-showcase
        if: steps.state.outputs.state == 'success' && inputs.service != '__select_a_service__' && env.SLACK_WEBHOOK_TS != ''
        uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0
        with:
          webhook: ${{ secrets.SLACK_WEBHOOK_TEAM_SHOWCASE }}
          webhook-type: incoming-webhook
          # Newlines via fromJSON('"\n"') (real LF), not a literal '\n' — see the
          # #oss-alerts step above for why a literal '\n' renders verbatim.
          payload: |
            {
              "text": ${{ toJSON(format(
                '{0} *Showcase Promoted to Prod*{5}Services: `{1}`{5}<{2}/{3}/actions/runs/{4}|View run>',
                steps.state.outputs.icon,
                steps.state.outputs.csv,
                github.server_url,
                github.repository,
                github.run_id,
                fromJSON('"\n"')
              )) }}
            }
      - name: Log (no Slack — webhook unset)
        if: steps.state.outputs.state == 'failure' && inputs.service != '__select_a_service__' && env.SLACK_WEBHOOK == ''
        env:
          CSV: ${{ steps.state.outputs.csv }}
        run: |
          echo "::warning::Showcase promote failed for '$CSV' but SLACK_WEBHOOK_OSS_ALERTS is not set; no Slack notification sent."
      - name: Log (no Slack — team-showcase webhook unset)
        if: steps.state.outputs.state == 'success' && inputs.service != '__select_a_service__' && env.SLACK_WEBHOOK_TS == ''
        env:
          CSV: ${{ steps.state.outputs.csv }}
          ICON: ${{ steps.state.outputs.icon }}
        run: |
          echo "::notice::$ICON Showcase promoted to prod for '$CSV' but SLACK_WEBHOOK_TEAM_SHOWCASE is not set; no #team-showcase notification sent."