copilotkit__copilotkit/.github/workflows/showcase_promote.yml

name: "Showcase: Promote (staging → prod)"

# Promotes the staging-tested digest of one or more services to prod.
# Workflow_dispatch only. Humans trigger. No automatic prod promotes.
#
# Order:
#   0. resolve-targets              → expand the workflow_dispatch `service`
#                                     input (SSOT key, dispatch_name, or
#                                     'all') into the canonical services_csv
#                                     consumed by every downstream job.
#   1. verify-staging-precondition  → live-probe staging for the target service(s).
#                                     Refuse on red (matches bin/railway
#                                     --require-staging-green default).
#   2. promote                      → bin/railway promote <service>; runs the
#                                     spec §7 hardening (P1..P6).
#   3. verify-prod                  → verify-deploy.ts --env prod for the
#                                     target service(s). Feature-level probes,
#                                     not naked 200. Then the prod equivalence
#                                     re-sweep gate (§6.2) and the pinned-ness
#                                     gate (lint-prod, §8.2 — every prod service
#                                     must be on an immutable @sha256: digest).
#   4. notify                       → Slack #oss-alerts on any red. Never #engr.
#                                     success → #team-showcase.

on:
  workflow_dispatch:
    inputs:
      service:
        description: "Service to promote (dispatch_name or SSOT key). 'all' = whole fleet. Leave the placeholder to abort."
        required: true
        type: choice
        # >>> BEGIN GENERATED service options (showcase/scripts/sync-promote-service-options.ts) — DO NOT EDIT
        default: __select_a_service__
        options:
          - __select_a_service__
          - all
          - ag2
          - agno
          - built-in-agent
          - claude-sdk-python
          - claude-sdk-typescript
          - crewai-crews
          - google-adk
          - langgraph-fastapi
          - langgraph-python
          - langgraph-typescript
          - langroid
          - llamaindex
          - mastra
          - ms-agent-dotnet
          - ms-agent-harness-dotnet
          - ms-agent-python
          - pydantic-ai
          - shell
          - shell-dashboard
          - shell-docs
          - shell-dojo
          - showcase-aimock
          - showcase-harness
          - showcase-pocketbase
          - spring-ai
          - strands
          - webhooks
        # <<< END GENERATED service options
      digest:
        description: "Optional digest override (default: snapshot from staging)"
        required: false
        type: string

# Serialize ALL promotes on a single input-agnostic group so concurrent runs
# (e.g. `all` + a single-service promote) can't pin the same Railway service
# at once; `cancel-in-progress: false` ensures an in-flight prod promote is
# never cancelled by a newer run queued behind it.
concurrency:
  group: showcase-promote
  cancel-in-progress: false

permissions:
  contents: read

jobs:
  resolve-targets:
    runs-on: ubuntu-latest
    timeout-minutes: 3
    permissions:
      contents: read
    outputs:
      # Leaf-set CSV — the BACKWARD-COMPAT promote target the promote job still
      # drives off (Phase 1 behavior is unchanged).
      services_csv: ${{ steps.resolve.outputs.services_csv }}
      # Tier-ordered promote closure (requested ∪ transitive runtimeDeps ∪
      # Tier-1 verification, §4.2). Phase 1 SURFACES it (operators see the
      # closure in the step summary) but the actual promote still uses
      # services_csv; U4 enforces tier ordering / dependent-gating off these.
      closure_csv: ${{ steps.resolve.outputs.closure_csv }}
      # Machine-readable `tier:name` form of the closure for U4 to consume.
      closure_plan: ${{ steps.resolve.outputs.closure_plan }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          persist-credentials: false
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 22.x
      - name: Generate SSOT artifact
        working-directory: showcase/scripts
        run: |
          npm ci
          npx tsx emit-railway-envs-json.ts
      - name: Resolve target service set
        id: resolve
        # Resolution + the tiered-closure derivation live in the bats-tested
        # showcase/scripts/resolve-promote-targets.sh so they can't drift from
        # their test (see __tests__/resolve-promote-targets.bats), mirroring how
        # promote-fleet.sh / verify-prod-display.sh were extracted from this same
        # workflow. The script preserves the existing guards (--digest+all
        # reject, empty-`all` fail-loud, unknown/ambiguous/not-prod-eligible) and
        # emits services_csv (leaf, backward-compat) + closure_csv/closure_plan
        # (tiered closure) into $GITHUB_OUTPUT, plus the closure plan + skips into
        # $GITHUB_STEP_SUMMARY.
        env:
          INPUT: ${{ inputs.service }}
          DIGEST: ${{ inputs.digest }}
          GENERATED: showcase/scripts/railway-envs.generated.json
        run: showcase/scripts/resolve-promote-targets.sh

  verify-staging-precondition:
    needs: [resolve-targets]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    environment: railway
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          persist-credentials: false
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 22.x
      - working-directory: showcase/scripts
        run: npm ci
      - name: Live-probe staging for promote precondition
        working-directory: showcase/scripts
        env:
          RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
          SERVICES_CSV: ${{ needs.resolve-targets.outputs.services_csv }}
        run: |
          if [ -z "$RAILWAY_TOKEN" ]; then
            echo "::error::RAILWAY_TOKEN is not set"
            exit 1
          fi
          # Spec §7.2 P3: the live staging probe at promote time is
          # authoritative. CI verify history is a leading indicator only.
          # Run from showcase/scripts (where `npm ci` installed tsx) so npx
          # uses the local install instead of network-fetching it.
          npx tsx verify-deploy.ts --env staging --services "$SERVICES_CSV"

  promote:
    needs: [resolve-targets, verify-staging-precondition]
    # Do NOT hard-depend on verify-staging-precondition's RESULT. That job
    # probes the FULL requested set all-or-nothing, so a single staging-red
    # service (e.g. a chronically-broken integration in `all`) fails it and
    # — under the default `if: success()` — would SKIP promote entirely,
    # re-blocking the whole fleet. bin/railway enforces staging-green
    # PER-SERVICE (spec §7 P2 = latest staging deploy must be SUCCESS, P3 =
    # live staging probe, default-on), so a red service is refused on its own
    # and lands in promote-fleet.sh's failed set (reds the run) without
    # taking the greens down with it. verify-staging-precondition therefore
    # stays as an advisory early signal surfaced in the notify payload, not a
    # hard blocker. `!cancelled()` keeps a human-cancelled run from promoting;
    # mirrors the verify-prod gate idiom below.
    if: ${{ !cancelled() && needs.resolve-targets.result == 'success' }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    environment: railway
    permissions:
      contents: read
      packages: read
    outputs:
      # CSV of the services that ACTUALLY promoted (best-effort: a partial
      # failure still exposes the succeeded subset). verify-prod scopes its
      # prod verification to exactly this set so a single failed service can't
      # red the verification of the services that did promote.
      succeeded_csv: ${{ steps.promote.outputs.succeeded_csv }}
      # Aggregated staging-drift payload: non-empty when any promoted service's
      # staging RUNNING digest differed from the current GHCR :latest (i.e.
      # staging was NOT serving :latest). Folded into the Slack notify payload
      # so operators see "what shipped to prod is not current :latest".
      staging_drift: ${{ steps.promote.outputs.staging_drift }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          persist-credentials: false
      - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
        with:
          ruby-version: "3.3"
          bundler-cache: false
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 22.x
      - working-directory: showcase/scripts
        run: |
          npm ci
          npx tsx emit-railway-envs-json.ts
      - name: bin/railway promote
        id: promote
        env:
          RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SERVICES_CSV: ${{ needs.resolve-targets.outputs.services_csv }}
          DIGEST: ${{ inputs.digest }}
        run: |
          set -euo pipefail
          if [ -z "$RAILWAY_TOKEN" ]; then
            echo "::error::RAILWAY_TOKEN is not set"
            exit 1
          fi
          # promote-fleet.sh runs each service in turn BEST-EFFORT: a single
          # red service (e.g. a chronically-broken integration in the `all`
          # set) must not abort promotion of the rest of the fleet. The script
          # attempts every service, aggregates the succeeded/failed sets, emits
          # a step summary, and exits non-zero iff ANY service failed (so the
          # notify job still fires) — but only AFTER attempting all of them.
          # bin/railway itself handles spec §7 preconditions (P1..P6);
          # --require-staging-green is default-on and the prior job already
          # established staging is green (defense in depth). We deliberately do
          # NOT pass --confirm-divergence: WARN-divergence refusals are a real
          # signal that must fail the run.
          RAILWAY_BIN="showcase/bin/railway" \
            showcase/scripts/promote-fleet.sh

  verify-prod:
    needs: [resolve-targets, promote]
    # Run whenever promote actually RAN — success OR partial failure — so the
    # services that DID promote still get prod verification. `!cancelled()`
    # excludes a human-cancelled run; `needs.promote.result != 'skipped'`
    # excludes the case where promote never ran (e.g. an upstream abort/skip).
    # Under the default `if: success()` this job was SKIPPED on any partial
    # promote failure, leaving the promoted services with zero prod
    # verification — that is the bug this gate fixes.
    if: ${{ !cancelled() && needs.promote.result != 'skipped' }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    environment: railway
    permissions:
      contents: read
    outputs:
      # Distinguishes a job that actually PROBED prod (`success`) from one that
      # SKIPPED probing because nothing promoted (`skipped`). The GitHub job
      # `result` is `success` in BOTH cases (the skip path exits 0), so the
      # notify job must read THIS output — not `needs.verify-prod.result` — to
      # avoid reporting a misleading `verify-prod=success` when prod was never
      # touched. A real probe failure / contract violation exits non-zero, so
      # the job `result` becomes `failure` and this output is never written
      # (notify falls back to the job result for that case).
      status: ${{ steps.verify.outputs.status }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          persist-credentials: false
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 22.x
      - working-directory: showcase/scripts
        run: npm ci
      - name: Run verify-deploy --env prod
        id: verify
        working-directory: showcase/scripts
        env:
          RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
          # Scope verification to the services that ACTUALLY promoted (from the
          # promote job's best-effort succeeded set), NOT the full requested set
          # (resolve-targets) — verifying a service that failed to promote would
          # guarantee a red verify and mask the health of the ones that did
          # promote.
          SERVICES_CSV: ${{ needs.promote.outputs.succeeded_csv }}
          # The promote job's result, so the empty-CSV branch can tell a genuine
          # all-failed run (empty succeeded set is expected) apart from a
          # contract violation (promote reported success yet emitted no CSV).
          PROMOTE_RESULT: ${{ needs.promote.result }}
        run: |
          set -euo pipefail
          if [ -z "$RAILWAY_TOKEN" ]; then
            echo "::error::RAILWAY_TOKEN is not set"
            exit 1
          fi
          # If NOTHING promoted (every requested service failed, so the
          # succeeded set is empty), there is nothing to verify — skip with a
          # clear log line rather than calling verify-deploy.ts with an empty
          # --services (which would either error or vacuously pass). The promote
          # job already exited non-zero in that case, so the overall run is red
          # via the notify state machine regardless.
          #
          # BUT: an empty succeeded set is only legitimate when promote did NOT
          # succeed. If promote reported success and STILL emitted no CSV, the
          # "promote already failed" assumption that justifies the vacuous skip
          # is violated — fail loud instead of silently exiting 0.
          if [ -z "$SERVICES_CSV" ]; then
            if [ "$PROMOTE_RESULT" = "success" ]; then
              echo "::error::promote reported success but succeeded_csv is empty — contract violation"
              exit 1
            fi
            echo "::notice::succeeded_csv is empty (no services promoted, or promote did not run); skipping prod verification. The run is red via the promote job result if anything failed."
            # Record that prod was SKIPPED, not verified. The job still exits 0
            # (its `result` is `success`), so the notify job reads this `status`
            # output to render `verify-prod=skipped` instead of a misleading
            # `verify-prod=success`.
            echo "status=skipped" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          # Run from showcase/scripts (where `npm ci` installed tsx) so npx uses
          # the local install instead of network-fetching it. A non-zero exit
          # here fails the step (job `result` = failure) and `status` is never
          # written, so notify falls back to the job result.
          npx tsx verify-deploy.ts --env prod --services "$SERVICES_CSV"
          # Prod was actually probed and passed.
          echo "status=success" >> "$GITHUB_OUTPUT"
      # ── Prod equivalence gate (UNIT U10, spec §6.2) ──────────────────────
      # After the closure pins (promote job) AND the per-service prod probe
      # (above), re-sweep the promoted integration closure on the PROD control
      # plane (prod harness scheduler + prod harness-workers, or scheduler
      # INLINE fallback when workers are unprovisioned — §4.4) and run U9's
      # equivalence gate over the FRESH prod rows vs current staging. Promote
      # success flips to the equivalence definition (fail only on
      # staging-green / prod-not-green, gray/stale excluded, one-directional).
      #
      # GATED ON CONFIGURATION: the equivalence gate needs prod + staging
      # PocketBase creds and the prod Railway environment id. Those are an
      # out-of-PR operational prerequisite (§8.1) — until the secrets are wired
      # the step ANNOTATES "not configured" and is a no-op, so this job's
      # existing per-service prod verification (above) remains the gate. Once
      # configured, a gate FAILURE fails the step (and the run).
      # Install the pnpm workspace ONLY when the equivalence gate is configured
      # (the gate step's enqueue dynamically imports the harness producer graph,
      # which lives in the pnpm workspace — `npm ci` in showcase/scripts alone
      # does not resolve it). `--ignore-scripts` skips postinstall/Playwright
      # browser downloads: the host only ENQUEUES + POLLS prod PocketBase; the
      # prod harness-workers own the browser. Gated on the same config check as
      # the gate step so the no-op (unconfigured) path stays cheap.
      - name: Set up pnpm (equivalence gate only)
        if: ${{ steps.verify.outputs.status == 'success' && vars.SHOWCASE_PROD_POCKETBASE_URL != '' && vars.SHOWCASE_STAGING_POCKETBASE_URL != '' && vars.SHOWCASE_RAILWAY_ENV_ID_PROD != '' }}
        uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
      - name: pnpm install (equivalence gate only)
        if: ${{ steps.verify.outputs.status == 'success' && vars.SHOWCASE_PROD_POCKETBASE_URL != '' && vars.SHOWCASE_STAGING_POCKETBASE_URL != '' && vars.SHOWCASE_RAILWAY_ENV_ID_PROD != '' }}
        run: pnpm install --ignore-scripts
      - name: Prod equivalence re-sweep gate
        if: ${{ steps.verify.outputs.status == 'success' }}
        working-directory: showcase/scripts
        env:
          # The promoted closure subset that ACTUALLY pinned (best-effort) —
          # the set to re-sweep + compare. Same scope as the prod probe above.
          PROMOTED_CLOSURE_CSV: ${{ needs.promote.outputs.succeeded_csv }}
          RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
          RAILWAY_PROJECT_ID: ${{ vars.SHOWCASE_RAILWAY_PROJECT_ID }}
          RAILWAY_ENVIRONMENT_ID_PROD: ${{ vars.SHOWCASE_RAILWAY_ENV_ID_PROD }}
          PROD_POCKETBASE_URL: ${{ vars.SHOWCASE_PROD_POCKETBASE_URL }}
          PROD_POCKETBASE_SUPERUSER_EMAIL: ${{ secrets.SHOWCASE_PROD_POCKETBASE_SUPERUSER_EMAIL }}
          PROD_POCKETBASE_SUPERUSER_PASSWORD: ${{ secrets.SHOWCASE_PROD_POCKETBASE_SUPERUSER_PASSWORD }}
          STAGING_POCKETBASE_URL: ${{ vars.SHOWCASE_STAGING_POCKETBASE_URL }}
          STAGING_POCKETBASE_SUPERUSER_EMAIL: ${{ secrets.SHOWCASE_STAGING_POCKETBASE_SUPERUSER_EMAIL }}
          STAGING_POCKETBASE_SUPERUSER_PASSWORD: ${{ secrets.SHOWCASE_STAGING_POCKETBASE_SUPERUSER_PASSWORD }}
          # §4.4 annotation: flip to "false" once prod harness-workers are
          # provisioned-but-degraded is the default-true full-throughput path.
          PROD_HARNESS_WORKERS_PROVISIONED: ${{ vars.SHOWCASE_PROD_HARNESS_WORKERS_PROVISIONED }}
        run: |
          set -euo pipefail
          # OUT-OF-PR PREREQ GATE: the equivalence gate is inert until the prod
          # + staging PocketBase + prod Railway env-id config is wired (§8.1).
          # Absence is NOT a failure — the per-service prod probe above already
          # gated the run; this step just annotates that the equivalence gate
          # was skipped for lack of configuration.
          if [ -z "${PROD_POCKETBASE_URL:-}" ] || [ -z "${STAGING_POCKETBASE_URL:-}" ] || [ -z "${RAILWAY_ENVIRONMENT_ID_PROD:-}" ]; then
            echo "::notice::Prod equivalence gate not configured (prod/staging PocketBase + prod Railway env-id absent) — skipping the re-sweep gate. This is the §8.1 out-of-PR prerequisite; the per-service prod probe remains the gate."
            exit 0
          fi
          if [ -z "${PROMOTED_CLOSURE_CSV:-}" ]; then
            echo "::notice::no promoted closure to re-sweep — equivalence gate vacuously passes."
            exit 0
          fi
          # A gate FAILURE (a genuine staging-green / prod-not-green regression)
          # or a re-sweep timeout (REFUSE) exits non-zero → fails this step and
          # the run, exactly like the per-service probe above. `pnpm exec` (not
          # `npx`) so the dynamically-imported harness producer graph resolves
          # through the workspace node_modules the install step above created.
          pnpm exec tsx verify-prod-resweep.ts
      # ── Prod pinned-ness gate (UNIT U12, spec §8.2) ──────────────────────
      # COMPLEMENTS the equivalence gate above — it asserts PINNED-NESS, not
      # equivalence. After the promote pins the closure and prod is verified
      # healthy + equivalent, assert that EVERY prod service is on an immutable
      # `@sha256:` digest, not a mutable `:latest` tag. A born-on-:latest prod
      # service (deploy-to-railway.ts provisions an unpinned source.image, spec
      # R-E) can be healthy AND equivalent while still floating on a mutable tag
      # — a latent rollback/repro hazard the other gates do not catch. The gate
      # logic lives in the bats-tested showcase/scripts/lint-prod-gate.sh (a thin
      # `bin/railway lint-prod` wrapper) so it can't drift from its test (see
      # __tests__/lint-prod-gate.bats). Runs whenever prod was actually probed
      # (same `status == 'success'` guard as the equivalence gate); a non-zero
      # exit (an unpinned prod service, or a hard lint-prod error) fails the step
      # and the run. No `--exit-zero`: an unpinned prod service must red the run.
      - name: Prod pinned-ness gate (lint-prod)
        if: ${{ steps.verify.outputs.status == 'success' }}
        env:
          RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
        run: |
          set -euo pipefail
          if [ -z "$RAILWAY_TOKEN" ]; then
            echo "::error::RAILWAY_TOKEN is not set"
            exit 1
          fi
          showcase/scripts/lint-prod-gate.sh

  notify:
    # Slack #oss-alerts only. Never #engr (engr is sacred — release alerts only).
    needs: [resolve-targets, verify-staging-precondition, promote, verify-prod]
    if: always()
    runs-on: ubuntu-latest
    timeout-minutes: 3
    permissions:
      contents: read
      actions: read
    env:
      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
      SLACK_WEBHOOK_TS: ${{ secrets.SLACK_WEBHOOK_TEAM_SHOWCASE }}
    steps:
      # Best-effort promote invariant (do NOT reintroduce a PRE gate below):
      #   resolve-targets              = HARD precondition (must succeed).
      #   verify-staging-precondition  = ADVISORY only — surfaced in the Slack
      #                                  payload (`pre-staging`), never gates
      #                                  promote or run success. bin/railway
      #                                  enforces staging-green per-service, so
      #                                  PRE is a leading indicator, not a gate.
      #   promote                      = best-effort (exits non-zero iff a
      #                                  service failed).
      #   verify-prod                  = verifies the succeeded subset.
      #   run success                  = PROMOTE == success AND PROD == success.
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          persist-credentials: false
      - name: Compute state
        id: state
        env:
          RESOLVE: ${{ needs.resolve-targets.result }}
          PRE: ${{ needs.verify-staging-precondition.result }}
          PROMOTE: ${{ needs.promote.result }}
          PROD: ${{ needs.verify-prod.result }}
          # The verify-prod job's own status output: `success` (prod actually
          # probed + passed) or `skipped` (nothing promoted, so prod was never
          # touched). Empty when the job failed/cancelled or never wrote it.
          PROD_STATUS: ${{ needs.verify-prod.outputs.status }}
          CSV: ${{ needs.resolve-targets.outputs.services_csv }}
          INPUT: ${{ inputs.service }}
          # Aggregated staging-drift payload from the promote job (empty when no
          # service drifted). Surfaced in BOTH Slack payloads below so a drifted
          # promote is loud even on a successful run.
          DRIFT: ${{ needs.promote.outputs.staging_drift }}
        run: |
          set -euo pipefail
          # Displayed verify-prod value for the Slack line. The job `result` is
          # `success` for BOTH a real pass AND the empty-CSV skip (which exits
          # 0), so a bare `result` would read `verify-prod=success` even when
          # prod was never probed. The mapping (prefer the job's `status` output
          # when it ran cleanly, else fall back to the raw result) lives in the
          # bats-tested showcase/scripts/verify-prod-display.sh so it can't
          # drift from its test (see __tests__/verify-prod-display.bats).
          PROD_DISPLAY="$(PROD="$PROD" PROD_STATUS="$PROD_STATUS" showcase/scripts/verify-prod-display.sh)"
          if [ "$INPUT" = "__select_a_service__" ]; then
            # Deliberate no-op abort: a human clicked Run without picking a
            # service. resolve-targets exits 1 and the downstream jobs skip.
            # This is NOT a failed promote — emit a neutral state so no red
            # alert pages #oss-alerts.
            STATE="aborted"; ICON=":heavy_minus_sign:"
          elif [ "$PROMOTE" = "success" ] && [ "$PROD" = "success" ]; then
            # PRE (verify-staging-precondition) is ADVISORY and intentionally
            # NOT in this gate: bin/railway enforces staging-green per-service,
            # so a service that was staging-red at precondition time but still
            # promoted+verified cleanly must not red the run. PRE remains
            # surfaced in the Slack payload as the `pre-staging` field.
            STATE="success"; ICON=":white_check_mark:"
          elif [ "$RESOLVE" = "cancelled" ] || [ "$PRE" = "cancelled" ] || [ "$PROMOTE" = "cancelled" ] || [ "$PROD" = "cancelled" ]; then
            # A human cancelled the run (e.g. via the Actions UI). This is a
            # deliberate stop, not a failed promote — emit a neutral state so
            # no red alert pages #oss-alerts. Cancelling DURING resolve-targets
            # leaves the three downstream jobs `skipped` (not `cancelled`), so
            # we must consult RESOLVE here too or the cancel would fall through
            # to `failure` and fire a false red page.
            STATE="cancelled"; ICON=":heavy_minus_sign:"
          else
            STATE="failure"; ICON=":x:"
          fi
          # Human-facing drift suffix for the Slack message. Empty string when
          # there is no drift (renders as nothing); otherwise a loud prefix line
          # naming the drifted services + digests. MUST be single-line (no
          # embedded LF) so it survives the GITHUB_OUTPUT key=value contract: a
          # newline anywhere in $DRIFT (upstream free-form text) would otherwise
          # split the value across lines and corrupt/inject the whole output map.
          # We ENFORCE that here via `tr -d '\n'` rather than trusting upstream.
          DRIFT_LINE=""
          if [ -n "${DRIFT:-}" ]; then
            DRIFT_LINE="$(printf '%s' ":warning: STAGING DRIFT — prod pinned to staging's RUNNING digest, NOT current :latest — $DRIFT" | tr -d '\n')"
          fi
          {
            echo "state=$STATE"
            echo "icon=$ICON"
            echo "csv=$CSV"
            echo "prod_display=$PROD_DISPLAY"
            echo "drift_line=$DRIFT_LINE"
          } >> "$GITHUB_OUTPUT"
      - name: Post to #oss-alerts
        if: steps.state.outputs.state == 'failure' && inputs.service != '__select_a_service__' && env.SLACK_WEBHOOK != ''
        uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0
        with:
          webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
          webhook-type: incoming-webhook
          # Newlines are injected via fromJSON('"\n"') (a real LF char), NOT a
          # literal '\n' in the template: GitHub Actions expression string
          # literals do not interpret backslash escapes, so a literal '\n' would
          # survive toJSON as the two chars \\n and Slack would render it
          # verbatim as "\n" instead of a line break.
          payload: |
            {
              "text": ${{ toJSON(format(
                '{0} *Showcase Promote Failed*{9}Services: `{1}`{9}resolve-targets={2} pre-staging={3} promote={4} verify-prod={5}{10}{9}<{6}/{7}/actions/runs/{8}|View run>',
                steps.state.outputs.icon,
                steps.state.outputs.csv,
                needs.resolve-targets.result,
                needs.verify-staging-precondition.result,
                needs.promote.result,
                steps.state.outputs.prod_display,
                github.server_url,
                github.repository,
                github.run_id,
                fromJSON('"\n"'),
                steps.state.outputs.drift_line != '' && format('{0}{1}', fromJSON('"\n"'), steps.state.outputs.drift_line) || ''
              )) }}
            }
      - name: Post to #team-showcase
        if: steps.state.outputs.state == 'success' && inputs.service != '__select_a_service__' && env.SLACK_WEBHOOK_TS != ''
        uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0
        with:
          webhook: ${{ secrets.SLACK_WEBHOOK_TEAM_SHOWCASE }}
          webhook-type: incoming-webhook
          # Newlines via fromJSON('"\n"') (real LF), not a literal '\n' — see the
          # #oss-alerts step above for why a literal '\n' renders verbatim.
          payload: |
            {
              "text": ${{ toJSON(format(
                '{0} *Showcase Promoted to Prod*{5}Services: `{1}`{6}{5}<{2}/{3}/actions/runs/{4}|View run>',
                steps.state.outputs.icon,
                steps.state.outputs.csv,
                github.server_url,
                github.repository,
                github.run_id,
                fromJSON('"\n"'),
                steps.state.outputs.drift_line != '' && format('{0}{1}', fromJSON('"\n"'), steps.state.outputs.drift_line) || ''
              )) }}
            }
      - name: Log (no Slack — webhook unset)
        if: steps.state.outputs.state == 'failure' && inputs.service != '__select_a_service__' && env.SLACK_WEBHOOK == ''
        env:
          CSV: ${{ steps.state.outputs.csv }}
          DRIFT_LINE: ${{ steps.state.outputs.drift_line }}
        run: |
          echo "::warning::Showcase promote failed for '$CSV' but SLACK_WEBHOOK_OSS_ALERTS is not set; no Slack notification sent."
          # Preserve the staging-drift signal on the webhook-unset path (it would
          # otherwise only survive in the promote job's step summary, not here).
          if [ -n "$DRIFT_LINE" ]; then
            echo "::warning::$DRIFT_LINE"
          fi
      - name: Log (no Slack — team-showcase webhook unset)
        if: steps.state.outputs.state == 'success' && inputs.service != '__select_a_service__' && env.SLACK_WEBHOOK_TS == ''
        env:
          CSV: ${{ steps.state.outputs.csv }}
          ICON: ${{ steps.state.outputs.icon }}
          DRIFT_LINE: ${{ steps.state.outputs.drift_line }}
        run: |
          echo "::notice::$ICON Showcase promoted to prod for '$CSV' but SLACK_WEBHOOK_TEAM_SHOWCASE is not set; no #team-showcase notification sent."
          # Preserve the staging-drift signal on the webhook-unset path (it would
          # otherwise only survive in the promote job's step summary, not here).
          if [ -n "$DRIFT_LINE" ]; then
            echo "::warning::$DRIFT_LINE"
          fi