mirror of
https://github.com/CopilotKit/CopilotKit.git
synced 2026-09-14 16:26:20 +08:00
Jordan Ritter
bf6933a436
The showcase_promote.yml resolve-targets job runs `emit-railway-envs-json.ts`, which shells out to the repo-root `node_modules/.bin/oxfmt` to produce oxfmt-canonical JSON. That job's `npm ci` runs in showcase/scripts only and never installs the root oxfmt binary, so every promote dispatch died at "Generate SSOT artifact" with `spawnSync .../node_modules/.bin/oxfmt ENOENT` (exit 1) — blocking ALL promotes, including the team's regular shell-docs promote, since the last green run on 2026-06-18. The emitted JSON on the resolve-targets / promote path is EPHEMERAL: it is parsed in-memory by jq (resolve-promote-targets.sh) and bin/railway to pick the promote target and is NEVER committed, so oxfmt-canonical formatting is irrelevant there. Add an explicit `EMIT_SKIP_OXFMT=1` opt-out that returns the raw `JSON.stringify` form, and set it on both ephemeral workflow steps. The DEFAULT (committed-artifact) path is unchanged: oxfmt stays REQUIRED and fails loud if the binary is absent, because the committed railway-envs.generated.json must stay oxfmt-canonical or CI's static_quality.yml `oxfmt --check` auto-format bot fires on the drift. This is opt-IN-to-skip, never silent-on-absence. Call sites of emit-railway-envs-json.ts: - showcase_promote.yml resolve-targets — EMIT_SKIP_OXFMT=1 (this fix). - showcase_promote.yml promote — EMIT_SKIP_OXFMT=1 (this fix). - static_quality.yml committed-artifact `--check` — unset, oxfmt required. - resolve-verify-matrix.ts (showcase_deploy.yml) — only invokes the emitter when the committed JSON is absent; the checkout always has it, so the default (oxfmt) path is correct and unchanged. Tests: 2037 showcase/scripts tests pass; 2 new EMIT_SKIP_OXFMT unit tests assert the skip path emits valid (raw) JSON; the existing oxfmt-canonical golden tests still gate the committed path.
611 lines
32 KiB
YAML
611 lines
32 KiB
YAML
name: "Showcase: Promote (staging → prod)"
|
||
|
||
# Promotes the staging-tested digest of one or more services to prod.
|
||
# Workflow_dispatch only. Humans trigger. No automatic prod promotes.
|
||
#
|
||
# Order:
|
||
# 0. resolve-targets → expand the workflow_dispatch `service`
|
||
# input (SSOT key, dispatch_name, or
|
||
# 'all') into the canonical services_csv
|
||
# consumed by every downstream job.
|
||
# 1. verify-staging-precondition → live-probe staging for the target service(s).
|
||
# Refuse on red (matches bin/railway
|
||
# --require-staging-green default).
|
||
# 2. promote → bin/railway promote <service>; runs the
|
||
# spec §7 hardening (P1..P6).
|
||
# 3. verify-prod → verify-deploy.ts --env prod for the
|
||
# target service(s). Feature-level probes,
|
||
# not naked 200. Then the prod equivalence
|
||
# re-sweep gate (§6.2) and the pinned-ness
|
||
# gate (lint-prod, §8.2 — every prod service
|
||
# must be on an immutable @sha256: digest).
|
||
# 4. notify → Slack #oss-alerts on any red. Never #engr.
|
||
# success → #team-showcase.
|
||
|
||
on:
|
||
workflow_dispatch:
|
||
inputs:
|
||
service:
|
||
description: "Service to promote (dispatch_name or SSOT key). 'all' = whole fleet. Leave the placeholder to abort."
|
||
required: true
|
||
type: choice
|
||
# >>> BEGIN GENERATED service options (showcase/scripts/sync-promote-service-options.ts) — DO NOT EDIT
|
||
default: __select_a_service__
|
||
options:
|
||
- __select_a_service__
|
||
- all
|
||
- ag2
|
||
- agno
|
||
- built-in-agent
|
||
- claude-sdk-python
|
||
- claude-sdk-typescript
|
||
- crewai-crews
|
||
- google-adk
|
||
- langgraph-fastapi
|
||
- langgraph-python
|
||
- langgraph-typescript
|
||
- langroid
|
||
- llamaindex
|
||
- mastra
|
||
- ms-agent-dotnet
|
||
- ms-agent-harness-dotnet
|
||
- ms-agent-python
|
||
- pydantic-ai
|
||
- shell
|
||
- shell-dashboard
|
||
- shell-docs
|
||
- shell-dojo
|
||
- showcase-aimock
|
||
- showcase-harness
|
||
- showcase-pocketbase
|
||
- spring-ai
|
||
- starter-adk
|
||
- starter-agno
|
||
- starter-crewai-crews
|
||
- starter-langgraph-fastapi
|
||
- starter-langgraph-js
|
||
- starter-langgraph-python
|
||
- starter-llamaindex
|
||
- starter-mastra
|
||
- starter-ms-agent-framework-dotnet
|
||
- starter-ms-agent-framework-python
|
||
- starter-pydantic-ai
|
||
- starter-strands-python
|
||
- strands
|
||
- webhooks
|
||
# <<< END GENERATED service options
|
||
digest:
|
||
description: "Optional digest override (default: snapshot from staging)"
|
||
required: false
|
||
type: string
|
||
|
||
# Serialize ALL promotes on a single input-agnostic group so concurrent runs
|
||
# (e.g. `all` + a single-service promote) can't pin the same Railway service
|
||
# at once; `cancel-in-progress: false` ensures an in-flight prod promote is
|
||
# never cancelled by a newer run queued behind it.
|
||
concurrency:
|
||
group: showcase-promote
|
||
cancel-in-progress: false
|
||
|
||
permissions:
|
||
contents: read
|
||
|
||
jobs:
|
||
resolve-targets:
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 3
|
||
permissions:
|
||
contents: read
|
||
outputs:
|
||
# Leaf-set CSV — the BACKWARD-COMPAT promote target the promote job still
|
||
# drives off (Phase 1 behavior is unchanged).
|
||
services_csv: ${{ steps.resolve.outputs.services_csv }}
|
||
# Tier-ordered promote closure (requested ∪ transitive runtimeDeps ∪
|
||
# Tier-1 verification, §4.2). Phase 1 SURFACES it (operators see the
|
||
# closure in the step summary) but the actual promote still uses
|
||
# services_csv; U4 enforces tier ordering / dependent-gating off these.
|
||
closure_csv: ${{ steps.resolve.outputs.closure_csv }}
|
||
# Machine-readable `tier:name` form of the closure for U4 to consume.
|
||
closure_plan: ${{ steps.resolve.outputs.closure_plan }}
|
||
steps:
|
||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||
with:
|
||
persist-credentials: false
|
||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||
with:
|
||
node-version: 22.x
|
||
- name: Generate SSOT artifact
|
||
working-directory: showcase/scripts
|
||
# The emitted JSON is EPHEMERAL here: resolve-promote-targets.sh reads it
|
||
# with jq to resolve which service(s) to promote and it is NEVER
|
||
# committed, so oxfmt-canonical formatting is irrelevant. This job's
|
||
# `npm ci` runs in showcase/scripts only and does NOT install the
|
||
# repo-root oxfmt binary the committed-artifact path shells out to, so
|
||
# leave it canonical-free via EMIT_SKIP_OXFMT (else spawnSync ENOENT
|
||
# fails the step and blocks EVERY promote). The committed-artifact path
|
||
# (static_quality.yml) keeps oxfmt REQUIRED + fail-loud.
|
||
env:
|
||
EMIT_SKIP_OXFMT: "1"
|
||
run: |
|
||
npm ci
|
||
npx tsx emit-railway-envs-json.ts
|
||
- name: Resolve target service set
|
||
id: resolve
|
||
# Resolution + the tiered-closure derivation live in the bats-tested
|
||
# showcase/scripts/resolve-promote-targets.sh so they can't drift from
|
||
# their test (see __tests__/resolve-promote-targets.bats), mirroring how
|
||
# promote-fleet.sh / verify-prod-display.sh were extracted from this same
|
||
# workflow. The script preserves the existing guards (--digest+all
|
||
# reject, empty-`all` fail-loud, unknown/ambiguous/not-prod-eligible) and
|
||
# emits services_csv (leaf, backward-compat) + closure_csv/closure_plan
|
||
# (tiered closure) into $GITHUB_OUTPUT, plus the closure plan + skips into
|
||
# $GITHUB_STEP_SUMMARY.
|
||
env:
|
||
INPUT: ${{ inputs.service }}
|
||
DIGEST: ${{ inputs.digest }}
|
||
GENERATED: showcase/scripts/railway-envs.generated.json
|
||
run: showcase/scripts/resolve-promote-targets.sh
|
||
|
||
verify-staging-precondition:
|
||
needs: [resolve-targets]
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 15
|
||
environment: railway
|
||
permissions:
|
||
contents: read
|
||
steps:
|
||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||
with:
|
||
persist-credentials: false
|
||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||
with:
|
||
node-version: 22.x
|
||
- working-directory: showcase/scripts
|
||
run: npm ci
|
||
- name: Live-probe staging for promote precondition
|
||
working-directory: showcase/scripts
|
||
env:
|
||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
|
||
SERVICES_CSV: ${{ needs.resolve-targets.outputs.services_csv }}
|
||
run: |
|
||
if [ -z "$RAILWAY_TOKEN" ]; then
|
||
echo "::error::RAILWAY_TOKEN is not set"
|
||
exit 1
|
||
fi
|
||
# Spec §7.2 P3: the live staging probe at promote time is
|
||
# authoritative. CI verify history is a leading indicator only.
|
||
# Run from showcase/scripts (where `npm ci` installed tsx) so npx
|
||
# uses the local install instead of network-fetching it.
|
||
npx tsx verify-deploy.ts --env staging --services "$SERVICES_CSV"
|
||
|
||
promote:
|
||
needs: [resolve-targets, verify-staging-precondition]
|
||
# Do NOT hard-depend on verify-staging-precondition's RESULT. That job
|
||
# probes the FULL requested set all-or-nothing, so a single staging-red
|
||
# service (e.g. a chronically-broken integration in `all`) fails it and
|
||
# — under the default `if: success()` — would SKIP promote entirely,
|
||
# re-blocking the whole fleet. bin/railway enforces staging-green
|
||
# PER-SERVICE (spec §7 P2 = latest staging deploy must be SUCCESS, P3 =
|
||
# live staging probe, default-on), so a red service is refused on its own
|
||
# and lands in promote-fleet.sh's failed set (reds the run) without
|
||
# taking the greens down with it. verify-staging-precondition therefore
|
||
# stays as an advisory early signal surfaced in the notify payload, not a
|
||
# hard blocker. `!cancelled()` keeps a human-cancelled run from promoting;
|
||
# mirrors the verify-prod gate idiom below.
|
||
if: ${{ !cancelled() && needs.resolve-targets.result == 'success' }}
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 20
|
||
environment: railway
|
||
permissions:
|
||
contents: read
|
||
packages: read
|
||
outputs:
|
||
# CSV of the services that ACTUALLY promoted (best-effort: a partial
|
||
# failure still exposes the succeeded subset). verify-prod scopes its
|
||
# prod verification to exactly this set so a single failed service can't
|
||
# red the verification of the services that did promote.
|
||
succeeded_csv: ${{ steps.promote.outputs.succeeded_csv }}
|
||
# Aggregated staging-drift payload: non-empty when any promoted service's
|
||
# staging RUNNING digest differed from the current GHCR :latest (i.e.
|
||
# staging was NOT serving :latest). Folded into the Slack notify payload
|
||
# so operators see "what shipped to prod is not current :latest".
|
||
staging_drift: ${{ steps.promote.outputs.staging_drift }}
|
||
steps:
|
||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||
with:
|
||
persist-credentials: false
|
||
- uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
|
||
with:
|
||
ruby-version: "3.3"
|
||
bundler-cache: false
|
||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||
with:
|
||
node-version: 22.x
|
||
- working-directory: showcase/scripts
|
||
# Same EPHEMERAL regeneration as resolve-targets: bin/railway reads this
|
||
# JSON to derive EXPECTED_DOMAINS and it is never committed, so skip the
|
||
# oxfmt-canonical pass (repo-root oxfmt is not installed by this job's
|
||
# showcase/scripts-scoped `npm ci`; an ENOENT here would abort promote).
|
||
env:
|
||
EMIT_SKIP_OXFMT: "1"
|
||
run: |
|
||
npm ci
|
||
npx tsx emit-railway-envs-json.ts
|
||
- name: bin/railway promote
|
||
id: promote
|
||
env:
|
||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
|
||
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||
SERVICES_CSV: ${{ needs.resolve-targets.outputs.services_csv }}
|
||
DIGEST: ${{ inputs.digest }}
|
||
run: |
|
||
set -euo pipefail
|
||
if [ -z "$RAILWAY_TOKEN" ]; then
|
||
echo "::error::RAILWAY_TOKEN is not set"
|
||
exit 1
|
||
fi
|
||
# promote-fleet.sh runs each service in turn BEST-EFFORT: a single
|
||
# red service (e.g. a chronically-broken integration in the `all`
|
||
# set) must not abort promotion of the rest of the fleet. The script
|
||
# attempts every service, aggregates the succeeded/failed sets, emits
|
||
# a step summary, and exits non-zero iff ANY service failed (so the
|
||
# notify job still fires) — but only AFTER attempting all of them.
|
||
# bin/railway itself handles spec §7 preconditions (P1..P6);
|
||
# --require-staging-green is default-on and the prior job already
|
||
# established staging is green (defense in depth). We deliberately do
|
||
# NOT pass --confirm-divergence: WARN-divergence refusals are a real
|
||
# signal that must fail the run.
|
||
RAILWAY_BIN="showcase/bin/railway" \
|
||
showcase/scripts/promote-fleet.sh
|
||
|
||
verify-prod:
|
||
needs: [resolve-targets, promote]
|
||
# Run whenever promote actually RAN — success OR partial failure — so the
|
||
# services that DID promote still get prod verification. `!cancelled()`
|
||
# excludes a human-cancelled run; `needs.promote.result != 'skipped'`
|
||
# excludes the case where promote never ran (e.g. an upstream abort/skip).
|
||
# Under the default `if: success()` this job was SKIPPED on any partial
|
||
# promote failure, leaving the promoted services with zero prod
|
||
# verification — that is the bug this gate fixes.
|
||
if: ${{ !cancelled() && needs.promote.result != 'skipped' }}
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 20
|
||
environment: railway
|
||
permissions:
|
||
contents: read
|
||
outputs:
|
||
# Distinguishes a job that actually PROBED prod (`success`) from one that
|
||
# SKIPPED probing because nothing promoted (`skipped`). The GitHub job
|
||
# `result` is `success` in BOTH cases (the skip path exits 0), so the
|
||
# notify job must read THIS output — not `needs.verify-prod.result` — to
|
||
# avoid reporting a misleading `verify-prod=success` when prod was never
|
||
# touched. A real probe failure / contract violation exits non-zero, so
|
||
# the job `result` becomes `failure` and this output is never written
|
||
# (notify falls back to the job result for that case).
|
||
status: ${{ steps.verify.outputs.status }}
|
||
steps:
|
||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||
with:
|
||
persist-credentials: false
|
||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||
with:
|
||
node-version: 22.x
|
||
- working-directory: showcase/scripts
|
||
run: npm ci
|
||
- name: Run verify-deploy --env prod
|
||
id: verify
|
||
working-directory: showcase/scripts
|
||
env:
|
||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
|
||
# Scope verification to the services that ACTUALLY promoted (from the
|
||
# promote job's best-effort succeeded set), NOT the full requested set
|
||
# (resolve-targets) — verifying a service that failed to promote would
|
||
# guarantee a red verify and mask the health of the ones that did
|
||
# promote.
|
||
SERVICES_CSV: ${{ needs.promote.outputs.succeeded_csv }}
|
||
# The promote job's result, so the empty-CSV branch can tell a genuine
|
||
# all-failed run (empty succeeded set is expected) apart from a
|
||
# contract violation (promote reported success yet emitted no CSV).
|
||
PROMOTE_RESULT: ${{ needs.promote.result }}
|
||
run: |
|
||
set -euo pipefail
|
||
if [ -z "$RAILWAY_TOKEN" ]; then
|
||
echo "::error::RAILWAY_TOKEN is not set"
|
||
exit 1
|
||
fi
|
||
# If NOTHING promoted (every requested service failed, so the
|
||
# succeeded set is empty), there is nothing to verify — skip with a
|
||
# clear log line rather than calling verify-deploy.ts with an empty
|
||
# --services (which would either error or vacuously pass). The promote
|
||
# job already exited non-zero in that case, so the overall run is red
|
||
# via the notify state machine regardless.
|
||
#
|
||
# BUT: an empty succeeded set is only legitimate when promote did NOT
|
||
# succeed. If promote reported success and STILL emitted no CSV, the
|
||
# "promote already failed" assumption that justifies the vacuous skip
|
||
# is violated — fail loud instead of silently exiting 0.
|
||
if [ -z "$SERVICES_CSV" ]; then
|
||
if [ "$PROMOTE_RESULT" = "success" ]; then
|
||
echo "::error::promote reported success but succeeded_csv is empty — contract violation"
|
||
exit 1
|
||
fi
|
||
echo "::notice::succeeded_csv is empty (no services promoted, or promote did not run); skipping prod verification. The run is red via the promote job result if anything failed."
|
||
# Record that prod was SKIPPED, not verified. The job still exits 0
|
||
# (its `result` is `success`), so the notify job reads this `status`
|
||
# output to render `verify-prod=skipped` instead of a misleading
|
||
# `verify-prod=success`.
|
||
echo "status=skipped" >> "$GITHUB_OUTPUT"
|
||
exit 0
|
||
fi
|
||
# Run from showcase/scripts (where `npm ci` installed tsx) so npx uses
|
||
# the local install instead of network-fetching it. A non-zero exit
|
||
# here fails the step (job `result` = failure) and `status` is never
|
||
# written, so notify falls back to the job result.
|
||
npx tsx verify-deploy.ts --env prod --services "$SERVICES_CSV"
|
||
# Prod was actually probed and passed.
|
||
echo "status=success" >> "$GITHUB_OUTPUT"
|
||
# ── Prod equivalence gate (UNIT U10, spec §6.2) ──────────────────────
|
||
# After the closure pins (promote job) AND the per-service prod probe
|
||
# (above), re-sweep the promoted integration closure on the PROD control
|
||
# plane (prod harness scheduler + prod harness-workers, or scheduler
|
||
# INLINE fallback when workers are unprovisioned — §4.4) and run U9's
|
||
# equivalence gate over the FRESH prod rows vs current staging. Promote
|
||
# success flips to the equivalence definition (fail only on
|
||
# staging-green / prod-not-green, gray/stale excluded, one-directional).
|
||
#
|
||
# GATED ON CONFIGURATION: the equivalence gate needs prod + staging
|
||
# PocketBase creds and the prod Railway environment id. Those are an
|
||
# out-of-PR operational prerequisite (§8.1) — until the secrets are wired
|
||
# the step ANNOTATES "not configured" and is a no-op, so this job's
|
||
# existing per-service prod verification (above) remains the gate. Once
|
||
# configured, a gate FAILURE fails the step (and the run).
|
||
# Install the pnpm workspace ONLY when the equivalence gate is configured
|
||
# (the gate step's enqueue dynamically imports the harness producer graph,
|
||
# which lives in the pnpm workspace — `npm ci` in showcase/scripts alone
|
||
# does not resolve it). `--ignore-scripts` skips postinstall/Playwright
|
||
# browser downloads: the host only ENQUEUES + POLLS prod PocketBase; the
|
||
# prod harness-workers own the browser. Gated on the same config check as
|
||
# the gate step so the no-op (unconfigured) path stays cheap.
|
||
- name: Set up pnpm (equivalence gate only)
|
||
if: ${{ steps.verify.outputs.status == 'success' && vars.SHOWCASE_PROD_POCKETBASE_URL != '' && vars.SHOWCASE_STAGING_POCKETBASE_URL != '' && vars.SHOWCASE_RAILWAY_ENV_ID_PROD != '' }}
|
||
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
|
||
- name: pnpm install (equivalence gate only)
|
||
if: ${{ steps.verify.outputs.status == 'success' && vars.SHOWCASE_PROD_POCKETBASE_URL != '' && vars.SHOWCASE_STAGING_POCKETBASE_URL != '' && vars.SHOWCASE_RAILWAY_ENV_ID_PROD != '' }}
|
||
run: pnpm install --ignore-scripts
|
||
- name: Prod equivalence re-sweep gate
|
||
if: ${{ steps.verify.outputs.status == 'success' }}
|
||
working-directory: showcase/scripts
|
||
env:
|
||
# The promoted closure subset that ACTUALLY pinned (best-effort) —
|
||
# the set to re-sweep + compare. Same scope as the prod probe above.
|
||
PROMOTED_CLOSURE_CSV: ${{ needs.promote.outputs.succeeded_csv }}
|
||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
|
||
RAILWAY_PROJECT_ID: ${{ vars.SHOWCASE_RAILWAY_PROJECT_ID }}
|
||
RAILWAY_ENVIRONMENT_ID_PROD: ${{ vars.SHOWCASE_RAILWAY_ENV_ID_PROD }}
|
||
PROD_POCKETBASE_URL: ${{ vars.SHOWCASE_PROD_POCKETBASE_URL }}
|
||
PROD_POCKETBASE_SUPERUSER_EMAIL: ${{ secrets.SHOWCASE_PROD_POCKETBASE_SUPERUSER_EMAIL }}
|
||
PROD_POCKETBASE_SUPERUSER_PASSWORD: ${{ secrets.SHOWCASE_PROD_POCKETBASE_SUPERUSER_PASSWORD }}
|
||
STAGING_POCKETBASE_URL: ${{ vars.SHOWCASE_STAGING_POCKETBASE_URL }}
|
||
STAGING_POCKETBASE_SUPERUSER_EMAIL: ${{ secrets.SHOWCASE_STAGING_POCKETBASE_SUPERUSER_EMAIL }}
|
||
STAGING_POCKETBASE_SUPERUSER_PASSWORD: ${{ secrets.SHOWCASE_STAGING_POCKETBASE_SUPERUSER_PASSWORD }}
|
||
# §4.4 annotation: flip to "false" once prod harness-workers are
|
||
# provisioned-but-degraded is the default-true full-throughput path.
|
||
PROD_HARNESS_WORKERS_PROVISIONED: ${{ vars.SHOWCASE_PROD_HARNESS_WORKERS_PROVISIONED }}
|
||
run: |
|
||
set -euo pipefail
|
||
# OUT-OF-PR PREREQ GATE: the equivalence gate is inert until the prod
|
||
# + staging PocketBase + prod Railway env-id config is wired (§8.1).
|
||
# Absence is NOT a failure — the per-service prod probe above already
|
||
# gated the run; this step just annotates that the equivalence gate
|
||
# was skipped for lack of configuration.
|
||
if [ -z "${PROD_POCKETBASE_URL:-}" ] || [ -z "${STAGING_POCKETBASE_URL:-}" ] || [ -z "${RAILWAY_ENVIRONMENT_ID_PROD:-}" ]; then
|
||
echo "::notice::Prod equivalence gate not configured (prod/staging PocketBase + prod Railway env-id absent) — skipping the re-sweep gate. This is the §8.1 out-of-PR prerequisite; the per-service prod probe remains the gate."
|
||
exit 0
|
||
fi
|
||
if [ -z "${PROMOTED_CLOSURE_CSV:-}" ]; then
|
||
echo "::notice::no promoted closure to re-sweep — equivalence gate vacuously passes."
|
||
exit 0
|
||
fi
|
||
# A gate FAILURE (a genuine staging-green / prod-not-green regression)
|
||
# or a re-sweep timeout (REFUSE) exits non-zero → fails this step and
|
||
# the run, exactly like the per-service probe above. `pnpm exec` (not
|
||
# `npx`) so the dynamically-imported harness producer graph resolves
|
||
# through the workspace node_modules the install step above created.
|
||
pnpm exec tsx verify-prod-resweep.ts
|
||
# ── Prod pinned-ness gate (UNIT U12, spec §8.2) ──────────────────────
|
||
# COMPLEMENTS the equivalence gate above — it asserts PINNED-NESS, not
|
||
# equivalence. After the promote pins the closure and prod is verified
|
||
# healthy + equivalent, assert that EVERY prod service is on an immutable
|
||
# `@sha256:` digest, not a mutable `:latest` tag. A born-on-:latest prod
|
||
# service (deploy-to-railway.ts provisions an unpinned source.image, spec
|
||
# R-E) can be healthy AND equivalent while still floating on a mutable tag
|
||
# — a latent rollback/repro hazard the other gates do not catch. The gate
|
||
# logic lives in the bats-tested showcase/scripts/lint-prod-gate.sh (a thin
|
||
# `bin/railway lint-prod` wrapper) so it can't drift from its test (see
|
||
# __tests__/lint-prod-gate.bats). Runs whenever prod was actually probed
|
||
# (same `status == 'success'` guard as the equivalence gate); a non-zero
|
||
# exit (an unpinned prod service, or a hard lint-prod error) fails the step
|
||
# and the run. No `--exit-zero`: an unpinned prod service must red the run.
|
||
- name: Prod pinned-ness gate (lint-prod)
|
||
if: ${{ steps.verify.outputs.status == 'success' }}
|
||
env:
|
||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
|
||
run: |
|
||
set -euo pipefail
|
||
if [ -z "$RAILWAY_TOKEN" ]; then
|
||
echo "::error::RAILWAY_TOKEN is not set"
|
||
exit 1
|
||
fi
|
||
showcase/scripts/lint-prod-gate.sh
|
||
|
||
notify:
|
||
# Slack #oss-alerts only. Never #engr (engr is sacred — release alerts only).
|
||
needs: [resolve-targets, verify-staging-precondition, promote, verify-prod]
|
||
if: always()
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 3
|
||
permissions:
|
||
contents: read
|
||
actions: read
|
||
env:
|
||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
||
SLACK_WEBHOOK_TS: ${{ secrets.SLACK_WEBHOOK_TEAM_SHOWCASE }}
|
||
steps:
|
||
# Best-effort promote invariant (do NOT reintroduce a PRE gate below):
|
||
# resolve-targets = HARD precondition (must succeed).
|
||
# verify-staging-precondition = ADVISORY only — surfaced in the Slack
|
||
# payload (`pre-staging`), never gates
|
||
# promote or run success. bin/railway
|
||
# enforces staging-green per-service, so
|
||
# PRE is a leading indicator, not a gate.
|
||
# promote = best-effort (exits non-zero iff a
|
||
# service failed).
|
||
# verify-prod = verifies the succeeded subset.
|
||
# run success = PROMOTE == success AND PROD == success.
|
||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||
with:
|
||
persist-credentials: false
|
||
- name: Compute state
|
||
id: state
|
||
env:
|
||
RESOLVE: ${{ needs.resolve-targets.result }}
|
||
PRE: ${{ needs.verify-staging-precondition.result }}
|
||
PROMOTE: ${{ needs.promote.result }}
|
||
PROD: ${{ needs.verify-prod.result }}
|
||
# The verify-prod job's own status output: `success` (prod actually
|
||
# probed + passed) or `skipped` (nothing promoted, so prod was never
|
||
# touched). Empty when the job failed/cancelled or never wrote it.
|
||
PROD_STATUS: ${{ needs.verify-prod.outputs.status }}
|
||
CSV: ${{ needs.resolve-targets.outputs.services_csv }}
|
||
INPUT: ${{ inputs.service }}
|
||
# Aggregated staging-drift payload from the promote job (empty when no
|
||
# service drifted). Surfaced in BOTH Slack payloads below so a drifted
|
||
# promote is loud even on a successful run.
|
||
DRIFT: ${{ needs.promote.outputs.staging_drift }}
|
||
run: |
|
||
set -euo pipefail
|
||
# Displayed verify-prod value for the Slack line. The job `result` is
|
||
# `success` for BOTH a real pass AND the empty-CSV skip (which exits
|
||
# 0), so a bare `result` would read `verify-prod=success` even when
|
||
# prod was never probed. The mapping (prefer the job's `status` output
|
||
# when it ran cleanly, else fall back to the raw result) lives in the
|
||
# bats-tested showcase/scripts/verify-prod-display.sh so it can't
|
||
# drift from its test (see __tests__/verify-prod-display.bats).
|
||
PROD_DISPLAY="$(PROD="$PROD" PROD_STATUS="$PROD_STATUS" showcase/scripts/verify-prod-display.sh)"
|
||
if [ "$INPUT" = "__select_a_service__" ]; then
|
||
# Deliberate no-op abort: a human clicked Run without picking a
|
||
# service. resolve-targets exits 1 and the downstream jobs skip.
|
||
# This is NOT a failed promote — emit a neutral state so no red
|
||
# alert pages #oss-alerts.
|
||
STATE="aborted"; ICON=":heavy_minus_sign:"
|
||
elif [ "$PROMOTE" = "success" ] && [ "$PROD" = "success" ]; then
|
||
# PRE (verify-staging-precondition) is ADVISORY and intentionally
|
||
# NOT in this gate: bin/railway enforces staging-green per-service,
|
||
# so a service that was staging-red at precondition time but still
|
||
# promoted+verified cleanly must not red the run. PRE remains
|
||
# surfaced in the Slack payload as the `pre-staging` field.
|
||
STATE="success"; ICON=":white_check_mark:"
|
||
elif [ "$RESOLVE" = "cancelled" ] || [ "$PRE" = "cancelled" ] || [ "$PROMOTE" = "cancelled" ] || [ "$PROD" = "cancelled" ]; then
|
||
# A human cancelled the run (e.g. via the Actions UI). This is a
|
||
# deliberate stop, not a failed promote — emit a neutral state so
|
||
# no red alert pages #oss-alerts. Cancelling DURING resolve-targets
|
||
# leaves the three downstream jobs `skipped` (not `cancelled`), so
|
||
# we must consult RESOLVE here too or the cancel would fall through
|
||
# to `failure` and fire a false red page.
|
||
STATE="cancelled"; ICON=":heavy_minus_sign:"
|
||
else
|
||
STATE="failure"; ICON=":x:"
|
||
fi
|
||
# Human-facing drift suffix for the Slack message. Empty string when
|
||
# there is no drift (renders as nothing); otherwise a loud prefix line
|
||
# naming the drifted services + digests. MUST be single-line (no
|
||
# embedded LF) so it survives the GITHUB_OUTPUT key=value contract: a
|
||
# newline anywhere in $DRIFT (upstream free-form text) would otherwise
|
||
# split the value across lines and corrupt/inject the whole output map.
|
||
# We ENFORCE that here via `tr -d '\n'` rather than trusting upstream.
|
||
DRIFT_LINE=""
|
||
if [ -n "${DRIFT:-}" ]; then
|
||
DRIFT_LINE="$(printf '%s' ":warning: STAGING DRIFT — prod pinned to staging's RUNNING digest, NOT current :latest — $DRIFT" | tr -d '\n')"
|
||
fi
|
||
{
|
||
echo "state=$STATE"
|
||
echo "icon=$ICON"
|
||
echo "csv=$CSV"
|
||
echo "prod_display=$PROD_DISPLAY"
|
||
echo "drift_line=$DRIFT_LINE"
|
||
} >> "$GITHUB_OUTPUT"
|
||
- name: Post to #oss-alerts
|
||
if: steps.state.outputs.state == 'failure' && inputs.service != '__select_a_service__' && env.SLACK_WEBHOOK != ''
|
||
uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0
|
||
with:
|
||
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
||
webhook-type: incoming-webhook
|
||
# Newlines are injected via fromJSON('"\n"') (a real LF char), NOT a
|
||
# literal '\n' in the template: GitHub Actions expression string
|
||
# literals do not interpret backslash escapes, so a literal '\n' would
|
||
# survive toJSON as the two chars \\n and Slack would render it
|
||
# verbatim as "\n" instead of a line break.
|
||
payload: |
|
||
{
|
||
"text": ${{ toJSON(format(
|
||
'{0} *Showcase Promote Failed*{9}Services: `{1}`{9}resolve-targets={2} pre-staging={3} promote={4} verify-prod={5}{10}{9}<{6}/{7}/actions/runs/{8}|View run>',
|
||
steps.state.outputs.icon,
|
||
steps.state.outputs.csv,
|
||
needs.resolve-targets.result,
|
||
needs.verify-staging-precondition.result,
|
||
needs.promote.result,
|
||
steps.state.outputs.prod_display,
|
||
github.server_url,
|
||
github.repository,
|
||
github.run_id,
|
||
fromJSON('"\n"'),
|
||
steps.state.outputs.drift_line != '' && format('{0}{1}', fromJSON('"\n"'), steps.state.outputs.drift_line) || ''
|
||
)) }}
|
||
}
|
||
- name: Post to #team-showcase
|
||
if: steps.state.outputs.state == 'success' && inputs.service != '__select_a_service__' && env.SLACK_WEBHOOK_TS != ''
|
||
uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0
|
||
with:
|
||
webhook: ${{ secrets.SLACK_WEBHOOK_TEAM_SHOWCASE }}
|
||
webhook-type: incoming-webhook
|
||
# Newlines via fromJSON('"\n"') (real LF), not a literal '\n' — see the
|
||
# #oss-alerts step above for why a literal '\n' renders verbatim.
|
||
payload: |
|
||
{
|
||
"text": ${{ toJSON(format(
|
||
'{0} *Showcase Promoted to Prod*{5}Services: `{1}`{6}{5}<{2}/{3}/actions/runs/{4}|View run>',
|
||
steps.state.outputs.icon,
|
||
steps.state.outputs.csv,
|
||
github.server_url,
|
||
github.repository,
|
||
github.run_id,
|
||
fromJSON('"\n"'),
|
||
steps.state.outputs.drift_line != '' && format('{0}{1}', fromJSON('"\n"'), steps.state.outputs.drift_line) || ''
|
||
)) }}
|
||
}
|
||
- name: Log (no Slack — webhook unset)
|
||
if: steps.state.outputs.state == 'failure' && inputs.service != '__select_a_service__' && env.SLACK_WEBHOOK == ''
|
||
env:
|
||
CSV: ${{ steps.state.outputs.csv }}
|
||
DRIFT_LINE: ${{ steps.state.outputs.drift_line }}
|
||
run: |
|
||
echo "::warning::Showcase promote failed for '$CSV' but SLACK_WEBHOOK_OSS_ALERTS is not set; no Slack notification sent."
|
||
# Preserve the staging-drift signal on the webhook-unset path (it would
|
||
# otherwise only survive in the promote job's step summary, not here).
|
||
if [ -n "$DRIFT_LINE" ]; then
|
||
echo "::warning::$DRIFT_LINE"
|
||
fi
|
||
- name: Log (no Slack — team-showcase webhook unset)
|
||
if: steps.state.outputs.state == 'success' && inputs.service != '__select_a_service__' && env.SLACK_WEBHOOK_TS == ''
|
||
env:
|
||
CSV: ${{ steps.state.outputs.csv }}
|
||
ICON: ${{ steps.state.outputs.icon }}
|
||
DRIFT_LINE: ${{ steps.state.outputs.drift_line }}
|
||
run: |
|
||
echo "::notice::$ICON Showcase promoted to prod for '$CSV' but SLACK_WEBHOOK_TEAM_SHOWCASE is not set; no #team-showcase notification sent."
|
||
# Preserve the staging-drift signal on the webhook-unset path (it would
|
||
# otherwise only survive in the promote job's step summary, not here).
|
||
if [ -n "$DRIFT_LINE" ]; then
|
||
echo "::warning::$DRIFT_LINE"
|
||
fi
|