Files
copilotkit__copilotkit/.github/workflows/showcase_build.yml
2026-08-20 13:58:50 +00:00

1691 lines
94 KiB
YAML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
name: "Showcase: Build & Push"
# Decoupled from the old "Build & Deploy" workflow. This workflow builds
# Docker images, pushes them to GHCR, and triggers Railway to redeploy.
# The separate "Showcase: Verify Deploy" workflow (showcase_deploy.yml)
# handles health verification.
#
# Critical design property: NO concurrency group with cancel-in-progress.
# Every push to main runs to completion so that rapid-fire PR merges never
# cancel in-flight builds. This was the #1 operational pain point with the
# old combined workflow.
#
# This is NOT merely a convenience: `detect-changes` builds a per-push,
# path-filtered matrix, so concurrent runs build OVERLAPPING BUT NON-IDENTICAL
# service sets. Cancelling an older run therefore drops the services only IT
# builds — trading a stale-image bug for a never-shipped bug. Concurrent runs
# are not redundant and must not be cancelled.
#
# The corollary, and the thing that bit us on 2026-07-26: if runs are never
# serialized, they RACE on the one shared mutable resource — the `:latest`
# tag. Three merges landed in 34s, the newest commit's build finished first,
# and the two older builds overwrote its `:latest` on every shared service.
# All three runs reported `success`. Staging served pre-#6158 code while CI,
# the redeploy gate, and deploy verification all looked clean.
#
# The fix is NOT a concurrency group (see above — that breaks the matrix
# contract). Instead the mutable pointer is made monotonic: the build step
# pushes ONLY the immutable `:<sha>` tag, and a guarded step moves `:latest`
# unless the tag already holds a DESCENDANT of the commit being built. See
# showcase/scripts/advance-latest-tag.ts for the incident reconstruction, the
# decision table, and the residual sub-second TOCTOU this does not close.
on:
push:
branches: [main]
paths:
- "showcase/**"
- "examples/integrations/**"
- "packages/a2ui-renderer/**"
- "packages/angular/**"
- "packages/core/**"
- "packages/shared/**"
- "packages/web-components/**"
- ".github/workflows/showcase_build.yml"
- ".github/workflows/showcase_build_check.yml"
workflow_dispatch:
inputs:
service:
description: "Service to build"
required: false
default: "all"
type: choice
options:
- all
- shell
- langgraph-python
- mastra
- crewai-crews
- crewai-conversational-flows
- pydantic-ai
- google-adk
- ag2
- agno
- llamaindex
- langgraph-fastapi
- langgraph-typescript
- langroid
- spring-ai
- strands
- strands-typescript
- ms-agent-python
- claude-sdk-typescript
- ms-agent-dotnet
- ms-agent-harness-dotnet
- claude-sdk-python
- built-in-agent
- shell-dojo
- shell-dashboard
- shell-docs
- showcase-harness
- showcase-aimock
- showcase-pocketbase
- webhooks
# Per-starter image builds (model B, §b stage 1). These map to the
# build-starters job below, NOT the showcase `build` job. "all"
# builds the full showcase fleet AND all 12 starters; a specific
# starter slug narrows to that one starter via STARTER_DISPATCH.
- starter-langgraph-python
- starter-mastra
- starter-langgraph-js
- starter-crewai-crews
- starter-pydantic-ai
- starter-adk
- starter-agno
- starter-llamaindex
- starter-langgraph-fastapi
- starter-strands-python
- starter-ms-agent-framework-python
- starter-ms-agent-framework-dotnet
# No top-level concurrency group. Every build run completes. This is the
# whole point of the decoupling: rapid pushes to main no longer cancel
# in-flight builds.
# Top-level env intentionally empty: env IDs are resolved inside
# showcase/scripts/redeploy-env.ts from the showcase/scripts/railway-envs.ts
# SSOT (and its emitted railway-envs.generated.json), not by this workflow.
permissions:
contents: read
jobs:
detect-changes:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
outputs:
matrix: ${{ steps.build-matrix.outputs.matrix }}
has_changes: ${{ steps.build-matrix.outputs.has_changes }}
needs_angular: ${{ steps.build-matrix.outputs.needs_angular }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Detect changed paths
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
id: filter
with:
# All filter values use list form for consistency. `shell`
# genuinely needs multiple paths; the others could collapse to
# single-line strings, but mixing styles (list vs. string)
# in the same filters block is easy to misread during review.
filters: |
workflow_config:
- '.github/workflows/showcase_build.yml'
- '.github/workflows/showcase_build_check.yml'
angular:
- 'showcase/angular/**'
- 'packages/a2ui-renderer/**'
- 'packages/angular/**'
- 'packages/core/**'
- 'packages/shared/**'
- 'packages/web-components/**'
shell:
- 'showcase/shell/**'
- 'showcase/shared/**'
- 'showcase/scripts/**'
- 'showcase/integrations/*/manifest.yaml'
langgraph_python:
- 'showcase/integrations/langgraph-python/**'
mastra:
- 'showcase/integrations/mastra/**'
crewai_crews:
- 'showcase/integrations/crewai-crews/**'
crewai_conversational_flows:
- 'showcase/integrations/crewai-conversational-flows/**'
pydantic_ai:
- 'showcase/integrations/pydantic-ai/**'
google_adk:
- 'showcase/integrations/google-adk/**'
ag2:
- 'showcase/integrations/ag2/**'
agno:
- 'showcase/integrations/agno/**'
llamaindex:
- 'showcase/integrations/llamaindex/**'
langgraph_fastapi:
- 'showcase/integrations/langgraph-fastapi/**'
langgraph_typescript:
- 'showcase/integrations/langgraph-typescript/**'
langroid:
- 'showcase/integrations/langroid/**'
spring_ai:
- 'showcase/integrations/spring-ai/**'
strands:
- 'showcase/integrations/strands/**'
strands_typescript:
- 'showcase/integrations/strands-typescript/**'
ms_agent_python:
- 'showcase/integrations/ms-agent-python/**'
claude_sdk_typescript:
- 'showcase/integrations/claude-sdk-typescript/**'
ms_agent_dotnet:
- 'showcase/integrations/ms-agent-dotnet/**'
ms_agent_harness_dotnet:
- 'showcase/integrations/ms-agent-harness-dotnet/**'
claude_sdk_python:
- 'showcase/integrations/claude-sdk-python/**'
built_in_agent:
- 'showcase/integrations/built-in-agent/**'
shell_dojo:
- 'showcase/shell-dojo/**'
- 'showcase/shared/**'
- 'showcase/scripts/**'
- 'showcase/integrations/*/manifest.yaml'
shell_dashboard:
- 'showcase/shell-dashboard/**'
- 'showcase/shared/**'
- 'showcase/scripts/**'
- 'showcase/integrations/*/manifest.yaml'
# The dashboard re-exports the harness's shared cell-model fold
# (src/lib/{cell-model,live-status,staleness,format-ts}.ts →
# ../../../harness/src/shared/cell-model/*), so those files are
# compiled INTO the dashboard's `next build`. A change to the
# fold that only touches showcase/harness/** would otherwise
# select `showcase_harness` but NOT this slot — exactly how
# #5952 shipped an unbuilt dashboard. Gate the dashboard on the
# fold so a fold change can never again bypass its build.
- 'showcase/harness/src/shared/**'
shell_docs:
- 'showcase/shell-docs/**'
- 'showcase/shared/**'
- 'showcase/scripts/**'
- 'showcase/integrations/*/manifest.yaml'
- 'showcase/integrations/*/docs-links.json'
- 'showcase/integrations/*/docs/setup/**'
- 'showcase/integrations/*/src/**'
showcase_harness:
- 'showcase/harness/**'
- 'showcase/shared/**'
- 'showcase/scripts/**'
- 'showcase/integrations/*/manifest.yaml'
showcase_aimock:
- 'showcase/aimock/**'
pocketbase:
# PB image is self-contained: PB binary + pb_migrations +
# pb_hooks + Dockerfile. No shared-module copy, so the slot
# is gated purely to its own subtree — it does NOT rebuild
# on every showcase push, only when migrations/hooks/Dockerfile
# change.
- 'showcase/pocketbase/**'
webhooks:
# Sentinel pattern that cannot match any in-tree path.
# The webhooks GHCR image is built by the showcase-eval-webhook
# repo's own release workflow, so we never want a push-driven
# build run to include it. workflow_dispatch can still target
# webhooks explicitly via the service input — that path skips
# paths-filter entirely.
- 'showcase/__no_match_webhooks_built_out_of_band__'
- name: Build service matrix
id: build-matrix
env:
DISPATCH_SERVICE: ${{ github.event.inputs.service }}
GITHUB_SHA_ENV: ${{ github.sha }}
GITHUB_REF_NAME_ENV: ${{ github.ref_name }}
FILTER_CHANGES: ${{ steps.filter.outputs.changes }}
run: |
# Full service config as JSON
# Fields: dispatch_name, filter_key, context, image, railway_id, timeout, lfs, build_args, build_args_sha, build_args_branch, dockerfile, health_path, skip_build
# skip_build (optional, boolean): when true, the Docker build step
# is skipped for this slot (image is built out-of-band by another
# workflow/repo). Currently used by `webhooks` (built by the
# showcase-eval-webhook repo's own release workflow).
# health_path: historical field retained in the matrix for human
# reference. The actual verify probe is driven by per-service
# drivers in verify-deploy.ts (showcase/scripts/verify-deploy.ts),
# NOT by this field — it is informational only at this layer.
#
# timeout: the per-slot `timeout-minutes` budget. It is a BACKSTOP
# against a hung build, not a schedule — GitHub kills the job when
# it is exceeded and reports the kill as conclusion `cancelled`
# (Depot surfaces it as "Step canceled by GitHub"), which is a
# cancelled slot that pushed no image.
#
# The `context: "."` shell slots build the whole monorepo root and
# are the HEAVIEST builds in the fleet, yet they used to carry the
# SMALLEST budgets (10 min, vs 15 for each small per-integration
# build) — inverted, with no headroom for a cold Depot cache. When
# three full-fleet rebuilds ran concurrently on 2026-07-25 (~84
# simultaneous amd64 Depot builds), the shell builds were making
# real but slow progress — repeated docker.io base-image pull
# stalls, and the final `generate-registry` layer alone took 54s —
# and were still 2 steps from done when the 10-minute budget killed
# them. Measured evidence, same slots:
# warm/uncontended ......... 25 min
# contended, killed at ..... 10 min (runs 30162773601, 30162770765)
# showcase-pocketbase ...... 8.8 min of a 10 min budget (near-miss)
# So the four shell slots get 20, showcase-aimock (killed at its 5
# min budget) gets 12, and showcase-pocketbase gets 15. `webhooks`
# stays at 5: it is `skip_build`, and measured at 0.6 min.
#
# NOTE this is mitigation, not the root fix. The root cause is
# Depot builder contention from concurrent full-fleet rebuilds; the
# durable fix is to not run three of them at once (this workflow
# deliberately has NO concurrency group — see the header — so a
# non-cancelling `concurrency` queue would be the lever, at the
# cost of serializing main builds). Left as a follow-up.
ALL_SERVICES='[
{"dispatch_name":"shell","filter_key":"shell","context":".","image":"showcase-shell","railway_id":"40eea0da-6071-4ea8-bdb9-39afb19225ec","timeout":20,"build_args_sha":"__GH_SHA__","build_args_branch":"__GH_REF_NAME__","dockerfile":"showcase/shell/Dockerfile","health_path":"/"},
{"dispatch_name":"langgraph-python","filter_key":"langgraph_python","context":"showcase/integrations/langgraph-python","image":"showcase-langgraph-python","railway_id":"90d03214-4569-41b0-b4c1-6438a8a7b203","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"mastra","filter_key":"mastra","context":"showcase/integrations/mastra","image":"showcase-mastra","railway_id":"d7979eb7-2405-4aab-ad21-438f4a1b08af","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"crewai-crews","filter_key":"crewai_crews","context":"showcase/integrations/crewai-crews","image":"showcase-crewai-crews","railway_id":"0e9c284d-8d87-4fcf-9f82-6b704d7e4bd4","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"crewai-conversational-flows","filter_key":"crewai_conversational_flows","context":"showcase/integrations/crewai-conversational-flows","image":"showcase-crewai-conversational-flows","railway_id":"11859593-da4e-486c-a810-6cdffeff9750","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"pydantic-ai","filter_key":"pydantic_ai","context":"showcase/integrations/pydantic-ai","image":"showcase-pydantic-ai","railway_id":"0a106173-2282-4887-a994-0ca276a99d69","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"google-adk","filter_key":"google_adk","context":"showcase/integrations/google-adk","image":"showcase-google-adk","railway_id":"87f60507-5a3d-4b8a-9e23-2b1de85d939c","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"ag2","filter_key":"ag2","context":"showcase/integrations/ag2","image":"showcase-ag2","railway_id":"4a37481b-f264-4eb7-a9cd-0a9ebb9ac05c","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"agno","filter_key":"agno","context":"showcase/integrations/agno","image":"showcase-agno","railway_id":"32cab80b-e329-45bd-9c73-c4e1ddc94305","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"llamaindex","filter_key":"llamaindex","context":"showcase/integrations/llamaindex","image":"showcase-llamaindex","railway_id":"285386e8-492d-4cb8-b632-0a7d4607378f","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"langgraph-fastapi","filter_key":"langgraph_fastapi","context":"showcase/integrations/langgraph-fastapi","image":"showcase-langgraph-fastapi","railway_id":"06cccb5c-59f4-46b5-8adc-7113e77011a4","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"langgraph-typescript","filter_key":"langgraph_typescript","context":"showcase/integrations/langgraph-typescript","image":"showcase-langgraph-typescript","railway_id":"66246d3b-a18e-46f0-be51-5f3ff7a36e5a","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"langroid","filter_key":"langroid","context":"showcase/integrations/langroid","image":"showcase-langroid","railway_id":"6dd9cb0a-66cc-46f1-972e-7cd74756157d","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"spring-ai","filter_key":"spring_ai","context":"showcase/integrations/spring-ai","image":"showcase-spring-ai","railway_id":"eed5d041-91be-4282-b414-beea00843401","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"strands","filter_key":"strands","context":"showcase/integrations/strands","image":"showcase-strands","railway_id":"92e1cfad-ad53-403f-ab2b-5ab380832232","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"strands-typescript","filter_key":"strands_typescript","context":"showcase/integrations/strands-typescript","image":"showcase-strands-typescript","railway_id":"d6f47c8c-a0a1-4dbe-991c-50f8463fd68d","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"ms-agent-python","filter_key":"ms_agent_python","context":"showcase/integrations/ms-agent-python","image":"showcase-ms-agent-python","railway_id":"655db75a-af8d-427d-a4f9-441570ae5003","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"claude-sdk-typescript","filter_key":"claude_sdk_typescript","context":"showcase/integrations/claude-sdk-typescript","image":"showcase-claude-sdk-typescript","railway_id":"18a98727-5700-44aa-b497-b60795dbbd6a","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"ms-agent-dotnet","filter_key":"ms_agent_dotnet","context":"showcase/integrations/ms-agent-dotnet","image":"showcase-ms-agent-dotnet","railway_id":"beeb2dd6-87a4-4599-aa07-0578f7bd6519","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"ms-agent-harness-dotnet","filter_key":"ms_agent_harness_dotnet","context":"showcase/integrations/ms-agent-harness-dotnet","image":"showcase-ms-agent-harness-dotnet","railway_id":"6343d7f9-6c3f-4c8d-9a6e-79f03d2f1e37","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"claude-sdk-python","filter_key":"claude_sdk_python","context":"showcase/integrations/claude-sdk-python","image":"showcase-claude-sdk-python","railway_id":"b122ab65-9854-4cb2-a68e-b50ff13f7481","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"built-in-agent","filter_key":"built_in_agent","context":"showcase/integrations/built-in-agent","image":"showcase-built-in-agent","railway_id":"f4f8371a-bc46-45b2-b6d4-9c9af608bdbf","timeout":15,"build_args":"","dockerfile":"","health_path":"/api/health"},
{"dispatch_name":"shell-dojo","filter_key":"shell_dojo","context":".","image":"showcase-shell-dojo","railway_id":"7ad1ece7-2228-49cd-8a78-bddf30322907","timeout":20,"build_args":"","dockerfile":"showcase/shell-dojo/Dockerfile","health_path":"/"},
{"dispatch_name":"shell-dashboard","filter_key":"shell_dashboard","context":".","image":"showcase-shell-dashboard","railway_id":"4d5dfd74-be61-40b2-8564-b53b7dd4c15b","timeout":20,"build_args_sha":"__GH_SHA__","build_args_branch":"__GH_REF_NAME__","dockerfile":"showcase/shell-dashboard/Dockerfile","health_path":"/"},
{"dispatch_name":"shell-docs","filter_key":"shell_docs","context":".","image":"showcase-shell-docs","railway_id":"7badfb8d-4228-414c-9145-b4026803714f","timeout":20,"build_args_sha":"__GH_SHA__","build_args_branch":"__GH_REF_NAME__","dockerfile":"showcase/shell-docs/Dockerfile","health_path":"/"},
{"dispatch_name":"showcase-harness","filter_key":"showcase_harness","context":".","image":"showcase-harness","railway_id":"3a14bfed-0537-4d71-897b-7c593dca161d","timeout":20,"build_args":"","dockerfile":"showcase/harness/Dockerfile","health_path":"/health"},
{"dispatch_name":"showcase-aimock","filter_key":"showcase_aimock","context":"showcase/aimock","image":"showcase-aimock","railway_id":"0fa0435d-8a66-46f0-84fd-e4250b580013","timeout":12,"build_args":"","dockerfile":"showcase/aimock/Dockerfile","health_path":"/health"},
{"dispatch_name":"showcase-pocketbase","filter_key":"pocketbase","context":"showcase/pocketbase","image":"showcase-pocketbase","railway_id":"ba11e854-d695-4738-9a45-2b0776788824","timeout":15,"build_args":"","dockerfile":"showcase/pocketbase/Dockerfile","health_path":"/api/health"},
{"dispatch_name":"webhooks","filter_key":"webhooks","context":".","image":"showcase-eval-webhook","railway_id":"ba6acc13-7585-41fe-a5ee-585b34a58fcd","timeout":5,"build_args":"","dockerfile":"","health_path":"/health","skip_build":true}
]'
DISPATCH="$DISPATCH_SERVICE"
CHANGES="${FILTER_CHANGES:-[]}"
# Filter services based on three dispatch modes:
# dispatch == "all": manual "deploy all" — include every service unconditionally.
# This re-pulls :latest for every matrix slot (~38 services); intentional for
# drift-rebuild runs and full-fleet restarts. Do NOT try to short-circuit
# unchanged services here — operators invoke "all" precisely when they want
# the fleet re-deployed regardless of git state (cache poisoning, base-image CVE).
# (paths-filter is unreliable on workflow_dispatch because there is no 'before' SHA,
# so we must NOT consult $changes here — doing so silently produces an empty matrix).
# dispatch == <specific service>: narrow to that service only (skips paths-filter so
# drift-rebuild and manual single-service dispatches work regardless of $changes).
# dispatch == "": push event — include services whose filter_key appears in paths-filter CHANGES.
MATRIX=$(echo "$ALL_SERVICES" | jq -c --arg dispatch "$DISPATCH" --argjson changes "$CHANGES" --arg sha "$GITHUB_SHA_ENV" --arg ref "$GITHUB_REF_NAME_ENV" '
[.[] |
if .build_args_sha == "__GH_SHA__" then .build_args_sha = $sha else . end |
if .build_args_branch == "__GH_REF_NAME__" then .build_args_branch = $ref else . end |
(.filter_key as $fk | select(
$dispatch == "all" or
($dispatch != "" and $dispatch != "all" and $dispatch == .dispatch_name) or
($dispatch == "" and (
($changes | index("workflow_config") != null) or
($changes | index($fk) != null) or
(($changes | index("angular") != null) and (
(.context | startswith("showcase/integrations/")) or
.dispatch_name == "shell"
))
))
))]
')
# Fail loudly on typo'd workflow_dispatch inputs. If a user types a
# service name that doesn't exist in ALL_SERVICES, the jq filter
# silently produces [] and the run shows green with zero work
# done — a common "did my dispatch deploy?" footgun. The `all` and
# empty (push-event) modes legitimately produce [] when there are
# no changes and must still succeed.
#
# The `starter-*` namespace is OWNED by the detect-starter-changes
# job (which scopes its OWN fail-loud to `starter-*`). A
# `service=starter-<slug>` dispatch legitimately yields [] here
# (no showcase service is named `starter-*`), so it must SKIP — an
# empty showcase matrix, NOT a fail-loud exit 1. Mirrors how the
# starter job scopes its fail-loud to the `starter-*` namespace.
case "$DISPATCH" in
starter-*) ;; # starter namespace → empty showcase matrix + skip
"" | "all") ;; # push / full-fleet → [] is legitimate
*)
if [ "$MATRIX" = "[]" ]; then
echo "::error::workflow_dispatch service='$DISPATCH' did not match any entry in ALL_SERVICES — check the dispatch_name spelling"
exit 1
fi
;;
esac
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
echo "needs_angular=$(echo "$MATRIX" | jq -r 'any(.[]; (.context | startswith("showcase/integrations/")))')" >> $GITHUB_OUTPUT
if [ "$MATRIX" = "[]" ]; then
echo "has_changes=false" >> $GITHUB_OUTPUT
else
echo "has_changes=true" >> $GITHUB_OUTPUT
fi
check-lockfile:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
# Omit `version:` so pnpm/action-setup inherits from the repo's
# `packageManager` field in package.json (enforced via corepack).
# Earlier revisions hard-pinned `version: 10.13.1` which silently
# drifted from package.json whenever the repo bumped pnpm —
# resulting in lockfile-vs-engine mismatches that only surfaced on
# the slow `--frozen-lockfile` path.
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22.x
- run: pnpm install --frozen-lockfile --ignore-scripts
verify-image-refs:
needs: [detect-changes]
if: needs.detect-changes.outputs.has_changes == 'true'
runs-on: ubuntu-latest
timeout-minutes: 3
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22.x
- name: Verify Railway image refs
env:
RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
run: npx tsx showcase/scripts/verify-railway-image-refs.ts
# Build one browser bundle for every selected integration image. Staging
# redeploys these GHCR images directly; production promotion pins the same
# tested image digest, so both environments receive this exact artifact.
build-angular:
name: Build canonical Angular browser artifact
needs: [detect-changes]
if: needs.detect-changes.outputs.has_changes == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 30
permissions:
contents: read
steps:
- name: Checkout
if: needs.detect-changes.outputs.needs_angular == 'true'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Setup pnpm
if: needs.detect-changes.outputs.needs_angular == 'true'
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
- name: Setup Node
if: needs.detect-changes.outputs.needs_angular == 'true'
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22.x
- name: Install
if: needs.detect-changes.outputs.needs_angular == 'true'
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Build
if: needs.detect-changes.outputs.needs_angular == 'true'
run: pnpm nx build @copilotkit/showcase-angular-host
- name: Upload canonical Angular browser artifact
if: needs.detect-changes.outputs.needs_angular == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: showcase-angular-browser-${{ github.sha }}
path: showcase/angular/dist/showcase-angular/browser
if-no-files-found: error
retention-days: 1
build:
needs: [detect-changes, check-lockfile, verify-image-refs, build-angular]
if: needs.detect-changes.outputs.has_changes == 'true'
runs-on: depot-ubuntu-24.04-4
timeout-minutes: ${{ fromJSON(matrix.service.timeout) }}
permissions:
id-token: write
contents: read
packages: write
strategy:
fail-fast: false
matrix:
service: ${{ fromJSON(needs.detect-changes.outputs.matrix) }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# Always pull LFS. Nearly every integration ships LFS-tracked demo
# assets under public/ (public/demo-files/*.png|*.pdf,
# public/demo-audio/*.wav per root .gitattributes). With lfs:false
# these check out as 130-byte LFS pointer stubs and get COPYed into
# the image as text — the deployed image then serves the pointer
# with HTTP 200 and the frontend's magic-bytes guard rejects it,
# breaking the multimodal test pill. This used to be driven by a
# per-slot `lfs` flag in ALL_SERVICES that was true only for
# shell/shell-dashboard/shell-docs, leaving every framework
# integration shipping pointer stubs. That flag has since been
# deleted outright: uniform lfs:true is the least-error-prone fix,
# because a new integration is covered automatically with no
# per-slot flag to forget.
lfs: true
persist-credentials: false
- name: Download canonical Angular browser artifact
if: startsWith(matrix.service.context, 'showcase/integrations/')
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: showcase-angular-browser-${{ github.sha }}
path: ${{ runner.temp }}/showcase-angular-browser
- name: Setup Depot
uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1
- name: Login to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Prepare build args
id: build-args
env:
BUILD_ARGS_SHA: ${{ matrix.service.build_args_sha }}
BUILD_ARGS_BRANCH: ${{ matrix.service.build_args_branch }}
run: |
# set -euo pipefail: without `-e`, a transient `$GITHUB_OUTPUT`
# write failure (disk pressure, ENOSPC) could silently produce
# empty build-args and we'd ship an image without COMMIT_SHA /
# BRANCH baked in — invisible drift between build label and
# what's actually running.
set -euo pipefail
ARGS=""
if [ -n "$BUILD_ARGS_SHA" ]; then
ARGS="COMMIT_SHA=${BUILD_ARGS_SHA}"
ARGS="${ARGS}"$'\n'"BRANCH=${BUILD_ARGS_BRANCH}"
fi
# Use delimiter to safely pass multiline value
echo "args<<BUILDARGS_EOF" >> $GITHUB_OUTPUT
echo "$ARGS" >> $GITHUB_OUTPUT
echo "BUILDARGS_EOF" >> $GITHUB_OUTPUT
- name: Copy shared modules into build context
env:
ANGULAR_BROWSER: ${{ runner.temp }}/showcase-angular-browser
run: |
set -euo pipefail
CONTEXT="${{ matrix.service.context }}"
# Idempotent copy: if a stale `shared_python`/`shared_typescript`
# already exists (previous failed run on the same runner, or a
# checkout artifact), remove it first. `cp -r src dst` into an
# existing directory nests source-inside-destination, which
# would silently produce a broken build context.
if [ -d "showcase/shared/python" ] && [ -d "$CONTEXT" ]; then
rm -rf "$CONTEXT/shared_python"
cp -r showcase/shared/python "$CONTEXT/shared_python"
fi
if [ -d "showcase/shared/typescript/tools" ] && [ -d "$CONTEXT" ]; then
rm -rf "$CONTEXT/shared_typescript"
mkdir -p "$CONTEXT/shared_typescript"
cp -r showcase/shared/typescript/tools "$CONTEXT/shared_typescript/tools"
fi
# Dereference tools/, shared-tools/, data/, and _shared/ symlinks for the
# Docker context. Integration directories use symlinks pointing to
# ../../shared/python/tools etc., and _shared -> ../_shared for the
# CVDIAG bootstrap modules. Docker cannot follow symlinks outside
# the build context (buildkit fails the checksum with "too many
# symlinks: /_shared"), so we replace each symlink with a real copy
# of its target. Mirrors stage_shared() in
# showcase/scripts/cli/_common.sh (the local bin/showcase path).
for link_name in tools shared-tools data _shared; do
link_path="$CONTEXT/$link_name"
if [ -L "$link_path" ]; then
target="$(readlink -f "$link_path")"
if [ -d "$target" ]; then
rm "$link_path"
cp -r "$target" "$link_path"
fi
fi
done
if [ -L "$CONTEXT/public/angular" ]; then
source showcase/scripts/cli/_common.sh
stage_angular "$CONTEXT" "$ANGULAR_BROWSER"
fi
- name: Build and push
if: ${{ matrix.service.skip_build != true }}
uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.18.0
with:
project: m2kw2wmmcp
context: ${{ matrix.service.context }}
file: ${{ matrix.service.dockerfile != '' && matrix.service.dockerfile || format('{0}/Dockerfile', matrix.service.context) }}
# Pin amd64: Railway and GHCR serve x86 hosts. An arm64-only
# image crashes on pull with "does not have a linux/amd64
# variant available".
platforms: linux/amd64
push: true
# ONLY the immutable per-commit tag is pushed here. `:latest` is
# moved by the guarded step below, which refuses to roll staging
# back when a NEWER commit's build already claimed it. Pushing
# `:latest` from this step is what let an older concurrent build
# overwrite a newer one (2026-07-26; see advance-latest-tag.ts for
# the full incident reconstruction and run IDs).
tags: ghcr.io/copilotkit/${{ matrix.service.image }}:${{ github.sha }}
# The revision label is how the guard identifies which commit the
# image currently tagged `:latest` was built from. Without it the
# guard cannot prove a regression and (correctly) fails open.
labels: |
org.opencontainers.image.revision=${{ github.sha }}
build-args: ${{ steps.build-args.outputs.args }}
- name: Write per-slot build result
if: always()
env:
SERVICE: ${{ matrix.service.dispatch_name }}
BUILD_STATUS: ${{ job.status }}
run: |
# job.status is one of: success, failure, cancelled — mapped
# 1:1 onto the BuildOutcome contract in
# showcase/scripts/lib/build-outputs.ts. We deliberately do NOT
# write to $GITHUB_OUTPUT — matrix-slot outputs are not
# aggregable across slots in GitHub Actions, so we publish the
# per-slot result as an artifact instead. The downstream
# aggregator job downloads every `build-result-*` artifact.
#
# `cancelled` used to be laundered into `skipped` here. That one
# line is what made a partially-cancelled fleet build silent:
# this slot's result is the ONLY place the cancellation is
# recorded, because GitHub's own status functions cannot see it
# downstream. Proven on run 30166429073 (a purpose-built probe:
# matrix leg killed by `timeout-minutes` → job.status=cancelled,
# rollup `needs.*.result`=cancelled, yet BOTH `if: cancelled()`
# and `if: failure()` evaluated FALSE in dependent jobs). With
# `cancelled` collapsed to `skipped` it was indistinguishable
# from a slot that legitimately never built, so nothing could
# alert and nothing could red the run. Keep it distinct.
#
# The `*)` catch-all stays as a defensive fallback in case
# GitHub ever introduces a fourth job.status value; `skipped` is
# the conservative choice because it is excluded from both the
# redeploy success-set and the cancelled-slot alert.
case "$BUILD_STATUS" in
success) STATUS=success ;;
failure) STATUS=failure ;;
cancelled) STATUS=cancelled ;;
*) STATUS=skipped ;;
esac
mkdir -p "$RUNNER_TEMP/build-result"
printf '{"service":"%s","status":"%s"}\n' "$SERVICE" "$STATUS" \
> "$RUNNER_TEMP/build-result/result.json"
- name: Upload per-slot build-result artifact
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
# Canonical per-slot name (see buildResultArtifactName in
# showcase/scripts/lib/build-outputs.ts). The aggregator
# downloads every artifact matching `build-result-*`.
name: build-result-${{ matrix.service.dispatch_name }}
path: ${{ runner.temp }}/build-result/result.json
if-no-files-found: error
retention-days: 7
# ────────────────────────────────────────────────────────────────────
# Per-starter image publish (model B, §b stage 1 / Phase 1).
#
# Fully decoupled from the showcase `build`→`aggregate`→`redeploy` chain
# above: starters are self-contained npm projects (no monorepo-source
# build, no shared-module copy) and build from their own root Dockerfile
# at examples/integrations/<slug>/Dockerfile (the single-image deployable:
# Next.js frontend + agent, EXPOSE 3000, CMD entrypoint.sh — distinct from
# the docker/Dockerfile.app + docker/Dockerfile.agent split stack used by
# docker-compose.test.yml). Published to ghcr.io/copilotkit/starter-<slug>
# (the `starter-` prefix is disjoint from `showcase-*`; S2's harness
# discovery filters on namePrefix "starter-"). Railway deploy is the
# gated S5 — NOT done here.
#
# The 12 starter slugs are the matrix source of truth (the smoke matrix in
# test_smoke-starter.yml and STARTER_TO_COLUMN in
# showcase/harness/src/probes/helpers/starter-mapping.ts). The dashboard
# column remap lives in the harness (§a), so this layer uses raw starter
# slugs.
detect-starter-changes:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
outputs:
matrix: ${{ steps.starter-matrix.outputs.matrix }}
has_changes: ${{ steps.starter-matrix.outputs.has_changes }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Detect changed starter paths
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
id: filter
with:
filters: |
workflow_config:
- '.github/workflows/showcase_build.yml'
langgraph_python:
- 'examples/integrations/langgraph-python/**'
mastra:
- 'examples/integrations/mastra/**'
langgraph_js:
- 'examples/integrations/langgraph-js/**'
crewai_crews:
- 'examples/integrations/crewai-crews/**'
pydantic_ai:
- 'examples/integrations/pydantic-ai/**'
adk:
- 'examples/integrations/adk/**'
agno:
- 'examples/integrations/agno/**'
llamaindex:
- 'examples/integrations/llamaindex/**'
langgraph_fastapi:
- 'examples/integrations/langgraph-fastapi/**'
strands_python:
- 'examples/integrations/strands-python/**'
ms_agent_framework_python:
- 'examples/integrations/ms-agent-framework-python/**'
ms_agent_framework_dotnet:
- 'examples/integrations/ms-agent-framework-dotnet/**'
- name: Build starter matrix
id: starter-matrix
env:
DISPATCH_SERVICE: ${{ github.event.inputs.service }}
FILTER_CHANGES: ${{ steps.filter.outputs.changes }}
run: |
set -euo pipefail
# One slot per starter. `slug` is the examples/integrations/<slug>
# directory name; `image` is the published GHCR repo (starter-<slug>);
# `filter_key` matches the paths-filter key above.
ALL_STARTERS='[
{"slug":"langgraph-python","image":"starter-langgraph-python","filter_key":"langgraph_python"},
{"slug":"mastra","image":"starter-mastra","filter_key":"mastra"},
{"slug":"langgraph-js","image":"starter-langgraph-js","filter_key":"langgraph_js"},
{"slug":"crewai-crews","image":"starter-crewai-crews","filter_key":"crewai_crews"},
{"slug":"pydantic-ai","image":"starter-pydantic-ai","filter_key":"pydantic_ai"},
{"slug":"adk","image":"starter-adk","filter_key":"adk"},
{"slug":"agno","image":"starter-agno","filter_key":"agno"},
{"slug":"llamaindex","image":"starter-llamaindex","filter_key":"llamaindex"},
{"slug":"langgraph-fastapi","image":"starter-langgraph-fastapi","filter_key":"langgraph_fastapi"},
{"slug":"strands-python","image":"starter-strands-python","filter_key":"strands_python"},
{"slug":"ms-agent-framework-python","image":"starter-ms-agent-framework-python","filter_key":"ms_agent_framework_python"},
{"slug":"ms-agent-framework-dotnet","image":"starter-ms-agent-framework-dotnet","filter_key":"ms_agent_framework_dotnet"}
]'
# Dispatch modes (mirror the showcase detect-changes job):
# "all" → every starter (full-fleet rebuild).
# "starter-<slug>" → that one starter (strip "starter-" prefix to match .image).
# "<showcase service slug>" → no starters (this is a showcase-only dispatch).
# "" (push event) → starters whose filter_key appears in CHANGES
# (or workflow_config touched → rebuild all).
DISPATCH="${DISPATCH_SERVICE:-}"
CHANGES="${FILTER_CHANGES:-[]}"
# `.image` is exactly "starter-<slug>", which is also the
# workflow_dispatch choice value, so a specific-starter dispatch
# matches `$dispatch == .image` directly.
MATRIX=$(echo "$ALL_STARTERS" | jq -c --arg dispatch "$DISPATCH" --argjson changes "$CHANGES" '
[.[] |
(.filter_key as $fk | select(
$dispatch == "all" or
($dispatch != "" and $dispatch != "all" and $dispatch == .image) or
($dispatch == "" and (($changes | index("workflow_config") != null) or ($changes | index($fk) != null)))
))]
')
# Fail loudly on a typo'd starter dispatch (mirrors showcase job).
# Only applies to the starter-* dispatch namespace; a showcase
# service slug or "all" legitimately yields [] here.
case "$DISPATCH" in
starter-*)
if [ "$MATRIX" = "[]" ]; then
echo "::error::workflow_dispatch service='$DISPATCH' did not match any starter — check the slug"
exit 1
fi
;;
esac
echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"
if [ "$MATRIX" = "[]" ]; then
echo "has_changes=false" >> "$GITHUB_OUTPUT"
else
echo "has_changes=true" >> "$GITHUB_OUTPUT"
fi
build-starters:
needs: [detect-starter-changes]
if: needs.detect-starter-changes.outputs.has_changes == 'true'
runs-on: depot-ubuntu-24.04-4
timeout-minutes: 20
permissions:
id-token: write
contents: read
packages: write
strategy:
fail-fast: false
matrix:
starter: ${{ fromJSON(needs.detect-starter-changes.outputs.matrix) }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Setup Depot
uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1
- name: Login to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push starter image
uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.18.0
with:
project: m2kw2wmmcp
context: examples/integrations/${{ matrix.starter.slug }}
file: examples/integrations/${{ matrix.starter.slug }}/Dockerfile
# Pin amd64: Railway and GHCR serve x86 hosts. Depot defaults to
# the runner's native arch, so an arm64-only image would crash on
# pull with "does not have a linux/amd64 variant available".
platforms: linux/amd64
push: true
# Immutable tag only — `:latest` is moved by the guarded step
# below. Same rationale as the showcase `build` matrix above.
tags: ghcr.io/copilotkit/${{ matrix.starter.image }}:${{ github.sha }}
labels: |
org.opencontainers.image.revision=${{ github.sha }}
- name: Write per-slot starter build result
if: always()
env:
SERVICE: ${{ matrix.starter.slug }}
BUILD_STATUS: ${{ job.status }}
run: |
# Mirror of the main `build` matrix's per-slot result write
# (see "Write per-slot build result" above). job.status is one of
# success, failure, cancelled; normalize cancelled→skipped to match
# the same {service,status} shape. We publish per-slot via artifact
# because matrix-slot outputs aren't aggregable across slots.
#
# IMPORTANT: starter artifacts use the `starter-build-result-*`
# prefix, NOT `build-result-*`. The aggregate-build-results job
# downloads every `build-result-*` and feeds it to the showcase
# redeploy set (keyed by showcase dispatch_name). Starters are a
# separate namespace and must NOT pollute that intersection, so
# they get their own prefix. This artifact exists to give the
# `notify` job a durable per-starter failure surface.
case "$BUILD_STATUS" in
success) STATUS=success ;;
failure) STATUS=failure ;;
*) STATUS=skipped ;;
esac
mkdir -p "$RUNNER_TEMP/starter-build-result"
printf '{"service":"%s","status":"%s"}\n' "$SERVICE" "$STATUS" \
> "$RUNNER_TEMP/starter-build-result/result.json"
- name: Upload per-slot starter build-result artifact
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
# Distinct `starter-build-result-*` prefix so this never matches the
# aggregator's `build-result-*` download pattern.
name: starter-build-result-${{ matrix.starter.slug }}
path: ${{ runner.temp }}/starter-build-result/result.json
if-no-files-found: error
retention-days: 7
aggregate-build-results:
name: Aggregate build results
needs: [detect-changes, build]
if: ${{ !cancelled() && needs.detect-changes.outputs.has_changes == 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 3
permissions:
contents: read
outputs:
results: ${{ steps.collect.outputs.results }}
any_success: ${{ steps.collect.outputs.any_success }}
# `any_cancelled` / `cancelled_services` are the ONLY downstream
# signal that one or more build slots were cancelled (in practice:
# killed by `timeout-minutes`). They are derived from the per-slot
# artifacts, NOT from GitHub's status functions, because those
# cannot see a leg-level cancellation — see the comment on the
# `notify-cancelled-builds` job below for the measured proof.
any_cancelled: ${{ steps.collect.outputs.any_cancelled }}
cancelled_services: ${{ steps.collect.outputs.cancelled_services }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Setup pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
- name: Setup Node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22.x
cache: pnpm
- name: Install
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Download all per-slot build-result artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
# `pattern` matches every per-slot artifact emitted by the
# build matrix. `merge-multiple: false` keeps each artifact
# in its own subdirectory so we can iterate them deterministically.
pattern: build-result-*
path: ${{ runner.temp }}/build-results-in
merge-multiple: false
- name: Collect per-service build outcomes
id: collect
env:
INPUT_DIR: ${{ runner.temp }}/build-results-in
OUTPUT_DIR: ${{ runner.temp }}/build-results-out
run: |
set -euo pipefail
mkdir -p "$OUTPUT_DIR"
# Each per-slot artifact extracts to
# $INPUT_DIR/build-result-<dispatch_name>/result.json
# The aggregator script (showcase/scripts/aggregate-build-results.ts)
# reads them, merges via the shared helper (mergeBuildResultFiles),
# writes $OUTPUT_DIR/results.json, and appends `results` +
# `any_success` + `any_cancelled` + `cancelled_services` to
# $GITHUB_OUTPUT. The contract (service + status enum) is
# enforced in one place (build-outputs.ts).
npx tsx showcase/scripts/aggregate-build-results.ts
- name: Upload aggregated build-results artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: build-results
path: ${{ runner.temp }}/build-results-out/results.json
if-no-files-found: error
retention-days: 7
redeploy-staging:
name: Trigger Railway staging redeploy
needs: [detect-changes, build, aggregate-build-results]
# Run if the WORKFLOW RUN was not cancelled, at least one service was
# scheduled, AND at least one slot actually built
# (aggregate-build-results.outputs.any_success). any_success is the
# single source of truth for "should we redeploy?": it is derived from
# the real per-slot build-result artifacts (see aggregate-build-results),
# so it is 'true' on a PARTIAL build (fail-fast: false) and 'false' on a
# genuinely dead / entirely-skipped build — subsuming the old
# needs.build.result != 'skipped' check. It is the explicit "do not
# redeploy when nothing was pushed" guard: without it an all-failed run
# would still kick a redeploy that re-pulls the stale :latest and
# silently looks healthy.
#
# We deliberately do NOT gate on needs.build.result (the matrix rollup).
# GitHub Actions rolls a matrix job up to 'cancelled' when ANY single leg
# is cancelled (e.g. the LFS shell build under runner contention), so
# `needs.build.result != 'cancelled'` skipped the redeploy for the WHOLE
# fleet even when 27/28 legs succeeded. Mirror the sibling
# aggregate-build-results job instead: gate on !cancelled() + has_changes
# and let the artifact-derived any_success decide. !cancelled() (a status
# function) still overrides GH's default "skip when a needed job did not
# succeed", so this job runs even when the build rollup is cancelled/
# failure — as long as the RUN itself was not cancelled.
if: >-
${{ !cancelled()
&& needs.detect-changes.outputs.has_changes == 'true'
&& needs.aggregate-build-results.outputs.any_success == 'true' }}
runs-on: ubuntu-latest
# 20, not the historical 5. This job no longer just POSTs to Railway: the
# "Advance :latest" guard below walks EVERY successfully-built slot
# serially, and each slot costs up to three network round-trips (an
# `imagetools inspect` of `:latest`, a `gh api .../compare`, and an
# `imagetools create` retag). At full fleet width (~28 slots) that is ~84
# serial calls before the Railway redeploy loop even starts.
#
# Under-sizing this is not a benign "job takes longer" bug: an overrun
# concludes the job `cancelled`, and a cancelled conclusion is invisible to
# both `if: failure()` and `if: cancelled()` in dependent jobs (proven on
# run 30166429073 — see the "Write per-slot build result" step). The
# redeploy would silently not happen and nothing would alert.
timeout-minutes: 20
permissions:
contents: read
# The "Advance :latest" step performs an authenticated GHCR WRITE
# (`docker buildx imagetools create`). Without `packages: write` the
# retag 401s, the guard exits non-zero, the redeploy step never runs,
# and — because the build step no longer pushes `:latest` itself —
# `:latest` freezes permanently and staging stops redeploying at all.
packages: write # GHCR retag (imagetools create) in the guard below
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22.x
# Mirrors the `build` job's login (see "Login to GHCR" there). Required
# for BOTH directions of the guard, not just the retag: the showcase
# packages are `visibility: private`, so an unauthenticated
# `imagetools inspect` also 401s. The guard treats a failed read as
# "unknown revision" and advances anyway — i.e. without this step it
# would silently FAIL OPEN and never actually guard.
- name: Login to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Compute changed-service list from build matrix
id: changed
env:
MATRIX_JSON: ${{ needs.detect-changes.outputs.matrix }}
BUILD_RESULTS_JSON: ${{ needs.aggregate-build-results.outputs.results }}
run: |
# We feed the redeploy script with the INTERSECTION of:
# (a) the scheduled build matrix (detect-changes.outputs.matrix —
# JSON array of objects with `dispatch_name`), and
# (b) the SUCCESS set from the aggregator
# (aggregate-build-results.outputs.results — JSON array of
# `{service: <dispatch_name>, status: success|failure|skipped}`;
# the `service` field is the dispatch_name; shape defined in
# showcase/scripts/lib/build-outputs.ts).
# Without this intersection a service whose Docker build FAILED
# would still be in the redeploy CSV, Railway would re-pull its
# stale `:latest`, and the verify workflow would report it as a
# fresh, healthy deploy — a false green. Skipped slots are also
# excluded (only `status == "success"` qualifies).
set -euo pipefail
matrix_names="$(echo "$MATRIX_JSON" | jq -r '[.[] | .dispatch_name]')"
success_names="$(echo "$BUILD_RESULTS_JSON" | jq -r '[.[] | select(.status == "success") | .service]')"
csv="$(jq -rn --argjson m "$matrix_names" --argjson s "$success_names" \
'($m | map(select(. as $n | $s | index($n)))) | join(",")')"
if [ -z "$csv" ]; then
# This job only runs when aggregate-build-results.any_success == 'true'
# (see the job-level `if:` guard). So reaching an EMPTY intersection
# here does NOT mean "nothing to deploy" — it means at least one slot
# BUILT successfully yet none of those successes maps back into the
# scheduled build matrix. That is a dispatch_name↔service contract
# skew: the aggregator's `service` values and the matrix's
# `dispatch_name` values have drifted out of sync. Emitting an empty
# `services=` here would let the build go GREEN while redeploying
# NOTHING — a silent "we thought we shipped but didn't" hole. Fail
# LOUD instead, naming both sides of the skew so it can be diagnosed.
echo "::error::Build succeeded (any_success=true) but matrix ∩ success-set is EMPTY — dispatch_name/service contract skew; nothing would be redeployed."
echo "Successful build service values (from aggregate-build-results): $success_names"
echo "Scheduled matrix dispatch_name values (from detect-changes): $matrix_names"
echo "These two sets share no members — a successful build maps to no matrix entry. Reconcile showcase/scripts/lib/build-outputs.ts (service field) with the build matrix dispatch_name values."
exit 1
fi
echo "services=$csv" >> "$GITHUB_OUTPUT"
echo "Computed services CSV (matrix ∩ build-success): $csv"
# Same intersection, projected onto GHCR image names, MINUS the
# `skip_build` slots. This is the exact set whose `:latest` may move,
# so a service that failed to build can never have its tag advanced.
#
# The `skip_build` exclusion is load-bearing and applies to `images`
# ONLY — never to the `services` CSV above:
#
# * `webhooks` (the sole skip_build slot) is built and released by
# its OWN repo's workflow, not this one. Its build step is
# skipped by `if: matrix.service.skip_build != true`, yet the
# slot still concludes job.status == success, so it lands in
# success_names like any other slot.
# * It therefore has NO `:<sha>` tag in GHCR from this run. Handing
# it to the guard makes `imagetools create` fail with
# "manifest unknown"; the guard exits non-zero and blocks the
# redeploy for the ENTIRE fleet. This fires on a `service=all`
# dispatch and on any push matching the `workflow_config`
# paths-filter disjunct (which selects every slot) — including
# this workflow file's own merge commit.
# * It must nevertheless STAY in `services`: bouncing webhooks so
# Railway re-pulls its out-of-band `:latest` is the documented,
# intended behavior. See the `webhooks` entry in
# showcase/scripts/railway-envs.ts.
#
# `.skip_build` is absent (null) on every other slot, so
# `select(.skip_build != true)` keeps them all.
images="$(jq -rn --argjson m "$(echo "$MATRIX_JSON" | jq -c '[.[] | {dispatch_name, image, skip_build}]')" --argjson s "$success_names" \
'($m | map(select(.dispatch_name as $n | $s | index($n))) | map(select(.skip_build != true)) | map("ghcr.io/copilotkit/" + .image)) | join(",")')"
echo "images=$images" >> "$GITHUB_OUTPUT"
echo "Computed images CSV: $images"
- name: Advance :latest for successfully-built images
if: steps.changed.outputs.images != ''
env:
IMAGES: ${{ steps.changed.outputs.images }}
# GITHUB_SHA / GITHUB_REPOSITORY are deliberately NOT declared here:
# the runner exports both as default environment variables and
# advance-latest-tag.ts reads them straight off process.env, so
# restating them was pure shadowing. GH_TOKEN is NOT a default —
# without it every `gh api …/compare` fails and the guard degrades to
# advancing unconditionally.
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Runs immediately BEFORE the Railway redeploy below, which is what
# pulls `:latest`. Doing it here rather than per-build-slot keeps the
# decision as close as possible to the pull, so the read→retag window
# an older concurrent run could slip through is as narrow as it gets.
#
# Exits non-zero if a retag fails, which reds the job and stops the
# redeploy — the right outcome, since redeploying against a `:latest`
# that did not move would look like a fresh, healthy deploy of code
# that was never shipped.
#
# tsx is pinned to the exact version pnpm-lock.yaml resolves for the
# workspace. showcase_validate.yml enforces the same no-drift rule via
# `pnpm exec tsx`; that form is unavailable here because this job has
# no workspace install, and bare `npx tsx` would fetch a drifting
# version on a registry cache miss.
run: npx --yes tsx@4.21.0 showcase/scripts/advance-latest-tag.ts
- name: Redeploy changed services in staging
if: steps.changed.outputs.services != ''
env:
RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
SERVICES_CSV: ${{ steps.changed.outputs.services }}
# Bridge the per-service redeploy summary to showcase_deploy.yml's
# `enforce-redeploy-gate` (consumed via the `redeploy-summary`
# artifact, extracted to `.redeploy/summary.json`). redeploy-env.ts
# writes this path atomically (.tmp → rename) but does NOT create
# parent dirs, so the step below mkdir's `.redeploy` first.
REDEPLOY_SUMMARY_JSON: .redeploy/summary.json
run: |
# Staging is non-blocking by design: the script always exits 0
# and writes per-service failures into $GITHUB_STEP_SUMMARY. The
# verify-deploy workflow is the real release gate.
mkdir -p .redeploy
npx tsx showcase/scripts/redeploy-env.ts staging --services "$SERVICES_CSV"
- name: Upload redeploy summary
# Upload is MANDATORY whenever a redeploy was attempted (services
# != ''). Both failure modes red the build — no false-green path:
#
# (A) HARD crash inside redeploy-env.ts BEFORE summary.json is
# written. redeploy-env.ts writes the summary atomically
# (.tmp → rename) AFTER the per-service loop completes, so
# a crash leaves no file. The redeploy step itself exits
# non-zero on that crash and fails the redeploy-staging
# job; this upload step is then skipped entirely by
# step-failure propagation. Build → red.
#
# (B) redeploy step exits 0 but summary.json is absent (e.g. a
# logic bug skipped the write). `if-no-files-found: error`
# reds this step → reds the redeploy-staging job → reds the
# build. The deploy workflow's resolve-matrix.if
# (workflow_run.conclusion == 'success') then blocks the
# deploy run from starting at all.
#
# Do NOT add hashFiles() guards here: that would silently skip
# the upload on (B), the deploy workflow would see "artifact
# absent" via check-redeploy-summary, treat it as "nothing
# redeployed", skip the gate, and ship a false-green.
# The legitimate "services == '' → nothing redeployed → no upload"
# path is preserved by the services != '' guard.
#
# If a future change ever switches this step to `if: always()`,
# `if-no-files-found: error` STILL reds path (A): the redeploy
# step's non-zero exit on a HARD crash is independent of upload
# gating, and `if-no-files-found: error` on `always()` then trips
# because summary.json was never written. So neither relaxation
# alone opens a false-green window.
#
# However, swapping the guard to `if: always()` would ALSO red the
# legitimate `services == ''` (nothing-to-redeploy) path — no
# summary.json is written there either, so `if-no-files-found:
# error` would trip on every push that didn't redeploy anything.
# Net effect: trades the (already-closed) false-green risk for a
# false-red on every non-buildable push. Don't do it.
if: steps.changed.outputs.services != ''
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
# Artifact name MUST stay `redeploy-summary`: showcase_deploy.yml's
# `resolve-matrix` job downloads it by this exact name and reads
# `.redeploy/summary.json` inside.
name: redeploy-summary
path: .redeploy/summary.json
if-no-files-found: error
retention-days: 7
redeploy-staging-starters:
name: Trigger Railway staging redeploy (starters)
needs: [detect-starter-changes, build-starters]
# Starter analogue of `redeploy-staging`. The starter lane
# (detect-starter-changes → build-starters) previously ENDED at the GHCR
# push: a starter-only change (examples/integrations/<slug>/**) built a
# fresh `starter-<slug>:latest` image, but nothing ever told Railway to
# pull it, so the fix sat undeployed until someone manually redeployed
# (the PR #6061 agno incident — merged + built to GHCR, never deployed,
# starter-agno stayed crashed). The showcase `redeploy-staging` job only
# covers the showcase `build` lane (needs: [detect-changes, build, ...]),
# so the starter lane had NO redeploy step at all. This job closes that
# gap by redeploying — in STAGING only — exactly the starters that built
# successfully.
#
# Gate mirrors redeploy-staging:
# - !cancelled() → don't fire when the
# whole RUN was cancelled.
# - detect-starter-changes.has_changes=='true' → at least one starter in
# the build matrix.
#
# We deliberately do NOT gate on needs.build-starters.result (the matrix
# rollup). GitHub Actions rolls a matrix job up to 'cancelled' when ANY
# single leg is cancelled (e.g. under runner contention), so the old
# `build-starters.result != 'cancelled'` clause skipped the redeploy for
# the WHOLE starter fleet even when every other starter built fine — the
# same P0 defect that hit redeploy-staging.
#
# This is safe WITHOUT an any_success job guard: the "deploy on a dead
# build" footgun is closed NOT by the job `if:` but by the per-slot
# success intersection in the compute step below (only starters whose OWN
# build slot reported status:success are redeployed) plus the
# `if: steps.changed.outputs.services != ''` guard on the redeploy step.
# A partial success (fail-fast:false) redeploys exactly what built; a
# zero-success run (all failed / all cancelled / build hard-crashed before
# writing any result artifact) yields an EMPTY CSV → the redeploy step is
# skipped → nothing is deployed. The compute step's fail-loud checks only
# fire when build-starters.result == 'success', so a cancelled/failure
# rollup does not misfire them. This is the same net guarantee
# redeploy-staging gets from aggregate-build-results.any_success, enforced
# one level down to avoid standing up a second aggregator job.
if: >-
${{ !cancelled()
&& needs.detect-starter-changes.outputs.has_changes == 'true' }}
runs-on: ubuntu-latest
# 20, not the historical 5 — same reasoning as the sibling
# `redeploy-staging` job: the "Advance starter :latest" guard adds up to
# three serial registry/API round-trips per successfully-built starter,
# and an overrun concludes `cancelled`, which dependent jobs cannot see.
# Sized identically to the sibling because the two lanes run the same
# guard + redeploy shape.
timeout-minutes: 20
permissions:
contents: read
# Same rationale as `redeploy-staging` above.
packages: write # GHCR retag (imagetools create) in the guard below
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22.x
# Mirrors the `build` job's "Login to GHCR". Needed for the guard's
# `imagetools inspect` READ as well as its `imagetools create` WRITE —
# these packages are private, and an unauthenticated read makes the
# guard fail open instead of guarding.
- name: Login to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Download per-slot starter build-result artifacts
# `pattern` (not `name`) download: unlike a `name:` download, a
# pattern that matches ZERO artifacts logs a warning and SUCCEEDS
# rather than hard-failing. That is exactly the wanted behavior when
# build-starters hard-crashed before any slot wrote its result
# artifact — the compute step then sees an empty success set and the
# redeploy step is skipped (no deploy on a total build failure). These
# `starter-build-result-*` artifacts are already emitted per-slot by
# build-starters; before this job they were write-only.
# merge-multiple:false keeps each slot in its own subdirectory so the
# find below reads them deterministically.
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: starter-build-result-*
path: ${{ runner.temp }}/starter-build-results-in
merge-multiple: false
- name: Compute successfully-built starter services
id: changed
env:
MATRIX_JSON: ${{ needs.detect-starter-changes.outputs.matrix }}
RESULTS_DIR: ${{ runner.temp }}/starter-build-results-in
BUILD_STARTERS_RESULT: ${{ needs.build-starters.result }}
run: |
set -euo pipefail
# Each per-slot artifact extracts to
# $RESULTS_DIR/starter-build-result-<slug>/result.json
# with shape {"service":"<RAW slug>","status":"success|failure|skipped"}
# (see build-starters' "Write per-slot starter build result" step).
# Slurp ALL result records first. `find -exec cat {} +` runs cat zero
# times when nothing matches (empty or absent result dir) → jq -s
# slurps zero inputs → `[]`, so an all-failed / crashed build yields
# an empty record set with no xargs-on-empty hang.
#
# NO `2>/dev/null` on the cat (Finding 2): suppressing read errors
# would let a starter whose result.json exists-but-is-unreadable be
# silently dropped from the success set (a silent under-deploy). Let a
# read error surface and trip `pipefail` so the job fails loud.
records='[]'
if [ -d "$RESULTS_DIR" ]; then
records="$(find "$RESULTS_DIR" -name result.json -exec cat {} + \
| jq -sc '.')"
fi
# Fail-loud artifact-completeness check (Finding 2): every matrix slot
# writes a result.json ("Write per-slot starter build result" runs
# `if: always()`), so when ALL slots built (build-starters.result ==
# 'success') the parsed record count MUST equal the matrix slot count.
# A shortfall means a `starter-build-result-*` artifact is missing or
# expired → a successfully-built starter would be silently dropped
# from the redeploy set. Gated on 'success' so a partial/total build
# FAILURE (where a crashed slot may legitimately lack its artifact) is
# NOT double-reported here — that failure is already surfaced by the
# build-starters job itself.
matrix_slot_count="$(echo "$MATRIX_JSON" | jq 'length')"
record_count="$(echo "$records" | jq 'length')"
if [ "$BUILD_STARTERS_RESULT" = "success" ] && [ "$record_count" -ne "$matrix_slot_count" ]; then
echo "::error::All starter slots built (build-starters.result=success) but parsed $record_count build-result record(s) for $matrix_slot_count matrix slot(s) — a per-slot starter-build-result-* artifact is missing or expired. A successfully-built starter could be silently dropped from the redeploy set. Check the build-starters upload step and artifact retention (retention-days)."
echo "Scheduled starter matrix (detect-starter-changes): $MATRIX_JSON"
echo "Parsed build-result records (from RESULTS_DIR): $records"
exit 1
fi
success_slugs="$(echo "$records" | jq -c '[.[] | select(.status == "success") | .service]')"
# Map each successfully-built RAW slug → its SSOT key via the starter
# matrix `.image` field (image === "starter-<slug>" === the
# railway-envs.ts SERVICES key). redeploy-env.ts resolves that SSOT
# key directly. The RAW slug must NOT be passed: e.g. the raw slug
# "agno" collides with the SHOWCASE `agno` dispatch_name and would
# redeploy the wrong (showcase, not starter) Railway service.
csv="$(jq -rn --argjson m "$MATRIX_JSON" --argjson s "$success_slugs" \
'($m | map(select(.slug as $sl | $s | index($sl)) | .image)) | join(",")')"
# Fail-loud empty-deploy-set check (Finding 1): mirror the sibling
# redeploy-staging job's empty-intersection guard. When ALL slots
# built (build-starters.result == 'success') yet the matrix ∩ success
# CSV is EMPTY, everything built but nothing would be redeployed — a
# silent "we thought we shipped but didn't" hole (a slug↔.image
# contract skew, or the success set maps to no matrix entry). Fail
# loud. Do NOT fail when build-starters merely FAILED (partial or
# total): an empty CSV there is the legitimate "no slot succeeded →
# nothing to deploy" path, and the build failure is already surfaced
# by build-starters itself.
if [ -z "$csv" ] && [ "$BUILD_STARTERS_RESULT" = "success" ]; then
echo "::error::All starter slots built (build-starters.result=success) but matrix ∩ success-set is EMPTY — nothing would be redeployed. Likely a slug↔.image contract skew between detect-starter-changes' matrix and the per-slot result artifacts' service values."
echo "Successful starter slugs (status:success records): $success_slugs"
echo "Scheduled starter matrix (detect-starter-changes): $MATRIX_JSON"
echo "Reconcile the starter matrix (.slug/.image) with build-starters' per-slot result service values so every built starter maps to a Railway service."
exit 1
fi
if [ -z "$csv" ]; then
echo "build-starters.result=$BUILD_STARTERS_RESULT with an empty deploy set — no starter built successfully, nothing to redeploy (any build failure is surfaced by the build-starters job)."
fi
echo "services=$csv" >> "$GITHUB_OUTPUT"
echo "Computed starter services CSV (matrix ∩ build-success): $csv"
# The starter CSV is already a list of GHCR repo names (`.image`),
# so the image list is just that CSV qualified with the registry.
images="$(echo "$csv" | tr ',' '\n' | sed -e '/^$/d' -e 's#^#ghcr.io/copilotkit/#' | paste -sd, -)"
echo "images=$images" >> "$GITHUB_OUTPUT"
echo "Computed starter images CSV: $images"
- name: Advance starter :latest for successfully-built images
if: steps.changed.outputs.images != ''
env:
IMAGES: ${{ steps.changed.outputs.images }}
# No GITHUB_SHA / GITHUB_REPOSITORY shadowing; see the sibling job.
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Same placement rationale as the showcase redeploy job: advance the
# mutable tag immediately before the pull that consumes it. tsx pinned
# to the lockfile-resolved version for the same no-drift reason.
run: npx --yes tsx@4.21.0 showcase/scripts/advance-latest-tag.ts
- name: Redeploy changed starters in staging
# Empty CSV = no starter built successfully (or the build hard-crashed
# before writing any result artifact) → nothing to redeploy → skip.
# This is the "no deploy on failure" guard for the starter lane.
if: steps.changed.outputs.services != ''
env:
RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
SERVICES_CSV: ${{ steps.changed.outputs.services }}
run: |
# Staging is non-blocking by design: redeploy-env.ts always exits 0
# on staging and writes per-service failures to $GITHUB_STEP_SUMMARY.
# We deliberately do NOT set REDEPLOY_SUMMARY_JSON or upload a
# `redeploy-summary` artifact here: that artifact name is OWNED by
# the showcase `redeploy-staging` job and is downloaded by
# showcase_deploy.yml by EXACT name, so a second upload of the same
# name would collide on a combined (starter + showcase) push.
# Starter STAGING verification is intentionally out of scope for
# this deploy-gap fix — starters are already smoke-covered by
# test_smoke-starter.yml and the harness `starter_smoke` axis.
npx tsx showcase/scripts/redeploy-env.ts staging --services "$SERVICES_CSV"
notify-all-builds-failed:
name: Notify all builds failed (staging unchanged)
needs: [detect-changes, build, aggregate-build-results]
# Explicit "everything failed; nothing redeployed" signal — distinct
# from the `notify:` job below which fires on any build-job failure
# (some slots may still have succeeded in that case). Both jobs can
# fire; that's intentional and matches the Slack alert SOP.
if: >-
${{ !cancelled()
&& needs.detect-changes.outputs.has_changes == 'true'
&& needs.build.result != 'skipped'
&& needs.aggregate-build-results.outputs.any_success == 'false' }}
# Previously this keyed off `needs.build.result == 'failure'`, which shared
# the redeploy job's cancelled-rollup blind spot: GitHub rolls the build
# matrix up to 'cancelled' when ANY leg is cancelled (e.g. the LFS shell
# leg under runner contention), so an all-failed build with one cancelled
# leg rolled up to 'cancelled', the `== 'failure'` clause went false, and
# NO alert fired. `any_success == 'false'` (the artifact-derived signal the
# redeploy fix uses) is the authoritative "nothing built" check, so we fire
# on it directly. The `!= 'skipped'` clause still guards the case where the
# build job was SKIPPED (e.g. verify-image-refs failed upstream, so the
# matrix never executed) — the aggregator would report `any_success=false`
# for an empty run, and the sibling `notify` job already covers that
# upstream red, so firing here too would be a misleading "all builds
# failed" alert. `!cancelled()` keeps a user-cancelled RUN silent.
runs-on: ubuntu-latest
timeout-minutes: 3
permissions:
contents: read
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
steps:
- name: Mark workflow red (no service succeeded)
run: |
echo "::error::All builds failed for this run; staging redeploy skipped; :latest unchanged."
exit 1
- name: Slack #oss-alerts
if: always() && env.SLACK_WEBHOOK != ''
uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
with:
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
webhook-type: incoming-webhook
# NOTE: `\n` is NOT an escape sequence in GitHub Actions expression
# string literals — `format()` would emit the two literal characters
# backslash+n, which `toJSON` then encodes as `\\n`, so Slack renders
# a literal "\n". Inject real newlines via `fromJSON('"\n"')` ({4}) so
# `toJSON` encodes them as a single `\n` that Slack honors.
payload: |
{ "text": ${{ toJSON(format(':x: *Showcase: all builds failed*{4}staging unchanged (no redeploy){4}Commit: `{0}` by {1}{4}<https://github.com/{2}/actions/runs/{3}|View run>', github.sha, github.actor, github.repository, github.run_id, fromJSON('"\n"'))) }} }
notify-cancelled-builds:
name: Notify cancelled/timed-out build slots
needs: [detect-changes, build, aggregate-build-results]
# ── Why this job exists ───────────────────────────────────────────────
#
# A PARTIALLY-CANCELLED fleet build used to be indistinguishable from a
# clean one, in BOTH directions: no alert, and no post-deploy verify.
# Observed on build run 30162773601 (merge of #6160): 5 of 28 slots were
# killed by `timeout-minutes`, 23 built and were redeployed to staging,
# `redeploy-staging` SUCCEEDED and uploaded `redeploy-summary` — and
# then:
# - `notify` was SKIPPED, so no Slack alert fired, and
# - the run rolled up to conclusion `cancelled`, which failed
# showcase_deploy.yml's `conclusion == 'success'` gate, so the
# staging redeploy was never verified.
# A real, partial, unverified staging deploy looked exactly like a
# silent success.
#
# The reason no existing guard caught it is that GitHub's status
# functions are blind to a leg-level cancellation. Measured on purpose-
# built probe run 30166429073 (matrix leg killed by `timeout-minutes`,
# sibling leg green):
# job.status of the killed leg .... cancelled
# needs.<matrix>.result (rollup) .. cancelled
# `if: cancelled()` ............... SKIPPED (i.e. evaluated FALSE)
# `if: failure()` ................. SKIPPED (i.e. evaluated FALSE)
# pre-fix `notify` condition ...... SKIPPED ← the bug
# post-fix `notify` condition ..... RAN ← the fix
# workflow run conclusion ......... cancelled
# This matches the docs: `cancelled()` "returns true if the workflow was
# canceled" (workflow-scoped, not job-scoped), and `failure()` "returns
# true if any ancestor job fails" — a CANCELLED ancestor did not FAIL.
# So `failure() || cancelled()` would NOT have fixed this; the signal
# has to come from the per-slot build results.
#
# This job does the two things a cancelled slot must do:
# 1. RED THE RUN (step 1 exits 1) so the conclusion becomes `failure`
# instead of `cancelled`. That is honest — a slot killed by its
# timeout budget is a failure, not a cancellation — and it makes
# the incomplete build visible on the commit status. It does NOT
# block verification: showcase_deploy.yml no longer requires
# `conclusion == 'success'` (it self-gates on the presence of the
# redeploy-summary artifact instead), so the subset that really
# shipped still gets probed.
# 2. ALERT, naming the affected services so the run can be re-run
# with "Re-run failed jobs" without re-reading 28 job logs.
#
# `!cancelled()` is retained as the intentional-vs-flake discriminator:
# a human cancelling the whole RUN makes `cancelled()` TRUE and keeps
# this silent, while a leg-level timeout leaves it FALSE (proven above)
# so the alert still fires. Belt and braces: on a run-level cancel
# `aggregate-build-results` is itself skipped by its own `!cancelled()`
# guard, so `any_cancelled` would be '' rather than 'true' anyway.
if: >-
${{ !cancelled()
&& needs.detect-changes.outputs.has_changes == 'true'
&& needs.aggregate-build-results.outputs.any_cancelled == 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 3
permissions:
contents: read
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
steps:
- name: Mark workflow red (build incomplete)
env:
CANCELLED_SERVICES: ${{ needs.aggregate-build-results.outputs.cancelled_services }}
run: |
echo "::error::Build INCOMPLETE — these slots were cancelled (typically killed by timeout-minutes) and pushed NO image: ${CANCELLED_SERVICES}"
echo "Those services were correctly excluded from the staging redeploy (matrix ∩ build-success), so staging still serves their previous :latest."
echo "Re-run the failed jobs on this run to finish the fleet."
exit 1
- name: Slack #oss-alerts
# `always()` so the alert still goes out after step 1 deliberately
# exits non-zero to red the run.
if: always() && env.SLACK_WEBHOOK != ''
uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
with:
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
webhook-type: incoming-webhook
# NOTE: `\n` is NOT an escape sequence in GitHub Actions expression
# string literals — `format()` would emit the two literal characters
# backslash+n, which `toJSON` then encodes as `\\n`, so Slack renders
# a literal "\n". Inject real newlines via `fromJSON('"\n"')` ({5}) so
# `toJSON` encodes them as a single `\n` that Slack honors.
payload: |
{ "text": ${{ toJSON(format(':warning: *Showcase build INCOMPLETE — slots cancelled*{5}Cancelled (no image pushed): `{4}`{5}Staging keeps the previous :latest for those services.{5}Commit: `{0}` by {1}{5}<https://github.com/{2}/actions/runs/{3}|View run>', github.sha, github.actor, github.repository, github.run_id, needs.aggregate-build-results.outputs.cancelled_services, fromJSON('"\n"'))) }} }
notify:
name: Notify on failure
# Cover the whole detect→build→aggregate→redeploy pipeline. `needs: [build]`
# alone meant a failure in aggregate-build-results or redeploy-staging
# produced ZERO Slack signal (verify just never ran). Likewise, a red
# in detect-changes, check-lockfile, or verify-image-refs would skip
# `build` (skipped != failure) so the original `needs: [build]` form
# also missed those pre-build red paths. Adding the early-stage jobs
# to `needs` extends the alert surface to the full workflow, matching
# the workflow-level notify pattern in showcase_promote.yml.
# The guard is `!cancelled() && (failure() || any_success == 'false')`
# rather than a bare `failure()`. Bare `failure()` shared the redeploy
# job's cancelled-rollup blind spot: when every real build leg failed but
# one leg was CANCELLED (runner contention), the matrix rolled up to
# 'cancelled' — which is NOT a `failure()` — so no needs job "failed" and
# the alert stayed silent on a genuinely dead build. The extra
# `any_success == 'false'` clause (the artifact-derived signal the redeploy
# fix uses) fires on "nothing built" regardless of the rollup. `failure()`
# is retained so a failure DOWNSTREAM of a successful build (e.g.
# redeploy-staging, on which any_success == 'true') still alerts. `!cancelled()`
# keeps a user-cancelled RUN silent, and the "no changes → build skipped"
# path stays a no-op (aggregate is skipped, so any_success is '' not 'false',
# and nothing failed). If `notify-all-builds-failed` also fires (genuine
# all-failed case), both alerts firing for the same event is acceptable per
# the alert SOP.
#
# `detect-starter-changes` + `build-starters` + `redeploy-staging-starters`
# are included so a FAILED starter image build OR a failed starter staging
# redeploy (examples/integrations/*) also triggers the Slack alert + PR
# comment. Previously the starter lane had NO alert surface: its jobs were
# absent from `needs`, so `if: failure()` could never see a starter build
# failure and it shipped silently. `if: failure()` still no-ops when the
# starter jobs are skipped (no starter changes) — skipped != failure — so
# this only fires on a genuine starter build or starter redeploy failure.
needs:
[
detect-changes,
detect-starter-changes,
check-lockfile,
verify-image-refs,
build,
build-starters,
aggregate-build-results,
redeploy-staging,
redeploy-staging-starters,
]
#
# The third clause, `any_cancelled == 'true'`, closes the partial-cancel
# hole: with SOME slots green, `failure()` is false (a cancelled ancestor
# did not fail) and `any_success` is 'true', so the first two clauses both
# go false and this job was SKIPPED on a genuinely incomplete build — see
# `notify-cancelled-builds` above for the measured proof (probe run
# 30166429073) and the production incident (run 30162773601). This clause
# also restores the PR comment on that path, which is the only place the
# merge author gets told. It cannot add noise to a clean run: a build with
# no cancelled slot emits `any_cancelled=false`, and a build with no
# changes skips the aggregator entirely so the output is '' — neither
# equals 'true'. `notify-cancelled-builds` fires alongside this job on
# that path; two alerts for one event is the same convention already used
# by `notify-all-builds-failed` + this job, and matches the alert SOP.
if: >-
${{ !cancelled()
&& (failure()
|| needs.aggregate-build-results.outputs.any_success == 'false'
|| needs.aggregate-build-results.outputs.any_cancelled == 'true') }}
runs-on: ubuntu-latest
permissions:
pull-requests: write
issues: write
timeout-minutes: 5
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
steps:
- name: Slack alert
if: env.SLACK_WEBHOOK != ''
uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
with:
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
webhook-type: incoming-webhook
# NOTE: `\n` is NOT an escape sequence in GitHub Actions expression
# string literals — `format()` would emit the two literal characters
# backslash+n, which `toJSON` then encodes as `\\n`, so Slack renders
# a literal "\n". Inject real newlines via `fromJSON('"\n"')` ({4}) so
# `toJSON` encodes them as a single `\n` that Slack honors.
payload: |
{ "text": ${{ toJSON(format(':x: *Showcase Build Failed*{4}Commit: `{0}` by {1}{4}<https://github.com/{2}/actions/runs/{3}|View run>', github.sha, github.actor, github.repository, github.run_id, fromJSON('"\n"'))) }} }
- name: Comment on PR
env:
# Non-empty only on the partial-cancel path, where the generic
# "the build failed" wording would be wrong (most slots built and
# were redeployed; a subset was killed mid-build).
CANCELLED_SERVICES: ${{ needs.aggregate-build-results.outputs.cancelled_services }}
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
with:
script: |
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
owner: context.repo.owner,
repo: context.repo.repo,
commit_sha: context.sha,
});
const merged = prs.find(pr => pr.merged_at);
if (!merged) {
console.log('No merged PR found for this commit — skipping comment');
return;
}
const cancelled = (process.env.CANCELLED_SERVICES || '')
.split(',')
.map(s => s.trim())
.filter(Boolean);
const marker = '<!-- showcase-build-failure -->';
const body = [
marker,
cancelled.length > 0
? `### :warning: Showcase Build Incomplete`
: `### :x: Showcase Build Failed`,
``,
cancelled.length > 0
? [
`The Docker build triggered by this PR's merge did not finish for every service.`,
`These slots were cancelled (typically killed by \`timeout-minutes\`) and pushed **no** image, so staging still serves their previous \`:latest\`:`,
``,
cancelled.map(s => `- \`${s}\``).join('\n'),
``,
`Everything else built and was redeployed. Use **Re-run failed jobs** on the run to finish the fleet.`,
].join('\n')
: `The Docker build triggered by this PR's merge failed.`,
``,
`**Run:** ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
`**Commit:** \`${context.sha.slice(0, 8)}\``,
``,
`@${merged.user?.login ?? 'unknown'} — please check the build logs and fix the issue.`,
].join('\n');
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: merged.number,
});
const existing = comments.find(c => c.body?.includes(marker));
if (existing) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
body,
});
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: merged.number,
body,
});
}