mirror of
https://github.com/CopilotKit/CopilotKit.git
synced 2026-09-14 16:26:20 +08:00
15222d8589
The previous suite was 37/37 green against a workflow that could not have
run at all: the guard step had no `packages: write` and no GHCR login, so
every `imagetools create` would have 401'd. Asserting that a step EXISTS
proves nothing about whether it can succeed. Each change below was checked
by reintroducing the defect and confirming the test goes red.
Registry-auth precondition (the miss that let the above ship)
- Both redeploy jobs must declare `packages: write` AND a `docker/login-action`
step for ghcr.io, ordered BEFORE the guard. Dropping either reds the suite.
The intersection, executed rather than restated
- The old test only checked that the string `images=` appears in the compute
step; swapping its jq for the full matrix — so a FAILED build moves
`:latest` — kept it green. The real `changed` shell now RUNS, against the
real ALL_SERVICES matrix read out of the workflow, and the emitted
$GITHUB_OUTPUT is asserted: a failed build is in neither set, and the
`skip_build` slot is in `services` but NOT in `images` (handing it to the
guard fails "manifest unknown" and blocks the redeploy for the whole fleet).
Same treatment for the starter lane.
Fixtures joined to real registry output
- `extractRevisionLabel` was pinned to hand-written payloads that were never
compared with reality. Since `readLatestRevision` maps every failure to
null, and null ADVANCES, a parser that silently never matches yields a
permanently-blind guard with a fully green suite. Both fixtures are now
verbatim `docker buildx imagetools inspect --format '{{json .Image}}'`
output (buildx v0.35.0), unformatted so key order survives: a platform-keyed
multi-arch image carrying the label, and our own `:latest`, which turns out
to be a bare config object with NO labels at all. The label key used in the
injection test is read from the workflow's own `labels:` input, so parser
and producer cannot drift apart silently.
Closed vacuities
- The `:latest` ban read one step's `tags` and was vacuously green on an empty
list. It now covers every tagged step plus hand-rolled `docker push`/`docker
tag`/`imagetools create`, and fails on an empty `tags`.
- The `cancel-in-progress` ban read only top-level config; a job-level
`concurrency` on `build` reproduced the exact harm and stayed green. All
jobs are checked now.
- `already-current` is keyed on the digest, matching the script.
- The unreachable `(null, "ahead")` row is labelled as the defensive
input-space case it is, and a new test pins that the real flow never calls
compare() with an unknown revision.
New coverage: classifyProbeFailure (incl. a REAL execFileSync timeout, and
the two precedence traps — gh's rate-limited 403 is throttling, not auth; a
registry's 404-with-denied is auth, not absent), readFlag, escapeAnnotationData
(a forged `\n::error::` stays inert), isDirectInvocation through a symlink,
the `::error` annotation on fleet failure, and the digest-mismatch advance.
Also: GITHUB_SHA / GITHUB_REPOSITORY are no longer shadowed in the workflow.
The runner exports both and the script reads process.env, so the old
assertions pinned a redundancy rather than a capability — removed together,
as the comment there required. Workflow-reading scaffolding duplicated with
redeploy-guard.test.ts is extracted and the YAML parse memoized.
131 lines
4.1 KiB
TypeScript
131 lines
4.1 KiB
TypeScript
import { readFileSync } from "node:fs";
|
|
import { fileURLToPath } from "node:url";
|
|
import { dirname, join } from "node:path";
|
|
import { parse as parseYaml } from "yaml";
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Shared read-side scaffolding for the tests that assert against the LIVE
|
|
// `.github/workflows/showcase_build.yml`.
|
|
//
|
|
// Two suites (advance-latest-tag.test.ts, redeploy-guard.test.ts) previously
|
|
// carried byte-identical copies of the path constant and the parse helper, and
|
|
// each re-read + re-parsed the 1,700-line YAML on EVERY helper call. The parse
|
|
// is memoized here: the workflow cannot change mid-run, so one parse per test
|
|
// process is both correct and ~2 orders of magnitude cheaper.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export const WORKFLOW_PATH = join(
|
|
dirname(fileURLToPath(import.meta.url)),
|
|
"..",
|
|
"..",
|
|
"..",
|
|
".github",
|
|
"workflows",
|
|
"showcase_build.yml",
|
|
);
|
|
|
|
export interface WorkflowStep {
|
|
name?: string;
|
|
id?: string;
|
|
if?: string;
|
|
uses?: string;
|
|
run?: string;
|
|
with?: Record<string, unknown>;
|
|
env?: Record<string, string>;
|
|
}
|
|
|
|
export interface WorkflowJob {
|
|
name?: string;
|
|
if?: string;
|
|
concurrency?: unknown;
|
|
permissions?: Record<string, string> | string;
|
|
steps?: WorkflowStep[];
|
|
}
|
|
|
|
export interface WorkflowDoc {
|
|
concurrency?: unknown;
|
|
permissions?: Record<string, string> | string;
|
|
jobs: Record<string, WorkflowJob>;
|
|
}
|
|
|
|
let cached: WorkflowDoc | undefined;
|
|
|
|
/** The parsed workflow. Parsed once per process, then reused. */
|
|
export function readWorkflow(): WorkflowDoc {
|
|
cached ??= parseYaml(readFileSync(WORKFLOW_PATH, "utf8")) as WorkflowDoc;
|
|
return cached;
|
|
}
|
|
|
|
export function jobOf(jobId: string): WorkflowJob {
|
|
const job = readWorkflow().jobs[jobId];
|
|
if (!job) throw new Error(`Job '${jobId}' not found in ${WORKFLOW_PATH}`);
|
|
return job;
|
|
}
|
|
|
|
export function stepsOf(jobId: string): WorkflowStep[] {
|
|
const job = jobOf(jobId);
|
|
if (!Array.isArray(job.steps)) {
|
|
throw new Error(`Job '${jobId}' has no steps`);
|
|
}
|
|
return job.steps;
|
|
}
|
|
|
|
/** Every step of every job, flattened — for workflow-wide bans. */
|
|
export function allSteps(): Array<{ jobId: string; step: WorkflowStep }> {
|
|
const out: Array<{ jobId: string; step: WorkflowStep }> = [];
|
|
for (const [jobId, job] of Object.entries(readWorkflow().jobs)) {
|
|
for (const step of job.steps ?? []) out.push({ jobId, step });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/** The single step in `jobId` with the given `id:`. */
|
|
export function stepById(jobId: string, stepId: string): WorkflowStep {
|
|
const step = stepsOf(jobId).find((s) => s.id === stepId);
|
|
if (!step) throw new Error(`Job '${jobId}' has no step with id '${stepId}'`);
|
|
return step;
|
|
}
|
|
|
|
/**
|
|
* Pull a single-quoted shell heredoc-style JSON literal (`NAME='[...]'`) out of
|
|
* a step's `run:` script and parse it.
|
|
*
|
|
* The service and starter matrices are defined as inline JSON inside
|
|
* `detect-changes` / `detect-starter-changes`. Reading them from the workflow
|
|
* rather than restating them in a fixture is what keeps the intersection tests
|
|
* joined to the real fleet — a new `skip_build` slot is picked up automatically.
|
|
*/
|
|
export function parseJsonLiteralFromRun<T>(run: string, name: string): T {
|
|
const match = run.match(new RegExp(`${name}='([\\s\\S]*?)'`));
|
|
if (!match)
|
|
throw new Error(`No ${name}='…' literal found in the step script`);
|
|
return JSON.parse(match[1]) as T;
|
|
}
|
|
|
|
export interface ServiceSlot {
|
|
dispatch_name: string;
|
|
image: string;
|
|
skip_build?: boolean;
|
|
}
|
|
|
|
export interface StarterSlot {
|
|
slug: string;
|
|
image: string;
|
|
}
|
|
|
|
/** The live showcase service matrix (`ALL_SERVICES` in `detect-changes`). */
|
|
export function allServiceSlots(): ServiceSlot[] {
|
|
return parseJsonLiteralFromRun<ServiceSlot[]>(
|
|
stepById("detect-changes", "build-matrix").run ?? "",
|
|
"ALL_SERVICES",
|
|
);
|
|
}
|
|
|
|
/** The live starter matrix (`ALL_STARTERS` in `detect-starter-changes`). */
|
|
export function allStarterSlots(): StarterSlot[] {
|
|
return parseJsonLiteralFromRun<StarterSlot[]>(
|
|
stepById("detect-starter-changes", "starter-matrix").run ?? "",
|
|
"ALL_STARTERS",
|
|
);
|
|
}
|