mirror of
https://github.com/vercel/workflow.git
synced 2026-09-14 19:59:43 +08:00
54ba1888cd
* fix: compare benchmarks against PR base branch instead of main - Use github.event.pull_request.base.ref instead of hardcoded main - Remove search_artifacts: true to ensure most recent baseline is used - For stacked PRs, this compares against the parent PR's baseline 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * fix: group failed e2e tests by category and app in summary Instead of listing each failed test as a separate item, group them by: 1. Category (world): e.g., "Community Worlds", "Vercel Production" 2. App (framework): e.g., "mongodb", "turso", "nextjs-turbopack" This makes the summary much more readable when there are many failures. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * fix: ensure local E2E tests always produce JSON output - Add 'fastify' to app detection list in aggregate-e2e-results.js - Change && to ; so e2e tests run even if dev.test.ts fails - This ensures local-dev, local-prod, and local-postgres categories appear in the E2E summary comment 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * fix: ensure local E2E tests always produce JSON output - Add 'fastify' to app detection list in aggregate-e2e-results.js - Change && to ; so e2e tests run even if dev.test.ts fails - This ensures local-dev, local-prod, and local-postgres categories appear in the E2E summary comment 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * feat: publish CI results to GitHub Pages for docs - Add generate-docs-data.js script to create JSON summaries from CI artifacts - Add publish-results job to tests.yml and benchmarks.yml workflows - Update docs/lib/worlds-data.ts to fetch from GitHub Pages URLs - Results published to https://vercel.github.io/workflow/ci/ This allows the docs worlds page to display actual test/benchmark results without requiring a GITHUB_TOKEN. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * fix: correct outputFile path for local E2E test artifacts The --outputFile path was using ../../ which placed files outside the repo because pnpm run test:e2e executes from workspace root, not from the cd'd workbench directory. This prevented local-dev, local-prod, and local-postgres test results from being uploaded as artifacts. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * fix: show green checkmark for skipped tests instead of warning Skipped tests are intentional and shouldn't show as warnings in the E2E test summary comments. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * feat: use collapsible sections in benchmark PR comment Wrap each benchmark, stream benchmarks section, and summary tables in <details> toggles to make the PR comment more compact and readable. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * feat: add Vercel observability links to benchmark PR comments - Store runId in benchmark timing data - Add project-slug to Vercel benchmark matrix - Pass WORKFLOW_VERCEL_PROJECT_SLUG env var to benchmarks - Store Vercel metadata (teamSlug, projectSlug, environment) in timing files - Generate observability deep links for each Vercel world benchmark - Show observability links below Production (Vercel) tables 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * fix: use correct Vercel project slugs for observability links - nextjs-turbopack → example-nextjs-workflow-turbopack - nitro-v3 → workbench-nitro-workflow 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> --------- Co-authored-by: Claude <noreply@anthropic.com>
338 lines
8.5 KiB
TypeScript
338 lines
8.5 KiB
TypeScript
/**
|
|
* GitHub API utilities for fetching CI workflow results
|
|
*/
|
|
|
|
const GITHUB_API = 'https://api.github.com';
|
|
const OWNER = 'vercel';
|
|
const REPO = 'workflow';
|
|
|
|
// Artifact names we're looking for
|
|
const E2E_ARTIFACT_PATTERNS = [
|
|
'e2e-results-local',
|
|
'e2e-results-postgres',
|
|
'e2e-results-vercel',
|
|
'e2e-results-starter',
|
|
'e2e-results-turso',
|
|
'e2e-results-mongodb',
|
|
'e2e-results-redis',
|
|
];
|
|
|
|
const BENCH_ARTIFACT_PATTERNS = [
|
|
'bench-results-nextjs-turbopack-local',
|
|
'bench-results-nextjs-turbopack-postgres',
|
|
'bench-results-nextjs-turbopack-vercel',
|
|
'bench-results-nextjs-turbopack-starter',
|
|
'bench-results-nextjs-turbopack-turso',
|
|
'bench-results-nextjs-turbopack-mongodb',
|
|
'bench-results-nextjs-turbopack-redis',
|
|
];
|
|
|
|
interface GitHubWorkflowRun {
|
|
id: number;
|
|
head_sha: string;
|
|
head_branch: string;
|
|
status: string;
|
|
conclusion: string;
|
|
created_at: string;
|
|
updated_at: string;
|
|
}
|
|
|
|
interface GitHubArtifact {
|
|
id: number;
|
|
name: string;
|
|
archive_download_url: string;
|
|
size_in_bytes: number;
|
|
created_at: string;
|
|
}
|
|
|
|
interface WorkflowRunsResponse {
|
|
total_count: number;
|
|
workflow_runs: GitHubWorkflowRun[];
|
|
}
|
|
|
|
interface ArtifactsResponse {
|
|
total_count: number;
|
|
artifacts: GitHubArtifact[];
|
|
}
|
|
|
|
interface E2ETestResult {
|
|
numTotalTests: number;
|
|
numPassedTests: number;
|
|
numFailedTests: number;
|
|
numPendingTests: number;
|
|
testResults: Array<{
|
|
assertionResults: Array<{
|
|
fullName: string;
|
|
status: 'passed' | 'failed' | 'skipped';
|
|
duration?: number;
|
|
}>;
|
|
}>;
|
|
}
|
|
|
|
interface BenchmarkResult {
|
|
files: Array<{
|
|
groups: Array<{
|
|
benchmarks: Array<{
|
|
name: string;
|
|
mean: number;
|
|
min: number;
|
|
max: number;
|
|
sampleCount: number;
|
|
}>;
|
|
}>;
|
|
}>;
|
|
}
|
|
|
|
async function fetchGitHub<T>(
|
|
path: string,
|
|
options?: RequestInit
|
|
): Promise<T | null> {
|
|
const url = `${GITHUB_API}${path}`;
|
|
const headers: HeadersInit = {
|
|
Accept: 'application/vnd.github+json',
|
|
'X-GitHub-Api-Version': '2022-11-28',
|
|
};
|
|
|
|
// Use GITHUB_TOKEN if available (for higher rate limits)
|
|
const token = process.env.GITHUB_TOKEN;
|
|
if (token) {
|
|
headers['Authorization'] = `Bearer ${token}`;
|
|
}
|
|
|
|
try {
|
|
const res = await fetch(url, {
|
|
...options,
|
|
headers: { ...headers, ...options?.headers },
|
|
next: { revalidate: 300 }, // Cache for 5 minutes
|
|
});
|
|
|
|
if (!res.ok) {
|
|
console.error(`GitHub API error: ${res.status} ${res.statusText}`);
|
|
return null;
|
|
}
|
|
|
|
return res.json();
|
|
} catch (error) {
|
|
console.error('Failed to fetch from GitHub:', error);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get the latest successful workflow run for a specific workflow on main branch
|
|
*/
|
|
export async function getLatestWorkflowRun(
|
|
workflowFileName: string
|
|
): Promise<GitHubWorkflowRun | null> {
|
|
const params = new URLSearchParams({
|
|
branch: 'main',
|
|
status: 'completed',
|
|
per_page: '1',
|
|
});
|
|
|
|
const data = await fetchGitHub<WorkflowRunsResponse>(
|
|
`/repos/${OWNER}/${REPO}/actions/workflows/${workflowFileName}/runs?${params}`
|
|
);
|
|
|
|
return data?.workflow_runs?.[0] ?? null;
|
|
}
|
|
|
|
/**
|
|
* Get artifacts from a workflow run
|
|
*/
|
|
export async function getWorkflowArtifacts(
|
|
runId: number
|
|
): Promise<GitHubArtifact[]> {
|
|
const data = await fetchGitHub<ArtifactsResponse>(
|
|
`/repos/${OWNER}/${REPO}/actions/runs/${runId}/artifacts?per_page=100`
|
|
);
|
|
|
|
return data?.artifacts ?? [];
|
|
}
|
|
|
|
/**
|
|
* Download and parse an artifact's JSON content
|
|
* Note: This requires authentication for private repos
|
|
*/
|
|
export async function downloadArtifact<T>(
|
|
artifactId: number
|
|
): Promise<T | null> {
|
|
const token = process.env.GITHUB_TOKEN;
|
|
if (!token) {
|
|
console.warn('GITHUB_TOKEN not set, cannot download artifacts');
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
// Get the download URL (this redirects to a blob storage URL)
|
|
const res = await fetch(
|
|
`${GITHUB_API}/repos/${OWNER}/${REPO}/actions/artifacts/${artifactId}/zip`,
|
|
{
|
|
headers: {
|
|
Authorization: `Bearer ${token}`,
|
|
Accept: 'application/vnd.github+json',
|
|
},
|
|
redirect: 'follow',
|
|
}
|
|
);
|
|
|
|
if (!res.ok) {
|
|
console.error(`Failed to download artifact: ${res.status}`);
|
|
return null;
|
|
}
|
|
|
|
// The response is a ZIP file - we need to extract the JSON
|
|
const JSZip = (await import('jszip')).default;
|
|
const arrayBuffer = await res.arrayBuffer();
|
|
const zip = await JSZip.loadAsync(arrayBuffer);
|
|
|
|
// Find and parse the JSON file
|
|
const files = Object.keys(zip.files);
|
|
const jsonFile = files.find((f) => f.endsWith('.json'));
|
|
if (!jsonFile) {
|
|
console.error('No JSON file found in artifact');
|
|
return null;
|
|
}
|
|
|
|
const content = await zip.files[jsonFile].async('string');
|
|
return JSON.parse(content);
|
|
} catch (error) {
|
|
console.error('Failed to download/parse artifact:', error);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Parse E2E results into the WorldE2E format
|
|
*/
|
|
export function parseE2EResults(results: E2ETestResult | null): {
|
|
status: 'passing' | 'partial' | 'failing' | 'pending';
|
|
total: number;
|
|
passed: number;
|
|
failed: number;
|
|
skipped: number;
|
|
progress: number;
|
|
tests?: Array<{
|
|
name: string;
|
|
status: 'passed' | 'failed' | 'skipped';
|
|
duration?: number;
|
|
}>;
|
|
} | null {
|
|
if (!results) return null;
|
|
|
|
const total = results.numTotalTests;
|
|
const passed = results.numPassedTests;
|
|
const failed = results.numFailedTests;
|
|
const skipped = results.numPendingTests;
|
|
const progress = total > 0 ? (passed / total) * 100 : 0;
|
|
|
|
let status: 'passing' | 'partial' | 'failing' | 'pending';
|
|
if (passed === total) {
|
|
status = 'passing';
|
|
} else if (passed > 0) {
|
|
status = 'partial';
|
|
} else if (failed > 0) {
|
|
status = 'failing';
|
|
} else {
|
|
status = 'pending';
|
|
}
|
|
|
|
// Extract individual test results
|
|
const tests: Array<{
|
|
name: string;
|
|
status: 'passed' | 'failed' | 'skipped';
|
|
duration?: number;
|
|
}> = [];
|
|
for (const testFile of results.testResults) {
|
|
for (const assertion of testFile.assertionResults) {
|
|
tests.push({
|
|
name: assertion.fullName,
|
|
status:
|
|
assertion.status === 'pending'
|
|
? 'skipped'
|
|
: (assertion.status as 'passed' | 'failed'),
|
|
duration: assertion.duration,
|
|
});
|
|
}
|
|
}
|
|
|
|
return { status, total, passed, failed, skipped, progress, tests };
|
|
}
|
|
|
|
/**
|
|
* Parse benchmark results into the WorldBenchmark format
|
|
*/
|
|
export function parseBenchmarkResults(results: BenchmarkResult | null): {
|
|
status: 'measured' | 'pending';
|
|
metrics: Record<
|
|
string,
|
|
{ mean: number; min: number; max: number; samples?: number }
|
|
> | null;
|
|
} | null {
|
|
if (!results?.files?.[0]?.groups?.[0]?.benchmarks) return null;
|
|
|
|
const metrics: Record<
|
|
string,
|
|
{ mean: number; min: number; max: number; samples?: number }
|
|
> = {};
|
|
|
|
for (const bench of results.files[0].groups[0].benchmarks) {
|
|
metrics[bench.name] = {
|
|
mean: bench.mean,
|
|
min: bench.min,
|
|
max: bench.max,
|
|
samples: bench.sampleCount,
|
|
};
|
|
}
|
|
|
|
return {
|
|
status: Object.keys(metrics).length > 0 ? 'measured' : 'pending',
|
|
metrics: Object.keys(metrics).length > 0 ? metrics : null,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Map artifact name to world ID
|
|
* Artifact naming conventions:
|
|
* - E2E: e2e-results-{category}-{app} where category maps to world
|
|
* - vercel-prod → vercel
|
|
* - local-dev, local-prod → local
|
|
* - local-postgres → postgres
|
|
* - community-{world} → {world}
|
|
* - windows → local (windows tests use local world)
|
|
* - Benchmarks: bench-results-{app}-{world}
|
|
*/
|
|
export function artifactToWorldId(artifactName: string): string | null {
|
|
// E2E results for community worlds: e2e-results-community-{world}
|
|
if (artifactName.startsWith('e2e-results-community-')) {
|
|
return artifactName.replace('e2e-results-community-', '');
|
|
}
|
|
|
|
// E2E results: e2e-results-{category}-{app}
|
|
if (artifactName.startsWith('e2e-results-')) {
|
|
const rest = artifactName.replace('e2e-results-', '');
|
|
if (rest.startsWith('vercel-prod-')) return 'vercel';
|
|
if (rest.startsWith('local-dev-') || rest.startsWith('local-prod-'))
|
|
return 'local';
|
|
if (rest.startsWith('local-postgres-')) return 'postgres';
|
|
if (rest.startsWith('windows-')) return 'local';
|
|
return null;
|
|
}
|
|
|
|
// Benchmark results: bench-results-{app}-{world}
|
|
if (artifactName.startsWith('bench-results-')) {
|
|
const parts = artifactName.replace('bench-results-', '').split('-');
|
|
return parts[parts.length - 1];
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export {
|
|
type GitHubWorkflowRun,
|
|
type GitHubArtifact,
|
|
type E2ETestResult,
|
|
type BenchmarkResult,
|
|
E2E_ARTIFACT_PATTERNS,
|
|
BENCH_ARTIFACT_PATTERNS,
|
|
};
|