Files
Pranay Prakash 54ba1888cd fix: compare benchmarks against PR base branch instead of main (#560)
* fix: compare benchmarks against PR base branch instead of main

- Use github.event.pull_request.base.ref instead of hardcoded main
- Remove search_artifacts: true to ensure most recent baseline is used
- For stacked PRs, this compares against the parent PR's baseline

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

* fix: group failed e2e tests by category and app in summary

Instead of listing each failed test as a separate item, group them by:
1. Category (world): e.g., "Community Worlds", "Vercel Production"
2. App (framework): e.g., "mongodb", "turso", "nextjs-turbopack"

This makes the summary much more readable when there are many failures.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

* fix: ensure local E2E tests always produce JSON output

- Add 'fastify' to app detection list in aggregate-e2e-results.js
- Change && to ; so e2e tests run even if dev.test.ts fails
- This ensures local-dev, local-prod, and local-postgres categories
  appear in the E2E summary comment

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

* fix: ensure local E2E tests always produce JSON output

- Add 'fastify' to app detection list in aggregate-e2e-results.js
- Change && to ; so e2e tests run even if dev.test.ts fails
- This ensures local-dev, local-prod, and local-postgres categories
  appear in the E2E summary comment

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

* feat: publish CI results to GitHub Pages for docs

- Add generate-docs-data.js script to create JSON summaries from CI artifacts
- Add publish-results job to tests.yml and benchmarks.yml workflows
- Update docs/lib/worlds-data.ts to fetch from GitHub Pages URLs
- Results published to https://vercel.github.io/workflow/ci/

This allows the docs worlds page to display actual test/benchmark
results without requiring a GITHUB_TOKEN.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

* fix: correct outputFile path for local E2E test artifacts

The --outputFile path was using ../../ which placed files outside the
repo because pnpm run test:e2e executes from workspace root, not from
the cd'd workbench directory. This prevented local-dev, local-prod, and
local-postgres test results from being uploaded as artifacts.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

* fix: show green checkmark for skipped tests instead of warning

Skipped tests are intentional and shouldn't show as warnings in the
E2E test summary comments.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

* feat: use collapsible sections in benchmark PR comment

Wrap each benchmark, stream benchmarks section, and summary tables in
<details> toggles to make the PR comment more compact and readable.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

* feat: add Vercel observability links to benchmark PR comments

- Store runId in benchmark timing data
- Add project-slug to Vercel benchmark matrix
- Pass WORKFLOW_VERCEL_PROJECT_SLUG env var to benchmarks
- Store Vercel metadata (teamSlug, projectSlug, environment) in timing files
- Generate observability deep links for each Vercel world benchmark
- Show observability links below Production (Vercel) tables

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

* fix: use correct Vercel project slugs for observability links

- nextjs-turbopack → example-nextjs-workflow-turbopack
- nitro-v3 → workbench-nitro-workflow

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

---------

Co-authored-by: Claude <noreply@anthropic.com>
2025-12-06 18:30:25 -08:00

338 lines
8.5 KiB
TypeScript

/**
* GitHub API utilities for fetching CI workflow results
*/
const GITHUB_API = 'https://api.github.com';
const OWNER = 'vercel';
const REPO = 'workflow';
// Artifact names we're looking for
const E2E_ARTIFACT_PATTERNS = [
'e2e-results-local',
'e2e-results-postgres',
'e2e-results-vercel',
'e2e-results-starter',
'e2e-results-turso',
'e2e-results-mongodb',
'e2e-results-redis',
];
const BENCH_ARTIFACT_PATTERNS = [
'bench-results-nextjs-turbopack-local',
'bench-results-nextjs-turbopack-postgres',
'bench-results-nextjs-turbopack-vercel',
'bench-results-nextjs-turbopack-starter',
'bench-results-nextjs-turbopack-turso',
'bench-results-nextjs-turbopack-mongodb',
'bench-results-nextjs-turbopack-redis',
];
interface GitHubWorkflowRun {
id: number;
head_sha: string;
head_branch: string;
status: string;
conclusion: string;
created_at: string;
updated_at: string;
}
interface GitHubArtifact {
id: number;
name: string;
archive_download_url: string;
size_in_bytes: number;
created_at: string;
}
interface WorkflowRunsResponse {
total_count: number;
workflow_runs: GitHubWorkflowRun[];
}
interface ArtifactsResponse {
total_count: number;
artifacts: GitHubArtifact[];
}
interface E2ETestResult {
numTotalTests: number;
numPassedTests: number;
numFailedTests: number;
numPendingTests: number;
testResults: Array<{
assertionResults: Array<{
fullName: string;
status: 'passed' | 'failed' | 'skipped';
duration?: number;
}>;
}>;
}
interface BenchmarkResult {
files: Array<{
groups: Array<{
benchmarks: Array<{
name: string;
mean: number;
min: number;
max: number;
sampleCount: number;
}>;
}>;
}>;
}
async function fetchGitHub<T>(
path: string,
options?: RequestInit
): Promise<T | null> {
const url = `${GITHUB_API}${path}`;
const headers: HeadersInit = {
Accept: 'application/vnd.github+json',
'X-GitHub-Api-Version': '2022-11-28',
};
// Use GITHUB_TOKEN if available (for higher rate limits)
const token = process.env.GITHUB_TOKEN;
if (token) {
headers['Authorization'] = `Bearer ${token}`;
}
try {
const res = await fetch(url, {
...options,
headers: { ...headers, ...options?.headers },
next: { revalidate: 300 }, // Cache for 5 minutes
});
if (!res.ok) {
console.error(`GitHub API error: ${res.status} ${res.statusText}`);
return null;
}
return res.json();
} catch (error) {
console.error('Failed to fetch from GitHub:', error);
return null;
}
}
/**
* Get the latest successful workflow run for a specific workflow on main branch
*/
export async function getLatestWorkflowRun(
workflowFileName: string
): Promise<GitHubWorkflowRun | null> {
const params = new URLSearchParams({
branch: 'main',
status: 'completed',
per_page: '1',
});
const data = await fetchGitHub<WorkflowRunsResponse>(
`/repos/${OWNER}/${REPO}/actions/workflows/${workflowFileName}/runs?${params}`
);
return data?.workflow_runs?.[0] ?? null;
}
/**
* Get artifacts from a workflow run
*/
export async function getWorkflowArtifacts(
runId: number
): Promise<GitHubArtifact[]> {
const data = await fetchGitHub<ArtifactsResponse>(
`/repos/${OWNER}/${REPO}/actions/runs/${runId}/artifacts?per_page=100`
);
return data?.artifacts ?? [];
}
/**
* Download and parse an artifact's JSON content
* Note: This requires authentication for private repos
*/
export async function downloadArtifact<T>(
artifactId: number
): Promise<T | null> {
const token = process.env.GITHUB_TOKEN;
if (!token) {
console.warn('GITHUB_TOKEN not set, cannot download artifacts');
return null;
}
try {
// Get the download URL (this redirects to a blob storage URL)
const res = await fetch(
`${GITHUB_API}/repos/${OWNER}/${REPO}/actions/artifacts/${artifactId}/zip`,
{
headers: {
Authorization: `Bearer ${token}`,
Accept: 'application/vnd.github+json',
},
redirect: 'follow',
}
);
if (!res.ok) {
console.error(`Failed to download artifact: ${res.status}`);
return null;
}
// The response is a ZIP file - we need to extract the JSON
const JSZip = (await import('jszip')).default;
const arrayBuffer = await res.arrayBuffer();
const zip = await JSZip.loadAsync(arrayBuffer);
// Find and parse the JSON file
const files = Object.keys(zip.files);
const jsonFile = files.find((f) => f.endsWith('.json'));
if (!jsonFile) {
console.error('No JSON file found in artifact');
return null;
}
const content = await zip.files[jsonFile].async('string');
return JSON.parse(content);
} catch (error) {
console.error('Failed to download/parse artifact:', error);
return null;
}
}
/**
* Parse E2E results into the WorldE2E format
*/
export function parseE2EResults(results: E2ETestResult | null): {
status: 'passing' | 'partial' | 'failing' | 'pending';
total: number;
passed: number;
failed: number;
skipped: number;
progress: number;
tests?: Array<{
name: string;
status: 'passed' | 'failed' | 'skipped';
duration?: number;
}>;
} | null {
if (!results) return null;
const total = results.numTotalTests;
const passed = results.numPassedTests;
const failed = results.numFailedTests;
const skipped = results.numPendingTests;
const progress = total > 0 ? (passed / total) * 100 : 0;
let status: 'passing' | 'partial' | 'failing' | 'pending';
if (passed === total) {
status = 'passing';
} else if (passed > 0) {
status = 'partial';
} else if (failed > 0) {
status = 'failing';
} else {
status = 'pending';
}
// Extract individual test results
const tests: Array<{
name: string;
status: 'passed' | 'failed' | 'skipped';
duration?: number;
}> = [];
for (const testFile of results.testResults) {
for (const assertion of testFile.assertionResults) {
tests.push({
name: assertion.fullName,
status:
assertion.status === 'pending'
? 'skipped'
: (assertion.status as 'passed' | 'failed'),
duration: assertion.duration,
});
}
}
return { status, total, passed, failed, skipped, progress, tests };
}
/**
* Parse benchmark results into the WorldBenchmark format
*/
export function parseBenchmarkResults(results: BenchmarkResult | null): {
status: 'measured' | 'pending';
metrics: Record<
string,
{ mean: number; min: number; max: number; samples?: number }
> | null;
} | null {
if (!results?.files?.[0]?.groups?.[0]?.benchmarks) return null;
const metrics: Record<
string,
{ mean: number; min: number; max: number; samples?: number }
> = {};
for (const bench of results.files[0].groups[0].benchmarks) {
metrics[bench.name] = {
mean: bench.mean,
min: bench.min,
max: bench.max,
samples: bench.sampleCount,
};
}
return {
status: Object.keys(metrics).length > 0 ? 'measured' : 'pending',
metrics: Object.keys(metrics).length > 0 ? metrics : null,
};
}
/**
* Map artifact name to world ID
* Artifact naming conventions:
* - E2E: e2e-results-{category}-{app} where category maps to world
* - vercel-prod → vercel
* - local-dev, local-prod → local
* - local-postgres → postgres
* - community-{world} → {world}
* - windows → local (windows tests use local world)
* - Benchmarks: bench-results-{app}-{world}
*/
export function artifactToWorldId(artifactName: string): string | null {
// E2E results for community worlds: e2e-results-community-{world}
if (artifactName.startsWith('e2e-results-community-')) {
return artifactName.replace('e2e-results-community-', '');
}
// E2E results: e2e-results-{category}-{app}
if (artifactName.startsWith('e2e-results-')) {
const rest = artifactName.replace('e2e-results-', '');
if (rest.startsWith('vercel-prod-')) return 'vercel';
if (rest.startsWith('local-dev-') || rest.startsWith('local-prod-'))
return 'local';
if (rest.startsWith('local-postgres-')) return 'postgres';
if (rest.startsWith('windows-')) return 'local';
return null;
}
// Benchmark results: bench-results-{app}-{world}
if (artifactName.startsWith('bench-results-')) {
const parts = artifactName.replace('bench-results-', '').split('-');
return parts[parts.length - 1];
}
return null;
}
export {
type GitHubWorkflowRun,
type GitHubArtifact,
type E2ETestResult,
type BenchmarkResult,
E2E_ARTIFACT_PATTERNS,
BENCH_ARTIFACT_PATTERNS,
};