mirror of
https://github.com/nexu-io/open-design.git
synced 2026-09-20 06:15:06 +08:00
open-design-crew[bot]
8be9b0fcaf
* fix(daemon): deliver artifacts with syntax warning telemetry * docs: record main-based syntax warning acceptance * fix(daemon): surface internal syntax failures at delivery boundary --------- Co-authored-by: zhouzheng <zhouzheng@refly.ai>
566 lines
36 KiB
TypeScript
566 lines
36 KiB
TypeScript
/** Explicit local acceptance, NOT a hermetic CI test or remote batch creator. */
|
|
import { execFile, spawn } from 'node:child_process';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { chmod, mkdir, mkdtemp, open, readFile, writeFile } from 'node:fs/promises';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { parseArgs, promisify } from 'node:util';
|
|
import { DELIVERABLE_SYNTAX_FINALIZATION_REASONS, DELIVERABLE_SYNTAX_SAFE_FIX_RULES } from '@open-design/contracts';
|
|
import { createToolsDevSuite, e2eWorkspaceRoot } from '../lib/tools-dev/runtime.ts';
|
|
import { runToolsDevJson } from '../lib/tools-dev/cli.ts';
|
|
|
|
type Json = Record<string, any>;
|
|
type ReplayFixture = {
|
|
id: string; source: string; expected?: string; action: 'allow' | 'warn'; attempts?: number;
|
|
status?: 'pass' | 'repairable' | 'incomplete';
|
|
appliedRepairRules?: string[]; finalizationReason?: string; refusal?: string;
|
|
};
|
|
const hash = (value: string | Buffer) => createHash('sha256').update(value).digest('hex');
|
|
const sourceScopes = ['apps/daemon', 'packages/contracts', 'pnpm-lock.yaml'];
|
|
|
|
/** Explicit manual canary only: never accepts user datasets or inherited account identity. */
|
|
export function resolveSyntaxTelemetryCanary(input: {
|
|
enabled: boolean; mode: string; externalInputs: boolean; profile: string;
|
|
isolatedRoot: string; repeat?: string | undefined; token?: string | undefined; relayUrl?: string | undefined;
|
|
}) {
|
|
if (!input.enabled) return null;
|
|
if (input.mode !== 'replay' || input.externalInputs || input.profile !== 'test'
|
|
|| (input.repeat !== undefined && input.repeat !== '1')) {
|
|
throw new Error('Telemetry canary requires replay, test profile, one repeat and built-in synthetic fixtures only');
|
|
}
|
|
if (input.relayUrl !== 'https://telemetry-test.open-design.ai/api/langfuse') {
|
|
throw new Error('Telemetry canary requires explicit OPEN_DESIGN_TELEMETRY_RELAY_URL pointing to the official test relay');
|
|
}
|
|
if (!input.token || !/^[a-z0-9]{8}$/.test(input.token)) throw new Error('Missing isolated canary token');
|
|
const html = (script: string) => `<!doctype html><script>${script}</script>`;
|
|
const fixtures: ReplayFixture[] = [
|
|
{ id: 'synthetic-clean', source: html('const items = [1, 2];'), expected: html('const items = [1, 2];'), action: 'allow', status: 'pass', attempts: 0 },
|
|
{ id: 'synthetic-repaired', source: html('const items = [1, 2;'), expected: html('const items = [1, 2];'), action: 'allow', status: 'pass', attempts: 1, appliedRepairRules: ['insert_missing_closing_delimiter'] },
|
|
{ id: 'synthetic-warning', source: html('const value = ;'), expected: html('const value = ;'), action: 'warn', status: 'repairable', attempts: 0, finalizationReason: 'no_safe_fix', refusal: 'unsupported_syntax_error' },
|
|
];
|
|
return {
|
|
fixtures,
|
|
prefs: { metrics: true, content: true, artifactManifest: false },
|
|
env: {
|
|
// Do not disable Vela priority. With an empty AMR home and no keys the
|
|
// real resolver naturally selects the anonymous test relay.
|
|
AMR_HOME: path.join(input.isolatedRoot, 'amr'),
|
|
OD_INSTALLATION_DIR: '', OD_LEGACY_DATA_DIR: '',
|
|
VELA_CONTROL_KEY: '', VELA_RUNTIME_KEY: '',
|
|
LANGFUSE_PUBLIC_KEY: '', LANGFUSE_SECRET_KEY: '',
|
|
POSTHOG_KEY: '', NEXT_PUBLIC_POSTHOG_KEY: '',
|
|
OPEN_DESIGN_TELEMETRY_RELAY_URL: input.relayUrl,
|
|
OPEN_DESIGN_OBJECT_RELAY_URL: '',
|
|
OD_TELEMETRY_ENV: `synthetic-test-${input.token}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
/** A Run terminal is not an exporter receipt; consent skips are not accepted writes. */
|
|
export function syntaxTelemetryDeliveryOutcome(state: Json): 'pending' | 'failed' | 'accepted' {
|
|
const delivery = state.telemetryDelivery;
|
|
if (delivery?.status === 'failed' || delivery?.status === 'not_expected') return 'failed';
|
|
if (!delivery || !Number.isFinite(delivery.finalizedAt)) return 'pending';
|
|
return delivery.status === 'accepted' && Number.isInteger(delivery.attemptCount)
|
|
&& delivery.attemptCount > 0 ? 'accepted' : 'failed';
|
|
}
|
|
|
|
/** Mirror the real client's final-message PUT without inventing content or metrics. */
|
|
export function syntaxTelemetryFinalizationMessage(messages: Json[], run: Json): Json {
|
|
const message = messages.find(entry => entry.id === run.assistantMessageId && entry.runId === run.id);
|
|
if (!message || message.role !== 'assistant' || message.runStatus !== run.status
|
|
|| !['succeeded', 'failed', 'canceled'].includes(run.status)) {
|
|
throw new Error('Missing matching durable terminal assistant message; telemetry was not finalized');
|
|
}
|
|
return { ...message, telemetryFinalized: true };
|
|
}
|
|
|
|
/** git diff excludes new files: record their scoped paths and bytes alongside the tracked diff. */
|
|
export async function captureUntrackedSourceIdentity(workspace: string) {
|
|
const { stdout } = await promisify(execFile)('git', ['ls-files', '--others', '--exclude-standard', '-z', '--', ...sourceScopes], { cwd: workspace });
|
|
const paths = stdout.split('\0').filter(Boolean).sort();
|
|
const untrackedFiles = await Promise.all(paths.map(async relativePath => ({
|
|
path: relativePath, sha256: hash(await readFile(path.join(workspace, relativePath))),
|
|
})));
|
|
return { untrackedFiles, untrackedFilesSha256: hash(JSON.stringify(untrackedFiles)) };
|
|
}
|
|
|
|
/** File-only fixture input: never imports daemon implementation or executes fixture JavaScript. */
|
|
export async function loadReplayFixtures(manifestPath: string): Promise<ReplayFixture[]> {
|
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) as Json;
|
|
if (!Array.isArray(manifest.fixtures) || manifest.fixtures.length === 0) throw new Error('Fixture manifest must have a nonempty fixtures array');
|
|
const seen = new Set<string>();
|
|
return await Promise.all(manifest.fixtures.map(async (entry: Json) => {
|
|
if (typeof entry.id !== 'string' || !/^[a-zA-Z0-9_-]+$/.test(entry.id) || seen.has(entry.id)) throw new Error('Fixture IDs must be unique and path-safe');
|
|
seen.add(entry.id);
|
|
if (entry.action !== 'allow' && entry.action !== 'warn') throw new Error(`${entry.id}: invalid fixture action (syntax-only failure must warn)`);
|
|
if (entry.status !== undefined && !['pass', 'repairable', 'incomplete'].includes(entry.status)) throw new Error(`${entry.id}: invalid expected checker status`);
|
|
if (typeof entry.before !== 'string' || (entry.action === 'allow' && typeof entry.after !== 'string')) throw new Error(`${entry.id}: before and exact expected after are required for allow`);
|
|
if (entry.attempts !== undefined && (!Number.isInteger(entry.attempts) || entry.attempts < 0)) throw new Error(`${entry.id}: invalid attempts`);
|
|
const source = await readFile(path.resolve(path.dirname(manifestPath), entry.before), 'utf8');
|
|
const expected = typeof entry.after === 'string' ? await readFile(path.resolve(path.dirname(manifestPath), entry.after), 'utf8') : source;
|
|
if (entry.action === 'warn' && expected !== source) throw new Error(`${entry.id}: warning fixture must preserve original bytes`);
|
|
return { id: entry.id, source, expected, action: entry.action, status: entry.status, attempts: entry.attempts,
|
|
appliedRepairRules: entry.appliedRepairRules, finalizationReason: entry.finalizationReason, refusal: entry.refusal };
|
|
}));
|
|
}
|
|
|
|
/** Evidence oracle only: no syntax parsing or repair logic is implemented by the harness. */
|
|
function hostTerminalOutcome(runStatus: unknown, validation: Json) {
|
|
const finalization = validation.finalization;
|
|
const summaryComplete = finalization?.summaryVersion === 1
|
|
&& finalization.repairEngine === 'host-safe-fixer@2'
|
|
&& ['pass', 'repairable', 'incomplete', 'skipped'].includes(finalization.initialStatus)
|
|
&& Number.isInteger(finalization.stagedPatchCount)
|
|
&& finalization.stagedPatchCount >= 0 && finalization.stagedPatchCount <= 8
|
|
&& Number.isInteger(finalization.committedPatchCount)
|
|
&& finalization.committedPatchCount >= 0 && finalization.committedPatchCount <= finalization.stagedPatchCount
|
|
&& Array.isArray(finalization.committedRepairRules)
|
|
&& finalization.committedRepairRules.every((rule: any) => DELIVERABLE_SYNTAX_SAFE_FIX_RULES.includes(rule))
|
|
&& (finalization.committedPatchCount > 0 ? finalization.committedRepairRules.length > 0 : finalization.committedRepairRules.length === 0);
|
|
const deliveredWithWarning = summaryComplete && runStatus === 'succeeded'
|
|
&& finalization.action === 'warn' && finalization.committedPatchCount === 0
|
|
&& ['pass', 'repairable', 'incomplete'].includes(validation.status)
|
|
&& DELIVERABLE_SYNTAX_FINALIZATION_REASONS.includes(finalization.reason);
|
|
const repairVerified = summaryComplete && runStatus === 'succeeded'
|
|
&& finalization.initialStatus === 'repairable' && finalization.committedPatchCount > 0
|
|
&& validation.status === 'pass' && finalization.action === 'allow';
|
|
return { summaryComplete, deliveredWithWarning, repairVerified };
|
|
}
|
|
|
|
export function replayCaseVerdict(fixture: ReplayFixture, run: Json, content: string) {
|
|
const validation = run.deliverableSyntaxValidation ?? {};
|
|
const beforeAfterEqual = content === fixture.source;
|
|
const expectedAfterEqual = fixture.expected === undefined ? null : content === fixture.expected;
|
|
const artifactMatches = fixture.expected !== undefined ? expectedAfterEqual === true
|
|
: fixture.action === 'warn' || fixture.attempts === 0 ? beforeAfterEqual : !beforeAfterEqual;
|
|
const appliedRepairRules = validation.metrics?.appliedRepairRules ?? [];
|
|
const finalization = validation.finalization;
|
|
const outcome = hostTerminalOutcome(run.status, validation);
|
|
const expectedCommitted = fixture.action === 'allow' ? (fixture.attempts ?? 0) : 0;
|
|
const terminalSummaryValid = outcome.summaryComplete
|
|
&& finalization.committedPatchCount === expectedCommitted
|
|
&& (fixture.attempts === undefined || finalization.stagedPatchCount === fixture.attempts)
|
|
&& (fixture.action !== 'allow' || finalization.initialStatus === (expectedCommitted > 0 ? 'repairable' : 'pass'))
|
|
&& JSON.stringify(finalization.committedRepairRules) === JSON.stringify(expectedCommitted > 0 ? appliedRepairRules : [])
|
|
&& (fixture.action !== 'warn' || outcome.deliveredWithWarning);
|
|
const deliveredWithWarning = terminalSummaryValid && outcome.deliveredWithWarning && beforeAfterEqual && artifactMatches;
|
|
const repairVerified = terminalSummaryValid && outcome.repairVerified && artifactMatches;
|
|
const terminalDurationMs = validation.metrics?.repairToTerminalDurationMs ?? validation.metrics?.repairToDeliveryDurationMs ?? null;
|
|
return {
|
|
passed: run.status === 'succeeded'
|
|
&& terminalSummaryValid
|
|
&& validation.finalization?.action === fixture.action
|
|
&& (fixture.attempts === undefined || (validation.repairState?.attempt ?? 0) === fixture.attempts)
|
|
&& (fixture.action !== 'allow' || validation.status === 'pass')
|
|
&& (fixture.status === undefined || validation.status === fixture.status)
|
|
&& (fixture.action !== 'warn' || beforeAfterEqual)
|
|
&& artifactMatches
|
|
&& (fixture.appliedRepairRules === undefined || JSON.stringify(appliedRepairRules) === JSON.stringify(fixture.appliedRepairRules))
|
|
&& (fixture.finalizationReason === undefined || validation.finalization?.reason === fixture.finalizationReason)
|
|
&& (fixture.refusal === undefined || validation.finalization?.refusal === fixture.refusal),
|
|
terminalSummaryValid, deliveredWithWarning, repairVerified,
|
|
beforeAfterEqual, expectedAfterEqual, beforeSha256: hash(fixture.source), afterSha256: hash(content),
|
|
expectedAfterSha256: fixture.expected === undefined ? null : hash(fixture.expected), appliedRepairRules,
|
|
// Historical repairToDeliveryDurationMs also contains failed terminal time; keep outcomes distinct.
|
|
discoveryToDeliveryMs: run.status === 'succeeded' ? terminalDurationMs : null,
|
|
discoveryToWarningDeliveryMs: deliveredWithWarning ? terminalDurationMs : null,
|
|
discoveryToRepairedDeliveryMs: repairVerified ? terminalDurationMs : null,
|
|
discoveryToFailedTerminalMs: run.status === 'failed' ? terminalDurationMs : null,
|
|
};
|
|
}
|
|
|
|
/** Recover completed records even if node-job exits before writing result.json. Never invent failed cases. */
|
|
export async function collectRealEvidence(output: string, expectedIds: string[]) {
|
|
const evidenceErrors: string[] = [];
|
|
const entries = new Map<string, Json>();
|
|
const started = new Set<string>();
|
|
const finishedEvents = new Set<string>();
|
|
const duplicateFinishedCaseIds: string[] = [];
|
|
let hasResult = false;
|
|
let hasEvents = false;
|
|
try {
|
|
const result = JSON.parse(await readFile(path.join(output, 'result.json'), 'utf8')) as Json;
|
|
if (!Array.isArray(result.cases)) throw new Error('missing cases array');
|
|
hasResult = true;
|
|
for (const entry of result.cases) {
|
|
if (typeof entry.evalId !== 'string') { evidenceErrors.push('result case missing evalId'); continue; }
|
|
if (entries.has(entry.evalId)) evidenceErrors.push(`duplicate result case ${entry.evalId}`);
|
|
entries.set(entry.evalId, entry);
|
|
}
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') evidenceErrors.push(`result.json: ${String(error)}`);
|
|
}
|
|
try {
|
|
const lines = (await readFile(path.join(output, 'events.jsonl'), 'utf8')).split('\n');
|
|
hasEvents = true;
|
|
for (const [index, line] of lines.entries()) {
|
|
if (!line.trim()) continue;
|
|
let event: Json;
|
|
try { event = JSON.parse(line) as Json; } catch { evidenceErrors.push(`events.jsonl: invalid JSON at line ${index + 1}`); continue; }
|
|
if (event.type === 'case_started' && typeof event.evalId === 'string') started.add(event.evalId);
|
|
if (event.type !== 'case_finished') continue;
|
|
if (typeof event.evalId !== 'string') { evidenceErrors.push(`case_finished missing evalId at line ${index + 1}`); continue; }
|
|
if (finishedEvents.has(event.evalId)) duplicateFinishedCaseIds.push(event.evalId);
|
|
finishedEvents.add(event.evalId);
|
|
if (!entries.has(event.evalId)) entries.set(event.evalId, event);
|
|
}
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') evidenceErrors.push(`events.jsonl: ${String(error)}`);
|
|
}
|
|
const cases = [...entries.values()].map(entry => {
|
|
const syntax = entry.metrics?.deliverableSyntax ?? {};
|
|
const outcome = hostTerminalOutcome(syntax.terminalRunStatus, syntax);
|
|
const deliveredWithWarning = entry.status === 'succeeded' && outcome.deliveredWithWarning
|
|
&& syntax.deliveredWithSyntaxWarningCount === 1 && syntax.recoveredDeliveryCount === 0
|
|
&& syntax.blockedBrokenDeliveryCount === 0;
|
|
const repairVerified = entry.status === 'succeeded' && outcome.repairVerified && syntax.recoveredDeliveryCount === 1;
|
|
const deliveredWithoutRepair = outcome.summaryComplete && syntax.terminalRunStatus === 'succeeded'
|
|
&& syntax.status === 'pass' && syntax.finalization.action === 'allow'
|
|
&& syntax.finalization.initialStatus === 'pass' && syntax.finalization.stagedPatchCount === 0
|
|
&& syntax.finalization.committedPatchCount === 0 && syntax.recoveredDeliveryCount === 0;
|
|
return { ...entry, deliveredWithWarning, repairVerified,
|
|
passed: entry.status === 'succeeded' && entry.metrics?.strategyRoute === 'od-next'
|
|
&& entry.metrics?.agent === 'open-design:amr' && entry.metrics?.model === 'deepseek-v4-flash'
|
|
&& (deliveredWithWarning || repairVerified || deliveredWithoutRepair),
|
|
};
|
|
});
|
|
const missingCaseIds = expectedIds.filter(id => !entries.has(id));
|
|
const unexpectedCaseIds = [...entries.keys()].filter(id => !expectedIds.includes(id));
|
|
return { cases, collection: {
|
|
source: hasResult ? hasEvents ? 'result+events' : 'result' : hasEvents ? 'events' : 'missing',
|
|
expectedCaseCount: expectedIds.length, finishedCaseCount: entries.size,
|
|
startedCaseIds: [...started], missingCaseIds, unexpectedCaseIds, duplicateFinishedCaseIds, evidenceErrors,
|
|
unfinishedCaseIds: missingCaseIds.filter(id => started.has(id)),
|
|
notStartedCaseIds: missingCaseIds.filter(id => !started.has(id)),
|
|
complete: expectedIds.length > 0 && missingCaseIds.length === 0 && unexpectedCaseIds.length === 0
|
|
&& duplicateFinishedCaseIds.length === 0 && evidenceErrors.length === 0,
|
|
} };
|
|
}
|
|
|
|
export async function runSyntaxAcceptance(args = process.argv.slice(2)) {
|
|
const exec = promisify(execFile);
|
|
const workspace = e2eWorkspaceRoot();
|
|
const { values } = parseArgs({ args, options: {
|
|
mode: { type: 'string' }, dataset: { type: 'string' }, sha256: { type: 'string' },
|
|
runner: { type: 'string' }, vela: { type: 'string' }, profile: { type: 'string', default: 'test' },
|
|
'runner-version': { type: 'string', default: '0.9.24' },
|
|
'expected-rows': { type: 'string', default: '24' }, repeat: { type: 'string' },
|
|
'fixture-manifest': { type: 'string' }, 'timeout-ms': { type: 'string', default: String(3 * 60 * 60 * 1000) },
|
|
'upload-telemetry': { type: 'boolean', default: false },
|
|
} });
|
|
if (values.mode !== 'real' && values.mode !== 'replay') {
|
|
throw new Error('Choose --mode replay or --mode real (real also requires --dataset --sha256 --runner --vela).');
|
|
}
|
|
const mode = values.mode;
|
|
const root = await mkdtemp(path.join(os.tmpdir(), 'od-syntax-acceptance-'));
|
|
await chmod(root, 0o700);
|
|
const canary = resolveSyntaxTelemetryCanary({
|
|
enabled: values['upload-telemetry'] === true, mode, profile: values.profile!,
|
|
externalInputs: Boolean(values.dataset || values['fixture-manifest'] || values.runner || values.vela || values.sha256),
|
|
repeat: values.repeat, isolatedRoot: root, token: randomUUID().slice(0, 8),
|
|
relayUrl: process.env.OPEN_DESIGN_TELEMETRY_RELAY_URL,
|
|
});
|
|
if (canary) await mkdir(canary.env.AMR_HOME, { mode: 0o700 });
|
|
const telemetryPrefs = canary?.prefs ?? { metrics: false, content: false, artifactManifest: false };
|
|
const runtime = createToolsDevSuite({
|
|
root, namespace: `syntax-${randomUUID().slice(0, 8)}`, ownerPid: process.pid,
|
|
toolsDevRoot: path.join(root, 'runtime'), dataDir: path.join(root, 'data'),
|
|
codexHomeDir: path.join(root, 'codex'),
|
|
}, {
|
|
// Repository dotenv files must not override this harness's isolated identity,
|
|
// data root or consent configuration (including in the default offline lane).
|
|
runJson: (workspace, suite, args, env, options) =>
|
|
runToolsDevJson(workspace, suite, [...args, '--no-env-file'], env, options),
|
|
});
|
|
const env: Record<string, string | undefined> = {
|
|
OD_INSTALLATION_DIR: '', OD_LEGACY_DATA_DIR: '',
|
|
OD_NEXT_STRATEGY_ROLLOUT: mode === 'real' ? 'active' : 'off',
|
|
OD_DELIVERABLE_SYNTAX_FINALIZER: '1',
|
|
OPEN_DESIGN_AMR_PROFILE: values.profile, VELA_PROFILE: values.profile,
|
|
...(values.vela ? { VELA_BIN: path.resolve(values.vela) } : {}),
|
|
...canary?.env,
|
|
};
|
|
const report: Json = {
|
|
mode, status: 'RUNNING', startedAt: new Date().toISOString(), root,
|
|
boundary: mode === 'real' ? 'real AMR / OD Next generation' : 'fake CLI / real deployed daemon terminal chain; NOT AMR acceptance',
|
|
cases: [],
|
|
...(canary ? { telemetry: {
|
|
uploadEnabled: true, destination: 'official-test-relay', environment: canary.env.OD_TELEMETRY_ENV,
|
|
syntheticOnly: true, readbackVerified: false,
|
|
} } : {}),
|
|
};
|
|
async function save(name: string, data: unknown) {
|
|
await writeFile(path.join(root, name), JSON.stringify(data, null, 2) + '\n', { mode: 0o600 });
|
|
}
|
|
async function command(command: string, args: string[], logName: string, timeout = 180_000) {
|
|
const log = await open(path.join(root, logName), 'w', 0o600);
|
|
try {
|
|
await new Promise<void>((resolve, reject) => {
|
|
const child = spawn(command, args, {
|
|
cwd: workspace, env: { ...process.env, ...env }, stdio: ['ignore', log.fd, log.fd],
|
|
timeout,
|
|
});
|
|
child.on('error', reject);
|
|
child.on('exit', (code, signal) => code === 0 ? resolve() : reject(new Error(`${logName}: exit=${code}, signal=${signal}`)));
|
|
});
|
|
} finally { await log.close(); }
|
|
}
|
|
async function request(url: string, method = 'GET', body?: unknown): Promise<Json> {
|
|
let response: Response;
|
|
try {
|
|
response = await fetch(url, {
|
|
method, signal: AbortSignal.timeout(30_000),
|
|
...(body !== undefined ? { headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) } : {}),
|
|
});
|
|
} catch (error) {
|
|
throw new Error(`${method} ${new URL(url).pathname}: ${error instanceof Error ? error.message : 'request failed'}`);
|
|
}
|
|
if (!response.ok) throw new Error(`${method} ${new URL(url).pathname}: HTTP ${response.status}`);
|
|
return await response.json() as Json;
|
|
}
|
|
async function sourceIdentity() {
|
|
const git = async (...args: string[]) => (await exec('git', args, { cwd: workspace })).stdout.trim();
|
|
return {
|
|
branch: await git('branch', '--show-current'), commit: await git('rev-parse', 'HEAD'),
|
|
diffSha256: hash(await git('diff', 'HEAD', '--', ...sourceScopes)),
|
|
...await captureUntrackedSourceIdentity(workspace),
|
|
harnessSha256: hash(await readFile(new URL(import.meta.url))),
|
|
node: process.version,
|
|
};
|
|
}
|
|
const fixtures = [
|
|
{ id: 'valid', script: 'const items = [1, 2];', action: 'allow', attempts: 0 },
|
|
{ id: 'array', script: 'const items = [1, 2;', action: 'allow', attempts: 1 },
|
|
{ id: 'division', script: 'const ratio = 10 / 2; const items = [1, 2;', action: 'allow', attempts: 1 },
|
|
{ id: 'regex', script: 'const re = /[{}()]/; const items = [1, 2;', action: 'allow', attempts: 1 },
|
|
{ id: 'three', script: 'function f() { if (true) { const items = [1, 2;', action: 'allow', attempts: 3 },
|
|
{ id: 'budget', script: 'function f() { if (true) { const items = [[[[[[[1, 2;', action: 'warn', status: 'repairable', attempts: 8 },
|
|
{ id: 'expression-hole', script: 'const value = ;', action: 'warn', status: 'repairable', attempts: 0 },
|
|
{ id: 'comment', script: 'const ready = true; /* unfinished', action: 'allow', attempts: 1 },
|
|
{ id: 'string', script: 'const label = "hello', action: 'allow', attempts: 1 },
|
|
{ id: 'oversize', script: ' '.repeat(2 * 1024 * 1024), action: 'warn', status: 'incomplete', attempts: 0 },
|
|
] as const;
|
|
let activeFixtures: ReplayFixture[] = fixtures.map(fixture => ({ ...fixture, source: `<!doctype html><script>${fixture.script}</script>` }));
|
|
if (canary) activeFixtures = canary.fixtures;
|
|
let expectedIds: string[] = [];
|
|
const output = path.join(root, 'evaluation');
|
|
|
|
async function replay() {
|
|
const repeat = Number(values.repeat ?? (canary ? '1' : '3'));
|
|
if (!Number.isInteger(repeat) || repeat < 1 || repeat > 10) throw new Error('--repeat must be 1..10');
|
|
for (let round = 1; round <= repeat; round++) {
|
|
for (const fixture of activeFixtures) {
|
|
const id = `${fixture.id}-${round}`;
|
|
const source = fixture.source;
|
|
// The fake CLI supplies fixed output; it does not pretend to be an AMR model.
|
|
const bin = path.join(root, `fixture-${id}.cjs`);
|
|
await writeFile(bin, `#!/usr/bin/env node
|
|
const fs = require('node:fs');
|
|
if (process.argv.includes('--version')) { console.log('claude-code 1.0.0-acceptance'); process.exit(0); }
|
|
if (process.argv.includes('--help')) { console.log('Usage: claude -p [--add-dir DIR]'); process.exit(0); }
|
|
const content = ${JSON.stringify(source)};
|
|
fs.writeFileSync('index.html', content);
|
|
const emit = value => fs.writeSync(1, JSON.stringify(value) + '\\n');
|
|
emit({ type:'system', subtype:'init', model:'fixed-fixture', session_id:'${id}' });
|
|
emit({ type:'assistant', message:{ id:'write', content:[{type:'tool_use',id:'write-index',name:'Write',input:{file_path:'index.html'}}],stop_reason:'tool_use' } });
|
|
emit({ type:'user', message:{content:[{type:'tool_result',tool_use_id:'write-index',content:'File written',is_error:false}]} });
|
|
emit({ type:'assistant',message:{id:'final',content:[{type:'text',text:'Done'}],stop_reason:'end_turn'} });
|
|
emit({ type:'result',subtype:'success',is_error:false,session_id:'${id}',stop_reason:'end_turn',usage:{input_tokens:2,output_tokens:1},duration_ms:1 });
|
|
`, { mode: 0o700 });
|
|
await request(runtime.url.api('/api/app-config'), 'PUT', {
|
|
agentId: 'claude', agentCliEnv: { claude: { CLAUDE_BIN: bin } },
|
|
telemetry: telemetryPrefs, privacyDecisionAt: Date.now(),
|
|
});
|
|
const projectId = `acceptance_${randomUUID()}`;
|
|
const project = await request(runtime.url.api('/api/projects'), 'POST', {
|
|
id: projectId, name: id, metadata: { kind: 'prototype', entryFile: 'index.html' }, skipDiscoveryBrief: true,
|
|
});
|
|
const started = await request(runtime.url.api('/api/runs'), 'POST', {
|
|
projectId, conversationId: project.conversationId, agentId: 'claude',
|
|
assistantMessageId: randomUUID(), clientRequestId: randomUUID(),
|
|
message: 'Create the page', currentPrompt: 'Create the page',
|
|
});
|
|
const start = Date.now();
|
|
let run: Json;
|
|
do {
|
|
run = await request(runtime.url.api(`/api/runs/${started.runId}`));
|
|
if (['succeeded', 'failed', 'canceled'].includes(run.status)) break;
|
|
if (Date.now() - start > 60_000) throw new Error(`${id}: Run ${started.runId} timeout`);
|
|
await new Promise(resolve => setTimeout(resolve, 100));
|
|
} while (true);
|
|
let telemetry: Json | undefined;
|
|
if (canary) {
|
|
// Successful Runs report after final message persistence, not at child
|
|
// process exit. Save the actual daemon-owned message through the same
|
|
// route as the client; the real bridge derives all telemetry itself.
|
|
const messagesPath = `/api/projects/${projectId}/conversations/${project.conversationId}/messages`;
|
|
const messages = await request(runtime.url.api(messagesPath));
|
|
const message = syntaxTelemetryFinalizationMessage(messages.messages ?? [], run);
|
|
const saved = await request(runtime.url.api(`${messagesPath}/${message.id}`), 'PUT', message);
|
|
syntaxTelemetryFinalizationMessage(saved.message ? [saved.message] : [], run);
|
|
// The exporter is detached from the HTTP Run response. Its durable
|
|
// checkpoint has no HTTP projection; observe only this isolated Run's
|
|
// state file, with a bounded wait rather than a fixed post-run sleep.
|
|
const deliveryStartedAt = Date.now();
|
|
const statePath = path.join(runtime.dataDir, 'runs', started.runId, 'state.json');
|
|
while (true) {
|
|
let state: Json = {};
|
|
try { state = JSON.parse(await readFile(statePath, 'utf8')); }
|
|
catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error; }
|
|
const outcome = syntaxTelemetryDeliveryOutcome(state);
|
|
if (outcome !== 'pending') {
|
|
const delivery = state.telemetryDelivery;
|
|
telemetry = { status: outcome, attemptCount: delivery.attemptCount, finalizedAt: delivery.finalizedAt,
|
|
...(delivery.dropReason ? { dropReason: delivery.dropReason } : {}),
|
|
waitDurationMs: Date.now() - deliveryStartedAt, traceIdCandidate: run.id };
|
|
await save(`${id}-delivery.json`, telemetry);
|
|
break;
|
|
}
|
|
if (Date.now() - deliveryStartedAt >= 60_000) throw new Error(`${id}: telemetry delivery checkpoint timeout`);
|
|
await new Promise(resolve => setTimeout(resolve, 200));
|
|
}
|
|
}
|
|
await save(`${id}-run.json`, run);
|
|
const response = await fetch(runtime.url.api(`/api/projects/${projectId}/raw/index.html`), { signal: AbortSignal.timeout(10_000) });
|
|
if (!response.ok) throw new Error(`${id}: cannot read artifact`);
|
|
const content = await response.text();
|
|
await writeFile(path.join(root, `${id}-before.html`), source, { mode: 0o600 });
|
|
await writeFile(path.join(root, `${id}-after.html`), content, { mode: 0o600 });
|
|
const validation = run.deliverableSyntaxValidation ?? {};
|
|
if (fixture.expected !== undefined) await writeFile(path.join(root, `${id}-expected.html`), fixture.expected, { mode: 0o600 });
|
|
const verdict = replayCaseVerdict(fixture, run, content);
|
|
const passed = verdict.passed && (!canary || telemetry?.status === 'accepted');
|
|
report.cases.push({
|
|
id, ...verdict, passed,
|
|
runId: run.id, projectId, status: run.status, validation, ...(telemetry ? { telemetry } : {}),
|
|
elapsedMs: Date.now() - start,
|
|
});
|
|
await save('report.json', report);
|
|
console.log(`[replay] ${id}: ${passed ? 'PASS' : 'FAIL'} (${run.status})`);
|
|
if (canary && telemetry?.status !== 'accepted') throw new Error(`${id}: exporter did not accept telemetry; no further canary Runs started`);
|
|
}
|
|
}
|
|
}
|
|
|
|
try {
|
|
report.sourceBefore = await sourceIdentity();
|
|
await save('report.json', report);
|
|
console.log(`Local evidence: ${root}`);
|
|
let dataset: Buffer | undefined;
|
|
const timeoutMs = Number(values['timeout-ms']);
|
|
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1) throw new Error('--timeout-ms must be a positive integer (node-job total budget, not per-case)');
|
|
report.evaluationTimeoutMs = timeoutMs;
|
|
if (values['fixture-manifest']) {
|
|
if (mode !== 'replay') throw new Error('--fixture-manifest only applies to replay');
|
|
const manifestPath = path.resolve(values['fixture-manifest']);
|
|
activeFixtures = await loadReplayFixtures(manifestPath);
|
|
report.fixtures = { manifestPath, manifestSha256: hash(await readFile(manifestPath)),
|
|
entries: activeFixtures.map(fixture => ({ id: fixture.id, beforeSha256: hash(fixture.source), expectedSha256: hash(fixture.expected!), action: fixture.action })) };
|
|
}
|
|
if (mode === 'real') {
|
|
if (values.profile !== 'test') throw new Error('Real local syntax acceptance requires --profile test; no production wallet fallback');
|
|
if (!values.dataset || !values.sha256 || !values.runner || !values.vela) throw new Error('Missing real-lane arguments');
|
|
dataset = await readFile(path.resolve(values.dataset));
|
|
const rows = dataset.toString('utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line));
|
|
if (hash(dataset) !== values.sha256.replace(/^sha256:/, '')) throw new Error('Dataset hash mismatch');
|
|
if (rows.length !== Number(values['expected-rows'])) throw new Error('Dataset row count mismatch');
|
|
expectedIds = rows.map(row => row.eval_id);
|
|
if (expectedIds.some(id => typeof id !== 'string' || !id) || new Set(expectedIds).size !== rows.length) throw new Error('Dataset eval_id must be unique and nonempty');
|
|
report.dataset = { sha256: hash(dataset), rows: rows.length };
|
|
report.runner = { path: path.resolve(values.runner), sha256: hash(await readFile(path.resolve(values.runner))) };
|
|
report.runner.version = (await exec(process.execPath, [path.resolve(values.runner), '--version'], { timeout: 30_000 })).stdout.trim();
|
|
if (report.runner.version.replace(/^(?:od-evals\s+)?v?/, '') !== values['runner-version']) {
|
|
throw new Error(`RUNNER_VERSION_MISMATCH: expected ${values['runner-version']}, got ${report.runner.version}`);
|
|
}
|
|
report.profile = values.profile;
|
|
report.vela = { path: path.resolve(values.vela), version: (await exec(path.resolve(values.vela), ['--version'])).stdout.trim() };
|
|
try {
|
|
// Never log whoami's personal data or credentials and never silently change profile/model.
|
|
await exec(path.resolve(values.vela), ['whoami'], { env: { ...process.env, ...env }, timeout: 30_000 });
|
|
} catch { throw new Error(`AMR_AUTH_BLOCKED: Vela profile ${values.profile} is not authenticated/reachable`); }
|
|
}
|
|
console.log('Building current worktree and dependencies (no --skip-build).');
|
|
await command('pnpm', ['--filter', '@open-design/daemon...', '--workspace-concurrency=4', 'build'], 'build.log', 600_000);
|
|
report.build = {
|
|
serverSha256: hash(await readFile(path.join(workspace, 'apps/daemon/dist/server.js'))),
|
|
finalizerSha256: hash(await readFile(path.join(workspace, 'apps/daemon/dist/artifacts/deliverable-syntax-finalization.js'))),
|
|
fixerSha256: hash(await readFile(path.join(workspace, 'apps/daemon/dist/artifacts/deliverable-syntax-safe-fix.js'))),
|
|
quotesSha256: hash(await readFile(path.join(workspace, 'apps/daemon/dist/artifacts/deliverable-syntax-quotes.js'))),
|
|
};
|
|
report.runtime = await runtime.startWeb(env);
|
|
report.runtimeCheck = await runtime.check(env);
|
|
await request(runtime.url.api('/api/app-config'), 'PUT', {
|
|
agentId: mode === 'real' ? 'amr' : 'claude',
|
|
telemetry: telemetryPrefs, privacyDecisionAt: Date.now(),
|
|
...(mode === 'real' ? { agentCliEnv: { amr: {
|
|
VELA_BIN: path.resolve(values.vela!), VELA_PROFILE: values.profile, OPEN_DESIGN_AMR_PROFILE: values.profile,
|
|
} } } : {}),
|
|
});
|
|
if (mode === 'replay') {
|
|
await replay();
|
|
} else {
|
|
const login = await request(runtime.url.api('/api/integrations/vela/status'));
|
|
report.daemonAuth = { loggedIn: login.loggedIn, profile: login.profile, sessionState: login.sessionState };
|
|
if (login.loggedIn !== true || login.profile !== values.profile) throw new Error('DAEMON_AMR_AUTH_MISMATCH');
|
|
await writeFile(path.join(root, 'dataset.jsonl'), dataset!, { mode: 0o600 });
|
|
await save('arm.json', {
|
|
id: 'local-syntax-acceptance', cli: 'amr', model: 'deepseek-v4-flash',
|
|
strategyRoute: 'od-next', keepDiscovery: true, autoAnswerForms: false,
|
|
env: { kind: 'daemon', url: runtime.daemonUrl },
|
|
});
|
|
await mkdir(output);
|
|
await command(process.execPath, [path.resolve(values.runner!), 'node-job',
|
|
'--dataset', path.join(root, 'dataset.jsonl'), '--arm-file', path.join(root, 'arm.json'),
|
|
'--out-dir', output, '--concurrency', '2'], 'evaluation.log', timeoutMs);
|
|
const evidence = await collectRealEvidence(output, expectedIds);
|
|
report.cases = evidence.cases;
|
|
report.collection = evidence.collection;
|
|
if (!evidence.collection.complete) throw new Error('Incomplete evaluation evidence; see collection for unfinished/missing cases');
|
|
}
|
|
report.status = report.cases.length > 0 && report.cases.every((entry: Json) => entry.passed) ? 'PASS' : 'FAIL';
|
|
report.note = mode === 'real' ? 'Generation/terminal acceptance only; visual quality is not scored.' : 'Replay PASS is not real AMR / OD Next acceptance.';
|
|
if (canary) report.note += ' Canary PASS proves local execution and exporter acceptance only; exact remote Trace readback is still required. elapsedMs includes telemetry wait; checker metrics do not.';
|
|
} catch (error) {
|
|
report.status = 'BLOCKED';
|
|
report.error = error instanceof Error ? error.message : String(error);
|
|
} finally {
|
|
if (mode === 'real' && expectedIds.length > 0) {
|
|
try {
|
|
const evidence = await collectRealEvidence(output, expectedIds);
|
|
report.cases = evidence.cases;
|
|
report.collection = evidence.collection;
|
|
if (!evidence.collection.complete) report.status = 'BLOCKED';
|
|
} catch (error) { report.collectionError = String(error); report.status = 'BLOCKED'; }
|
|
}
|
|
try {
|
|
report.sourceAfter = await sourceIdentity();
|
|
report.sourceStable = JSON.stringify(report.sourceBefore) === JSON.stringify(report.sourceAfter);
|
|
if (!report.sourceStable) report.status = 'BLOCKED';
|
|
} catch (error) { report.sourceAfterError = String(error); report.status = 'BLOCKED'; }
|
|
// Preserve local evidence, but never leave a temporary background runtime running.
|
|
try { await save('runtime-logs.json', await runtime.logs(env)); } catch { /* startup may not have happened */ }
|
|
report.cleanup = { attempted: true, stopped: false };
|
|
try {
|
|
report.cleanup.stopResult = await runtime.stopWeb(env);
|
|
report.cleanup.status = await runtime.status(env);
|
|
report.cleanup.stopped = ['daemon', 'web'].every(app => {
|
|
const state = report.cleanup.status.apps?.[app];
|
|
return state && state.state !== 'running' && state.pid == null;
|
|
});
|
|
if (!report.cleanup.stopped) { report.status = 'BLOCKED'; report.cleanupError = 'Runtime stop was not confirmed by status'; }
|
|
} catch (error) {
|
|
report.cleanupError = String(error); report.status = 'BLOCKED';
|
|
}
|
|
report.finishedAt = new Date().toISOString();
|
|
await save('report.json', report);
|
|
console.log(`${report.status}: ${path.join(root, 'report.json')}`);
|
|
if (report.error) console.log(report.error);
|
|
if (report.status !== 'PASS') process.exitCode = 1;
|
|
}
|
|
return report;
|
|
}
|
|
|
|
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await runSyntaxAcceptance();
|