mirror of
https://github.com/callstack/agent-device.git
synced 2026-09-14 20:06:34 +08:00
Michał Pierzchała
250e30a578
* test(bench): falsification fixtures for oracles + typed runner outcomes
Two deterministic PR-time quality gates for the help-conformance bench
(the repo's single non-gating small-model planning oracle):
- Every EXPECTATION_SCORERS entry in help-conformance-case-checks.mjs
now has a falsification fixture (a minimal passing witness plus at
least one known-bad counterexample, and a metamorphic variant where
useful) in the new help-conformance-expectation-fixtures.ts, run
through the real validatePlanCommands/scoreExpectations pipeline.
help-conformance-expectation-falsification.test.ts is the "what
enumerates N" completeness gate: a new named expectation with no
fixture fails it. Counterexamples cover swallowed lifecycle command
prefixes, unsupported flags/selectors, pseudo refs, shell operators,
and invalid positional ordering.
- help-conformance-runner-output.mjs now returns a discriminated
RunnerOutcome ({kind:'success',commands}|{kind:'runner-error',
message,reason}) instead of a raw-string success inference. Only a
'success' outcome ever reaches validatePlanCommands/scoreExpectations
in runCase, so a runner-error result can no longer also carry
model-validation checks, and an all-runner-error aggregate now
reports passRate: null (rendered as "N/A") instead of "0/0 (0%)".
Fixes #1481
* refactor(bench): dedupe RunnerOutcome construction, drop leftover narrowing
Thermo-nuclear pass over 4b2df0a38's diff:
- help-conformance-bench.mjs's runOutcome() catch block was hand-building
the exact {kind:'runner-error', raw, message, reason} shape that
runner-output.mjs's private runnerError() helper already constructs for
its own two error paths. Export it as runnerErrorOutcome so the
discriminated union has exactly one constructor for its error variant,
reused by both error sources instead of duplicated.
- runCase's two return branches repeated the same
{runner, caseId, trial, outputPath} fields; pulled into a shared `base`
object.
- Reverted bench.test.ts's rateLimitedOutcome block: it had an explicit
`: RunnerOutcome` annotation and an if/throw narrowing guard, added only
to give fallow's dead-code checker a "real consumer" of the type before
the actual fix (adding the .d.mts to .fallowrc.json's ignorePatterns,
matching the existing sample-outputs.d.mts precedent) was found. That
workaround is now unnecessary scaffolding — replaced with the same
flat assert.deepEqual style the surrounding assertions already use.
48 lines
1.5 KiB
JavaScript
48 lines
1.5 KiB
JavaScript
export function summarizeResults(results) {
|
|
const groups = new Map();
|
|
for (const result of results) {
|
|
const key = `${result.runner}\0${result.caseId}`;
|
|
const group = groups.get(key) ?? {
|
|
runner: result.runner,
|
|
caseId: result.caseId,
|
|
trials: 0,
|
|
evaluatedTrials: 0,
|
|
passed: 0,
|
|
failedChecks: {},
|
|
validationIssues: {},
|
|
runnerErrors: 0,
|
|
};
|
|
group.trials += 1;
|
|
if (result.runnerError) {
|
|
group.runnerErrors += 1;
|
|
} else {
|
|
group.evaluatedTrials += 1;
|
|
if (result.passed) group.passed += 1;
|
|
countFailedChecks(group.failedChecks, result.checks);
|
|
countValidationIssues(group.validationIssues, result.commandValidation);
|
|
}
|
|
groups.set(key, group);
|
|
}
|
|
// A group with zero evaluated trials (every trial was a runner error) has
|
|
// no pass rate to report — 0 would silently read as "0% correct" instead
|
|
// of "the model was never actually graded".
|
|
return [...groups.values()].map((group) => ({
|
|
...group,
|
|
passRate: group.evaluatedTrials === 0 ? null : group.passed / group.evaluatedTrials,
|
|
}));
|
|
}
|
|
|
|
function countFailedChecks(counts, checks = {}) {
|
|
for (const [id, passed] of Object.entries(checks)) {
|
|
if (!passed) counts[id] = (counts[id] ?? 0) + 1;
|
|
}
|
|
}
|
|
|
|
function countValidationIssues(counts, commandValidation = []) {
|
|
for (const command of commandValidation) {
|
|
for (const issue of command.issues ?? []) {
|
|
counts[issue.kind] = (counts[issue.kind] ?? 0) + 1;
|
|
}
|
|
}
|
|
}
|