mirror of
https://github.com/callstack/agent-device.git
synced 2026-09-14 20:06:34 +08:00
Michał Pierzchała
423927fdd8
* chore(mutation): shrink the lane to report-only (#1457, #1781 wave 2) The mutation harness's two real catches (#1474, #1475) both came from humans reading the weekly score report. The ratchet half never operated: the baseline was committed exactly twice (8cce0ef6b,60400d04b), both times with `stableRuns: 0, gating: false`, and was never updated after the very fixes it triggered — the weekly job computed a new baseline and then `git checkout --`d it, uploading a proposal nobody applied in 3+ weeks. A gate nobody arms is harness weight; the report is the part that paid. Deletes ratchet.ts + ratchet.test.ts, mutation-baselines/, and every baseline/graduation/gating path in run.ts (`--update`, `mutation:baseline`). run.ts now exits non-zero only on a harness failure, never on a score. The report renders the per-kernel table (kernel, score, killed, survived, total, timeouts) plus the surviving mutants a strengthening PR works from. Kernel scoping stays: stryker.config.json and KERNEL_MODULES are untouched. * fix(mutation): restore denominator coverage and publish the table before judging the shard set Review of #1828: - `report.test.ts` re-asserts that Ignored/CompileError/RuntimeError leave the denominator — the one behaviour `ratchet.test.ts` covered and nothing replaced. A `tally()` edit that counted tool noise would have deflated every published score with a green `mutation:test`. - `assertShardsCoverModules` now runs after `emit()`, so an incomplete shard set still publishes the kernels that completed instead of only an error string. This makes the workflow comments' claim about the job summary true rather than re-wording them down. * chore(mutation): trigger the affected lane on exactly the paths that can select mutants The PR lane returns an empty matrix unless the diff touches the harness, so the kernel-source and `**/*.test.ts` triggers only bought a 1-4 min no-op job on ~96% of PRs. `on.pull_request.paths` is now exactly `LANE_TOOLING` plus the workflow file, asserted in both directions by workflow.test.ts against the exported constant — a missing path would let a harness change merge unproven, an extra one starts a job that can only answer `[]`. Also drops the workflow header's contradictory scope paragraph: it claimed the lane selects on kernel sources and any test reaching one, which has not been true since the ratchet went. * fix(mutation): score and publish a short shard set before failing on the count The expected-count check ran inside readShardedReports, before anything was summarized, so on the weekly's real `--expect-shards 10` one dead shard threw away the nine that had reported — the earlier reorder only moved the zero-mutants check. The merge now returns the shard count, and both verdicts run after emit() with the same exit code and `score` stage. Regression uses the weekly argument shape (`--expect-shards 10`, one shard present) and asserts the reporting kernel's row reaches stdout while the run still fails.
91 lines
3.2 KiB
TypeScript
91 lines
3.2 KiB
TypeScript
// Selection is what the PR lane spends money on, so both halves of the rule are
|
|
// asserted end to end through the real CLI: a kernel diff selects nothing (the
|
|
// weekly sweep is the kernel report), and a lane-tooling diff selects real
|
|
// mutants rather than an empty matrix that proves nothing.
|
|
|
|
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { after, test } from 'node:test';
|
|
import { runCmdSync } from '../../src/utils/exec.ts';
|
|
import { LANE_CANARY, shardMatrix, type ShardSpec } from './modules.ts';
|
|
import { affectedMatrixFor } from './run.ts';
|
|
|
|
const repoRoot = path.resolve(import.meta.dirname, '../..');
|
|
const worktrees: string[] = [];
|
|
|
|
after(() => {
|
|
for (const dir of worktrees) {
|
|
runCmdSync('git', ['worktree', 'remove', '--force', dir], { cwd: repoRoot });
|
|
}
|
|
});
|
|
|
|
/**
|
|
* A throwaway worktree holding one commit that touches only the given files, so
|
|
* `--list-affected` runs against a real `git diff` rather than a stubbed list.
|
|
*/
|
|
function worktreeWithCommit(name: string, files: readonly string[]): string {
|
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), `mutation-${name}-`));
|
|
fs.rmSync(dir, { recursive: true });
|
|
runCmdSync('git', ['worktree', 'add', '--detach', '--quiet', dir, 'HEAD'], { cwd: repoRoot });
|
|
worktrees.push(dir);
|
|
for (const file of files) {
|
|
fs.appendFileSync(path.join(dir, file), '\n');
|
|
}
|
|
runCmdSync('git', ['add', ...files], { cwd: dir });
|
|
// CI runners have no committer identity configured, and this commit is a
|
|
// fixture, so it carries its own rather than depending on the environment.
|
|
runCmdSync(
|
|
'git',
|
|
[
|
|
'-c',
|
|
'user.name=mutation-selection-test',
|
|
'-c',
|
|
'user.email=mutation-selection-test@invalid',
|
|
'commit',
|
|
'--quiet',
|
|
'--no-verify',
|
|
'-m',
|
|
`touch ${name}`,
|
|
],
|
|
{ cwd: dir },
|
|
);
|
|
return dir;
|
|
}
|
|
|
|
function listAffected(cwd: string): ShardSpec[] {
|
|
const result = runCmdSync(
|
|
'node',
|
|
[
|
|
'--experimental-strip-types',
|
|
path.join(repoRoot, 'scripts/mutation/run.ts'),
|
|
'--list-affected',
|
|
'--base',
|
|
'HEAD~1',
|
|
],
|
|
{ cwd },
|
|
);
|
|
assert.equal(result.exitCode, 0, result.stderr);
|
|
return JSON.parse(result.stdout.trim().split('\n').at(-1)!) as ShardSpec[];
|
|
}
|
|
|
|
test('a lane-tooling diff selects real mutants', () => {
|
|
const dir = worktreeWithCommit('tooling', ['scripts/mutation/run.ts']);
|
|
// The lane's own sources own no kernel, so derivation alone yields nothing:
|
|
// without the canary a harness change would run zero mutants.
|
|
assert.deepEqual(listAffected(dir), shardMatrix([LANE_CANARY]));
|
|
});
|
|
|
|
// The weekly sweep is the kernel report; selecting on derived ownership would
|
|
// run the full ten-shard sweep on most PRs for a report nobody gates on.
|
|
test('a kernel diff selects nothing — only a harness diff spends mutants', () => {
|
|
const dir = worktreeWithCommit('kernel', ['src/utils/scroll-edge-state.ts']);
|
|
assert.deepEqual(listAffected(dir), []);
|
|
assert.deepEqual(affectedMatrixFor(['src/utils/scroll-edge-state.ts']), []);
|
|
});
|
|
|
|
test('a docs-only diff selects nothing', () => {
|
|
assert.deepEqual(affectedMatrixFor(['docs/agents/testing.md']), []);
|
|
});
|