mirror of
https://github.com/callstack/agent-device.git
synced 2026-09-14 20:06:34 +08:00
8cce0ef6b8
* test: ratchet mutation score over enumerated decision kernels Adds a Stryker (vitest runner) mutation lane scoped to the decision kernels, a per-module baseline with tool/config provenance, and a ratchet that only lets scores rise. Non-gating until two consecutive stable weekly sweeps. Refs #1415 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore: declare the mutation test-scope seam for production-export analysis Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: own kernel tests in the mutation registry and ship the #1430 lane envelope - restore bench:help-conformance, broken by a formatting-path edit - kernel test files select their module on PRs (registry `tests` + workflow paths), asserted to reach the kernel through the import graph - every mutation run writes the standard scheduled-lane artifact envelope - move src/utils/__tests__/errors.test.ts beside its source per the mirror rule Refs #1415, #1430 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: derive kernel test ownership and land the scheduled-lane health monitor Ownership of a kernel's tests is now computed from the static import graph (scripts/mutation/ownership.ts) instead of a hand-listed set, so a test that reaches a kernel indirectly -- src/__tests__/daemon-error.test.ts through src/daemon.ts -- selects that kernel on a PR. The PR lane triggers on every src test and shards the derived modules, keeping wall clock at one module. The lane envelope (#1430) is now written on every exit path with the stage it reached, so a crash before any mutant runs is distinguishable from a lane that never ran. Adds the derived cadence monitor (scripts/lane-health, daily workflow): scheduled lanes are enumerated from .github/workflows/ and reported dark, failing, or never-run against their own cron cadence. * fix: merge only Stryker reports from a shard directory The shard artifacts now carry the lane envelope beside mutation.json, and the merge globbed every .json under the download path, so the ratchet job fed the envelope to the report parser and died after the mutants had already run. * fix(mutation): fail on incomplete shard sets and envelope pre-run failures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(mutation): downgrade a passing envelope when a later lane step fails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(mutation): shard by registry, defer the PR lane, drop the bundled watcher Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs: describe registry sharding and the deferred PR mutation lane Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(mutation): make the pre-graduation tooling exception select real mutants Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(mutation): give the worktree fixture commits their own identity Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Michał Pierzchała <thymikee@gmail.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
154 lines
5.6 KiB
TypeScript
154 lines
5.6 KiB
TypeScript
// Enumerated decision kernels the mutation lane measures (issue #1415).
|
|
//
|
|
// Mutation score is the only mechanical answer to "is this test load-bearing or
|
|
// decorative", but a full-suite sweep is unaffordable. This registry is the
|
|
// single source of truth for what Stryker mutates: `stryker.config.json`'s
|
|
// `mutate` globs are asserted against it, and PR-affected gating maps changed
|
|
// files onto modules through it.
|
|
//
|
|
// Membership rule: pure decision kernels only — a surviving mutant here means a
|
|
// silently wrong agent-facing decision. Anything that spawns subprocesses or
|
|
// waits real time is out of scope by construction (its mutants would be timeout
|
|
// noise, not test-strength signal).
|
|
|
|
export type ModuleId =
|
|
| 'kernel-errors'
|
|
| 'daemon-ref-frame'
|
|
| 'interaction-settle'
|
|
| 'scroll-edge-state'
|
|
| 'selectors';
|
|
|
|
export type KernelModule = {
|
|
readonly id: ModuleId;
|
|
readonly label: string;
|
|
/** Globs handed to Stryker's `mutate`. */
|
|
readonly mutate: readonly string[];
|
|
/**
|
|
* Paths this module owns; a trailing `/` marks a directory prefix. Sources
|
|
* only — the tests whose strength the score measures are derived from the
|
|
* import graph in `ownership.ts`, never listed here.
|
|
*/
|
|
readonly owns: readonly string[];
|
|
/**
|
|
* How many parallel jobs the module's mutants are sliced across. One job per
|
|
* module is the default; a module big enough to outrun the lane's 30-minute
|
|
* budget declares more (`--shard i/n` picks the slice).
|
|
*/
|
|
readonly shards?: number;
|
|
};
|
|
|
|
export const KERNEL_MODULES: readonly KernelModule[] = [
|
|
{
|
|
id: 'kernel-errors',
|
|
label: 'Error retriability + hints',
|
|
mutate: ['src/kernel/errors.ts'],
|
|
owns: ['src/kernel/errors.ts'],
|
|
},
|
|
{
|
|
id: 'daemon-ref-frame',
|
|
label: 'Ref-frame admission matrix (ADR 0014)',
|
|
mutate: ['src/daemon/ref-frame.ts'],
|
|
owns: ['src/daemon/ref-frame.ts'],
|
|
},
|
|
{
|
|
id: 'interaction-settle',
|
|
label: 'Interaction settle decisions',
|
|
mutate: ['src/commands/interaction/runtime/settle.ts'],
|
|
owns: ['src/commands/interaction/runtime/settle.ts'],
|
|
},
|
|
{
|
|
id: 'scroll-edge-state',
|
|
label: 'Scroll edge-state detection',
|
|
mutate: ['src/utils/scroll-edge-state.ts'],
|
|
owns: ['src/utils/scroll-edge-state.ts'],
|
|
},
|
|
{
|
|
id: 'selectors',
|
|
label: 'Selector parsing + matching',
|
|
mutate: ['src/selectors/**/*.ts', '!src/selectors/**/*.test.ts', '!src/selectors/__tests__/**'],
|
|
// Selector tests live under the owned directory, so the prefix covers them.
|
|
owns: ['src/selectors/'],
|
|
// ~1,280 mutants at the observed ~3s/mutant on a 2-core runner is ~64
|
|
// minutes in one job — past the acceptance budget and its own timeout.
|
|
shards: 4,
|
|
},
|
|
];
|
|
|
|
export const ALL_MODULE_IDS: readonly ModuleId[] = KERNEL_MODULES.map((module) => module.id);
|
|
|
|
export function moduleById(id: ModuleId): KernelModule {
|
|
const found = KERNEL_MODULES.find((module) => module.id === id);
|
|
if (!found) throw new Error(`Unknown mutation module: ${id}`);
|
|
return found;
|
|
}
|
|
|
|
export function isModuleId(value: string): value is ModuleId {
|
|
return ALL_MODULE_IDS.includes(value as ModuleId);
|
|
}
|
|
|
|
/** Mutate globs for a module subset, in registry order. */
|
|
export function mutateGlobs(ids: readonly ModuleId[] = ALL_MODULE_IDS): string[] {
|
|
return KERNEL_MODULES.filter((module) => ids.includes(module.id)).flatMap((module) => [
|
|
...module.mutate,
|
|
]);
|
|
}
|
|
|
|
/**
|
|
* The module a lane-tooling change proves itself against before graduation.
|
|
* `kernel-errors` is the cheapest real sweep in the registry (one file, ~183
|
|
* mutants), so a change to the ratchet, the config, or the baseline runs actual
|
|
* mutants end to end without paying for the full sweep.
|
|
*/
|
|
export const LANE_CANARY: ModuleId = 'kernel-errors';
|
|
|
|
/** One mutation job: a module, optionally one slice of it. */
|
|
export type ShardSpec = { name: string; module: ModuleId; shard?: string };
|
|
|
|
/**
|
|
* The jobs a module set expands into. Both workflows' matrices are this list, so
|
|
* a registry module (or a change to its shard count) can never leave the sweep
|
|
* without the workflow assertions noticing.
|
|
*/
|
|
export function shardMatrix(ids: readonly ModuleId[] = ALL_MODULE_IDS): ShardSpec[] {
|
|
return KERNEL_MODULES.filter((module) => ids.includes(module.id)).flatMap((module) => {
|
|
const count = module.shards ?? 1;
|
|
if (count === 1) return [{ name: module.id, module: module.id }];
|
|
return Array.from({ length: count }, (_unused, index) => ({
|
|
name: `${module.id}-${index + 1}`,
|
|
module: module.id,
|
|
shard: `${index + 1}/${count}`,
|
|
}));
|
|
});
|
|
}
|
|
|
|
export function normalizePath(filePath: string): string {
|
|
return filePath.replaceAll('\\', '/').replace(/^\.\//, '');
|
|
}
|
|
|
|
/**
|
|
* Which kernel module owns a repository-relative *source* path, if any.
|
|
*
|
|
* Test-file attribution is derived from the import graph — see
|
|
* `derivedAffectedModules` in `ownership.ts`, which is what the PR lane calls.
|
|
*/
|
|
export function moduleForFile(filePath: string): ModuleId | undefined {
|
|
const normalized = normalizePath(filePath);
|
|
for (const module of KERNEL_MODULES) {
|
|
for (const owned of module.owns) {
|
|
const match = owned.endsWith('/') ? normalized.startsWith(owned) : normalized === owned;
|
|
if (match) return module.id;
|
|
}
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
/** Kernel modules whose registry-owned paths a diff touches. */
|
|
export function affectedModules(changedFiles: readonly string[]): ModuleId[] {
|
|
const ids = new Set<ModuleId>();
|
|
for (const file of changedFiles) {
|
|
const id = moduleForFile(file);
|
|
if (id) ids.add(id);
|
|
}
|
|
return KERNEL_MODULES.filter((module) => ids.has(module.id)).map((module) => module.id);
|
|
}
|