mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Dragan Spiridonov
643 lines
27 KiB
TypeScript
643 lines
27 KiB
TypeScript
#!/usr/bin/env tsx
|
||
/**
|
||
* MinCut ON/OFF Benchmark
|
||
*
|
||
* Compares two MinCut applications with the feature ON vs OFF:
|
||
* 1. Test suite optimizer (ADR-047 / RVF) -- skip-redundant-tests vs run-all
|
||
* 2. Swarm health monitor (ADR-047) -- mincut-tick vs no-monitor
|
||
*
|
||
* Produces:
|
||
* - reports/mincut-onoff-benchmarks.json (raw numbers)
|
||
* - docs/benchmarks/mincut-onoff-report.md (human-readable report)
|
||
*
|
||
* Run: npx tsx scripts/benchmark-mincut-onoff.ts
|
||
*/
|
||
|
||
import { performance } from 'perf_hooks';
|
||
import { writeFileSync, mkdirSync, existsSync } from 'fs';
|
||
import { dirname, resolve } from 'path';
|
||
import * as os from 'os';
|
||
|
||
import { SwarmGraph } from '../src/coordination/mincut/swarm-graph';
|
||
import { MinCutCalculator } from '../src/coordination/mincut/mincut-calculator';
|
||
import { MinCutHealthMonitor } from '../src/coordination/mincut/mincut-health-monitor';
|
||
import { MinCutTestOptimizerImpl, type TestNode } from '../src/domains/test-execution/services/mincut-test-optimizer';
|
||
import type { SwarmVertex, SwarmEdge } from '../src/coordination/mincut/interfaces';
|
||
import type { DomainName } from '../src/shared/types';
|
||
|
||
// =============================================================================
|
||
// Config
|
||
// =============================================================================
|
||
|
||
const TEST_OPT_SIZES = [50, 200, 500, 1000]; // test counts
|
||
const FILES_PER_SUITE = (n: number) => Math.max(20, Math.floor(n / 3));
|
||
const FILES_PER_TEST_MIN = 2;
|
||
const FILES_PER_TEST_MAX = 15;
|
||
|
||
const HEALTH_MONITOR_SIZES = [5, 15, 50, 200, 500]; // agent counts
|
||
const HEALTH_TICKS = 1000;
|
||
const HEALTH_WARMUP_TICKS = 50;
|
||
const HEALTH_CHECK_INTERVAL_MS = 5000; // default config
|
||
|
||
const DOMAINS: DomainName[] = [
|
||
'test-generation',
|
||
'test-execution',
|
||
'coverage-analysis',
|
||
'quality-assessment',
|
||
'defect-intelligence',
|
||
];
|
||
|
||
// Deterministic PRNG so runs are comparable
|
||
function mulberry32(seed: number): () => number {
|
||
let a = seed >>> 0;
|
||
return () => {
|
||
a = (a + 0x6D2B79F5) >>> 0;
|
||
let t = a;
|
||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||
};
|
||
}
|
||
|
||
// =============================================================================
|
||
// Statistics helpers
|
||
// =============================================================================
|
||
|
||
function summarize(samples: number[]): {
|
||
count: number; avg: number; min: number; max: number; p50: number; p95: number; p99: number;
|
||
} {
|
||
const sorted = [...samples].sort((a, b) => a - b);
|
||
const n = sorted.length;
|
||
const pick = (q: number) => sorted[Math.min(n - 1, Math.floor(n * q))];
|
||
const sum = sorted.reduce((a, b) => a + b, 0);
|
||
return {
|
||
count: n,
|
||
avg: sum / n,
|
||
min: sorted[0],
|
||
max: sorted[n - 1],
|
||
p50: pick(0.5),
|
||
p95: pick(0.95),
|
||
p99: pick(0.99),
|
||
};
|
||
}
|
||
|
||
// =============================================================================
|
||
// Suite A: Test Optimizer ON vs OFF
|
||
// =============================================================================
|
||
|
||
interface TestOptResult {
|
||
testCount: number;
|
||
fileCount: number;
|
||
totalEdges: number;
|
||
off: {
|
||
totalRuntimeMs: number;
|
||
testsRun: number;
|
||
};
|
||
on: {
|
||
overheadMs: number; // time to compute the partition
|
||
criticalCount: number;
|
||
skippableCount: number;
|
||
criticalRuntimeMs: number; // sum of estimated durations of critical tests
|
||
estimatedTimeSavingsMs: number;
|
||
mincutValue: number;
|
||
coveragePreserved: number; // fraction (should be 1.0 by uniqueness guarantee)
|
||
netRuntimeOnMs: number; // criticalRuntimeMs + overheadMs
|
||
reductionPercent: number; // 100 * (1 - net/off)
|
||
};
|
||
greedy: {
|
||
overheadMs: number;
|
||
selectedCount: number;
|
||
selectedRuntimeMs: number;
|
||
coveragePreserved: number;
|
||
netRuntimeMs: number;
|
||
reductionPercent: number;
|
||
};
|
||
}
|
||
|
||
function generateSyntheticTests(testCount: number, fileCount: number, rng: () => number): TestNode[] {
|
||
const tests: TestNode[] = [];
|
||
for (let i = 0; i < testCount; i++) {
|
||
const coveredCount =
|
||
FILES_PER_TEST_MIN + Math.floor(rng() * (FILES_PER_TEST_MAX - FILES_PER_TEST_MIN + 1));
|
||
const covered = new Set<string>();
|
||
while (covered.size < coveredCount) {
|
||
covered.add(`src/file-${Math.floor(rng() * fileCount)}.ts`);
|
||
}
|
||
// Heavy-tailed test durations: most fast, a few slow
|
||
const durMs = rng() < 0.05
|
||
? 500 + Math.floor(rng() * 3000) // 5% slow tests
|
||
: 5 + Math.floor(rng() * 100); // bulk are fast
|
||
tests.push({
|
||
testId: `test-${i}`,
|
||
testFile: `tests/spec-${i}.test.ts`,
|
||
coveredFiles: Array.from(covered),
|
||
estimatedDurationMs: durMs,
|
||
});
|
||
}
|
||
return tests;
|
||
}
|
||
|
||
// Greedy set-cover baseline: pick tests that cover the most uncovered files first.
|
||
function greedySetCover(tests: readonly TestNode[]): {
|
||
selected: Set<string>;
|
||
overheadMs: number;
|
||
} {
|
||
const start = performance.now();
|
||
const allFiles = new Set<string>();
|
||
for (const t of tests) for (const f of t.coveredFiles) allFiles.add(f);
|
||
const remaining = new Set(allFiles);
|
||
const selected = new Set<string>();
|
||
const candidates = tests.map(t => ({ test: t, cov: new Set(t.coveredFiles) }));
|
||
|
||
while (remaining.size > 0) {
|
||
let best: typeof candidates[number] | null = null;
|
||
let bestGain = -1;
|
||
for (const c of candidates) {
|
||
if (selected.has(c.test.testId)) continue;
|
||
let gain = 0;
|
||
for (const f of c.cov) if (remaining.has(f)) gain++;
|
||
if (gain > bestGain) { bestGain = gain; best = c; }
|
||
}
|
||
if (!best || bestGain <= 0) break;
|
||
selected.add(best.test.testId);
|
||
for (const f of best.cov) remaining.delete(f);
|
||
}
|
||
return { selected, overheadMs: performance.now() - start };
|
||
}
|
||
|
||
function runTestOptimizerBench(): TestOptResult[] {
|
||
const rng = mulberry32(42);
|
||
const optimizer = new MinCutTestOptimizerImpl();
|
||
const results: TestOptResult[] = [];
|
||
|
||
for (const testCount of TEST_OPT_SIZES) {
|
||
const fileCount = FILES_PER_SUITE(testCount);
|
||
const tests = generateSyntheticTests(testCount, fileCount, rng);
|
||
|
||
// Cardinalities
|
||
const totalEdges = tests.reduce((s, t) => s + t.coveredFiles.length, 0);
|
||
const offRuntime = tests.reduce((s, t) => s + t.estimatedDurationMs, 0);
|
||
|
||
// === ON: mincut partition ===
|
||
// Warm up JIT
|
||
optimizer.optimize(tests, 1.0);
|
||
// Measure
|
||
const onStart = performance.now();
|
||
const opt = optimizer.optimize(tests, 1.0);
|
||
const overheadMs = performance.now() - onStart;
|
||
|
||
const criticalSet = new Set(opt.criticalTests);
|
||
const criticalRuntime = tests
|
||
.filter(t => criticalSet.has(t.testId))
|
||
.reduce((s, t) => s + t.estimatedDurationMs, 0);
|
||
|
||
// Verify coverage preserved
|
||
const allFiles = new Set<string>();
|
||
for (const t of tests) for (const f of t.coveredFiles) allFiles.add(f);
|
||
const coveredByCritical = new Set<string>();
|
||
for (const t of tests) {
|
||
if (criticalSet.has(t.testId)) {
|
||
for (const f of t.coveredFiles) coveredByCritical.add(f);
|
||
}
|
||
}
|
||
const coveragePreserved = allFiles.size > 0 ? coveredByCritical.size / allFiles.size : 1.0;
|
||
|
||
const netRuntimeOn = criticalRuntime + overheadMs;
|
||
const reductionPercent = offRuntime > 0 ? 100 * (1 - netRuntimeOn / offRuntime) : 0;
|
||
|
||
// === Greedy baseline ===
|
||
const greedy = greedySetCover(tests);
|
||
const greedyRuntime = tests
|
||
.filter(t => greedy.selected.has(t.testId))
|
||
.reduce((s, t) => s + t.estimatedDurationMs, 0);
|
||
const greedyCovered = new Set<string>();
|
||
for (const t of tests) {
|
||
if (greedy.selected.has(t.testId)) {
|
||
for (const f of t.coveredFiles) greedyCovered.add(f);
|
||
}
|
||
}
|
||
const greedyNet = greedyRuntime + greedy.overheadMs;
|
||
const greedyReduction = offRuntime > 0 ? 100 * (1 - greedyNet / offRuntime) : 0;
|
||
|
||
results.push({
|
||
testCount,
|
||
fileCount,
|
||
totalEdges,
|
||
off: { totalRuntimeMs: offRuntime, testsRun: tests.length },
|
||
on: {
|
||
overheadMs,
|
||
criticalCount: opt.criticalTests.length,
|
||
skippableCount: opt.skippableTests.length,
|
||
criticalRuntimeMs: criticalRuntime,
|
||
estimatedTimeSavingsMs: opt.estimatedTimeSavingsMs,
|
||
mincutValue: opt.graphStats.mincutValue,
|
||
coveragePreserved,
|
||
netRuntimeOnMs: netRuntimeOn,
|
||
reductionPercent,
|
||
},
|
||
greedy: {
|
||
overheadMs: greedy.overheadMs,
|
||
selectedCount: greedy.selected.size,
|
||
selectedRuntimeMs: greedyRuntime,
|
||
coveragePreserved: allFiles.size > 0 ? greedyCovered.size / allFiles.size : 1.0,
|
||
netRuntimeMs: greedyNet,
|
||
reductionPercent: greedyReduction,
|
||
},
|
||
});
|
||
}
|
||
return results;
|
||
}
|
||
|
||
// =============================================================================
|
||
// Suite B: Health Monitor ON vs OFF
|
||
// =============================================================================
|
||
|
||
interface HealthMonitorResult {
|
||
agentCount: number;
|
||
vertexCount: number;
|
||
edgeCount: number;
|
||
off: {
|
||
tickStats: ReturnType<typeof summarize>;
|
||
totalMs: number;
|
||
};
|
||
on: {
|
||
tickStats: ReturnType<typeof summarize>;
|
||
totalMs: number;
|
||
finalStatus: string;
|
||
finalMincutValue: number;
|
||
weakVertexCount: number;
|
||
};
|
||
overheadPerTickMs: number;
|
||
projectedCpuPercentAt5sInterval: number;
|
||
memoryGrowthBytes: number;
|
||
}
|
||
|
||
function buildRealisticSwarm(agentCount: number, rng: () => number): SwarmGraph {
|
||
const graph = new SwarmGraph();
|
||
|
||
// One coordinator per domain
|
||
for (const d of DOMAINS) {
|
||
graph.addVertex({
|
||
id: `coord-${d}`,
|
||
type: 'coordinator',
|
||
domain: d,
|
||
weight: 2.0,
|
||
createdAt: new Date(),
|
||
});
|
||
}
|
||
// Worker agents, evenly distributed across domains
|
||
for (let i = 0; i < agentCount; i++) {
|
||
const d = DOMAINS[i % DOMAINS.length];
|
||
graph.addVertex({
|
||
id: `agent-${i}`,
|
||
type: 'agent',
|
||
domain: d,
|
||
weight: 1.0,
|
||
createdAt: new Date(),
|
||
});
|
||
}
|
||
// Workers connect to their domain coordinator
|
||
for (let i = 0; i < agentCount; i++) {
|
||
const d = DOMAINS[i % DOMAINS.length];
|
||
graph.addEdge({
|
||
source: `agent-${i}`,
|
||
target: `coord-${d}`,
|
||
weight: 1.0,
|
||
type: 'coordination',
|
||
bidirectional: true,
|
||
});
|
||
}
|
||
// Domain coordinators form a mesh (workflow)
|
||
for (let i = 0; i < DOMAINS.length; i++) {
|
||
for (let j = i + 1; j < DOMAINS.length; j++) {
|
||
graph.addEdge({
|
||
source: `coord-${DOMAINS[i]}`,
|
||
target: `coord-${DOMAINS[j]}`,
|
||
weight: 0.8,
|
||
type: 'workflow',
|
||
bidirectional: true,
|
||
});
|
||
}
|
||
}
|
||
// Cross-agent peer edges (sparse, ~1.5 per agent on average)
|
||
const peerEdgeCount = Math.floor(agentCount * 1.5);
|
||
for (let k = 0; k < peerEdgeCount; k++) {
|
||
const a = Math.floor(rng() * agentCount);
|
||
const b = Math.floor(rng() * agentCount);
|
||
if (a === b) continue;
|
||
graph.addEdge({
|
||
source: `agent-${a}`,
|
||
target: `agent-${b}`,
|
||
weight: 0.3 + rng() * 0.4,
|
||
type: 'communication',
|
||
bidirectional: true,
|
||
});
|
||
}
|
||
return graph;
|
||
}
|
||
|
||
function runHealthMonitorBench(): HealthMonitorResult[] {
|
||
const rng = mulberry32(7);
|
||
const results: HealthMonitorResult[] = [];
|
||
|
||
for (const agentCount of HEALTH_MONITOR_SIZES) {
|
||
const graph = buildRealisticSwarm(agentCount, rng);
|
||
const stats = graph.getStats();
|
||
const monitor = new MinCutHealthMonitor(graph, {});
|
||
|
||
// --- Warmup ---
|
||
for (let i = 0; i < HEALTH_WARMUP_TICKS; i++) {
|
||
monitor.checkHealth();
|
||
}
|
||
|
||
if (global.gc) global.gc();
|
||
const memBefore = process.memoryUsage().heapUsed;
|
||
|
||
// --- ON: real health check loop ---
|
||
const onSamples: number[] = [];
|
||
const onLoopStart = performance.now();
|
||
for (let i = 0; i < HEALTH_TICKS; i++) {
|
||
const t = performance.now();
|
||
monitor.checkHealth();
|
||
onSamples.push(performance.now() - t);
|
||
}
|
||
const onTotal = performance.now() - onLoopStart;
|
||
|
||
const memAfter = process.memoryUsage().heapUsed;
|
||
const memDelta = memAfter - memBefore;
|
||
|
||
const finalHealth = monitor.getHealth();
|
||
|
||
// --- OFF: noop loop (cost of just iterating without any analysis) ---
|
||
// Stand-in for "swarm runs without the health monitor at all".
|
||
// We still walk the graph to make sure the OFF path is not a complete no-op
|
||
// -- it should represent the "the swarm graph exists, but nothing observes it"
|
||
// baseline, so all we do is touch the graph identity.
|
||
const offSamples: number[] = [];
|
||
const offLoopStart = performance.now();
|
||
for (let i = 0; i < HEALTH_TICKS; i++) {
|
||
const t = performance.now();
|
||
// touch graph to defeat dead-code elimination, but do no analysis
|
||
void graph.vertexCount;
|
||
offSamples.push(performance.now() - t);
|
||
}
|
||
const offTotal = performance.now() - offLoopStart;
|
||
|
||
const onStats = summarize(onSamples);
|
||
const offStats = summarize(offSamples);
|
||
const overheadPerTick = onStats.avg - offStats.avg;
|
||
const cpuPct = (overheadPerTick / HEALTH_CHECK_INTERVAL_MS) * 100;
|
||
|
||
results.push({
|
||
agentCount,
|
||
vertexCount: stats.vertexCount,
|
||
edgeCount: stats.edgeCount,
|
||
off: { tickStats: offStats, totalMs: offTotal },
|
||
on: {
|
||
tickStats: onStats,
|
||
totalMs: onTotal,
|
||
finalStatus: finalHealth.status,
|
||
finalMincutValue: finalHealth.minCutValue,
|
||
weakVertexCount: finalHealth.weakVertexCount,
|
||
},
|
||
overheadPerTickMs: overheadPerTick,
|
||
projectedCpuPercentAt5sInterval: cpuPct,
|
||
memoryGrowthBytes: memDelta,
|
||
});
|
||
|
||
monitor.stop();
|
||
}
|
||
return results;
|
||
}
|
||
|
||
// =============================================================================
|
||
// Report generation
|
||
// =============================================================================
|
||
|
||
interface BenchmarkReport {
|
||
timestamp: string;
|
||
system: {
|
||
platform: string;
|
||
arch: string;
|
||
cpu: string;
|
||
cpuCount: number;
|
||
nodeVersion: string;
|
||
totalMemMb: number;
|
||
};
|
||
testOptimizer: TestOptResult[];
|
||
healthMonitor: HealthMonitorResult[];
|
||
}
|
||
|
||
function formatMs(ms: number, digits = 3): string {
|
||
return `${ms.toFixed(digits)} ms`;
|
||
}
|
||
|
||
function formatBytes(b: number): string {
|
||
if (Math.abs(b) < 1024) return `${b.toFixed(0)} B`;
|
||
if (Math.abs(b) < 1024 * 1024) return `${(b / 1024).toFixed(1)} KB`;
|
||
return `${(b / 1024 / 1024).toFixed(2)} MB`;
|
||
}
|
||
|
||
function renderMarkdown(report: BenchmarkReport): string {
|
||
const lines: string[] = [];
|
||
lines.push(`# MinCut ON/OFF Benchmark Report`);
|
||
lines.push('');
|
||
lines.push(`**Generated:** ${report.timestamp}`);
|
||
lines.push('');
|
||
lines.push(`## System`);
|
||
lines.push('');
|
||
lines.push(`| Field | Value |`);
|
||
lines.push(`|---|---|`);
|
||
lines.push(`| Platform | \`${report.system.platform}\` (${report.system.arch}) |`);
|
||
lines.push(`| CPU | ${report.system.cpu} (${report.system.cpuCount} cores) |`);
|
||
lines.push(`| Node | ${report.system.nodeVersion} |`);
|
||
lines.push(`| RAM | ${report.system.totalMemMb} MB |`);
|
||
lines.push('');
|
||
lines.push(`## Suite A — Test Suite Optimizer (ADR-047 / RVF)`);
|
||
lines.push('');
|
||
lines.push(`**Question:** Does \`MinCutTestOptimizer\` actually save wall-clock test time, and at what overhead?`);
|
||
lines.push('');
|
||
lines.push(`**Setup:** Synthetic deterministic test suites (seed=42). Each test covers ${FILES_PER_TEST_MIN}–${FILES_PER_TEST_MAX} files from a pool of \`floor(testCount / 3)\` files. 5% of tests have heavy-tailed durations (500–3500 ms); the rest are 5–105 ms. The optimizer runs with \`coverageThreshold=1.0\` (no coverage loss allowed).`);
|
||
lines.push('');
|
||
lines.push(`**Baselines compared:**`);
|
||
lines.push(`- **OFF**: run all tests (sum of estimated durations)`);
|
||
lines.push(`- **ON (mincut)**: \`MinCutTestOptimizerImpl.optimize()\` → run only \`criticalTests\``);
|
||
lines.push(`- **Greedy set-cover**: simple "pick test covering most uncovered files" baseline, for honest comparison`);
|
||
lines.push('');
|
||
lines.push(`### Results`);
|
||
lines.push('');
|
||
lines.push(`| Tests | Files | OFF runtime | ON overhead | ON critical / skippable | ON net runtime | **ON reduction** | Greedy net | Greedy reduction | Coverage |`);
|
||
lines.push(`|---:|---:|---:|---:|---|---:|---:|---:|---:|---|`);
|
||
for (const r of report.testOptimizer) {
|
||
lines.push(
|
||
`| ${r.testCount} | ${r.fileCount} ` +
|
||
`| ${formatMs(r.off.totalRuntimeMs, 0)} ` +
|
||
`| ${formatMs(r.on.overheadMs)} ` +
|
||
`| ${r.on.criticalCount} / ${r.on.skippableCount} ` +
|
||
`| ${formatMs(r.on.netRuntimeOnMs, 0)} ` +
|
||
`| **${r.on.reductionPercent.toFixed(1)}%** ` +
|
||
`| ${formatMs(r.greedy.netRuntimeMs, 0)} ` +
|
||
`| ${r.greedy.reductionPercent.toFixed(1)}% ` +
|
||
`| ${(r.on.coveragePreserved * 100).toFixed(1)}% |`
|
||
);
|
||
}
|
||
lines.push('');
|
||
lines.push(`*Coverage is fraction of source files still covered after skipping. At \`coverageThreshold=1.0\` it should be 100% by the uniqueness guarantee in \`findUniquelyCriticalTests()\`.*`);
|
||
lines.push('');
|
||
lines.push(`## Suite B — Swarm Health Monitor (ADR-047)`);
|
||
lines.push('');
|
||
lines.push(`**Question:** What's the per-tick cost of \`MinCutHealthMonitor.checkHealth()\`, and how much steady-state CPU does it consume at the default 5 s check interval?`);
|
||
lines.push('');
|
||
lines.push(`**Setup:** Synthetic swarm topology (seed=7) with N worker agents distributed across ${DOMAINS.length} domains, one coordinator per domain, ~1.5 random peer edges per agent, plus a mesh of workflow edges between domain coordinators. ${HEALTH_WARMUP_TICKS} warmup ticks, then ${HEALTH_TICKS} measured ticks.`);
|
||
lines.push('');
|
||
lines.push(`**Baselines compared:**`);
|
||
lines.push(`- **OFF**: no monitor — just a tight loop touching the graph (represents "no analysis").`);
|
||
lines.push(`- **ON**: \`monitor.checkHealth()\` each tick (mincut value + weak-vertex scan + history + alert evaluation).`);
|
||
lines.push('');
|
||
lines.push(`### Results`);
|
||
lines.push('');
|
||
lines.push(`| Agents | V | E | OFF avg/tick | ON avg/tick | ON p99/tick | **Overhead/tick** | Projected CPU @ 5s | Mem growth | Mincut λ | Weak verts |`);
|
||
lines.push(`|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|`);
|
||
for (const r of report.healthMonitor) {
|
||
lines.push(
|
||
`| ${r.agentCount} | ${r.vertexCount} | ${r.edgeCount} ` +
|
||
`| ${formatMs(r.off.tickStats.avg, 4)} ` +
|
||
`| ${formatMs(r.on.tickStats.avg, 4)} ` +
|
||
`| ${formatMs(r.on.tickStats.p99, 4)} ` +
|
||
`| **${formatMs(r.overheadPerTickMs, 4)}** ` +
|
||
`| ${r.projectedCpuPercentAt5sInterval.toFixed(4)}% ` +
|
||
`| ${formatBytes(r.memoryGrowthBytes)} ` +
|
||
`| ${r.on.finalMincutValue.toFixed(2)} ` +
|
||
`| ${r.on.weakVertexCount} |`
|
||
);
|
||
}
|
||
lines.push('');
|
||
lines.push(`*Projected CPU at 5 s interval = overhead / 5000 ms. Memory growth is heap delta across 1000 ticks of history retention (\`MinCutHistoryEntry\` ring + alert map). Negative values indicate GC ran during measurement.*`);
|
||
lines.push('');
|
||
lines.push(`## Findings`);
|
||
lines.push('');
|
||
|
||
// Suite A findings: did mincut beat greedy?
|
||
const aWins = report.testOptimizer.filter(r => r.on.reductionPercent > r.greedy.reductionPercent).length;
|
||
const aTotal = report.testOptimizer.length;
|
||
const avgOnReduction = report.testOptimizer.reduce((s, r) => s + r.on.reductionPercent, 0) / aTotal;
|
||
const avgGreedyReduction = report.testOptimizer.reduce((s, r) => s + r.greedy.reductionPercent, 0) / aTotal;
|
||
|
||
lines.push(`### Suite A — Test Optimizer`);
|
||
lines.push('');
|
||
lines.push(`1. **MinCut works.** On these synthetic suites it cuts test wall-clock by **${avgOnReduction.toFixed(1)}% on average** while preserving 100% file coverage. Overhead is small (≤30 ms even at 1000 tests).`);
|
||
lines.push(`2. **But greedy set-cover beats it.** Greedy averages **${avgGreedyReduction.toFixed(1)}%** reduction — better than mincut in **${aTotal - aWins} of ${aTotal}** sizes. Why? Greedy directly optimizes the objective ("cover all files with fewest tests"); mincut optimizes a *bipartite-graph partition* that's a useful proxy but not the same thing. The \`findUniquelyCriticalTests\` safety net also forces inclusion of tests that greedy might skip via redundancy.`);
|
||
lines.push(`3. **The mincut approach is structurally conservative**, which is arguably what you want for a test optimizer: it includes tests that aren't strictly necessary for coverage but live on the partition boundary (any test whose unique-coverage file would otherwise be lost). That's a feature in production — fewer regressions slip past — but it makes the raw "% reduction" number look worse than greedy.`);
|
||
lines.push(`4. **Takeaway:** if pure runtime reduction is the goal, greedy set-cover with the same uniqueness guarantee would do better here. The mincut justification has to come from something else — e.g., the structural metadata (mincut value, weak vertices) being useful elsewhere, or robustness to coverage-data noise.`);
|
||
lines.push('');
|
||
|
||
// Suite B findings: scaling behavior
|
||
const smallest = report.healthMonitor[0];
|
||
const largest = report.healthMonitor[report.healthMonitor.length - 1];
|
||
const sizeRatio = largest.vertexCount / smallest.vertexCount;
|
||
const costRatio = largest.overheadPerTickMs / smallest.overheadPerTickMs;
|
||
|
||
lines.push(`### Suite B — Health Monitor`);
|
||
lines.push('');
|
||
lines.push(`1. **Overhead is negligible at production swarm sizes.** Even at 500 agents (505 vertices, 1257 edges), \`checkHealth()\` averages **${largest.overheadPerTickMs.toFixed(3)} ms** per tick. At the default 5 s check interval, that's **${largest.projectedCpuPercentAt5sInterval.toFixed(4)}% CPU**.`);
|
||
lines.push(`2. **Scaling is roughly linear in vertex count.** ${sizeRatio.toFixed(0)}× more vertices → ~${costRatio.toFixed(1)}× more time, which matches the **O(V)** weighted-degree heuristic plus **O(V+E)** Tarjan's articulation-point pass.`);
|
||
lines.push(`3. **ADR-047's <50 μs target is met for the mincut value alone**, but the full \`checkHealth()\` does more — weak-vertex find, history append, alert evaluation — so per-tick cost is 20–270 μs in this range. The ADR target was for the calc primitive; the full health check is still well under any reasonable budget.`);
|
||
lines.push(`4. **p99 stays well-behaved** until 500 agents where it jumps to ~2 ms (GC pauses dominate at that point, not algorithmic cost). For real swarms in the 10–50 agent range this is invisible.`);
|
||
lines.push(`5. **Memory retention is stable.** History is bounded; the negative deltas reflect GC reclaiming churn from the measurement loop, not net growth.`);
|
||
lines.push(`6. **Takeaway:** the health monitor is essentially free at realistic AQE swarm sizes (15–50 agents). The cost-benefit conversation here isn't about CPU — it's whether the *signal* (mincut value, weak vertices) drives correct self-healing actions, which this benchmark doesn't measure.`);
|
||
lines.push('');
|
||
lines.push(`### Combined takeaway`);
|
||
lines.push('');
|
||
lines.push(`The two applications behave very differently:`);
|
||
lines.push(`- The **health monitor** justifies itself on raw cost — it's cheap enough that you can leave it on always.`);
|
||
lines.push(`- The **test optimizer** justifies itself less obviously — a simpler greedy heuristic dominates it on the headline metric. Whether mincut is the right algorithm here depends on whether the bipartite-cut framing pays off on real coverage data (with hotspots and skewed distributions) in ways the synthetic benchmark doesn't surface.`);
|
||
lines.push('');
|
||
lines.push(`## Methodology notes & caveats`);
|
||
lines.push('');
|
||
lines.push(`- **Determinism:** PRNG-seeded test suites and topologies (seeds 42 and 7); rerun is reproducible.`);
|
||
lines.push(`- **Warmup:** All hot loops warm up the V8 JIT before measurement. Without warmup, p99 includes interpretation-tier samples.`);
|
||
lines.push(`- **Synthetic vs real:** Coverage data is uniform-random with a hot-tail duration distribution. Real test suites tend to have heavier skew (a handful of tests cover huge chunks of code) — that *helps* both mincut and greedy more than these results suggest, since redundancy is higher in practice.`);
|
||
lines.push(`- **OFF semantics:** "OFF" for the health monitor is "no observer at all", not "a cheaper observer". If the alternative is a hand-rolled health check (e.g., just count vertices), the marginal cost of MinCut is smaller than the numbers above.`);
|
||
lines.push(`- **Algorithm:** We're measuring the weighted-degree heuristic (\`approxMinCut\` / \`getMinCutValue\`), not Stoer-Wagner. The ADR's <50 μs claim is for this heuristic.`);
|
||
lines.push(`- **Single-process:** No I/O, no persistence layer, no event bus. Real systems pay extra to emit events.`);
|
||
lines.push('');
|
||
lines.push(`## How to reproduce`);
|
||
lines.push('');
|
||
lines.push(`\`\`\`bash`);
|
||
lines.push(`npx tsx scripts/benchmark-mincut-onoff.ts`);
|
||
lines.push(`\`\`\``);
|
||
lines.push('');
|
||
lines.push(`Outputs:`);
|
||
lines.push(`- \`reports/mincut-onoff-benchmarks.json\` — raw data`);
|
||
lines.push(`- \`docs/benchmarks/mincut-onoff-report.md\` — this file (regenerated each run)`);
|
||
lines.push('');
|
||
return lines.join('\n');
|
||
}
|
||
|
||
// =============================================================================
|
||
// Main
|
||
// =============================================================================
|
||
|
||
function main() {
|
||
console.log('Running MinCut ON/OFF benchmark...');
|
||
console.log(' Suite A: Test Optimizer (sizes:', TEST_OPT_SIZES.join(', '), ')');
|
||
console.log(' Suite B: Health Monitor (sizes:', HEALTH_MONITOR_SIZES.join(', '), ')');
|
||
console.log('');
|
||
|
||
console.log('[A] Running test optimizer benchmark...');
|
||
const tA = performance.now();
|
||
const testOptimizer = runTestOptimizerBench();
|
||
console.log(` done in ${(performance.now() - tA).toFixed(0)} ms`);
|
||
|
||
console.log('[B] Running health monitor benchmark...');
|
||
const tB = performance.now();
|
||
const healthMonitor = runHealthMonitorBench();
|
||
console.log(` done in ${(performance.now() - tB).toFixed(0)} ms`);
|
||
|
||
const cpus = os.cpus();
|
||
const report: BenchmarkReport = {
|
||
timestamp: new Date().toISOString(),
|
||
system: {
|
||
platform: process.platform,
|
||
arch: process.arch,
|
||
cpu: cpus[0]?.model ?? 'unknown',
|
||
cpuCount: cpus.length,
|
||
nodeVersion: process.version,
|
||
totalMemMb: Math.round(os.totalmem() / 1024 / 1024),
|
||
},
|
||
testOptimizer,
|
||
healthMonitor,
|
||
};
|
||
|
||
const jsonPath = resolve('reports/mincut-onoff-benchmarks.json');
|
||
const mdPath = resolve('docs/benchmarks/mincut-onoff-report.md');
|
||
for (const p of [jsonPath, mdPath]) {
|
||
const dir = dirname(p);
|
||
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
||
}
|
||
writeFileSync(jsonPath, JSON.stringify(report, null, 2));
|
||
writeFileSync(mdPath, renderMarkdown(report));
|
||
|
||
console.log('');
|
||
console.log(`Wrote: ${jsonPath}`);
|
||
console.log(`Wrote: ${mdPath}`);
|
||
console.log('');
|
||
console.log('Summary:');
|
||
for (const r of testOptimizer) {
|
||
console.log(
|
||
` [A] ${r.testCount} tests: ON reduction = ${r.on.reductionPercent.toFixed(1)}%, ` +
|
||
`greedy reduction = ${r.greedy.reductionPercent.toFixed(1)}%, ` +
|
||
`overhead = ${r.on.overheadMs.toFixed(2)} ms`
|
||
);
|
||
}
|
||
for (const r of healthMonitor) {
|
||
console.log(
|
||
` [B] ${r.agentCount} agents: avg tick = ${r.on.tickStats.avg.toFixed(4)} ms, ` +
|
||
`overhead = ${r.overheadPerTickMs.toFixed(4)} ms, ` +
|
||
`CPU @5s = ${r.projectedCpuPercentAt5sInterval.toFixed(4)}%`
|
||
);
|
||
}
|
||
}
|
||
|
||
main();
|