mirror of
https://github.com/ruvnet/ruflo.git
synced 2026-09-14 14:01:28 +08:00
rUv
aee3b99910
Addresses the biggest failure mode: the loop could overfit the self-supervised proxy while human relevance is merely PRESERVED, not improved. Fix makes that visible + falsifiable: 1. FROZEN PUBLIC HUMAN EVAL SET — .claude/eval/human-relevance-frozen-v1.json (10 ADR-081 labelled queries), loaded via harness-frozen-eval.ts which PINS the canonical content hash (sha256:6096e48e…) and THROWS on drift. Single source of truth for the red/blue anchor; supersede with a new versioned file, never edit. The daemon runtime now loads the anchor from it (was an inline duplicate). 2. PER-GENERATION HUMAN-RELEVANCE DELTA — every receipt records deltas.humanRelevance (candidate − baseline nDCG@3 on the frozen set) + humanEvalHash. flywheelStatus exposes cumulativeBenchmarkDelta (proxy) vs cumulativeHumanRelevanceDelta (human); the status script flags 'proxy up but human flat → OVERFITTING'. The overfitting mode is now observable in the lineage instead of hidden. 3. CLEAN-ROOM REPLAY ACCEPTANCE TEST — scripts/replay-generation.mjs: from a clean install, replay one PROMOTED generation from its receipt alone — recompute every hash bit-identically + re-run accept/v1+sig to reproduce promoted=true, with fetch trapped (offline). Wired into v3-ci alongside the proof guard (which now also verifies the frozen-eval pinned hash). Verified: real-generation-1 replays with identical hashes, offline. Additive + backwards-compatible (new optional receipt fields; old committed bundles still replay). 76 harness+config tests green; both CI guards pass. Claude-Session: https://claude.ai/code/session_01S7GYqnVUVxBfZ5W8znqry3
45 lines
2.9 KiB
JavaScript
45 lines
2.9 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Flywheel status endpoint (ADR-176 A-P3b) — reconstruct the persisted lineage
|
|
* for a project and answer: how many generations, promotions, cumulative
|
|
* improvement, rejection rate, has it plateaued, is it genuinely compounding?
|
|
*
|
|
* Usage: node scripts/flywheel-status.mjs [--dir <projectRoot>] [--json]
|
|
*/
|
|
import { fileURLToPath } from 'node:url';
|
|
import { dirname, join, resolve } from 'node:path';
|
|
|
|
const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
|
|
const CLI_ROOT = resolve(SCRIPT_DIR, '..');
|
|
const { flywheelStatus } = await import(`file://${join(CLI_ROOT, 'dist/src/services/harness-flywheel-generations.js')}`);
|
|
|
|
const argDir = process.argv.indexOf('--dir');
|
|
const root = argDir > -1 ? resolve(process.argv[argDir + 1]) : process.cwd();
|
|
const s = flywheelStatus(root);
|
|
|
|
if (process.argv.includes('--json')) { console.log(JSON.stringify(s, null, 2)); process.exit(0); }
|
|
|
|
console.log(`flywheel status — ${root}`);
|
|
console.log('='.repeat(64));
|
|
console.log(`generations (promotions) : ${s.generations}`);
|
|
console.log(`attempts (incl. refusals): ${s.attempts}`);
|
|
console.log(`cumulative held-out Δ : ${s.lineage.cumulativeHeldOutImprovement.toFixed(4)}`);
|
|
// Anti-overfitting view: proxy (self-supervised) gain vs FROZEN human-relevance gain.
|
|
const bench = s.cumulativeBenchmarkDelta ?? 0, human = s.cumulativeHumanRelevanceDelta ?? 0;
|
|
const overfit = bench > 0.02 && human <= 0.005;
|
|
console.log(`cumulative proxy Δ : ${bench.toFixed(4)} (self-supervised self-retrieval)`);
|
|
console.log(`cumulative HUMAN Δ : ${human.toFixed(4)} (frozen human-labeled eval)${overfit ? ' ⚠️ proxy up but human flat → OVERFITTING' : ''}`);
|
|
console.log(`frozen human eval : ${s.humanEvalHash ? s.humanEvalHash.slice(0, 26) + '…' : '(none)'}`);
|
|
console.log(`lineage intact : ${s.lineage.lineageIntact} replayable: ${s.lineage.allReplayable}`);
|
|
console.log(`immutable root : ${(s.lineage.rootHash || '(none yet)').slice(0, 26)}…`);
|
|
console.log(`plateau : ${s.plateau.status} — ${s.plateau.rationale}`);
|
|
console.log(`served champion : ${s.served.championHash ? s.served.championHash.slice(0, 26) + '…' : '(none — all in shadow)'}`);
|
|
console.log(`current champion config : ${JSON.stringify(s.champion.config)}`);
|
|
console.log('mutation effectiveness :');
|
|
for (const m of s.mutation) console.log(` ${m.mutationClass.padEnd(22)} attempts=${m.attempts} promotions=${m.promotions} meanΔ=${m.meanDelta.toFixed(4)}`);
|
|
console.log('axis payoff (meta-learn) :');
|
|
for (const a of s.axisEffectiveness) console.log(` ${a.axis.padEnd(18)} promotions=${a.promotions} meanΔ=${a.meanDelta.toFixed(4)}${a.meanDelta > 0 ? ' ← biased toward' : ''}`);
|
|
console.log('lineage:');
|
|
for (const n of s.lineage.nodes) console.log(` gen ${n.generation} [${n.branch}] ${n.promoted ? '✓' : '✗'} Δ=${n.delta.toFixed(4)} ${n.mutationClass} replay=${n.replayable}`);
|
|
process.exit(0);
|