mirror of
https://github.com/ruvnet/ruflo.git
synced 2026-09-14 14:01:28 +08:00
ruv
fdeacad7f8
Implements ADR-153 (Darwin Mode integration). Adds the WRITE layer that
closes the loop ADR-150's READ layer opens: score/genome describe a
harness; evolve changes one.
Three new surfaces, all honoring the four ADR-150 architectural constraints
(removable / optional / graceful degradation / CI-absent-path coverage):
- harness-evolve mutate 1 of 7 policy surfaces, sandbox-score, promote
- harness-security-bench upstream's "Darwin Shield" (their own ADR-155)
- harness-bench create/verify bench suites for evolve --bench
Wiring:
- plugins/ruflo-metaharness/scripts/_darwin.mjs — shared subprocess helper
- plugins/ruflo-metaharness/scripts/evolve.mjs — main verb (--confirm gate)
- plugins/ruflo-metaharness/scripts/security-bench.mjs — Darwin Shield wrapper
- plugins/ruflo-metaharness/scripts/bench.mjs — supporting verb
- plugins/ruflo-metaharness/skills/harness-{evolve,security-bench,bench}/SKILL.md
- v3/@claude-flow/cli/src/mcp-tools/metaharness-tools.ts — 3 new MCP tools:
metaharness_evolve, metaharness_security_bench, metaharness_bench
- v3/@claude-flow/cli/package.json optionalDependencies:
+ @metaharness/darwin ~0.3.1
bump metaharness ~0.1.11 -> ~0.2.6
- ruflo/package.json optionalDependencies: mirrored (per the #2112 lesson —
wrapper does NOT inherit root overrides; required for transitive pin)
Safety posture (matches mint.mjs convention):
- evolve requires --confirm; without it, returns a dry-run plan
- Ruflo caps: --generations 1..50, --children 1..20, --concurrency 1..8,
--population 1..20, --cycles 1..100 (upstream supports more)
- Upstream exit code 99 (safety-disqualified) propagates VERBATIM —
not remapped — so CI can distinguish "evolution failed" from
"evolution surfaced a safety-tripping mutation"
- All three scripts emit {degraded: true, reason: 'metaharness-darwin-not-available'}
and exit 0 when the optional dep is absent
Connection to ruflo's own ADR-155 (nightly self-learning security harness,
PR #2417, issue #2418): harness-security-bench is the closest reference
implementation for our Loop A reward-signal sanity check. Running it
periodically gives us the empirical floor — if Darwin Shield's champion
reaches TPR=1/FPR=0 on the seeded corpus, our loop A's gradient signal
is sound.
Tests:
- plugins/ruflo-metaharness/scripts/test-graceful-degradation.mjs extended
with the 3 new scripts; all 16 assertions pass (8 skills × 2 contracts:
exit 0 AND emit "degraded": true)
- npx tsc --noEmit clean on v3/@claude-flow/cli
Co-Authored-By: RuFlo <ruv@ruv.net>
96 lines
3.1 KiB
JavaScript
96 lines
3.1 KiB
JavaScript
#!/usr/bin/env node
|
|
// bench.mjs — wrapper around `metaharness-darwin bench <create|verify>`.
|
|
//
|
|
// Bench suites are JSON files describing a corpus of (input, expected-output)
|
|
// tasks. `evolve --bench <suite.json>` scores each variant against the
|
|
// corpus instead of (or in addition to) the project's test command, which
|
|
// lets you evolve a harness against a fixed evaluation set independent of
|
|
// the repo's natural tests. `bench create` scaffolds a suite from a repo;
|
|
// `bench verify` checks a suite parses and every task is well-formed.
|
|
//
|
|
// USAGE
|
|
// node scripts/bench.mjs --op create --repo . # scaffold suite.json from current dir
|
|
// node scripts/bench.mjs --op create --repo . --out /tmp/foo.json
|
|
// node scripts/bench.mjs --op verify --suite /tmp/foo.json
|
|
//
|
|
// EXIT CODES
|
|
// 0 ok (or degraded — Darwin not available)
|
|
// 1 --op verify and any task malformed
|
|
// 2 config error or upstream invocation failure
|
|
|
|
import { runDarwin, emitDarwinDegradedJsonAndExit } from './_darwin.mjs';
|
|
import { existsSync } from 'node:fs';
|
|
import { resolve } from 'node:path';
|
|
|
|
const ARGS = (() => {
|
|
const a = { op: null, repo: '.', suite: null, out: null, format: 'json' };
|
|
for (let i = 2; i < process.argv.length; i++) {
|
|
const v = process.argv[i];
|
|
if (v === '--op') a.op = process.argv[++i];
|
|
else if (v === '--repo') a.repo = process.argv[++i];
|
|
else if (v === '--suite') a.suite = process.argv[++i];
|
|
else if (v === '--out') a.out = process.argv[++i];
|
|
else if (v === '--format') a.format = process.argv[++i];
|
|
}
|
|
return a;
|
|
})();
|
|
|
|
function safetyChecks() {
|
|
if (!['create', 'verify'].includes(ARGS.op)) {
|
|
console.error('bench: --op must be create|verify');
|
|
process.exit(2);
|
|
}
|
|
if (ARGS.op === 'create' && !existsSync(resolve(ARGS.repo))) {
|
|
console.error(`bench: --repo path does not exist: ${ARGS.repo}`);
|
|
process.exit(2);
|
|
}
|
|
if (ARGS.op === 'verify' && !ARGS.suite) {
|
|
console.error('bench: --suite is required for verify');
|
|
process.exit(2);
|
|
}
|
|
if (ARGS.op === 'verify' && !existsSync(resolve(ARGS.suite))) {
|
|
console.error(`bench: --suite path does not exist: ${ARGS.suite}`);
|
|
process.exit(2);
|
|
}
|
|
}
|
|
|
|
function main() {
|
|
safetyChecks();
|
|
|
|
const cliArgs = ARGS.op === 'create'
|
|
? ['bench', 'create', resolve(ARGS.repo), ...(ARGS.out ? ['--out', resolve(ARGS.out)] : [])]
|
|
: ['bench', 'verify', resolve(ARGS.suite)];
|
|
|
|
const r = runDarwin(cliArgs, { timeoutMs: 60_000 });
|
|
|
|
if (r.degraded) {
|
|
emitDarwinDegradedJsonAndExit(r.reason);
|
|
return;
|
|
}
|
|
|
|
if (r.exitCode !== 0 && r.exitCode !== 1) {
|
|
const payload = {
|
|
success: false,
|
|
data: { exitCode: r.exitCode, stderrTail: r.stderr.slice(-400) },
|
|
generatedAt: new Date().toISOString(),
|
|
};
|
|
console.log(JSON.stringify(payload, null, 2));
|
|
process.exit(2);
|
|
}
|
|
|
|
const payload = {
|
|
success: r.exitCode === 0,
|
|
data: {
|
|
op: ARGS.op,
|
|
...(r.json ?? { stdoutTail: r.stdout.slice(-2000) }),
|
|
durationMs: r.durationMs,
|
|
},
|
|
generatedAt: new Date().toISOString(),
|
|
};
|
|
|
|
console.log(JSON.stringify(payload, null, 2));
|
|
process.exit(r.exitCode);
|
|
}
|
|
|
|
main();
|