mirror of
https://github.com/EvoMap/evolver.git
synced 2026-09-18 21:47:53 +08:00
autogame-17
89 lines
3.3 KiB
JavaScript
89 lines
3.3 KiB
JavaScript
// test/triggerShift.test.js
|
|
//
|
|
// Deterministic tests for the offline trigger-shift replay evaluator. No LLM,
|
|
// no subprocess, no network, and no live selection wiring.
|
|
'use strict';
|
|
|
|
const { describe, it } = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
|
|
const { evaluateTriggerShift, smallTriggerShiftSuite } = require('../src/experiment/triggerShift');
|
|
|
|
const pair = {
|
|
id: 'wrapper-timeout-retry',
|
|
objectiveId: 'safe-timeout-retry',
|
|
axis: 'wrapper_trigger',
|
|
expectedDecision: 'repair_timeout_source',
|
|
train: {
|
|
id: 'train-wrapper-timeout-retry',
|
|
prompt: 'EVOLVER_TRIGGER: timeout retry loop. Fix the timeout source instead of adding more retries.',
|
|
wrapperTrigger: 'EVOLVER_TRIGGER',
|
|
},
|
|
shifted: {
|
|
id: 'shifted-wrapper-timeout-retry',
|
|
prompt: 'A service keeps retrying after timeouts. Fix the timeout source instead of adding more retries.',
|
|
},
|
|
};
|
|
|
|
const stablePolicy = {
|
|
id: 'stable-policy',
|
|
predict: () => ({ label: 'repair_timeout_source' }),
|
|
};
|
|
|
|
function hasTrainMarker(task) {
|
|
return task && (task.wrapperTrigger === 'EVOLVER_TRIGGER' || String(task.prompt || '').includes('EVOLVER_TRIGGER'));
|
|
}
|
|
|
|
const triggerSensitivePolicy = {
|
|
id: 'trigger-sensitive-dummy',
|
|
predict: (task) => ({ label: hasTrainMarker(task) ? 'repair_timeout_source' : 'add_more_retries' }),
|
|
};
|
|
|
|
describe('trigger-shift replay evaluator', () => {
|
|
it('returns a zeroed report for an empty suite', () => {
|
|
const report = evaluateTriggerShift(stablePolicy, []);
|
|
assert.equal(report.policyId, 'stable-policy');
|
|
assert.equal(report.pairs, 0);
|
|
assert.equal(report.meanTrainReward, 0);
|
|
assert.equal(report.meanShiftedReward, 0);
|
|
assert.equal(report.meanGap, 0);
|
|
assert.equal(report.maxGap, 0);
|
|
assert.deepEqual(report.rows, []);
|
|
});
|
|
|
|
it('keeps equivalent train and shifted rewards aligned for a stable policy', () => {
|
|
const report = evaluateTriggerShift(stablePolicy, [pair]);
|
|
assert.equal(report.rows[0].trainReward, 1);
|
|
assert.equal(report.rows[0].shiftedReward, 1);
|
|
assert.equal(report.rows[0].gap, 0);
|
|
assert.equal(report.meanTrainReward, 1);
|
|
assert.equal(report.meanShiftedReward, 1);
|
|
assert.equal(report.meanGap, 0);
|
|
});
|
|
|
|
it('catches a deliberately trigger-sensitive dummy policy', () => {
|
|
const report = evaluateTriggerShift(triggerSensitivePolicy, [pair]);
|
|
assert.equal(report.rows[0].trainDecision, 'repair_timeout_source');
|
|
assert.equal(report.rows[0].shiftedDecision, 'add_more_retries');
|
|
assert.equal(report.rows[0].trainReward, 1);
|
|
assert.equal(report.rows[0].shiftedReward, 0);
|
|
assert.equal(report.rows[0].gap, 1);
|
|
assert.equal(report.maxGap, 1);
|
|
assert.equal(report.meanGap, 1);
|
|
});
|
|
|
|
it('ships one small calibration pair for each required shift axis', () => {
|
|
const axes = new Set(smallTriggerShiftSuite().map((p) => p.axis));
|
|
assert.deepEqual(axes, new Set(['wrapper_trigger', 'temporal_context', 'instruction_phrasing']));
|
|
});
|
|
|
|
it('logs train reward, shifted reward, and gap without raw prompt metadata', () => {
|
|
const encoded = JSON.stringify(evaluateTriggerShift(triggerSensitivePolicy, [pair]));
|
|
assert.match(encoded, /trainReward/);
|
|
assert.match(encoded, /shiftedReward/);
|
|
assert.match(encoded, /gap/);
|
|
assert.doesNotMatch(encoded, /metadata/);
|
|
assert.doesNotMatch(encoded, /EVOLVER_TRIGGER: timeout retry loop/);
|
|
});
|
|
});
|