evomap__evolver/test/experimentAgentRunner.test.js

// test/experimentAgentRunner.test.js
//
// Process-level tests for src/experiment/agentRunner.js — the ONE experiment
// module that intentionally spawns a subprocess. Kept out of
// experimentComparison.test.js, whose header promises zero-subprocess tests.
// The fake "agent CLI" is `node <script.js> <mode>`: runAgentTask appends
// `-p <prompt> --output-format json`, which land in the script's argv and are
// ignored (a bare `node -e` fake does NOT work — node itself rejects
// `--output-format` as a bad node option before any code runs).
'use strict';

const { describe, it, before, after } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('fs');
const os = require('os');
const path = require('path');

const { runAgentTask } = require('../src/experiment/agentRunner');

const FAKE_CLI = `
const envelope = JSON.stringify({
  result: 'done',
  is_error: false,
  usage: { input_tokens: 1, output_tokens: 1 },
  num_turns: 1,
});
const mode = process.argv[2];
if (mode === 'kill') {
  // Flush the clean envelope FIRST, then die by signal.
  process.stdout.write(envelope + '\\n', () => { process.kill(process.pid, 'SIGKILL'); });
  setInterval(() => {}, 1000); // keep alive until the kill lands
} else {
  process.stdout.write(envelope + '\\n');
}
`;

let tmpDir;
let cliPath;

before(() => {
  tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'evolver-agentrunner-'));
  cliPath = path.join(tmpDir, 'fake-agent-cli.js');
  fs.writeFileSync(cliPath, FAKE_CLI);
});

after(() => {
  try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch (_) { /* noop */ }
});

function fakeCli(mode) {
  return { command: process.execPath, extraArgs: [cliPath, mode], timeoutMs: 15000 };
}

describe('runAgentTask — exit-code semantics', () => {
  it('a signal-terminated arm is NEVER ok, even when stdout already holds a clean JSON envelope', async () => {
    // Regression: child 'exit' reports code=null for a signal death, and
    // Number(null) === 0 — the old num(code, -1) read that as a clean exit,
    // so a killed arm with parseable output scored ok:true.
    const res = await runAgentTask('noop', fakeCli('kill'));
    assert.equal(res.ok, false, 'a killed arm must not score ok');
    assert.notEqual(res.exitCode, 0, 'signal death must not read as exit 0');
    assert.ok(res.error, 'a killed arm carries an error label');
  });

  it('positive control: a clean exit 0 with the same envelope IS ok', async () => {
    const res = await runAgentTask('noop', fakeCli('clean'));
    assert.equal(res.ok, true);
    assert.equal(res.exitCode, 0);
    assert.equal(res.error, null);
  });
});