Files
proffesor-for-testing__agen…/tests/unit/init/browser-engine-installer.test.ts
Dragan Spiridonov 2988cbcc7c fix(qe-browser): Phase 2 high-severity fixes from devil's-advocate review
H1: detectVibium reads stderr too, extracts semver
  - Many Go CLIs (including some Vibium versions) write --version to stderr.
    detectVibium used to read only stdout and silently report 'unknown',
    polluting the init log with "vibium unknown (already installed)".
  - Fix: read both streams, prefer stdout, extract bare semver via regex
    so output is consistent regardless of where the binary printed it.
  - Tests: 5 new regression tests covering stderr fallback, stdout
    preference, verbose-format extraction, prerelease tags, fallback to
    first whitespace token.
  - Note: result.version is now bare semver (26.3.18) not v-prefixed.
    Two pre-existing tests updated to match the new contract.

H2: instructions_in_html_comment dead-code regex
  - TreeWalker(SHOW_COMMENT).textContent yields the inner text without
    the <!-- --> delimiters, so the pattern that requires the literal
    `<!--` prefix could never fire. Eval tc009 was passing only by
    accident (ignore_previous_instructions fired on the same fixture).
  - Fix: re-wrap each comment node in `<!-- ... -->` before adding to
    the hidden bucket. Pattern can now actually match.
  - Tests: 2 new regression tests asserting wrapped form fires AND
    unwrapped form does NOT fire (the bug condition).

H4: e2e-flow-verifier frontmatter regression — INVALID FINDING
  - Verified against `git show main:` — the skill always had minimal
    frontmatter on main (just name + description + user-invocable: true)
    and has no evals/, schemas/, or scripts/ directories. It is a
    user-invocable runbook-style skill, not trust-tier-3 validated.
    No metadata to restore. Marked closed-as-invalid.

H6: log-before-spawn for the 3-minute Vibium install
  - spawnSync(npm install -g vibium) blocks the event loop for up to
    3 minutes on a cold cache while Chrome for Testing downloads. The
    user saw zero output and would Ctrl-C thinking init hung.
  - Fix: pre-flight detect call (5s timeout) short-circuits when vibium
    is already installed (the common case, no banner). When the install
    is actually about to run, log a clear "this can take 1-3 minutes,
    downloads Chrome for Testing lazily" message BEFORE the spawn.

H7: scoped scripts/package.json survives npm pack — VERIFIED
  - `npm pack --dry-run` confirms both
    .claude/skills/qe-browser/scripts/package.json (225B) AND
    assets/skills/qe-browser/scripts/package.json (225B)
    are included in the published tarball. The CommonJS override that
    keeps the helper scripts loadable from an ESM project root will
    survive `npm install` after publication. No fix needed.

H8: eval yaml uses unsupported runner fields
  - The structured assertion fields I invented (exit_code, json_fields,
    severity_at_least, candidate_count_at_least) are NOT understood by
    src/validation/parallel-eval-runner.ts which expects must_contain
    keyword matching. The yaml as written is unrunnable.
  - Honest fix: relabel evals/qe-browser.yaml as `status: design-spec`
    with a HONESTY NOTE explaining it's a specification, not a runnable
    eval. Two paths to make it runnable documented (extend runner OR
    rewrite as must_contain).
  - Parallel artifact: scripts/smoke-test.sh — a runnable bash smoke
    test against the same pinned httpbin.org fixtures. THIS is the
    script that gates PR-reopen per ADR-091 Phase 3.

Bonus from B2/H5 already in Phase 1:
  - assert.js now uses ?? instead of || for the `expected` field

Verification:
  - 50/50 qe-browser tests pass (was 36 in initial PR; +14 regressions)
  - 392/392 init tests pass (was 387; +5 from H1)
  - npm run build: clean
  - npm pack --dry-run: scoped package.json confirmed in tarball
  - eslint on all touched source files: clean
  - bash -n smoke-test.sh: syntax OK

Phase 3 (real Vibium smoke test) still gating PR reopen per ADR-091.
Branch intentionally not pushed.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-08 18:50:39 +00:00

198 lines
7.5 KiB
TypeScript

/**
* Browser Engine Installer (Vibium) — unit tests
*
* TDD London school: we inject a spawner so tests never touch PATH, the
* network, or global npm. The spawner is a simple function that returns
* canned `SpawnSyncReturns` objects.
*/
import { describe, it, expect } from 'vitest';
import type { SpawnSyncReturns } from 'node:child_process';
import {
installBrowserEngine,
detectVibium,
type Spawner,
} from '../../../src/init/browser-engine-installer.js';
type MockCall = { bin: string; args: string[] };
function canned(overrides: Partial<SpawnSyncReturns<string>>): SpawnSyncReturns<string> {
return {
pid: 1,
status: 0,
signal: null,
stdout: '',
stderr: '',
output: [],
...overrides,
} as SpawnSyncReturns<string>;
}
function enoent(): SpawnSyncReturns<string> {
return canned({
status: null,
error: Object.assign(new Error('ENOENT'), { code: 'ENOENT' }),
});
}
function makeSpawner(
responses: (call: MockCall, index: number) => SpawnSyncReturns<string>
): { spawner: Spawner; calls: MockCall[] } {
const calls: MockCall[] = [];
const spawner: Spawner = (bin, args) => {
const call: MockCall = { bin, args };
calls.push(call);
return responses(call, calls.length - 1);
};
return { spawner, calls };
}
describe('browser-engine-installer', () => {
describe('detectVibium', () => {
it('extracts semver from stdout v-prefixed output', () => {
const { spawner, calls } = makeSpawner(() => canned({ stdout: 'v26.3.18\n' }));
const result = detectVibium(spawner);
expect(result).toBe('26.3.18');
expect(calls).toEqual([{ bin: 'vibium', args: ['--version'] }]);
});
it('returns null when vibium is not installed', () => {
const { spawner } = makeSpawner(() => enoent());
expect(detectVibium(spawner)).toBeNull();
});
it('returns "unknown" when vibium exits 0 with empty stdout AND empty stderr', () => {
const { spawner } = makeSpawner(() => canned({ stdout: ' \n', stderr: '' }));
expect(detectVibium(spawner)).toBe('unknown');
});
// H1 regression — devil's-advocate finding:
// Many Go CLIs (including some Vibium versions) write --version output to
// stderr. detectVibium used to read only stdout and silently report 'unknown'.
describe('H1 regression: stderr fallback', () => {
it('reads version from stderr when stdout is empty', () => {
const { spawner } = makeSpawner(() => canned({ stdout: '', stderr: 'v26.3.18\n' }));
expect(detectVibium(spawner)).toBe('26.3.18');
});
it('prefers stdout over stderr when both have content', () => {
const { spawner } = makeSpawner(() =>
canned({ stdout: 'v26.3.18\n', stderr: '26.0.0\n' })
);
expect(detectVibium(spawner)).toBe('26.3.18');
});
it('extracts semver from verbose output like "vibium version 26.3.18"', () => {
const { spawner } = makeSpawner(() => canned({ stdout: 'vibium version 26.3.18 (linux/amd64)\n' }));
expect(detectVibium(spawner)).toBe('26.3.18');
});
it('handles semver with prerelease tag', () => {
const { spawner } = makeSpawner(() => canned({ stdout: 'v27.0.0-rc.1\n' }));
expect(detectVibium(spawner)).toBe('27.0.0-rc.1');
});
it('falls back to first whitespace token when no semver match', () => {
const { spawner } = makeSpawner(() => canned({ stdout: 'unknown-build foo\n' }));
expect(detectVibium(spawner)).toBe('unknown-build');
});
});
});
describe('installBrowserEngine', () => {
it('returns skipped when options.skip is true', () => {
const result = installBrowserEngine({ skip: true });
expect(result.status).toBe('skipped');
expect(result.packageSpec).toBe('vibium');
});
it('returns already-installed when vibium is on PATH', () => {
const { spawner, calls } = makeSpawner(() => canned({ stdout: 'v26.3.18\n' }));
const result = installBrowserEngine({ spawner });
expect(result.status).toBe('already-installed');
// H1 fix: detectVibium now extracts the bare semver, dropping the v prefix.
expect(result.version).toBe('26.3.18');
// Only the detection call, no npm install attempt.
expect(calls).toHaveLength(1);
expect(calls[0]).toEqual({ bin: 'vibium', args: ['--version'] });
});
it('returns npm-unavailable when npm is missing', () => {
const { spawner, calls } = makeSpawner((call) => {
if (call.bin === 'vibium') return enoent();
if (call.bin === 'npm') return enoent();
throw new Error(`unexpected bin: ${call.bin}`);
});
const result = installBrowserEngine({ spawner });
expect(result.status).toBe('npm-unavailable');
expect(result.message).toMatch(/npm is not available/i);
// vibium check + npm check, no install attempt
expect(calls).toHaveLength(2);
});
it('installs via npm when vibium is missing but npm works', () => {
let vibiumChecks = 0;
const { spawner, calls } = makeSpawner((call) => {
if (call.bin === 'vibium') {
vibiumChecks += 1;
// First call: not installed. Second call (post-install): installed.
return vibiumChecks === 1 ? enoent() : canned({ stdout: 'v26.3.18' });
}
if (call.bin === 'npm' && call.args[0] === '--version') {
return canned({ stdout: '10.2.0' });
}
if (call.bin === 'npm' && call.args[0] === 'install') {
return canned({ stdout: 'added 1 package' });
}
throw new Error(`unexpected call: ${call.bin} ${call.args.join(' ')}`);
});
const result = installBrowserEngine({ spawner });
expect(result.status).toBe('installed');
// H1 fix: detectVibium now extracts the bare semver.
expect(result.version).toBe('26.3.18');
// vibium check, npm check, npm install, vibium re-check
expect(calls).toHaveLength(4);
expect(calls[2]).toEqual({ bin: 'npm', args: ['install', '-g', 'vibium'] });
});
it('returns install-failed when npm install exits non-zero', () => {
const { spawner } = makeSpawner((call) => {
if (call.bin === 'vibium') return enoent();
if (call.bin === 'npm' && call.args[0] === '--version') {
return canned({ stdout: '10.2.0' });
}
return canned({ status: 1, stderr: 'EACCES: permission denied' });
});
const result = installBrowserEngine({ spawner });
expect(result.status).toBe('install-failed');
expect(result.message).toContain('EACCES');
});
it('respects a custom packageSpec', () => {
const { spawner, calls } = makeSpawner(() => canned({ stdout: 'v26.3.18' }));
const result = installBrowserEngine({ spawner, packageSpec: 'vibium@26.3.18' });
expect(result.packageSpec).toBe('vibium@26.3.18');
// Already-installed detection path should not trigger npm install.
expect(calls).toHaveLength(1);
});
it('uses a custom npm binary when provided', () => {
let vibiumChecks = 0;
const { spawner, calls } = makeSpawner((call) => {
if (call.bin === 'vibium') {
vibiumChecks += 1;
return vibiumChecks === 1 ? enoent() : canned({ stdout: 'v26.3.18' });
}
return canned({ stdout: 'ok' });
});
const result = installBrowserEngine({ spawner, npmBin: '/usr/local/bin/npm' });
expect(result.status).toBe('installed');
const npmCalls = calls.filter((c) => c.bin === '/usr/local/bin/npm');
expect(npmCalls).toHaveLength(2);
});
});
});