mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
2988cbcc7c
H1: detectVibium reads stderr too, extracts semver
- Many Go CLIs (including some Vibium versions) write --version to stderr.
detectVibium used to read only stdout and silently report 'unknown',
polluting the init log with "vibium unknown (already installed)".
- Fix: read both streams, prefer stdout, extract bare semver via regex
so output is consistent regardless of where the binary printed it.
- Tests: 5 new regression tests covering stderr fallback, stdout
preference, verbose-format extraction, prerelease tags, fallback to
first whitespace token.
- Note: result.version is now bare semver (26.3.18) not v-prefixed.
Two pre-existing tests updated to match the new contract.
H2: instructions_in_html_comment dead-code regex
- TreeWalker(SHOW_COMMENT).textContent yields the inner text without
the <!-- --> delimiters, so the pattern that requires the literal
`<!--` prefix could never fire. Eval tc009 was passing only by
accident (ignore_previous_instructions fired on the same fixture).
- Fix: re-wrap each comment node in `<!-- ... -->` before adding to
the hidden bucket. Pattern can now actually match.
- Tests: 2 new regression tests asserting wrapped form fires AND
unwrapped form does NOT fire (the bug condition).
H4: e2e-flow-verifier frontmatter regression — INVALID FINDING
- Verified against `git show main:` — the skill always had minimal
frontmatter on main (just name + description + user-invocable: true)
and has no evals/, schemas/, or scripts/ directories. It is a
user-invocable runbook-style skill, not trust-tier-3 validated.
No metadata to restore. Marked closed-as-invalid.
H6: log-before-spawn for the 3-minute Vibium install
- spawnSync(npm install -g vibium) blocks the event loop for up to
3 minutes on a cold cache while Chrome for Testing downloads. The
user saw zero output and would Ctrl-C thinking init hung.
- Fix: pre-flight detect call (5s timeout) short-circuits when vibium
is already installed (the common case, no banner). When the install
is actually about to run, log a clear "this can take 1-3 minutes,
downloads Chrome for Testing lazily" message BEFORE the spawn.
H7: scoped scripts/package.json survives npm pack — VERIFIED
- `npm pack --dry-run` confirms both
.claude/skills/qe-browser/scripts/package.json (225B) AND
assets/skills/qe-browser/scripts/package.json (225B)
are included in the published tarball. The CommonJS override that
keeps the helper scripts loadable from an ESM project root will
survive `npm install` after publication. No fix needed.
H8: eval yaml uses unsupported runner fields
- The structured assertion fields I invented (exit_code, json_fields,
severity_at_least, candidate_count_at_least) are NOT understood by
src/validation/parallel-eval-runner.ts which expects must_contain
keyword matching. The yaml as written is unrunnable.
- Honest fix: relabel evals/qe-browser.yaml as `status: design-spec`
with a HONESTY NOTE explaining it's a specification, not a runnable
eval. Two paths to make it runnable documented (extend runner OR
rewrite as must_contain).
- Parallel artifact: scripts/smoke-test.sh — a runnable bash smoke
test against the same pinned httpbin.org fixtures. THIS is the
script that gates PR-reopen per ADR-091 Phase 3.
Bonus from B2/H5 already in Phase 1:
- assert.js now uses ?? instead of || for the `expected` field
Verification:
- 50/50 qe-browser tests pass (was 36 in initial PR; +14 regressions)
- 392/392 init tests pass (was 387; +5 from H1)
- npm run build: clean
- npm pack --dry-run: scoped package.json confirmed in tarball
- eslint on all touched source files: clean
- bash -n smoke-test.sh: syntax OK
Phase 3 (real Vibium smoke test) still gating PR reopen per ADR-091.
Branch intentionally not pushed.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
198 lines
7.5 KiB
TypeScript
198 lines
7.5 KiB
TypeScript
/**
|
|
* Browser Engine Installer (Vibium) — unit tests
|
|
*
|
|
* TDD London school: we inject a spawner so tests never touch PATH, the
|
|
* network, or global npm. The spawner is a simple function that returns
|
|
* canned `SpawnSyncReturns` objects.
|
|
*/
|
|
|
|
import { describe, it, expect } from 'vitest';
|
|
import type { SpawnSyncReturns } from 'node:child_process';
|
|
|
|
import {
|
|
installBrowserEngine,
|
|
detectVibium,
|
|
type Spawner,
|
|
} from '../../../src/init/browser-engine-installer.js';
|
|
|
|
type MockCall = { bin: string; args: string[] };
|
|
|
|
function canned(overrides: Partial<SpawnSyncReturns<string>>): SpawnSyncReturns<string> {
|
|
return {
|
|
pid: 1,
|
|
status: 0,
|
|
signal: null,
|
|
stdout: '',
|
|
stderr: '',
|
|
output: [],
|
|
...overrides,
|
|
} as SpawnSyncReturns<string>;
|
|
}
|
|
|
|
function enoent(): SpawnSyncReturns<string> {
|
|
return canned({
|
|
status: null,
|
|
error: Object.assign(new Error('ENOENT'), { code: 'ENOENT' }),
|
|
});
|
|
}
|
|
|
|
function makeSpawner(
|
|
responses: (call: MockCall, index: number) => SpawnSyncReturns<string>
|
|
): { spawner: Spawner; calls: MockCall[] } {
|
|
const calls: MockCall[] = [];
|
|
const spawner: Spawner = (bin, args) => {
|
|
const call: MockCall = { bin, args };
|
|
calls.push(call);
|
|
return responses(call, calls.length - 1);
|
|
};
|
|
return { spawner, calls };
|
|
}
|
|
|
|
describe('browser-engine-installer', () => {
|
|
describe('detectVibium', () => {
|
|
it('extracts semver from stdout v-prefixed output', () => {
|
|
const { spawner, calls } = makeSpawner(() => canned({ stdout: 'v26.3.18\n' }));
|
|
const result = detectVibium(spawner);
|
|
expect(result).toBe('26.3.18');
|
|
expect(calls).toEqual([{ bin: 'vibium', args: ['--version'] }]);
|
|
});
|
|
|
|
it('returns null when vibium is not installed', () => {
|
|
const { spawner } = makeSpawner(() => enoent());
|
|
expect(detectVibium(spawner)).toBeNull();
|
|
});
|
|
|
|
it('returns "unknown" when vibium exits 0 with empty stdout AND empty stderr', () => {
|
|
const { spawner } = makeSpawner(() => canned({ stdout: ' \n', stderr: '' }));
|
|
expect(detectVibium(spawner)).toBe('unknown');
|
|
});
|
|
|
|
// H1 regression — devil's-advocate finding:
|
|
// Many Go CLIs (including some Vibium versions) write --version output to
|
|
// stderr. detectVibium used to read only stdout and silently report 'unknown'.
|
|
describe('H1 regression: stderr fallback', () => {
|
|
it('reads version from stderr when stdout is empty', () => {
|
|
const { spawner } = makeSpawner(() => canned({ stdout: '', stderr: 'v26.3.18\n' }));
|
|
expect(detectVibium(spawner)).toBe('26.3.18');
|
|
});
|
|
|
|
it('prefers stdout over stderr when both have content', () => {
|
|
const { spawner } = makeSpawner(() =>
|
|
canned({ stdout: 'v26.3.18\n', stderr: '26.0.0\n' })
|
|
);
|
|
expect(detectVibium(spawner)).toBe('26.3.18');
|
|
});
|
|
|
|
it('extracts semver from verbose output like "vibium version 26.3.18"', () => {
|
|
const { spawner } = makeSpawner(() => canned({ stdout: 'vibium version 26.3.18 (linux/amd64)\n' }));
|
|
expect(detectVibium(spawner)).toBe('26.3.18');
|
|
});
|
|
|
|
it('handles semver with prerelease tag', () => {
|
|
const { spawner } = makeSpawner(() => canned({ stdout: 'v27.0.0-rc.1\n' }));
|
|
expect(detectVibium(spawner)).toBe('27.0.0-rc.1');
|
|
});
|
|
|
|
it('falls back to first whitespace token when no semver match', () => {
|
|
const { spawner } = makeSpawner(() => canned({ stdout: 'unknown-build foo\n' }));
|
|
expect(detectVibium(spawner)).toBe('unknown-build');
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('installBrowserEngine', () => {
|
|
it('returns skipped when options.skip is true', () => {
|
|
const result = installBrowserEngine({ skip: true });
|
|
expect(result.status).toBe('skipped');
|
|
expect(result.packageSpec).toBe('vibium');
|
|
});
|
|
|
|
it('returns already-installed when vibium is on PATH', () => {
|
|
const { spawner, calls } = makeSpawner(() => canned({ stdout: 'v26.3.18\n' }));
|
|
const result = installBrowserEngine({ spawner });
|
|
expect(result.status).toBe('already-installed');
|
|
// H1 fix: detectVibium now extracts the bare semver, dropping the v prefix.
|
|
expect(result.version).toBe('26.3.18');
|
|
// Only the detection call, no npm install attempt.
|
|
expect(calls).toHaveLength(1);
|
|
expect(calls[0]).toEqual({ bin: 'vibium', args: ['--version'] });
|
|
});
|
|
|
|
it('returns npm-unavailable when npm is missing', () => {
|
|
const { spawner, calls } = makeSpawner((call) => {
|
|
if (call.bin === 'vibium') return enoent();
|
|
if (call.bin === 'npm') return enoent();
|
|
throw new Error(`unexpected bin: ${call.bin}`);
|
|
});
|
|
const result = installBrowserEngine({ spawner });
|
|
expect(result.status).toBe('npm-unavailable');
|
|
expect(result.message).toMatch(/npm is not available/i);
|
|
// vibium check + npm check, no install attempt
|
|
expect(calls).toHaveLength(2);
|
|
});
|
|
|
|
it('installs via npm when vibium is missing but npm works', () => {
|
|
let vibiumChecks = 0;
|
|
const { spawner, calls } = makeSpawner((call) => {
|
|
if (call.bin === 'vibium') {
|
|
vibiumChecks += 1;
|
|
// First call: not installed. Second call (post-install): installed.
|
|
return vibiumChecks === 1 ? enoent() : canned({ stdout: 'v26.3.18' });
|
|
}
|
|
if (call.bin === 'npm' && call.args[0] === '--version') {
|
|
return canned({ stdout: '10.2.0' });
|
|
}
|
|
if (call.bin === 'npm' && call.args[0] === 'install') {
|
|
return canned({ stdout: 'added 1 package' });
|
|
}
|
|
throw new Error(`unexpected call: ${call.bin} ${call.args.join(' ')}`);
|
|
});
|
|
|
|
const result = installBrowserEngine({ spawner });
|
|
expect(result.status).toBe('installed');
|
|
// H1 fix: detectVibium now extracts the bare semver.
|
|
expect(result.version).toBe('26.3.18');
|
|
// vibium check, npm check, npm install, vibium re-check
|
|
expect(calls).toHaveLength(4);
|
|
expect(calls[2]).toEqual({ bin: 'npm', args: ['install', '-g', 'vibium'] });
|
|
});
|
|
|
|
it('returns install-failed when npm install exits non-zero', () => {
|
|
const { spawner } = makeSpawner((call) => {
|
|
if (call.bin === 'vibium') return enoent();
|
|
if (call.bin === 'npm' && call.args[0] === '--version') {
|
|
return canned({ stdout: '10.2.0' });
|
|
}
|
|
return canned({ status: 1, stderr: 'EACCES: permission denied' });
|
|
});
|
|
const result = installBrowserEngine({ spawner });
|
|
expect(result.status).toBe('install-failed');
|
|
expect(result.message).toContain('EACCES');
|
|
});
|
|
|
|
it('respects a custom packageSpec', () => {
|
|
const { spawner, calls } = makeSpawner(() => canned({ stdout: 'v26.3.18' }));
|
|
const result = installBrowserEngine({ spawner, packageSpec: 'vibium@26.3.18' });
|
|
expect(result.packageSpec).toBe('vibium@26.3.18');
|
|
// Already-installed detection path should not trigger npm install.
|
|
expect(calls).toHaveLength(1);
|
|
});
|
|
|
|
it('uses a custom npm binary when provided', () => {
|
|
let vibiumChecks = 0;
|
|
const { spawner, calls } = makeSpawner((call) => {
|
|
if (call.bin === 'vibium') {
|
|
vibiumChecks += 1;
|
|
return vibiumChecks === 1 ? enoent() : canned({ stdout: 'v26.3.18' });
|
|
}
|
|
return canned({ stdout: 'ok' });
|
|
});
|
|
|
|
const result = installBrowserEngine({ spawner, npmBin: '/usr/local/bin/npm' });
|
|
expect(result.status).toBe('installed');
|
|
const npmCalls = calls.filter((c) => c.bin === '/usr/local/bin/npm');
|
|
expect(npmCalls).toHaveLength(2);
|
|
});
|
|
});
|
|
});
|