mirror of
https://github.com/boshu2/agentops.git
synced 2026-09-14 15:08:13 +08:00
Bo
81e6380510
Registers the fully-built-but-never-wired eval module into the CLI spine and repairs its one real instrument bug. Three commits, RPI-shaped: wire + fix, drift guards + docs, fixture counts. ## What - **`ao eval` joins the spine** (cmd/ao/eval_composition.go): deterministic suite runs, run-record compare/baseline/scorecard/coverage, locked Tasks, holdout scenarios, suite A/B verdicts with power-derived n-required, and the holdout-safe Outcomes projection. This is the operating contract's *Learn* seat — a read/measure consumer of evidence. It reports numbers and owns no retry, scheduling, promotion, or lifecycle authority. The retired Aliases/Bench seats stay nil and their subcommands are omitted. - **Instrument bug fixed**: both scenario-ab arms were corpus-denied at runtime — the with-gold treatment was only a prompt injection through the retired `ao lookup`, so the A/B silently guaranteed a zero delta. Treatment is now environment-shaped: the with-gold arm runs with the corpus readable, the control arm stays sandbox-confined (the fail-closed empty-deny guard is untouched), and judges are always corpus-denied. - **Schema drift guards**: a run record persisted by the production writer must validate against eval-run.v1.schema.json (and the minimal suite against eval-suite.v1.schema.json) — closing the same silent-fork class the verdict.v2 golden corpus closed. - **Hermetic test builds hardened**: keep the real HOME's Go caches (TestMain isolation was forcing module re-downloads) and skip VCS stamping. - docs/architecture/go-cli.md gains the eval section; MIGRATION.md notes `ao eval`'s return; COMMANDS.md, cli-surface, and surface counts regenerated; new leaves carry honest allowlist reasons; new cmd/ao L2 tests drive eval run/compare/help through the full production wiring. ## Evidence - Full cli suite 2,801 green; race green on cmd/ao + the eval island; vet clean - Full-tier gate: 65/66 on the penultimate commit; the single failure (surface counts) is the final fixture commit, re-verified PASS standalone — no other check reads those fixtures - Fresh independent validation (Codex, frozen subject 86022a3c6, clean tree at start and end): **OVERALL PASS**, 6/6 — including a live end-to-end `ao eval run` producing a passing run record - Rubric-relocation decision honored: true holdout rubrics live in the external measurement register; the repo keeps only leak-guarded projections
89 lines
3.6 KiB
Bash
89 lines
3.6 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
setup() {
|
|
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)"
|
|
SCRIPT="$REPO_ROOT/scripts/check-removed-symbol-refs.sh"
|
|
|
|
TMP_DIR="$(mktemp -d)"
|
|
WORK_REPO="$TMP_DIR/repo"
|
|
|
|
git init -b main "$WORK_REPO" >/dev/null
|
|
git -C "$WORK_REPO" config user.name "Test User"
|
|
git -C "$WORK_REPO" config user.email "test@example.com"
|
|
mkdir -p "$WORK_REPO/scripts/lib" "$WORK_REPO/docs/releases" "$WORK_REPO/docs" "$WORK_REPO/skills/example" "$WORK_REPO/tests"
|
|
cp "$SCRIPT" "$WORK_REPO/scripts/check-removed-symbol-refs.sh"
|
|
cp "$BATS_TEST_DIRNAME/../../scripts/lib/repo-root.sh" "$WORK_REPO/scripts/lib/repo-root.sh"
|
|
chmod +x "$WORK_REPO/scripts/check-removed-symbol-refs.sh"
|
|
}
|
|
|
|
teardown() {
|
|
rm -rf "$TMP_DIR"
|
|
}
|
|
|
|
commit_fixture() {
|
|
git -C "$WORK_REPO" add .
|
|
git -C "$WORK_REPO" commit -m "fixture" >/dev/null
|
|
}
|
|
|
|
@test "check-removed-symbol-refs fails on shell doc skill and test callsites" {
|
|
printf 'ao defrag --old-flag\n' > "$WORK_REPO/scripts/run.sh"
|
|
printf 'Use --old-flag here.\n' > "$WORK_REPO/docs/usage.md"
|
|
printf 'Skill calls --old-flag.\n' > "$WORK_REPO/skills/example/SKILL.md"
|
|
printf 'assert --old-flag gone\n' > "$WORK_REPO/tests/old.bats"
|
|
printf 'historical --old-flag is allowed\n' > "$WORK_REPO/CHANGELOG.md"
|
|
printf 'historical --old-flag is allowed\n' > "$WORK_REPO/docs/releases/2026-05-16-v1.0.0-notes.md"
|
|
commit_fixture
|
|
|
|
run bash -c "cd '$WORK_REPO' && bash scripts/check-removed-symbol-refs.sh -- --old-flag"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"FAIL removed-symbol-refs"* ]]
|
|
[[ "$output" == *"scripts/run.sh"* ]]
|
|
[[ "$output" == *"docs/usage.md"* ]]
|
|
[[ "$output" == *"skills/example/SKILL.md"* ]]
|
|
[[ "$output" == *"tests/old.bats"* ]]
|
|
[[ "$output" != *"CHANGELOG.md"* ]]
|
|
[[ "$output" != *"docs/releases/2026-05-16-v1.0.0-notes.md"* ]]
|
|
}
|
|
|
|
@test "check-removed-symbol-refs passes when only historical release references remain" {
|
|
printf 'historical --old-flag is allowed\n' > "$WORK_REPO/CHANGELOG.md"
|
|
printf 'historical --old-flag is allowed\n' > "$WORK_REPO/docs/releases/2026-05-16-v1.0.0-notes.md"
|
|
printf 'no old flag here\n' > "$WORK_REPO/docs/usage.md"
|
|
commit_fixture
|
|
|
|
run bash -c "cd '$WORK_REPO' && bash scripts/check-removed-symbol-refs.sh -- --old-flag"
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"OK removed-symbol-refs"* ]]
|
|
}
|
|
|
|
@test "check-removed-symbol-refs supports regex mode and extra excludes" {
|
|
printf 'ao old-command --flag\n' > "$WORK_REPO/scripts/run.sh"
|
|
printf 'fixture old-command --flag\n' > "$WORK_REPO/tests/fixture.txt"
|
|
commit_fixture
|
|
|
|
# --regex is POSIX ERE (git grep -E); use a POSIX class, not the GNU/PCRE `\s`
|
|
# shorthand, which is out-of-contract and matches only on Linux git-grep (it
|
|
# silently fails to match on macOS, flipping the expected exit 1 -> 0).
|
|
run bash -c "cd '$WORK_REPO' && bash scripts/check-removed-symbol-refs.sh --regex --exclude 'tests/**' -- 'old-command[[:space:]]+--flag'"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"scripts/run.sh"* ]]
|
|
[[ "$output" != *"tests/fixture.txt"* ]]
|
|
}
|
|
|
|
@test "check-removed-symbol-refs requires -- before flag-like symbols" {
|
|
commit_fixture
|
|
|
|
run bash -c "cd '$WORK_REPO' && bash scripts/check-removed-symbol-refs.sh --old-flag"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"flag-like symbol before --"* ]]
|
|
}
|
|
|
|
@test "check-removed-symbol-refs reports invalid regex as an error" {
|
|
printf 'content\n' > "$WORK_REPO/docs/usage.md"
|
|
commit_fixture
|
|
|
|
run bash -c "cd '$WORK_REPO' && bash scripts/check-removed-symbol-refs.sh --regex -- '['"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"git grep failed for removed symbol"* ]]
|
|
}
|