mirror of
https://github.com/github/gh-stack.git
synced 2026-09-14 20:26:28 +08:00
Sameen Karim
cc2a40c084
Reorganize the public eval corpus into one self-contained directory per
scenario. Keep the exact agent prompt separate from the JSON contract so each
case's fixture, timeout, network scope, required status, and objective
assertions can be reviewed without reading the harness.
Add support for evaluating `skills/gh-stack` from any local commit, tag, or
branch with `--skill-ref`, without changing the working tree. Record the
resolved skill commit and Git tree, skill-directory and gh-stack binary
hashes, case contract hash, fixture seed, model, CLI versions, platform, and
repository state in every result.
Make batches repeatable with optional fixture seed SHA validation,
deterministically shuffled run plans, and per-case timeouts. Expand telemetry
to distinguish cumulative model input, model calls, all tool calls,
VC-related shell calls, failed tools, and output bytes.
Classify incorrect results into actionable failure categories and report
infrastructure failures separately from scored agent failures. Harden cleanup
by explicitly targeting the disposable repository, considering all matching
PRs when dissolving Stack metadata, and emitting an audit that fails when
namespaced refs or open PRs remain.
Rewrite the eval README around the public scenario layout, local execution,
commit-pinned comparisons, provenance, metric definitions, result
publication, cleanup, and adding new cases. Keep the suite local-first rather
than coupling privileged, nondeterministic agent runs to Actions.
Validation:
- python3 -m py_compile evals/*.py evals/tests/*.py
- python3 -m unittest discover -s evals/tests -v
- go vet ./...
- go test -race -count=1 ./...
- complete 12-case Mini suite from a fresh-context auditor
- commit-pinned smoke run with --skill-ref 14fc42e
- deterministic matrix plan, aggregation, and cleanup audit checks
- reran submit-prs after fixing origin/HEAD inheritance
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 03701648-d64b-477c-872a-84d2bd68c36d
49 lines
1.4 KiB
Python
49 lines
1.4 KiB
Python
"""Load declarative eval case contracts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
|
|
EVAL_ROOT = Path(__file__).resolve().parent
|
|
CASES_ROOT = EVAL_ROOT / "cases"
|
|
REQUIRED_FIELDS = {
|
|
"name",
|
|
"title",
|
|
"summary",
|
|
"tier",
|
|
"fixture",
|
|
"required",
|
|
"network",
|
|
"timeout_seconds",
|
|
"assertions",
|
|
}
|
|
|
|
|
|
def load_cases(root: Path = CASES_ROOT) -> dict[str, dict]:
|
|
cases = {}
|
|
for path in sorted(root.glob("*/case.json")):
|
|
case = json.loads(path.read_text())
|
|
missing = REQUIRED_FIELDS - set(case)
|
|
if missing:
|
|
raise ValueError(
|
|
f"{path} is missing fields: {', '.join(sorted(missing))}"
|
|
)
|
|
prompt_path = path.parent / "prompt.md"
|
|
if not prompt_path.is_file():
|
|
raise ValueError(f"missing prompt: {prompt_path}")
|
|
case["prompt"] = prompt_path.read_text().strip()
|
|
if not case["prompt"] or not case["assertions"]:
|
|
raise ValueError(f"{path} requires a prompt and assertions")
|
|
if int(case["timeout_seconds"]) <= 0:
|
|
raise ValueError(f"{path} timeout_seconds must be positive")
|
|
case["contract_path"] = str(path)
|
|
name = case["name"]
|
|
if name in cases:
|
|
raise ValueError(f"duplicate eval case: {name}")
|
|
cases[name] = case
|
|
if not cases:
|
|
raise ValueError(f"no eval cases found under {root}")
|
|
return cases
|