mirror of
https://github.com/boshu2/agentops.git
synced 2026-09-14 15:08:13 +08:00
Boden Fuller
8cdfa85cda
Wave 1 of the Day-2 deliverable per ~/.agents/evals/SCHEMA.md §9 row 2: - evalsubstrate package separate from legacy cli/internal/eval (deterministic RunRecord) — distinct concept, no cross-pollination. - §7 canonicalization: BOM strip, zero-width drop, NFC, YAML/JSON/MD per-format rules. Mac LF and bushido CRLF produce identical content_hashes. - §4 atomic-write contract: temp + fsync(temp) + rename + fsync(parent_dir). - Run lifecycle: pending->running->complete|failed|aborted|retracted with rc2-terminal retracted (no unretract). - Harness snapshot: dir-walk + per-file canonical hash + aggregate content_hash + harness.lock.json. VerifyHarnessLock backs gate #8. - ModelSpec capture with self-stable content_hash (zero-then-fill). - §6 Refusal format: GATE FAILED / Why / Evidence / Fix — exact whitespace matches doctor-#11 expectations. - Day-2 gates 1, 6, 7, 8, 9 (refusals only). Gates 2/3/4/5 deferred to Day 4 per §9 (need Judge + Suite history + Dolt burn ledger). - 51 unit tests across canonicalization, atomic write, harness, manifest, modelspec, refusal, gates. All green. Day-1 sealed (rc2 SCHEMA.md content_hash sha256:8cd3c9dd...d9a910d locked, 38 fixtures green). This commit is Wave 1 of Day 2; CLI wiring (eval task, eval cleanup) lands in the next commit. Stop condition not yet hit.
74 lines
2.0 KiB
Go
74 lines
2.0 KiB
Go
package evalsubstrate
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
func ModelSpecPath(evalsRoot, specID string) string {
|
|
return filepath.Join(evalsRoot, "models", specID, "spec.yaml")
|
|
}
|
|
|
|
// CaptureModelSpec writes the ModelSpec to disk after stamping content_hash.
|
|
// Returns (specID, contentHash) for Manifest.ModelSpecRef + Manifest.ModelSpecHash.
|
|
func CaptureModelSpec(evalsRoot string, spec *ModelSpec) (string, string, error) {
|
|
if spec.ID == "" {
|
|
return "", "", fmt.Errorf("CaptureModelSpec: empty id")
|
|
}
|
|
if spec.SchemaVersion == 0 {
|
|
spec.SchemaVersion = SchemaVersion
|
|
}
|
|
if spec.CapturedAt == "" {
|
|
spec.CapturedAt = timeNow().UTC().Format(time.RFC3339)
|
|
}
|
|
|
|
prev := spec.ContentHash
|
|
spec.ContentHash = ""
|
|
rawYAML, err := yaml.Marshal(spec)
|
|
if err != nil {
|
|
spec.ContentHash = prev
|
|
return "", "", fmt.Errorf("CaptureModelSpec: marshal: %w", err)
|
|
}
|
|
canon, err := CanonicalizeYAML(rawYAML)
|
|
if err != nil {
|
|
spec.ContentHash = prev
|
|
return "", "", fmt.Errorf("CaptureModelSpec: canonicalize: %w", err)
|
|
}
|
|
hash := ContentHash(canon)
|
|
spec.ContentHash = hash
|
|
|
|
finalYAML, err := yaml.Marshal(spec)
|
|
if err != nil {
|
|
return "", "", fmt.Errorf("CaptureModelSpec: re-marshal: %w", err)
|
|
}
|
|
finalCanon, err := CanonicalizeYAML(finalYAML)
|
|
if err != nil {
|
|
return "", "", fmt.Errorf("CaptureModelSpec: re-canonicalize: %w", err)
|
|
}
|
|
dest := ModelSpecPath(evalsRoot, spec.ID)
|
|
if err := WriteAtomic(dest, finalCanon); err != nil {
|
|
return "", "", fmt.Errorf("CaptureModelSpec: write: %w", err)
|
|
}
|
|
return spec.ID, hash, nil
|
|
}
|
|
|
|
func LoadModelSpec(evalsRoot, specID string) (*ModelSpec, error) {
|
|
raw, err := os.ReadFile(ModelSpecPath(evalsRoot, specID))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("LoadModelSpec: %w", err)
|
|
}
|
|
var spec ModelSpec
|
|
if err := yaml.Unmarshal(raw, &spec); err != nil {
|
|
return nil, fmt.Errorf("LoadModelSpec: parse: %w", err)
|
|
}
|
|
return &spec, nil
|
|
}
|
|
|
|
func ModelSpecHashEqual(a, b *ModelSpec) bool {
|
|
return a != nil && b != nil && a.ContentHash != "" && a.ContentHash == b.ContentHash
|
|
}
|