mirror of
https://github.com/boshu2/agentops.git
synced 2026-09-14 15:08:13 +08:00
boshu
313 lines
12 KiB
Bash
313 lines
12 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
setup() {
|
|
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)"
|
|
SCRIPT="$REPO_ROOT/scripts/resolve-release-artifacts.sh"
|
|
AUDIT_SCRIPT="$REPO_ROOT/scripts/validate-release-audit-artifacts.sh"
|
|
|
|
TMP_DIR="$(mktemp -d)"
|
|
FAKE_REPO="$TMP_DIR/repo"
|
|
mkdir -p "$FAKE_REPO/scripts" "$FAKE_REPO/.agents/releases/local-ci"
|
|
cp "$SCRIPT" "$FAKE_REPO/scripts/resolve-release-artifacts.sh"
|
|
cp "$AUDIT_SCRIPT" "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
chmod +x "$FAKE_REPO/scripts/resolve-release-artifacts.sh"
|
|
chmod +x "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
}
|
|
|
|
write_audit() {
|
|
local version="$1"
|
|
local run_id="$2"
|
|
local audit_date="${3:-2026-03-22}"
|
|
local dir="$FAKE_REPO/docs/releases"
|
|
mkdir -p "$dir"
|
|
cat > "$dir/${audit_date}-v${version}-audit.md" <<EOF
|
|
# Release v${version} - Audit
|
|
|
|
**Local CI Artifacts:** \`.agents/releases/local-ci/$run_id/\`
|
|
EOF
|
|
}
|
|
|
|
teardown() {
|
|
rm -rf "$TMP_DIR"
|
|
}
|
|
|
|
write_manifest() {
|
|
local run_id="$1"
|
|
local version="$2"
|
|
local fast_mode="$3"
|
|
local artifact_dir="${4:-.agents/releases/local-ci/$run_id}"
|
|
local schema_version="${5:-1}"
|
|
local dir="$FAKE_REPO/.agents/releases/local-ci/$run_id"
|
|
mkdir -p "$dir"
|
|
|
|
cat > "$dir/release-artifacts.json" <<EOF
|
|
{
|
|
"schema_version": $schema_version,
|
|
"run_id": "$run_id",
|
|
"generated_at": "2026-03-22T21:22:22Z",
|
|
"artifact_dir": "$artifact_dir",
|
|
"release_version": "$version",
|
|
"repo_version": "$version",
|
|
"fast_mode": $fast_mode,
|
|
"security_mode": "full",
|
|
"sbom_cyclonedx": "sbom-v$version.cyclonedx.json",
|
|
"sbom_spdx": "sbom-v$version.spdx.json",
|
|
"security_report": "security-gate-full.json",
|
|
"release_readiness": "release-readiness.json",
|
|
"hil_evidence": "hil-evidence.json",
|
|
"vil_evidence": "digital-twin-evidence.json",
|
|
"digital_twin_evidence": "digital-twin-evidence.json",
|
|
"eval_fast_report": "eval-agentops-fast.json",
|
|
"eval_baseline_audit": "eval-baseline-audit.json"
|
|
}
|
|
EOF
|
|
}
|
|
|
|
write_artifact_files() {
|
|
local run_id="$1"
|
|
local version="$2"
|
|
local dir="$FAKE_REPO/.agents/releases/local-ci/$run_id"
|
|
printf '{"bomFormat":"CycloneDX"}\n' > "$dir/sbom-v$version.cyclonedx.json"
|
|
printf '{"spdxVersion":"SPDX-2.3"}\n' > "$dir/sbom-v$version.spdx.json"
|
|
printf '{"gate_status":"PASS"}\n' > "$dir/security-gate-full.json"
|
|
cat > "$dir/release-readiness.json" <<'EOF'
|
|
{
|
|
"schema_version": 1,
|
|
"release_readiness_score": 10,
|
|
"release_status": "pass",
|
|
"dimensions": {
|
|
"sil": {"status": "pass"},
|
|
"vil": {"status": "pass"},
|
|
"hil": {"status": "pass"}
|
|
}
|
|
}
|
|
EOF
|
|
printf '{"schema_version":1,"status":"pass","targets":[{"name":"loop","status":"pass"}]}\n' > "$dir/hil-evidence.json"
|
|
cat > "$dir/digital-twin-evidence.json" <<'EOF'
|
|
{
|
|
"schema_version": 1,
|
|
"evidence_kind": "digital_twin",
|
|
"status": "pass",
|
|
"dimensions": {
|
|
"vil": {"status": "pass"},
|
|
"release_smoke": {"status": "pass"},
|
|
"hook_install_smoke": {"status": "pass"},
|
|
"init_smoke": {"status": "pass"}
|
|
}
|
|
}
|
|
EOF
|
|
cat > "$dir/eval-agentops-fast.json" <<'EOF'
|
|
{
|
|
"schema_version": 1,
|
|
"evidence_kind": "agentops_eval_fast",
|
|
"status": "not_applicable",
|
|
"reason": "legacy eval surface retired",
|
|
"baseline_audit": "eval-baseline-audit.json"
|
|
}
|
|
EOF
|
|
printf '{"suite_count":1,"baseline_count":0,"policy_mismatch_count":0,"stale_suite_hashes":[]}\n' > "$dir/eval-baseline-audit.json"
|
|
}
|
|
|
|
@test "resolve-release-artifacts picks the newest complete manifest for a version" {
|
|
write_manifest "20260322T204431Z" "2.29.0" false
|
|
write_artifact_files "20260322T204431Z" "2.29.0"
|
|
write_manifest "20260322T212222Z" "2.29.0" false
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
|
|
run "$FAKE_REPO/scripts/resolve-release-artifacts.sh" "v2.29.0"
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *'"run_id": "20260322T212222Z"'* ]]
|
|
[[ "$output" == *'"artifact_dir": ".agents/releases/local-ci/20260322T212222Z"'* ]]
|
|
}
|
|
|
|
@test "resolve-release-artifacts ignores fast-mode or incomplete manifests" {
|
|
write_manifest "20260322T204431Z" "2.29.0" false
|
|
write_artifact_files "20260322T204431Z" "2.29.0"
|
|
write_manifest "20260322T212222Z" "2.29.0" true
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
|
|
run "$FAKE_REPO/scripts/resolve-release-artifacts.sh" "2.29.0"
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *'"run_id": "20260322T204431Z"'* ]]
|
|
}
|
|
|
|
@test "resolve-release-artifacts fails when no full artifact set exists for the version" {
|
|
write_manifest "20260322T212222Z" "2.29.0" false
|
|
|
|
run "$FAKE_REPO/scripts/resolve-release-artifacts.sh" "2.29.0"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"no full local CI artifacts found for release version 2.29.0"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts accepts matching manifest versions" {
|
|
write_manifest "20260322T212222Z" "2.29.0" false
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
write_audit "2.29.0" "20260322T212222Z"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"release audit artifact validation passed"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts rejects mismatched manifest versions" {
|
|
write_manifest "20260322T212222Z" "2.29.0" false
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
write_audit "2.30.0" "20260322T212222Z"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"manifest release_version=2.29.0, expected 2.30.0"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts rejects manifest directories that do not match the audit" {
|
|
write_manifest "20260322T212222Z" "2.29.0" false ".agents/releases/local-ci/20260322T204431Z"
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
write_audit "2.29.0" "20260322T212222Z"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"manifest artifact_dir=.agents/releases/local-ci/20260322T204431Z, expected .agents/releases/local-ci/20260322T212222Z"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts rejects fast-mode manifest artifacts" {
|
|
write_manifest "20260322T212222Z" "2.29.0" true
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
write_audit "2.29.0" "20260322T212222Z"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"manifest fast_mode=true, expected false"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts rejects unsupported manifest schemas" {
|
|
write_manifest "20260322T212222Z" "2.29.0" false ".agents/releases/local-ci/20260322T212222Z" 2
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
write_audit "2.29.0" "20260322T212222Z"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"manifest schema_version=2, expected 1"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts rejects missing readiness evidence for latest audit" {
|
|
write_manifest "20260322T212222Z" "2.29.0" false
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
rm "$FAKE_REPO/.agents/releases/local-ci/20260322T212222Z/release-readiness.json"
|
|
write_audit "2.29.0" "20260322T212222Z" "2026-05-02"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"missing release readiness artifact release-readiness.json"* ]]
|
|
}
|
|
|
|
@test "resolve-release-artifacts rejects manifests without readiness evidence" {
|
|
write_manifest "20260322T212222Z" "2.29.0" false
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
rm "$FAKE_REPO/.agents/releases/local-ci/20260322T212222Z/hil-evidence.json"
|
|
|
|
run "$FAKE_REPO/scripts/resolve-release-artifacts.sh" "2.29.0"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"no full local CI artifacts found for release version 2.29.0"* ]]
|
|
}
|
|
|
|
@test "resolve-release-artifacts rejects manifests without eval evidence" {
|
|
write_manifest "20260322T212222Z" "2.29.0" false
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
rm "$FAKE_REPO/.agents/releases/local-ci/20260322T212222Z/eval-agentops-fast.json"
|
|
|
|
run "$FAKE_REPO/scripts/resolve-release-artifacts.sh" "2.29.0"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"no full local CI artifacts found for release version 2.29.0"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts rejects missing eval evidence for release audits" {
|
|
write_manifest "20260322T212222Z" "2.29.0" false
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
rm "$FAKE_REPO/.agents/releases/local-ci/20260322T212222Z/eval-agentops-fast.json"
|
|
write_audit "2.29.0" "20260322T212222Z" "2026-05-02"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"missing eval fast report artifact eval-agentops-fast.json"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts rejects missing security evidence for release audits" {
|
|
write_manifest "20260322T212222Z" "2.29.0" false
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
rm "$FAKE_REPO/.agents/releases/local-ci/20260322T212222Z/security-gate-full.json"
|
|
write_audit "2.29.0" "20260322T212222Z" "2026-05-02"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"missing security report artifact security-gate-full.json"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts rejects missing digital twin evidence for release audits" {
|
|
write_manifest "20260322T212222Z" "2.29.0" false
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
rm "$FAKE_REPO/.agents/releases/local-ci/20260322T212222Z/digital-twin-evidence.json"
|
|
write_audit "2.29.0" "20260322T212222Z" "2026-05-02"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"missing VIL evidence artifact digital-twin-evidence.json"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts rejects stale eval baseline audit evidence" {
|
|
write_manifest "20260322T212222Z" "2.29.0" false
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
printf '{"suite_count":1,"baseline_count":1,"policy_mismatch_count":0,"stale_suite_hashes":[{"suite_id":"core"}]}\n' > "$FAKE_REPO/.agents/releases/local-ci/20260322T212222Z/eval-baseline-audit.json"
|
|
write_audit "2.29.0" "20260322T212222Z" "2026-05-02"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"eval baseline audit has stale suite hashes or malformed output"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts accepts legacy versioned artifact dirs" {
|
|
mkdir -p "$FAKE_REPO/.agents/releases/local-ci/20260322T212222Z"
|
|
write_artifact_files "20260322T212222Z" "2.29.0"
|
|
write_audit "2.29.0" "20260322T212222Z"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"release audit artifact validation passed"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts skips older missing local artifact dirs" {
|
|
write_audit "2.29.0" "20260322T212222Z"
|
|
write_manifest "20260330T231401Z" "2.30.0" false
|
|
write_artifact_files "20260330T231401Z" "2.30.0"
|
|
write_audit "2.30.0" "20260330T231401Z"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh"
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"release audit artifact validation passed"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts changed mode skips missing latest when no audit changed" {
|
|
write_audit "2.40.0" "20260427T020612Z" "2026-05-03"
|
|
|
|
run env RELEASE_AUDIT_CHANGED_PATHS="scripts/pre-push-gate.sh" \
|
|
"$FAKE_REPO/scripts/validate-release-audit-artifacts.sh" --mode changed
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"no changed release audits to validate"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts changed mode rejects changed audit with missing artifacts" {
|
|
write_audit "2.40.0" "20260427T020612Z" "2026-05-03"
|
|
|
|
run env RELEASE_AUDIT_CHANGED_PATHS="docs/releases/2026-05-03-v2.40.0-audit.md" \
|
|
"$FAKE_REPO/scripts/validate-release-audit-artifacts.sh" --mode changed
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"no release-artifacts.json and no complete versioned artifact fallback"* ]]
|
|
}
|
|
|
|
@test "validate-release-audit-artifacts target mode remains strict for missing target artifacts" {
|
|
write_audit "2.40.0" "20260427T020612Z" "2026-05-03"
|
|
|
|
run "$FAKE_REPO/scripts/validate-release-audit-artifacts.sh" --mode target --target-release 2.40.0
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"no release-artifacts.json and no complete versioned artifact fallback"* ]]
|
|
}
|