mirror of
https://github.com/NVIDIA/skills.git
synced 2026-09-14 15:41:50 +08:00
Moshe Abramovitch
e6526d7c9c
SkillEvaluator's report format changed in July 2026. It stopped emitting `NVSkills-Eval profile` and `Pass threshold`, both of which the aggregator reads, so those columns in benchmarks.json have been emptying one skill at a time as each is re-evaluated: 0% null through June, 57% in July, 100% for every skill evaluated in August. Nothing failed. The regeneration succeeded, the schema stayed valid, --check passed, and the data quietly degraded for six weeks before anyone noticed. Neither field is recoverable from a v3 report. `profile` is gone entirely. The only "50%" left is a glossary sentence that is byte-identical across skills with 4, 5 and 18 tasks, so it is template prose rather than a measurement — scraping it would stamp an unverified pass threshold on every skill in the file whose purpose is recording provenance. Both stay null until SkillEvaluator emits them as real fields; a NO_FABRICATION note records why. What v3 does add is stronger provenance the parser was discarding: - evaluator_version which build produced the score - dataset_digest sha256 of the exact task set, unique per skill - validation_status pipeline outcome for the run Capturing these follows the existing multi-pattern idiom already used for `tasks`, so v1/v2 reports are unaffected and simply leave them null. The generic fix is null_rate_regressions(), which compares a regeneration against the committed benchmarks.json and refuses to write when any field lost values, counting only skills present in both so new additions cannot false-fire. --allow-null-regressions lands a deliberate retirement without requiring a code edit under pressure. Verified against the real catalog: it does not fire today, it does fire when the July drift is replayed over 20 reports, and it flags the exact fields lost in PR #466. Adds the first tests for .github/scripts (16 cases, two real BENCHMARK.md fixtures, one per format). Confirmed no regression: every pre-existing field is byte-identical across all 341 skills. Also adds the SPDX header aggregate_benchmarks.py was missing; it was the only .py in .github/scripts/ without one. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Signed-off-by: Moshe Abramovitch <moshea@nvidia.com>
89 lines
2.8 KiB
Python
89 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
"""The null-rate guard must actually block a regeneration, not just report.
|
|
|
|
Builds a throwaway repo with one skill, generates benchmarks.json from it,
|
|
then removes a field from the source report and regenerates. That second run
|
|
is the silent-degradation scenario and must fail loudly.
|
|
"""
|
|
|
|
import json
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
import aggregate_benchmarks as agg # noqa: E402
|
|
|
|
REPORT = """# Skill Benchmark: foo
|
|
|
|
> **Overall verdict: PASS**
|
|
|
|
## Evaluation Metadata
|
|
|
|
- Skill: `foo`
|
|
- Evaluation date: 2026-06-01
|
|
{profile_line}- Environment: `local`
|
|
- Tasks: 4 evaluation tasks
|
|
- Attempts per task: 1
|
|
|
|
## Results
|
|
|
|
| Dimension | claude-code |
|
|
|-----------|-------------|
|
|
| Accuracy | 90% (+10%) |
|
|
"""
|
|
|
|
PROFILE_LINE = "- NVSkills-Eval profile: `external`\n"
|
|
|
|
|
|
class TestGuardBlocksRegeneration(unittest.TestCase):
|
|
def setUp(self):
|
|
self.root = Path(tempfile.mkdtemp())
|
|
(self.root / "skills" / "foo").mkdir(parents=True)
|
|
(self.root / "components.d").mkdir()
|
|
self.report = self.root / "skills" / "foo" / "BENCHMARK.md"
|
|
# Baseline: the field is present, benchmarks.json records it.
|
|
self.report.write_text(REPORT.format(profile_line=PROFILE_LINE))
|
|
(self.root / "benchmarks.json").write_text(agg.generate(self.root))
|
|
self.addCleanup(shutil.rmtree, self.root)
|
|
|
|
def _run(self, *extra):
|
|
argv = sys.argv
|
|
sys.argv = ["aggregate_benchmarks.py", "--repo-root", str(self.root), *extra]
|
|
try:
|
|
return agg.main()
|
|
finally:
|
|
sys.argv = argv
|
|
|
|
def _drop_the_field(self):
|
|
self.report.write_text(REPORT.format(profile_line=""))
|
|
|
|
def test_regeneration_succeeds_when_nothing_empties(self):
|
|
self.assertEqual(self._run(), 0)
|
|
|
|
def test_regeneration_fails_when_a_field_empties(self):
|
|
self._drop_the_field()
|
|
self.assertEqual(self._run(), 1)
|
|
|
|
def test_failed_run_leaves_benchmarks_json_untouched(self):
|
|
before = (self.root / "benchmarks.json").read_text()
|
|
self._drop_the_field()
|
|
self._run()
|
|
self.assertEqual((self.root / "benchmarks.json").read_text(), before)
|
|
|
|
def test_escape_hatch_allows_a_deliberate_format_change(self):
|
|
"""An upstream format change must be landable without editing code."""
|
|
self._drop_the_field()
|
|
self.assertEqual(self._run("--allow-null-regressions"), 0)
|
|
written = json.loads((self.root / "benchmarks.json").read_text())
|
|
self.assertIsNone(written["skills"][0]["profile"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|