mirror of
https://github.com/dotnet/skills.git
synced 2026-09-20 09:49:54 +08:00
Abhitej John
39 lines
1.3 KiB
YAML
39 lines
1.3 KiB
YAML
# Skill-vs-baseline experiment for every dotnet skill.
|
|
#
|
|
# Each eval under tests/<plugin>/<skill>/eval.vally.yaml is run twice:
|
|
# - baseline: no skills loaded (a true skill-free control)
|
|
# - skilled: exactly the one skill under test, loaded via environment.skills
|
|
#
|
|
# The skill directory is derived from the eval's path:
|
|
# tests/<plugin>/<skill>/eval.vally.yaml -> plugins/<plugin>/skills/<skill>
|
|
# i.e. ${eval.grandparent}=<plugin> and ${eval.parent}=<skill>.
|
|
#
|
|
# The `agent.*` evals are excluded: they exercise orchestrator agents and do
|
|
# not map to a single plugins/<plugin>/skills/<skill> directory.
|
|
name: dotnet-skills
|
|
evals:
|
|
- tests/*/!(agent.*)/eval.vally.yaml
|
|
overrides:
|
|
model: claude-opus-4.8
|
|
judge_model: gpt-5.5
|
|
runs: 5
|
|
vary:
|
|
- /environment/skills
|
|
baseline: baseline
|
|
variants:
|
|
baseline:
|
|
environment:
|
|
skills: []
|
|
skilled:
|
|
environment:
|
|
skills:
|
|
- plugins/${eval.grandparent}/skills/${eval.parent}
|
|
# Trim study (eval-only, csharp-refactoring dispatch): a shorter rewrite of
|
|
# csharp-refactoring. The CI auto-adapt only compares baseline-vs-skilled; the
|
|
# trimmed/ trajectories are uploaded for local cross-family re-judging via
|
|
# adapt.mjs (baseline-vs-trimmed and skilled-vs-trimmed).
|
|
trimmed:
|
|
environment:
|
|
skills:
|
|
- plugins/dotnet/skills/dotnet-breaking-changes-v3
|