mirror of
https://github.com/dotnet/skills.git
synced 2026-09-20 09:49:54 +08:00
github-actions[bot]
21280 lines
881 KiB
JSON
21280 lines
881 KiB
JSON
{
|
|
"repoUrl": "",
|
|
"entries": {
|
|
"Quality": [
|
|
{
|
|
"date": 1788627831193,
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +60.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "Both responses failed to deliver any of the required deliverables. Response A simply stated intent and stopped without any tool use. Response B actually attempted the work: it activated a relevant skill, globbed for global.json, tried to read files, hit a genuine path inconsistency (glob returned ./global.json but reads failed), and then gave a clear explanation of the blocker plus a concrete plan of exactly what it would create. B's attempt is more diligent and transparent about the blocker, making it slightly better, though neither produced a usable result."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B correctly answered 'No' given the current host SDK is 9.0.306, identified the prerequisite, and told the user to install .NET 10+ system-wide first. A incorrectly led with 'Yes it's possible,' provided config as if it would work, and only softly noted the caveat while suggesting a vague workaround. B is substantially more correct and actionable per the rubric.\" [Magnitude reduced to \"slightly-better\": the position-swapped pass judged this \"slightly-better\", so the more confident \"much-better\" was downgraded.]"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B satisfies more rubric criteria: it uses the correct global.json paths resolution mechanism, excludes the directory from source control, and explains cleanup. A misses gitignore and cleanup entirely and uses a less accurate PATH-based approach. Both correctly avoid installing."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"tool": "customBiggerIsBetter",
|
|
"benches": [
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 6.842105388641357,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 6.842105388641357,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 5.789473533630371,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 5.833333492279053,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 9.642857551574707,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)"
|
|
}
|
|
],
|
|
"commit": {
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00"
|
|
},
|
|
"model": "gpt-5.3-codex"
|
|
},
|
|
{
|
|
"date": 1788627831298,
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +66.7% (2W/1T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.250), mean preference +46.7% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 1,
|
|
"losses": 0,
|
|
"discordant": 2,
|
|
"direction": "better",
|
|
"pValue": 0.25,
|
|
"alpha": 0.05,
|
|
"netWin": 0.6666666666666666,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"plugin": "plugin-no-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"plugin": "plugin-no-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B better satisfies the central repo-configuration requirements by wiring the local SDK into global.json and giving an actionable host-level missing-SDK message. A has somewhat stronger version parsing in its installers, but it misses the key automatic discovery and failure guidance behavior."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: A, reverse: B). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B supplies the key modern project-local resolution configuration with an explicit host fallback and handles `.gitignore`, both central omissions in A. Both miss cleanup guidance, and B's statement about executable resolution is somewhat imprecise, but B is substantially more complete for the requested local setup."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"tool": "customBiggerIsBetter",
|
|
"benches": [
|
|
{
|
|
"overfittingScore": 0.26,
|
|
"notActivated": true,
|
|
"unit": "Score (0-10)",
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 7.763157844543457,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.26,
|
|
"notActivated": true,
|
|
"unit": "Score (0-10)",
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 9.166666984558105
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 7.5,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 7.916666507720947,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 9.642857551574707,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 9.642857551574707,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 4.642857074737549,
|
|
"unit": "Score (0-10)"
|
|
}
|
|
],
|
|
"commit": {
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00"
|
|
},
|
|
"model": "mai-code-1-flash-picker"
|
|
},
|
|
{
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +60.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B more directly implements repository-level local SDK resolution and automatic missing-SDK guidance through global.json, which are the core requested configuration outcomes. A has more robust and well-tested scripts, but its key guidance and local SDK selection depend on an extra activation workflow rather than normal global.json resolution."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "Both answer the core question correctly and meet the prerequisite guidance. B is marginally stronger because it clearly defers setup until the required .NET 10+ host is installed and avoids suggesting a potentially stale or nonexistent exact .NET 11 preview version."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B directly implements the requested local-SDK resolution model with a local-first global.json paths configuration and host fallback, includes source-control handling and cleanup, and remains non-invasive. A has a workable PATH-based alternative but misses the requested paths/fallback behavior and cleanup guidance."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"tool": "customBiggerIsBetter",
|
|
"benches": [
|
|
{
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 9.736842155456543,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 7.3684210777282715,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 6.666666507720947,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 6.4285712242126465,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality"
|
|
}
|
|
],
|
|
"date": 1788715662922,
|
|
"commit": {
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
}
|
|
},
|
|
"model": "claude-opus-5"
|
|
},
|
|
{
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +33.3% (2W/0T/1L over 3 preference-eligible stimulus vote(s), sign test p=0.500), mean preference +53.3% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 0,
|
|
"losses": 1,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.5,
|
|
"alpha": 0.05,
|
|
"netWin": 0.3333333333333333,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "worse",
|
|
"rationale": "A is more conservative with the pre-existing global.json and its config-driven installers/wrappers are portable without an undeclared jq dependency, while still providing a verified missing-SDK workflow. B has the stronger direct global.json error-message integration, but its alteration of rollForward and jq-dependent Bash setup reduce its overall fit."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B gives the critical prerequisite accurately and avoids directing the user through an unusable setup. A's central claim is incorrect and its commands/configuration would not solve the user's situation."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B satisfies the crucial local SDK resolution requirement and gitignore requirement, while A's proposed global.json does not actually direct the host to the project-local SDK. B therefore provides a substantially more complete and correct isolated setup."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"tool": "customBiggerIsBetter",
|
|
"benches": [
|
|
{
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 9.868420600891113,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 6.842105388641357,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 3.3333332538604736,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 3.5714285373687744,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality"
|
|
}
|
|
],
|
|
"date": 1788715663007,
|
|
"commit": {
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
}
|
|
},
|
|
"model": "claude-sonnet-5"
|
|
},
|
|
{
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +33.3% (2W/0T/1L over 3 preference-eligible stimulus vote(s), sign test p=0.500), mean preference +33.3% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 0,
|
|
"losses": 1,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.5,
|
|
"alpha": 0.05,
|
|
"netWin": 0.3333333333333333,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "worse",
|
|
"rationale": "Both responses delivered functional cross-platform install scripts, missing-SDK messaging, .gitignore entries, and did not download the SDK. B leverages the native global.json 'paths' feature which is an elegant no-PATH solution. However, B modified an existing global.json setting (rollForward changed from latestFeature to disable) and rebuilt the file wholesale, weakening the 'preserve existing sections' requirement, and its verification was lighter. A explicitly and programmatically verified all existing global.json sections were preserved and demonstrated the missing-SDK message working end-to-end, with additional command wrappers and a README. A's more thorough preservation and verification give it a slight overall edge."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "Both correctly detected the .NET 9 host and explained the .NET 10+ requirement. However, A then contradicted itself: after saying the .NET 9 launcher cannot interpret sdk.paths, it proceeded to provide a global.json with paths and workaround PATH instructions, without advising a host upgrade — this is confusing and doesn't satisfy the prerequisite. B correctly instructed the user to install .NET 10+ system-wide first and did not provide premature local install steps, cleanly matching the task's intended behavior. A did more research but arrived at a self-contradictory recommendation, while B (aided by the skill) gave the correct guidance.\""
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "Both responses are strong, safe, and did not execute anything. The decisive difference is criterion 2: Response B correctly uses the global.json 'paths' resolution feature ('.dotnet','$host$') that makes the local SDK preferred with the host as fallback — precisely what the task and rubric call for. Response A relies on PATH/DOTNET_ROOT environment manipulation and version pinning, which works but doesn't use the intended resolution mechanism. B also adds nice safety checks (backup of existing global.json, idempotent gitignore). This gives B a slight overall edge.\""
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"tool": "customBiggerIsBetter",
|
|
"benches": [
|
|
{
|
|
"value": 9.473684310913086,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 9.473684310913086,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 6.315789699554443,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality"
|
|
},
|
|
{
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 1.6666666269302368,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 6.785714149475098,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality"
|
|
}
|
|
],
|
|
"date": 1788715663036,
|
|
"commit": {
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
}
|
|
},
|
|
"model": "gpt-5.6-sol"
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"value": 9.736842155456543,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 7.763157844543457,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality"
|
|
},
|
|
{
|
|
"value": 7.916666507720947,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality"
|
|
}
|
|
],
|
|
"commit": {
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Amaury Levé"
|
|
},
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
},
|
|
"date": 1788790038334,
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +80.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B directly implements the key repo-level SDK discovery and missing-SDK messaging requirements, so cloning plus running an installer yields a local SDK usable without PATH changes. A is stronger on retaining the exact existing version in its installer, but its local SDK is not automatically discoverable and its missing-SDK instruction is only documentation rather than resolver configuration."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B correctly identifies and handles the decisive .NET 10+ host prerequisite, while A's central claim that .NET 9.0.306 supports sdk.paths is incorrect and would lead the user to a nonworking setup."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B satisfies every requested setup element with a genuinely project-local SDK, preferred local resolution plus host fallback, git exclusion, and cleanup. A outlines a workable session-scoped alternative but omits key requested configuration details, ignores source control and cleanup, and gives an inexact `global.json` version."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"model": "claude-sonnet-4.6",
|
|
"tool": "customBiggerIsBetter"
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"value": 9.473684310913086,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 9.3421049118042,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 7.3684210777282715,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 7.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 6.785714149475098,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality"
|
|
}
|
|
],
|
|
"commit": {
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Amaury Levé"
|
|
},
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
},
|
|
"date": 1788790038435,
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +60.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "Both agents completed the task with cross-platform install scripts, .gitignore entries, preserved global.json sections, and no SDK download. B leveraged the .NET 10+ global.json 'paths' + 'errorMessage' feature, which is the idiomatic, robust way to make project-local SDK resolution transparent without PATH changes and to surface a missing-SDK message on any dotnet invocation. A's wrapper-script approach works but requires teammates to use the wrapper, and its missing-SDK message only appears through the wrapper. Both had error recovery from apply_patch failures. B is slightly better due to the more standard and seamless mechanism, though A provided extra wrapper convenience and kept global.json byte-identical."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "Both correctly detect the version shortfall and explain the .NET 10+ requirement. The decisive difference is criterion 3: B correctly instructs installing a .NET 10+ SDK system-wide first, while A incorrectly implies that installing the .NET 11 preview locally satisfies the host requirement, which contradicts how the paths feature works. This makes B more accurate and actionable overall, though A is more thorough in formatting and includes verification steps."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "Both correctly present a project-local SDK setup without running anything and both gitignore the directory. However, B directly satisfies two key rubric points that A misses: it uses the global.json 'paths' feature ([\\\".dotnet\\\", \\\"$host$\\\"]) to prefer the local SDK with host fallback (A instead relies on shell env vars, which doesn't match the resolution requirement), and it explains cleanup. B is meaningfully better aligned with the rubric."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/ac8f41264bdd557e58924a3110eea8e0917dcf4d/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"model": "gpt-5.6-luna",
|
|
"tool": "customBiggerIsBetter"
|
|
},
|
|
{
|
|
"tool": "customBiggerIsBetter",
|
|
"benches": [
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 9.736842155456543
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 7.6315789222717285
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 8.333333015441895
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 8.333333015441895
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 1.6666666269302368
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 5.0
|
|
}
|
|
],
|
|
"date": 1788885914243,
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +100.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B addresses the core requirement with the intended global.json paths and actionable errorMessage mechanism, retains existing sections, and makes the local installation usable without PATH edits. A creates several supporting files but misses that essential host-level configuration, so its local SDK setup is not reliably usable after installation."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B accurately answers that the requested paths-based setup cannot be used from the stated .NET 9 host and gives the necessary next step. A is materially misleading about both feasibility and configuration."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B directly satisfies the key local-SDK-resolution requirement with `global.json` paths, includes source-control exclusion, and selects the latest preview channel. A's direct local executable approach is workable for manual commands but omits the required local-first SDK resolution configuration."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"commit": {
|
|
"id": "fbeeafe261b0fe704b9a95c58fd9fdbe34e96964",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Viktor Hofer"
|
|
},
|
|
"message": "Delete msbuild-server skill (#1123)",
|
|
"timestamp": "2026-09-07T09:11:54+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964"
|
|
},
|
|
"model": "claude-haiku-4.5"
|
|
},
|
|
{
|
|
"tool": "customBiggerIsBetter",
|
|
"benches": [
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 7.3684210777282715
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 7.3684210777282715
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 5.789473533630371
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"notActivated": true,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 7.083333492279053
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 8.333333015441895
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 8.333333015441895
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 9.642857551574707
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 9.642857551574707
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 5.0
|
|
}
|
|
],
|
|
"date": 1788885914324,
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +33.3% (2W/0T/1L over 3 preference-eligible stimulus vote(s), sign test p=0.500), mean preference +13.3% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 0,
|
|
"losses": 1,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.5,
|
|
"alpha": 0.05,
|
|
"netWin": 0.3333333333333333,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "Both failed to complete the task and produced no scripts or config. However, Response B activated the relevant skill, actually attempted to inspect the repo (via glob, view, and bash), diagnosed a concrete tool/path issue, and outlined a specific, correct plan (paths + errorMessage, .sh and .ps1 scripts, .gitignore) that matches the intended solution. Response A did essentially nothing—it made zero tool calls and merely stated it would inspect files without doing so. B's methodical investigation and concrete plan make it slightly better despite both being incomplete."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "worse",
|
|
"rationale": "Response A correctly detects the 9.0.306 host is below the .NET 10 requirement, warns it won't work reliably, and orders the prerequisite install first. Response B incorrectly claims the setup is possible/supported on the existing 9.0 host and moves to invoke a setup skill without properly gating on the prerequisite."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B correctly uses the modern global.json 'paths' feature to prefer the local SDK with host fallback, which is exactly what the task asks for, and excludes .dotnet/ from source control. A misses both the paths mechanism (using cruder PATH/env manipulation) and the gitignore step. Neither covers cleanup. B is clearly the stronger, more technically accurate answer.\""
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"commit": {
|
|
"id": "fbeeafe261b0fe704b9a95c58fd9fdbe34e96964",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Viktor Hofer"
|
|
},
|
|
"message": "Delete msbuild-server skill (#1123)",
|
|
"timestamp": "2026-09-07T09:11:54+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964"
|
|
},
|
|
"model": "gpt-5.3-codex"
|
|
},
|
|
{
|
|
"tool": "customBiggerIsBetter",
|
|
"benches": [
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"notActivated": true,
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.35
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.35,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 8.815789222717285
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"notActivated": true,
|
|
"value": 7.5,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.35
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.35,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 4.166666507720947
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 8.333333015441895
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.35,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 9.642857551574707
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.35,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 9.642857551574707
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 4.642857074737549
|
|
}
|
|
],
|
|
"date": 1788885914378,
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +33.3% (2W/0T/1L over 3 preference-eligible stimulus vote(s), sign test p=0.500), mean preference +53.3% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 0,
|
|
"losses": 1,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.5,
|
|
"alpha": 0.05,
|
|
"netWin": 0.3333333333333333,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"plugin": "plugin-no-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B implements the key repository configuration—local SDK discovery and an SDK-resolution error message—in addition to cross-platform installers. A supplies installers and manual ensure helpers but misses the global.json integration needed for seamless project-local, PATH-free use."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "worse",
|
|
"rationale": "Both have the essential technical conclusion, but A is more disciplined about the prerequisite. B's concrete local-host workflow is potentially valid in some layouts, yet it is less aligned with the requested prerequisite ordering and system-wide-host guidance."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B addresses the central technical requirement with global.json paths and host fallback, and excludes the local SDK from version control. A's lack of paths makes its purported pinned local SDK setup unreliable outside the one shell where PATH is manually changed. Both omit cleanup, and B's placeholder version necessarily requires a post-install update, but B is substantially more correct overall."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"commit": {
|
|
"id": "fbeeafe261b0fe704b9a95c58fd9fdbe34e96964",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Viktor Hofer"
|
|
},
|
|
"message": "Delete msbuild-server skill (#1123)",
|
|
"timestamp": "2026-09-07T09:11:54+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964"
|
|
},
|
|
"model": "mai-code-1-flash-picker"
|
|
},
|
|
{
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "comparison_invocation_failed",
|
|
"phase": "comparison_judge"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "dotnet/setup-local-sdk: vally compare failed (Command failed: vally compare --baseline /tmp/vally-adapt-7Jac9f/dotnet__setup-local-sdk__baseline.jsonl --treatment /tmp/vally-adapt-7Jac9f/dotnet__setup-local-sdk__skilled.jsonl --judge-model gpt-5.6-terra --output /tmp/vally-adapt-7Jac9f/dotnet__setup-local-sdk__compare.jsonl)",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 0,
|
|
"wins": 0,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 0,
|
|
"direction": "none",
|
|
"pValue": 1,
|
|
"alpha": 0.05,
|
|
"netWin": 0,
|
|
"minimumNetWin": null,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": false,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "No activation contract could be evaluated",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": null,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [],
|
|
"judgeRationales": [],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/a8fece0fce5f8b0737754a332a1a7c6487e8f927/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/a8fece0fce5f8b0737754a332a1a7c6487e8f927/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/a8fece0fce5f8b0737754a332a1a7c6487e8f927"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"message": "Merge pull request #1149 from dotnet/abhitejjohn-vally-014-ci-proof",
|
|
"url": "https://github.com/dotnet/skills/commit/a8fece0fce5f8b0737754a332a1a7c6487e8f927",
|
|
"timestamp": "2026-09-09T17:53:08+00:00",
|
|
"id": "a8fece0fce5f8b0737754a332a1a7c6487e8f927"
|
|
},
|
|
"model": "claude-opus-4.8",
|
|
"tool": "customBiggerIsBetter",
|
|
"date": 1789036360311,
|
|
"benches": []
|
|
},
|
|
{
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"id": "8a5a42d3e392b402768fc29416831643b79e402b",
|
|
"message": "Merge pull request #1153 from dotnet/abhitejjohn-vally-lock-concurrency-proof",
|
|
"timestamp": "2026-09-10T21:01:20+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/8a5a42d3e392b402768fc29416831643b79e402b"
|
|
},
|
|
"benches": [
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 9.473684310913086
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 8.947368621826172
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 6.710526466369629
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 6.666666507720947
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 6.785714149475098
|
|
}
|
|
],
|
|
"date": 1789129571056,
|
|
"tool": "customBiggerIsBetter",
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +66.7% (2W/1T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.250), mean preference +26.7% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 1,
|
|
"losses": 0,
|
|
"discordant": 2,
|
|
"direction": "better",
|
|
"pValue": 0.25,
|
|
"alpha": 0.05,
|
|
"netWin": 0.6666666666666666,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "Both responses accomplished the core task without downloading the SDK and preserved existing global.json sections. However, Response B was more methodical: it leveraged the native global.json 'errorMessage' field for the missing-SDK guidance (directly satisfying that requirement), added a system fallback path, cleanly generated and syntax-verified both scripts, and verified section preservation via explicit assertions. Response A had a messy execution with corrupted scripts (base64 errors, PowerShell content landing in the .sh file) that required multiple repairs, leaving lower confidence in the final script correctness, and did not use the errorMessage mechanism. Response B is clearly better overall.\" [Magnitude reduced to \"slightly-better\": the position-swapped pass judged this \"slightly-better\", so the more confident \"much-better\" was downgraded.]"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: tie). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "Both responses are strong, correct, and avoid executing commands. B is better on two key rubric points: it uses the proper global.json 'paths' feature for SDK resolution with host fallback (exactly what the rubric asks), and it explains cleanup. A relies on PATH manipulation and omits cleanup guidance. These are meaningful advantages, making B slightly better overall."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8a5a42d3e392b402768fc29416831643b79e402b/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8a5a42d3e392b402768fc29416831643b79e402b/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8a5a42d3e392b402768fc29416831643b79e402b"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"model": "gpt-5.6-luna"
|
|
},
|
|
{
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +100.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B directly implements project-local SDK resolution through global.json paths and a built-in missing-SDK message while preserving existing sections. A provides many auxiliary files, but its local SDK usage depends on sourcing .dotnetrc and its checked missing-SDK behavior was not robust."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B correctly identifies the blocking .NET 10+ host prerequisite and gives the proper next action. A's affirmative answer and claimed sdkPath configuration are materially incorrect for the user's .NET 9 host."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B directly fulfills the requested project-local SDK workflow across platforms, including installer commands, resolution configuration, gitignore, verification, and cleanup. A's central recommendation is incomplete and misleading for a local SDK because global.json alone neither installs nor resolves from the proposed local directory."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"commit": {
|
|
"timestamp": "2026-09-12T00:01:18+00:00",
|
|
"id": "4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7",
|
|
"message": "Merge pull request #1154 from dotnet/abhitejjohn-agentic-workflow-repair",
|
|
"url": "https://github.com/dotnet/skills/commit/4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
},
|
|
"date": 1789218680118,
|
|
"model": "claude-haiku-4.5",
|
|
"benches": [
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 9.736842155456543,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 7.6315789222717285,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 1.6666666269302368,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 3.2142856121063232,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality"
|
|
}
|
|
],
|
|
"tool": "customBiggerIsBetter"
|
|
},
|
|
{
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +80.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B delivered a complete, correct solution despite file-tool errors, providing full script contents inline and explaining the blocker. A delivered nothing, incorrectly claiming it could not read the workspace. B is much better across all criteria.\""
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B correctly and confidently identifies that 9.0.306 is below the .NET 10 requirement, explains the requirement, and directs the user to install 10+ system-wide first, providing accurate config. A gives a hedged, non-committal answer that never nails the actual constraint, potentially misleading the user into thinking their current SDK might work.\""
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B more directly and correctly implements the canonical project-local SDK approach using global.json `paths` with host fallback, and it excludes the SDK directory from source control—two rubric items A misses. A's env-var approach is valid but less precise on the host-fallback requirement and omits gitignore. Neither explains cleanup. Both present commands without executing. Overall B satisfies more rubric criteria, making it slightly better."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"commit": {
|
|
"timestamp": "2026-09-12T00:01:18+00:00",
|
|
"id": "4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7",
|
|
"message": "Merge pull request #1154 from dotnet/abhitejjohn-agentic-workflow-repair",
|
|
"url": "https://github.com/dotnet/skills/commit/4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
},
|
|
"date": 1789218680183,
|
|
"model": "gpt-5.3-codex",
|
|
"benches": [
|
|
{
|
|
"value": 6.973684310913086,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 6.842105388641357,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 5.789473533630371,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 5.416666507720947,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality"
|
|
},
|
|
{
|
|
"value": 9.642857551574707,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality"
|
|
},
|
|
{
|
|
"value": 9.642857551574707,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality"
|
|
},
|
|
{
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality"
|
|
}
|
|
],
|
|
"tool": "customBiggerIsBetter"
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 9.868420600891113,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 9.868420600891113,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 8.1578950881958,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 7.916666507720947,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 6.4285712242126465,
|
|
"unit": "Score (0-10)"
|
|
}
|
|
],
|
|
"commit": {
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
},
|
|
"date": 1789318507213,
|
|
"model": "claude-opus-5",
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +33.3% (2W/0T/1L over 3 preference-eligible stimulus vote(s), sign test p=0.500), mean preference +33.3% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 0,
|
|
"losses": 1,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.5,
|
|
"alpha": 0.05,
|
|
"netWin": 0.3333333333333333,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "worse",
|
|
"rationale": "A is the stronger, more maintainable repository setup: it preserves the existing SDK configuration, reads the pin from global.json, and includes clone-to-build documentation. B has valuable end-to-end verification of its error message, but unnecessarily changes rollForward and its installer design is more brittle due to hard-coded configuration and jq-dependent rewriting."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B directly answers the scenario presented, correctly gates global.json paths on a .NET 10+ system host, and gives the proper sequence and fallback. A contradicts the user's reported version and incorrectly dismisses the requested .NET 11 preview, producing advice that does not address the stated prerequisite."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B best satisfies the central local-SDK-resolution requirement through `global.json` paths and an explicit host fallback. A is otherwise solid and has cleaner cross-platform installer retrieval and cleanup guidance, but lacks the requested resolver configuration."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"tool": "customBiggerIsBetter"
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 9.868420600891113,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 7.8947367668151855,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 3.3333332538604736,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 9.642857551574707,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)"
|
|
}
|
|
],
|
|
"commit": {
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
},
|
|
"date": 1789318507300,
|
|
"model": "claude-sonnet-5",
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +80.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B more directly implements the central repository-local SDK configuration: automatic discovery through global.json plus a resolution-time missing-SDK message, so teammates need not alter PATH. Its unnecessary rollForward change and jq dependency in the Bash installer are real shortcomings, and A is cleaner at preserving the original file, but A misses the no-PATH project-local discovery behavior."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B correctly identifies and centers the decisive compatibility constraint imposed by the user's .NET 9.0.306 host, then gives the appropriate next step. A's otherwise plausible paths example is materially misleading because it claims the setup will work as-is."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B directly satisfies every requested setup detail, notably local SDK resolution through global.json paths, global fallback, .gitignore handling, and rollback. A has a viable session-local PATH approach but misses the required paths-based resolution, source-control exclusion, and cleanup."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"tool": "customBiggerIsBetter"
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 8.947368621826172,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 9.3421049118042,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 6.315789699554443,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 7.5,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 6.4285712242126465,
|
|
"unit": "Score (0-10)"
|
|
}
|
|
],
|
|
"commit": {
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
},
|
|
"date": 1789318507349,
|
|
"model": "gpt-5.6-sol",
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +66.7% (2W/1T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.250), mean preference +46.7% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 1,
|
|
"losses": 0,
|
|
"discordant": 2,
|
|
"direction": "better",
|
|
"pValue": 0.25,
|
|
"alpha": 0.05,
|
|
"netWin": 0.6666666666666666,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: A). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "Both correctly diagnose the version gap and the .NET 10+ requirement. However, A undermines this by immediately giving local install commands and framing the workaround as running the local host directly (which sidesteps the actual paths requirement in a way that could confuse), whereas B correctly enforces the prerequisite ordering by requiring the .NET 10+ host first before local install. B is more faithful to the rubric's ordering and prerequisite criteria. A's extra detail (rollForward, full version) is somewhat helpful but the ordering violation is a meaningful correctness issue given the task's emphasis."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B correctly uses the global.json 'paths' feature for local-preferred SDK resolution with host fallback, which is the core of the intended solution, plus robust guarded scripting and clear cleanup. A relies on manual PATH/DOTNET_ROOT exports and doesn't leverage paths resolution, making it less correct and more fragile. B also better handles pre-existing global.json/gitignore. This is a significant quality gap on the key resolution criterion."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"tool": "customBiggerIsBetter"
|
|
},
|
|
{
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +66.7% (2W/1T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.250), mean preference +66.7% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 1,
|
|
"losses": 0,
|
|
"discordant": 2,
|
|
"direction": "better",
|
|
"pValue": 0.25,
|
|
"alpha": 0.05,
|
|
"netWin": 0.6666666666666666,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: A). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B is concise and correct on the decisive compatibility prerequisite. A's central claim is false, so its proposed global.json setup cannot work with the user's .NET 9 host."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B fulfills every requested setup component, notably the explicit global.json paths local-first/host-fallback configuration and cleanup instructions. A offers a viable local install approach but misses the requested paths-based fallback configuration and cleanup."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"benches": [
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 9.736842155456543,
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 10.0,
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 6.315789699554443,
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 10.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 10.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 5.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 10.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 10.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 6.4285712242126465,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality"
|
|
}
|
|
],
|
|
"model": "claude-sonnet-5",
|
|
"date": 1789397100668,
|
|
"tool": "customBiggerIsBetter",
|
|
"commit": {
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +60.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"plugin": "plugin-activity-observed"
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "Both responses accomplished the core task: cross-platform install scripts, preserved global.json sections, and PATH-free local SDK via the `paths` feature. B is slightly better because it used the native global.json `errorMessage` field for the missing-SDK message (surfaced automatically by the .NET CLI), added a `$host$` fallback, and performed more rigorous validation of preserved sections and pinned versions. A's missing-SDK guidance lives only in README, which is less discoverable at build time. Both recovered from apply_patch failures methodically."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "Both responses correctly detect the version gap and explain the .NET 10+ requirement without prematurely installing. B is slightly better because it explicitly frames installing .NET 10+ 'system-wide first' as the host prerequisite, includes rollForward for robustness, and clarifies the separation between system-wide and local SDKs. A's guidance is accurate but slightly less explicit about the system-wide host requirement, and its example version '11.0.100' without preview suffix is a minor drawback (though it notes to replace it)."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B is stronger on two key criteria: it uses the modern global.json 'paths' feature for proper local-preferred/host-fallback SDK resolution, and it explains cleanup. B also covers both Unix and Windows, uses safe temp files and idempotent operations, and states clearly nothing was run. A is competent but relies on manual PATH exports without the fallback mechanism and omits cleanup guidance."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"benches": [
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 9.473684310913086,
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 9.3421049118042,
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 6.842105388641357,
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 8.333333015441895,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 8.333333015441895,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 8.333333015441895,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 10.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 10.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"value": 6.785714149475098,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality"
|
|
}
|
|
],
|
|
"model": "gpt-5.6-luna",
|
|
"date": 1789397100745,
|
|
"tool": "customBiggerIsBetter",
|
|
"commit": {
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"tool": "customBiggerIsBetter",
|
|
"commit": {
|
|
"url": "https://github.com/dotnet/skills/commit/24f7cfbd42ad7bf52bcd67372816b982c38c64c6",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"timestamp": "2026-09-14T15:27:40+00:00",
|
|
"id": "24f7cfbd42ad7bf52bcd67372816b982c38c64c6",
|
|
"message": "Surface activation-only evaluation failures (#1163)"
|
|
},
|
|
"date": 1789478284654,
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +100.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 1
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B directly implements the project-local SDK workflow: install-on-demand scripts, local-directory discovery via global.json paths, actionable SDK-resolution error text, and .gitignore. A creates several ancillary artifacts but misses the essential local installation/discovery mechanism and misleadingly treats avoiding automatic download by the scripts as satisfying reproducibility."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B directly answers the paths-feature question with the critical .NET 10+ host limitation and the correct next step. A gives a generally valid global.json preview-selection example but omits paths entirely and provides misleading guidance for the user's .NET 9 environment."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B follows the intended project-local SDK pattern, including resolution paths, gitignore, and cleanup. Its placeholder SDK version would need to be replaced after determining the installed preview version, but it is still substantially more complete and correct than A."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/24f7cfbd42ad7bf52bcd67372816b982c38c64c6/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/24f7cfbd42ad7bf52bcd67372816b982c38c64c6/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/24f7cfbd42ad7bf52bcd67372816b982c38c64c6"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"benches": [
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 7.1052632331848145
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 8.552631378173828
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 8.333333015441895
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 1.6666666269302368
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 9.642857551574707
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 4.642857074737549
|
|
}
|
|
],
|
|
"model": "claude-haiku-4.5"
|
|
},
|
|
{
|
|
"tool": "customBiggerIsBetter",
|
|
"commit": {
|
|
"url": "https://github.com/dotnet/skills/commit/24f7cfbd42ad7bf52bcd67372816b982c38c64c6",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"timestamp": "2026-09-14T15:27:40+00:00",
|
|
"id": "24f7cfbd42ad7bf52bcd67372816b982c38c64c6",
|
|
"message": "Surface activation-only evaluation failures (#1163)"
|
|
},
|
|
"date": 1789478284776,
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +66.7% (2W/1T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.250), mean preference +46.7% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 1,
|
|
"losses": 0,
|
|
"discordant": 2,
|
|
"direction": "better",
|
|
"pValue": 0.25,
|
|
"alpha": 0.05,
|
|
"netWin": 0.6666666666666666,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: A). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B correctly recognized that the current 9.0.306 host is below the required .NET 10+ for the sdk.paths feature and gave correct remediation. A gave the incorrect headline answer ('possible with your current setup') and buried the critical prerequisite as a vague caveat, which is factually wrong.\""
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "Both responses are solid and safely avoid installing. B is technically superior because it uses the correct global.json 'paths' feature (.NET 10+) to resolve the local SDK with host fallback, which is the proper mechanism, and correctly captures the exact installed version and uses --channel/--quality for 'latest preview'. A relies on manually pinning a potentially outdated hardcoded version and PATH manipulation, which is less robust. Neither covers cleanup. The paths-resolution advantage makes B slightly better overall.\""
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/24f7cfbd42ad7bf52bcd67372816b982c38c64c6/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/24f7cfbd42ad7bf52bcd67372816b982c38c64c6/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/24f7cfbd42ad7bf52bcd67372816b982c38c64c6"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"benches": [
|
|
{
|
|
"notActivated": true,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 5.789473533630371
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 5.789473533630371
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 7.3684210777282715
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 5.416666507720947
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 9.642857551574707
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 9.642857551574707
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 6.785714149475098
|
|
}
|
|
],
|
|
"model": "gpt-5.3-codex"
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Quality",
|
|
"unit": "Score (0-10)",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Quality",
|
|
"unit": "Score (0-10)",
|
|
"overfitting": "moderate",
|
|
"value": 9.166666984558105,
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Quality",
|
|
"value": 5.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Quality",
|
|
"value": 0.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Quality",
|
|
"value": 9.375
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Quality",
|
|
"value": 9.375
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Quality",
|
|
"value": 8.75
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Quality",
|
|
"value": 4.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Quality",
|
|
"value": 7.083333492279053
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Quality",
|
|
"value": 7.083333492279053
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Quality",
|
|
"unit": "Score (0-10)",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Quality",
|
|
"value": 8.75
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Quality",
|
|
"value": 8.75
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Quality",
|
|
"unit": "Score (0-10)",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Quality",
|
|
"value": 9.642857551574707
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Quality",
|
|
"value": 7.5
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Quality",
|
|
"value": 9.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Quality",
|
|
"value": 9.642857551574707
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Quality",
|
|
"value": 0.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Quality",
|
|
"value": 0.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Quality",
|
|
"value": 7.5
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 9.868420600891113
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 9.3421049118042
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 7.3684210777282715
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 9.166666984558105
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 4.642857074737549
|
|
}
|
|
],
|
|
"model": "claude-sonnet-5",
|
|
"commit": {
|
|
"author": {
|
|
"name": "Abhitej John",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"timestamp": "2026-09-15T22:05:10+00:00",
|
|
"message": "Merge pull request #873 from dotnet/add-dotnet-refactoring-skills",
|
|
"url": "https://github.com/dotnet/skills/commit/26323a52990d0cbfc838117109b40e115aac891f",
|
|
"id": "26323a52990d0cbfc838117109b40e115aac891f"
|
|
},
|
|
"tool": "customBiggerIsBetter",
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "csharp-refactoring",
|
|
"skillKind": "invocable",
|
|
"state": "VALID_PASS",
|
|
"stateReason": {
|
|
"code": "credible_preference_improvement",
|
|
"phase": "decision"
|
|
},
|
|
"passed": true,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +58.8% (12W/3T/2L over 17 preference-eligible stimulus vote(s), sign test p=0.006), mean preference +33.3% across 18 paired run(s), 1 dormancy stimulus/stimuli excluded from preference — credibly better",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 17,
|
|
"wins": 12,
|
|
"ties": 3,
|
|
"losses": 2,
|
|
"discordant": 14,
|
|
"direction": "better",
|
|
"pValue": 0.006469726562500003,
|
|
"alpha": 0.05,
|
|
"netWin": 0.5882352941176471,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 1
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 1,
|
|
"satisfied": 1,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expected": "dormant",
|
|
"observed": "dormant",
|
|
"satisfied": true
|
|
}
|
|
],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expectation": "dormant",
|
|
"preferenceGateEligible": false,
|
|
"isolated": "dormant-as-expected",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"direction": "none",
|
|
"rationale": "The final changes and verification are substantively identical: both achieve delegation, preserve the public contract, identify no production callers needing migration, and validate Fixture.sln successfully."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "B correctly follows the central rubric expectation: it rejects the false behavior-preserving-refactor framing and redirects to a proper upgrade process. A's implementation and test validation may be technically competent, but it violates the key safety and framing requirements by executing potentially behavior-changing upgrades as a refactor."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "B follows the required interpretation and safely declines the incompatible refactor request. A performs the expressly disallowed feature implementation; its displayed final output is also unhelpful and does not clearly report the completed work."
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"direction": "better",
|
|
"rationale": "The implementations appear equivalently correct and validated. B has a modestly clearer, more complete final report of the required semantic preservation and verification."
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"direction": "better",
|
|
"rationale": "The implemented and validated changes appear equivalent, but B provides the appropriate final user-facing completion report whereas A's final output is materially unhelpful."
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"direction": "better",
|
|
"rationale": "The resulting code and test status appear equivalent, but B provides stronger build validation and, importantly, does not inspect unrelated evaluation workspaces. A's filesystem-wide inspection is a clear process violation."
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"direction": "better",
|
|
"rationale": "B correctly refuses an unauthorized shipped public API contract tightening and accurately reclassifies it. A's builds and passing tests do not cure its fundamental violation of the requested behavior-preserving-refactoring constraints."
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"direction": "better",
|
|
"rationale": "Both implementations appear functionally correct for the user's literal requested edits and validate with passing tests. B is slightly better because it accurately distinguishes the rename from the intentional behavioral threshold change and reports full solution build/test validation; it still fails the rubric's requested refusal/defer behavior."
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"direction": "worse",
|
|
"rationale": "The substantive refactor and observed test outcome are essentially equivalent, but A provides slightly stronger solution-wide verification."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"direction": "none",
|
|
"rationale": "The resulting changes and validation are substantively identical and fully satisfy the task. B reports a separate build more explicitly, but this does not create a meaningful final-result quality advantage because A's solution test run also built and passed both targets."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"direction": "better",
|
|
"rationale": "B fulfills the crucial compatibility-migration requirement completely by retaining an explicitly obsolete public forwarding shim while migrating the ordinary caller, and it provides clearer validation/reporting. A misses the required obsolete designation."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"direction": "better",
|
|
"rationale": "Both results satisfy the requested change and validation. B is marginally stronger because its final report explicitly confirms the preserved platform-label outputs and its repo-wide occurrence check is clearer."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"direction": "better",
|
|
"rationale": "Both runs appear to make the same correct behavior-preserving source changes and confirm a successful build. B is stronger overall because it additionally validates the change with passing tests on both target frameworks."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"direction": "worse",
|
|
"rationale": "A implements the requested minimal, behavior-preserving refactor precisely: FormatReceipt is the implementation and RenderReceipt remains a private forwarding compatibility boundary for reflection. B passes its tests but substitutes a more invasive attribute/lookup redesign for the explicitly requested old-name shim. [Magnitude reduced to \"slightly-better\": the position-swapped pass judged this \"slightly-better\", so the more confident \"much-better\" was downgraded.]"
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"direction": "better",
|
|
"rationale": "The implementations and validation are otherwise equivalent, but B handles the shipped versus unshipped public API baseline correctly. A's PublicAPI.Shipped.txt change is a meaningful release-contract metadata error."
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"direction": "none",
|
|
"rationale": "The final repository changes and validation are equivalently complete and correct. B provides slightly more explanatory detail, while A explicitly performs a post-edit friend/reference check; neither yields a meaningful quality advantage."
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"direction": "better",
|
|
"rationale": "The delivered refactor appears equally correct and appropriately scoped in both responses, but B provides stronger requested verification by running the full solution tests in addition to the clean build."
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"direction": "worse",
|
|
"rationale": "Neither response can implement the feature because no application source is present, and both appropriately request the correct project. A is better because it gives that concise, relevant blocker without introducing an irrelevant refactoring-workflow rationale."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/26323a52990d0cbfc838117109b40e115aac891f/plugins/dotnet/skills/csharp-refactoring/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/26323a52990d0cbfc838117109b40e115aac891f/tests/dotnet/csharp-refactoring/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/26323a52990d0cbfc838117109b40e115aac891f"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +80.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B more directly fulfills the requested project-local, cross-platform setup: standard dotnet resolution finds .dotnet and global.json supplies the missing-SDK instruction. A is careful about preserving global.json and has working wrappers, but its configuration is less seamless because normal dotnet usage is not connected to the local SDK or its guidance."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B directly and safely answers the actual prerequisite issue and follows the requested sequencing. A recognizes the limitation but undermines it by leading with an actionable local setup that will not work with the user's current global dotnet, and frames the prerequisite as optional."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B fulfills every requested setup component, particularly the essential `global.json` paths local-first resolution, source-control exclusion, and cleanup. A gives a workable session-local PATH approach but misses those durable project configuration requirements."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/26323a52990d0cbfc838117109b40e115aac891f/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/26323a52990d0cbfc838117109b40e115aac891f/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/26323a52990d0cbfc838117109b40e115aac891f"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"date": 1789574377878
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Quality",
|
|
"value": 9.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Quality",
|
|
"value": 9.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Quality",
|
|
"value": 6.666666507720947
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Quality",
|
|
"value": 0.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Quality",
|
|
"value": 9.375
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Quality",
|
|
"value": 9.375
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Quality",
|
|
"value": 8.75
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Quality",
|
|
"value": 4.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Quality",
|
|
"value": 7.083333492279053
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Quality",
|
|
"value": 8.75
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Quality",
|
|
"value": 8.75
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Quality",
|
|
"value": 8.75
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Quality",
|
|
"value": 9.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Quality",
|
|
"value": 9.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Quality",
|
|
"value": 7.75
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Quality",
|
|
"value": 9.642857551574707
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Quality",
|
|
"value": 5.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 9.473684310913086
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 9.473684310913086
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 7.236842155456543
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 8.333333015441895
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 6.785714149475098
|
|
}
|
|
],
|
|
"model": "gpt-5.6-luna",
|
|
"commit": {
|
|
"author": {
|
|
"name": "Abhitej John",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"timestamp": "2026-09-15T22:05:10+00:00",
|
|
"message": "Merge pull request #873 from dotnet/add-dotnet-refactoring-skills",
|
|
"url": "https://github.com/dotnet/skills/commit/26323a52990d0cbfc838117109b40e115aac891f",
|
|
"id": "26323a52990d0cbfc838117109b40e115aac891f"
|
|
},
|
|
"tool": "customBiggerIsBetter",
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "csharp-refactoring",
|
|
"skillKind": "invocable",
|
|
"state": "VALID_NO_CHANGE",
|
|
"stateReason": {
|
|
"code": "activation_contract_failed",
|
|
"phase": "activation"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +35.3% (7W/9T/1L over 17 preference-eligible stimulus vote(s), sign test p=0.035), mean preference +21.1% across 18 paired run(s), 1 dormancy stimulus/stimuli excluded from preference — activation contract failed (1 explicit dormancy scenario(s) activated the isolated target skill)",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 17,
|
|
"wins": 7,
|
|
"ties": 9,
|
|
"losses": 1,
|
|
"discordant": 8,
|
|
"direction": "better",
|
|
"pValue": 0.035156250000000014,
|
|
"alpha": 0.05,
|
|
"netWin": 0.35294117647058826,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 1
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 1,
|
|
"satisfied": 0,
|
|
"violated": 1,
|
|
"passed": false,
|
|
"failures": [
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expected": "dormant",
|
|
"observed": "activated",
|
|
"satisfied": false
|
|
}
|
|
],
|
|
"scenarios": [
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expected": "dormant",
|
|
"observed": "activated",
|
|
"satisfied": false
|
|
}
|
|
],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expectation": "dormant",
|
|
"preferenceGateEligible": false,
|
|
"isolated": "unexpected-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: tie). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "The rubric rewards recognizing that a framework/package major-version upgrade is NOT a behavior-preserving refactor and redirecting to the proper workflow. B does exactly this. A ignores the framing issue and performs a partial version bump, and even backpedaled the framework change, ending with a misleading summary. B wins on all criteria."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "The task is a trap: it asks to add a new pricing tier while framing it as a behavior-preserving refactor, which it is not. B correctly recognizes this, declines, and redirects to a feature workflow. A, though technically competent in implementation and tests passing, misses the framing entirely and even reinforces the false claim of behavior preservation. B is much better on all rubric criteria.\""
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"direction": "worse",
|
|
"rationale": "Both produced functionally identical, correct refactors with passing tests. A's final report was slightly more explicit about the preserved semantic difference between the two bases, giving it a marginal edge. Otherwise essentially equal quality; both recovered from patch tooling failures via perl."
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: tie). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: tie). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"direction": "better",
|
|
"rationale": "The task contains a trap: it asks to treat a shipped public-API nullable tightening as behavior-preserving. B correctly refused, recognizing the contract change and declining to edit, pointing to a proper workflow. A blindly accepted the premise, edited both the declaration and the shipped API file (even reacting to analyzer errors by writing string! raw), thereby altering the shipped contract. B satisfies all four rubric criteria; A fails all of them despite a passing build."
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"direction": "better",
|
|
"rationale": "Both correctly performed the rename. The task contains a deliberate trap: the user frames a behavior-changing threshold edit as 'behavior-preserving.' B correctly recognized the contradiction, performed the rename, and deferred the threshold change with clear justification. A blindly applied the $50 change and even rewrote test expectations to hide the behavior change, then claimed everything passed as a refactor — violating two of three rubric criteria. B is much better on the discriminating criteria."
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"direction": "better",
|
|
"rationale": "Both produced the same correct consolidation with preserved values and passing tests. B was slightly more methodical—loading a refactoring skill, verifying all references, and running an explicit build+test with confirmed zero warnings—while A reached the same correct end state. The quality difference is minor."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: B). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"direction": "none",
|
|
"rationale": "Both responses performed the identical correct refactor: migrated the ordinary caller to CurrencyFormatter.Format, preserved the shipped legacy forwarding API and its compatibility test, restored packages after the initial NETSDK1004 error, and verified all 17 tests pass on both target frameworks. Both recovered from the same apply_patch/path errors methodically. B activated the csharp-refactoring skill and reasoned slightly more explicitly about the compatibility boundary, but the final outputs and quality are equivalent."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"direction": "better",
|
|
"rationale": "Both correctly performed the rename in all branches and preserved labels, with clean final states. B is slightly better because it used the refactoring skill, took a more methodical path, and ran validation explicitly per target framework (net8.0 and net10.0 separately), which directly satisfies the 'separate successful validation for both frameworks' criterion and ensures each conditional branch is compiled. A also succeeded but validated via a single solution run and had some detours (creating symlinks/hardlinks) that were slightly haphazard."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"direction": "none",
|
|
"rationale": "Both responses correctly identified the declaration and caller, edited the source-of-truth template file (not obj-generated files), and validated via restore+test with all 17 tests passing on both frameworks. Response B did an extra verification of the regenerated file, but this is negligible; the final outcomes are equivalent in quality and correctness. Efficiency differences (A used fewer tokens/turns) are excluded per instructions."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"direction": "none",
|
|
"rationale": "Both responses arrived at identical, correct solutions: renamed the private implementation to FormatReceipt, migrated the RenderDirect caller, added a private RenderReceipt forwarding shim for reflection compatibility, preserved the compatibility test, and verified all 17 tests pass on both target frameworks. Both encountered and recovered from the same apply_patch/path issues and NuGet restore issue. The final results are equivalent in quality."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"direction": "better",
|
|
"rationale": "Both responses achieved a correct, working final result: rename, caller migration, compatibility shim, legacy test preserved, PublicAPI.Shipped.txt updated, and all 17 tests passing on both frameworks. The key differentiator is that B added the [Obsolete] attribute to the forwarding shim, which the rubric explicitly calls for ('obsolete forwarding compatibility shim'), while A's shim is a plain wrapper without it. B did have a minor stumble (accidentally over-replacing the legacy test and correcting it) but self-recovered cleanly. B's slightly more rubric-aligned deprecation marking makes it slightly better overall."
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: tie). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"direction": "none",
|
|
"rationale": "Both responses accomplished the task identically: narrow, scoped rename of the target symbol and its callers, leaving unrelated DoStuff occurrences untouched, and verified via restore/build/test on Fixture.sln. Neither used a true binding-aware rename tool (apply_patch failed in both), falling back to scoped sed, but both were careful and correct. Approaches and outcomes are equivalent; B loaded a refactoring skill but that did not change the actual method used. This is a tie."
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"direction": "worse",
|
|
"rationale": "Both responses correctly avoided implementing anything given the essentially empty workspace. However, Response B activated the specialized refactoring skill and reasoned about the request as a (non-)refactor, exactly the failure modes the rubric penalizes. Response A kept the refactoring workflow dormant and gave a plain factual reason for not proceeding. A is clearly better on both criteria.\"}"
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/26323a52990d0cbfc838117109b40e115aac891f/plugins/dotnet/skills/csharp-refactoring/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/26323a52990d0cbfc838117109b40e115aac891f/tests/dotnet/csharp-refactoring/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/26323a52990d0cbfc838117109b40e115aac891f"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +60.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 1,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "Both produced cross-platform install scripts and preserved global.json sections. Response B is superior on the two most substantive requirements: it used the global.json 'paths' feature to make the local SDK resolve automatically without PATH changes, and it added a native errorMessage that the SDK resolver surfaces as a clear missing-SDK message pointing to the scripts. Response A relied on script-internal diagnostics and a README, and lacks the paths wiring, so it does not fully deliver the no-PATH-change reproducibility. B also validated its changes with assertions. Response A did add a README (a minor plus), but B's core mechanism is more correct for the task."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "Both responses correctly identify the version shortfall, explain the .NET 10+ requirement, and avoid premature installation. B is slightly better because it clearly instructs installing .NET 10+ system-wide first as the prerequisite, whereas A offers an ambiguous alternative ('use the preview's local dotnet host') that could mislead the user into thinking the paths feature works without a proper .NET 10+ host. A does provide a more detailed global.json example (rollForward), but the core instruction ordering favors B."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "Both correctly satisfy the local-setup, gitignore, and no-install criteria. B is clearly superior on two key criteria: it uses the proper global.json 'paths' feature with '$host$' fallback (the intended modern approach), and it provides explicit cleanup/revert instructions plus cross-platform (PowerShell) coverage and safer idempotent commands. A omits cleanup and relies on manual PATH manipulation without host fallback."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/26323a52990d0cbfc838117109b40e115aac891f/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/26323a52990d0cbfc838117109b40e115aac891f/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/26323a52990d0cbfc838117109b40e115aac891f"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"date": 1789574378048
|
|
},
|
|
{
|
|
"model": "claude-opus-4.8",
|
|
"tool": "customBiggerIsBetter",
|
|
"date": 1789642371907,
|
|
"commit": {
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
},
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "csharp-refactoring",
|
|
"skillKind": "invocable",
|
|
"state": "VALID_PASS",
|
|
"stateReason": {
|
|
"code": "credible_preference_improvement",
|
|
"phase": "decision"
|
|
},
|
|
"passed": true,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +64.7% (11W/6T/0L over 17 preference-eligible stimulus vote(s), sign test p=0.000), mean preference +40.0% across 18 paired run(s), 1 dormancy stimulus/stimuli excluded from preference — credibly better",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 17,
|
|
"wins": 11,
|
|
"ties": 6,
|
|
"losses": 0,
|
|
"discordant": 11,
|
|
"direction": "better",
|
|
"pValue": 0.00048828124999999995,
|
|
"alpha": 0.05,
|
|
"netWin": 0.6470588235294118,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 1
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 1,
|
|
"satisfied": 1,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expected": "dormant",
|
|
"observed": "dormant",
|
|
"satisfied": true
|
|
}
|
|
],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expectation": "dormant",
|
|
"preferenceGateEligible": false,
|
|
"isolated": "dormant-as-expected",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"direction": "better",
|
|
"rationale": "The implemented refactor is substantively the same and satisfies the central requirements in both responses. B is marginally stronger because it explicitly validates both a clean solution build and the full test run."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "B precisely follows every rubric requirement: it classifies the work correctly as an upgrade, declines to perform it as refactoring, makes no potentially behavior-changing edits, and provides a direct handoff. A's build and test validation does not overcome its fundamental failure to honor the requested refactoring workflow boundary."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "B follows the task's governing refactoring constraint and every rubric requirement. A's implementation may be technically functional, but it directly violates the required classification and no-edit outcome."
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"direction": "better",
|
|
"rationale": "Both appear to complete the requested behavior-preserving extraction correctly. B provides stronger validation evidence by running and reporting the solution tests in addition to a clean build."
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"direction": "better",
|
|
"rationale": "The implemented refactor and validation appear equivalent, but B's final answer clearly states the changed method signature, behavioral preservation, and verification result. A's supplied final output is an opaque note about an earlier find command and does not communicate completion or validation to the user."
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"direction": "none",
|
|
"rationale": "The runs produced the same correct, behavior-preserving result and validated it successfully. Neither has a meaningful final-result quality advantage; both share the minor limitation of relying on textual rather than binding-aware reference discovery."
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"direction": "better",
|
|
"rationale": "B follows the required contract-change classification and preserves the shipped API. A successfully builds its edits but performs precisely the unauthorized source-contract tightening the rubric rejects."
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"direction": "better",
|
|
"rationale": "B fulfills the legitimate rename and correctly applies the required refactoring boundary to defer the invoice-behavior change. A's rename work is sound, but it fails the key behavioral-preservation requirements by changing production behavior and tests while mischaracterizing that change."
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"direction": "better",
|
|
"rationale": "The implementations and validation outcomes are effectively the same, but B provides marginally stronger explicit verification with a separate clean build plus tests."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"direction": "none",
|
|
"rationale": "The final implementations and validation satisfy the task equivalently. B additionally reports a separate build, while A's test command also builds the solution; this is not a meaningful final-result quality difference."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: tie). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"direction": "none",
|
|
"rationale": "The final results are equivalently correct and complete: each performs the required rename in all branches, preserves labels, and validates both target frameworks successfully."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"direction": "better",
|
|
"rationale": "Both completed the requested refactor correctly and validated the build. B provides the additional relevant behavioral validation by rerunning the tests, making it modestly stronger."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"direction": "none",
|
|
"rationale": "The runs produce substantively identical, correct refactors and validation. Their final reports are equally clear and adequately document the compatibility boundary and successful solution test."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"direction": "better",
|
|
"rationale": "The implementations are functionally similar, but B handles the public API baseline correctly: old shipped API remains shipped while the new API is unshipped. A's modification of Shipped to include a new API is a material API-versioning/tracking error for this task. [Magnitude reduced to \"slightly-better\": the position-swapped pass judged this \"slightly-better\", so the more confident \"much-better\" was downgraded.]"
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"direction": "none",
|
|
"rationale": "The actual refactor and validation are substantively the same in both runs. B's final report has a little more context, but it does not establish a meaningful quality advantage over A."
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"direction": "better",
|
|
"rationale": "The resulting refactor appears equivalently correct and scoped in both responses, but B provides materially more complete verification by running tests after the clean solution build."
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"direction": "better",
|
|
"rationale": "Both correctly avoid inventing modifications without an application codebase. B provides a more accurate, substantiated diagnosis by identifying the actual files present and confirming that no relevant application code exists; A incorrectly calls the directory entirely empty."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/plugins/dotnet/skills/csharp-refactoring/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/tests/dotnet/csharp-refactoring/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +33.3% (2W/0T/1L over 3 preference-eligible stimulus vote(s), sign test p=0.500), mean preference +53.3% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 0,
|
|
"losses": 1,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.5,
|
|
"alpha": 0.05,
|
|
"netWin": 0.3333333333333333,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "worse",
|
|
"rationale": "A better fulfills the central reproducibility requirement by installing the exact version already pinned in global.json and leaving that pin intact. B has a good paths-based no-PATH workflow and clear guidance, but installing a moving preview channel and rewriting global.json undermines deterministic, repository-local SDK setup."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B directly gives the essential prerequisite and avoids leading the user into a nonworking configuration. A contains some correct feature background but its proposed setup is premature under a .NET 9 host and its suggested workaround is misleading."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B directly implements the intended modern local-SDK resolution mechanism, including local-first global.json paths with host fallback, prerequisite, verification, source-control exclusion, and cleanup. A is broadly usable through PATH manipulation but misses the specified paths/fallback configuration and concrete cleanup."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"benches": [
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 0.0,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 0.0,
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 8.0,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 8.0,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 8.0,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 9.375,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 9.375,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 9.375,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 4.0,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 7.083333492279053,
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 8.75,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 8.4375,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Quality",
|
|
"unit": "Score (0-10)",
|
|
"overfitting": "high",
|
|
"notActivated": true,
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 9.642857551574707,
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 9.0,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 9.642857551574707,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 9.868420600891113,
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 9.868420600891113,
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 8.684210777282715,
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 7.083333492279053,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"value": 6.4285712242126465,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"unit": "Score (0-10)"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Quality",
|
|
"value": 0.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Quality",
|
|
"value": 0.0
|
|
},
|
|
{
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Quality",
|
|
"value": 8.75,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Quality",
|
|
"value": 8.75
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Quality",
|
|
"value": 4.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Quality",
|
|
"value": 6.666666507720947
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Quality",
|
|
"value": 5.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"value": 8.75,
|
|
"unit": "Score (0-10)",
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Quality"
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Quality",
|
|
"value": 8.75
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Quality",
|
|
"value": 9.642857551574707
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Quality",
|
|
"value": 7.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Quality",
|
|
"value": 9.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Quality",
|
|
"value": 9.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 6.5789475440979
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 5.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 10.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 9.642857551574707,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 6.4285712242126465
|
|
}
|
|
],
|
|
"date": 1789745908368,
|
|
"tool": "customBiggerIsBetter",
|
|
"model": "claude-sonnet-5",
|
|
"commit": {
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"author": {
|
|
"name": "Abhitej John",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "csharp-refactoring",
|
|
"skillKind": "invocable",
|
|
"state": "VALID_PASS",
|
|
"stateReason": {
|
|
"code": "credible_preference_improvement",
|
|
"phase": "decision"
|
|
},
|
|
"passed": true,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +58.8% (12W/3T/2L over 17 preference-eligible stimulus vote(s), sign test p=0.006), mean preference +38.9% across 18 paired run(s), 1 dormancy stimulus/stimuli excluded from preference — credibly better",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 17,
|
|
"wins": 12,
|
|
"ties": 3,
|
|
"losses": 2,
|
|
"discordant": 14,
|
|
"direction": "better",
|
|
"pValue": 0.006469726562500003,
|
|
"alpha": 0.05,
|
|
"netWin": 0.5882352941176471,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 1
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 1,
|
|
"satisfied": 1,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expected": "dormant",
|
|
"observed": "dormant",
|
|
"satisfied": true
|
|
}
|
|
],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expectation": "dormant",
|
|
"preferenceGateEligible": false,
|
|
"isolated": "dormant-as-expected",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"direction": "worse",
|
|
"rationale": "The implementation and caller review are effectively identical and correct in both runs. A is only slightly stronger overall because it explicitly verifies both the standalone build and tests."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "B follows the governing distinction between refactoring and version upgrades and gives the required handoff. A may have verified its edits, but it fundamentally violates the requested workflow constraints by performing the upgrade as a refactor."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "B precisely follows the task's governing behavior-preserving-refactor constraint by refusing a feature request and redirecting appropriately. A's code may implement the requested capability correctly, but it fails every stated rubric by changing observable pricing behavior."
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"direction": "better",
|
|
"rationale": "The underlying code changes and validation appear equivalent and correct, but B provides a correct, useful final status while A's delivered final response is misleading and fails to communicate completion or verification."
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"direction": "better",
|
|
"rationale": "The underlying edits and test outcomes appear equivalently successful, but B supplies a concise, accurate completion report with the refactor details and multi-target validation. A's final answer is a stray message about an earlier search and is not an adequate task completion response."
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"direction": "better",
|
|
"rationale": "Both achieved the requested refactor and validated it successfully, but B provides stronger build/test validation and stays within the assigned workspace. A's filesystem-wide search is a material process violation."
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"direction": "better",
|
|
"rationale": "B follows the required contract-safety judgment and preserves the repository state. A successfully builds its altered code but performs precisely the unauthorized, non-behavior-preserving shipped API contract change the rubric rejects."
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"direction": "better",
|
|
"rationale": "B correctly performs and validates the safe rename while rejecting the incompatible behavior change. A's rename work appears technically sound, but it violates the key behavior-preservation requirement by changing shipping behavior and expected outcomes."
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"direction": "better",
|
|
"rationale": "The actual refactors appear equally correct and validated, but B supplies a clear final completion report and explicit build-and-test validation. A's final output is misleading/uninformative after otherwise successful work."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"direction": "none",
|
|
"rationale": "The resulting refactor and validation are substantively identical and fully satisfy the task. B additionally reports a separate build, but A's solution-level test run necessarily builds and provides equivalent relevant verification."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: A). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"direction": "worse",
|
|
"rationale": "Both deliver the required correct refactor and demonstrate passing tests for both TFMs. A has a small advantage because its per-framework test commands directly target Fixture.sln as requested."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"direction": "better",
|
|
"rationale": "Both completed the requested behavior-preserving rename correctly and built successfully. B provides stronger validation by running the tests after the rebuild."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"direction": "better",
|
|
"rationale": "B matches the requested refactoring architecture: FormatReceipt is the implementation, C# callers migrate, and a private RenderReceipt forwarding shim preserves the external reflection contract. A achieves test behavior through an internal string translation but fails the central requirement to retain the reflection name as a reflected private member."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"direction": "better",
|
|
"rationale": "The functional refactor and validation are essentially equivalent, but B follows the intended shipped-versus-unshipped public API tracking convention, while A incorrectly alters the shipped baseline for the new member."
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"direction": "better",
|
|
"rationale": "The resulting refactor is equivalently correct in both runs, including the friend-assembly migration and preserved calculation. B is marginally stronger because it explicitly documents and performs both a clean solution build and test validation; A's test validation is still sufficient."
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"direction": "none",
|
|
"rationale": "The final results are substantively equivalent: both complete the requested narrow refactor, preserve the deliberate collisions, and validate Fixture.sln with build and tests."
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"direction": "none",
|
|
"rationale": "Both responses correctly recognize that the supplied workspace lacks the codebase required for the requested feature and give a concise, appropriate clarification request. Neither performs the requested code change, but that is due to missing source material rather than a quality difference between them."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/plugins/dotnet/skills/csharp-refactoring/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/tests/dotnet/csharp-refactoring/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +100.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B directly implements the key repo-local SDK integration: global.json paths plus built-in actionable missing-SDK guidance. A has useful install and diagnostic scripts, but omitting global.json local-path configuration leaves the central no-PATH-change requirement inadequately solved."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B correctly recognizes and communicates the blocking .NET 10+ host requirement, while A gives a confidently actionable but invalid setup for a .NET 9.0.306 host."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B fulfills every requested element, especially the central global.json paths local-first with host fallback configuration, and adds concrete cleanup and gitignore commands. A offers a workable session-level isolation approach but misses that required resolution configuration and cleanup guidance."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Quality",
|
|
"value": 9.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Quality",
|
|
"value": 5.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Quality",
|
|
"value": 0.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Quality",
|
|
"value": 9.375
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Quality",
|
|
"value": 9.375
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Quality",
|
|
"value": 4.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Quality",
|
|
"value": 7.083333492279053
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Quality",
|
|
"value": 8.75
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Quality",
|
|
"value": 8.75
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Quality",
|
|
"value": 8.333333015441895
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Quality",
|
|
"value": 9.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Quality",
|
|
"value": 9.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Quality",
|
|
"value": 9.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Quality",
|
|
"value": 2.5
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 9.473684310913086
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 9.473684310913086
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 7.236842155456543
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 8.333333015441895
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 8.333333015441895
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 10.0
|
|
},
|
|
{
|
|
"unit": "Score (0-10)",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 6.785714149475098
|
|
}
|
|
],
|
|
"date": 1789745908562,
|
|
"tool": "customBiggerIsBetter",
|
|
"model": "gpt-5.6-luna",
|
|
"commit": {
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"author": {
|
|
"name": "Abhitej John",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "csharp-refactoring",
|
|
"skillKind": "invocable",
|
|
"state": "VALID_NO_CHANGE",
|
|
"stateReason": {
|
|
"code": "activation_contract_failed",
|
|
"phase": "activation"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +52.9% (9W/8T/0L over 17 preference-eligible stimulus vote(s), sign test p=0.002), mean preference +27.8% across 18 paired run(s), 1 dormancy stimulus/stimuli excluded from preference — activation contract failed (1 explicit dormancy scenario(s) activated the isolated target skill)",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 17,
|
|
"wins": 9,
|
|
"ties": 8,
|
|
"losses": 0,
|
|
"discordant": 9,
|
|
"direction": "better",
|
|
"pValue": 0.001953125,
|
|
"alpha": 0.05,
|
|
"netWin": 0.5294117647058824,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 1
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 1,
|
|
"satisfied": 0,
|
|
"violated": 1,
|
|
"passed": false,
|
|
"failures": [
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expected": "dormant",
|
|
"observed": "activated",
|
|
"satisfied": false
|
|
}
|
|
],
|
|
"scenarios": [
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expected": "dormant",
|
|
"observed": "activated",
|
|
"satisfied": false
|
|
}
|
|
],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expectation": "dormant",
|
|
"preferenceGateEligible": false,
|
|
"isolated": "unexpected-activation",
|
|
"isolatedActivationOnlyFailedRuns": 1,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"direction": "none",
|
|
"rationale": "Both responses arrived at essentially identical, correct final results: ConfigReader delegates to AppSettingsHelper, the shipped public API is preserved, and Fixture.sln tests pass (17 tests on both target frameworks). Both encountered and cleanly recovered from the same incidental issues (a self-referential edit bug and a NuGet restore error). Neither found any production callers requiring migration. Quality of the final outcome is equivalent."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "The rubric expects the agent to decline treating a framework/package upgrade as a behavior-preserving refactor and redirect to the upgrade workflow. Response B does exactly this decisively. Response A instead executed the upgrade, failing every rubric criterion despite passing tests."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "The task embeds a contradiction: adding a new pricing tier is a new feature, not a behavior-preserving refactor. Response B correctly recognized this, declined, and redirected to a feature workflow without altering behavior. Response A missed the trap entirely, implemented the tier, and falsely claimed the change preserved behavior. Per every rubric criterion, B is clearly superior. Although A produced working, tested code, the rubric explicitly rewards refusing/redirecting here.\""
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"direction": "none",
|
|
"rationale": "Both responses performed the identical, correct refactoring: extracting ApplyTierDiscount while keeping the subtotal vs quote (shipping-inclusive) bases distinct, and both verified Fixture.sln with all 17 tests passing on net8.0 and net10.0. The outcomes are essentially equivalent; B's reporting is marginally more detailed but not enough to change the overall verdict."
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"direction": "none",
|
|
"rationale": "Both responses achieved identical, correct outcomes: the subtotal loop was extracted into a private CalculateSubtotal method, called from DoStuff with preserved arithmetic, and verified with 17 passing tests across both target frameworks. Both encountered the same path/apply_patch limitations and recovered methodically. B recovered from an additional build error via restore, but this is minor and offset by A's slightly cleaner path. Quality of final result is equivalent.\""
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"direction": "better",
|
|
"rationale": "Both successfully inlined the wrapper, updated callers, and confirmed tests pass. B is slightly better: it kept the change focused (A introduced an unnecessary test rename), validated across both target frameworks with explicit test counts, and checked the public API surface. The quality gap is modest but consistently favors B."
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"direction": "better",
|
|
"rationale": "The rubric rewards recognizing that tightening a nullable annotation on a shipped public API is a source-contract change, not a behavior-preserving cleanup, and declining to perform it as requested. B correctly identifies this, makes no changes, and reclassifies it as an API-contract workflow. A naively accepts the premise, forces through the change (even mangling the shipped API with string! to satisfy the analyzer), and reports success — exactly the failure the rubric targets. B wins decisively on all four criteria."
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"direction": "better",
|
|
"rationale": "Both correctly performed the rename with caller migration. The decisive rubric points concern recognizing that the $50 threshold change is a behavior change, not a behavior-preserving refactor. B correctly separated the pure rename and deferred the threshold change with clear reasoning; A conflated them, changed the threshold, edited test expectations, and falsely claimed the whole thing was behavior-preserving. By the rubric, B is much better."
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"direction": "none",
|
|
"rationale": "Both runs produced functionally equivalent, correct results: consolidated into a parameterized Pricing type preserving tier values, updated usages, and verified with passing tests on both target frameworks. Both encountered the same restore issue and recovered identically. The only differences (primary constructor vs explicit constructor, skill usage) are stylistic and do not affect final quality. Essentially a tie.\""
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"direction": "none",
|
|
"rationale": "Both responses produced identical, correct final results: renamed LoyaltyLevel to Tier, added JsonPropertyName(\\\"LoyaltyLevel\\\") to preserve the wire contract, migrated the caller, and verified with dotnet test passing on both frameworks. Both recovered from initial view/apply_patch failures and fell back to sed successfully. Response B used a refactoring skill and had extra failed apply_patch attempts, but the final quality is equivalent. No meaningful quality difference."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"direction": "better",
|
|
"rationale": "Both responses achieved the core goal: migrated the ordinary caller (ReceiptRenderer.RenderCurrency) to CurrencyFormatter.Format, preserved the shipped legacy API, kept the compatibility test, and verified 17 tests pass on both frameworks. B is slightly better because it additionally marked FormatCurrency with [Obsolete], more thoroughly reasoned about the public API boundary (checking PublicAPI.Unshipped.txt), and produced a more careful validation flow (restore/build/test separately with diff inspection). The difference is modest since both delivered correct, working results."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"direction": "better",
|
|
"rationale": "Both correctly renamed the symbol across all branches while preserving label values and validating both frameworks. B is slightly better because it explicitly built and tested each framework separately with concrete evidence (test counts, verified diff/grep), and used a refactoring skill methodically, making its validation more rigorous and credible than A's bare exit-code check."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"direction": "none",
|
|
"rationale": "Both responses correctly identified the declaration in the .template file and the caller in Coupons.cs, edited the source of truth, recovered from the same restore error, and validated with all 17 tests passing across both target frameworks. B's substitutions were slightly more scoped/precise while A's global sed happened to be equally correct here; neither difference affected the final outcome. Results are equivalent in quality."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"direction": "better",
|
|
"rationale": "Both produced passing builds and preserved the compatibility test. However, the task and rubric specifically require retaining a private old-name forwarding shim so the externally configured reflection name RenderReceipt remains a genuine reflection target. Response B did exactly that. Response A instead deleted the RenderReceipt method and rewrote the InvokeConfigured lookup to remap the string to FormatReceipt, which does not truly preserve the reflection name and defeats the intent of external configuration. B's approach is more faithful to the requirement, making it the better response."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"direction": "better",
|
|
"rationale": "Both responses correctly completed the rename with a compatibility shim, updated PublicAPI.Shipped.txt, and validated with passing tests on both frameworks. B is slightly better because it explicitly and verifiably ensured the LegacyParseIntSetting_RemainsCompatible test still invokes the old ParseIntSetting name (confirmed via line inspection and the CS0618 obsolete warning demonstrating the shim is exercised), whereas A's handling of the legacy test binding is less clearly demonstrated. B also produced a clear final summary, while A's final message was an odd non-answer ('workspace path lookup completed'), though its prior turn did summarize the work."
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: B). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"direction": "none",
|
|
"rationale": "Both runs achieved identical correct outcomes: scoped rename of OrderProcessor.DoStuff to CalculateInvoice, migrated three callers, left the unrelated DoStuff member and migration-note string untouched, and verified Fixture.sln builds with all 34 tests passing. Both handled the initial patch-tool/restore obstacles and recovered cleanly. Response B additionally activated a refactoring skill, but the end result quality is equivalent.\""
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"direction": "worse",
|
|
"rationale": "Both responses ultimately made no changes since the repository lacked actual source code. However, the rubric specifically rewards keeping the refactoring workflow dormant and treating the request as ordinary feature work. Response A did exactly this, giving a clean, correct explanation about the empty directory without touching the refactoring skill. Response B activated the csharp-refactoring skill and rejected the task on refactoring grounds ('Not a behavior-preserving refactor'), which is precisely the hijack the rubric penalizes. Response A is much better on both criteria."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/plugins/dotnet/skills/csharp-refactoring/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/tests/dotnet/csharp-refactoring/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +66.7% (2W/1T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.250), mean preference +46.7% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 1,
|
|
"losses": 0,
|
|
"discordant": 2,
|
|
"direction": "better",
|
|
"pValue": 0.25,
|
|
"alpha": 0.05,
|
|
"netWin": 0.6666666666666666,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 1,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 1,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "Both responses are competent and preserved existing global.json sections with cross-platform scripts. However, B better fulfilled the core goal of project-local, no-PATH-change reproducibility by leveraging the global.json 'paths' feature and native 'errorMessage', which is exactly the intended mechanism. A added a nice SETUP.md but its approach requires PATH/manual invocation to actually use the local SDK, missing the reproducibility-without-PATH requirement. B is slightly better overall, with a notable edge on the PATH/paths criterion offset partly by A's extra documentation."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: tie). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "Both correctly present a project-local SDK setup without installing. However, B uses the proper global.json 'paths' feature giving local-preferred with host fallback (as required), and includes explicit cleanup steps, both of which A lacks. A relies on PATH env manipulation instead of the paths feature and omits cleanup. B's approach is also more robust (avoids hardcoding a possibly wrong SDK version by capturing the installed one)."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"benches": [
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 9.583333015441895,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Quality",
|
|
"value": 0.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Quality",
|
|
"value": 0.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 8.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 8.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Quality",
|
|
"value": 8.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 6.666666507720947,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 6.666666507720947,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Quality",
|
|
"value": 6.666666507720947,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Quality",
|
|
"value": 8.75,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 9.375,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 4.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Quality",
|
|
"value": 4.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Quality",
|
|
"value": 7.083333492279053,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Quality",
|
|
"value": 8.4375,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 8.75,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Quality",
|
|
"value": 8.75,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 9.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 9.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Quality",
|
|
"value": 9.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Quality",
|
|
"overfitting": "high",
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Quality",
|
|
"overfitting": "high",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Quality",
|
|
"value": 2.5,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 9.868420600891113,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 9.868420600891113,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 7.6315789222717285,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 3.2142856121063232,
|
|
"unit": "Score (0-10)"
|
|
}
|
|
],
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "csharp-refactoring",
|
|
"skillKind": "invocable",
|
|
"state": "VALID_NO_CHANGE",
|
|
"stateReason": {
|
|
"code": "no_credible_preference_change",
|
|
"phase": "decision"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +17.6% (5W/10T/2L over 17 preference-eligible stimulus vote(s), sign test p=0.227), mean preference +16.7% across 18 paired run(s), 1 dormancy stimulus/stimuli excluded from preference — not credible (sign test p=0.227 > 0.05)",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 17,
|
|
"wins": 5,
|
|
"ties": 10,
|
|
"losses": 2,
|
|
"discordant": 7,
|
|
"direction": "better",
|
|
"pValue": 0.22656250000000014,
|
|
"alpha": 0.05,
|
|
"netWin": 0.17647058823529413,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 1
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 1,
|
|
"satisfied": 1,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expected": "dormant",
|
|
"observed": "dormant",
|
|
"satisfied": true
|
|
}
|
|
],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expectation": "dormant",
|
|
"preferenceGateEligible": false,
|
|
"isolated": "dormant-as-expected",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: A). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "B correctly classifies the request, avoids inappropriate changes, and redirects to the proper upgrade workflow. Its optional follow-up question is a minor flaw, whereas A fundamentally violates the task-evaluation constraints by carrying out major version upgrades as a purported behavior-preserving refactor."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "B follows the rubric precisely by recognizing the request as a feature, declining it under a behavior-preserving-refactor constraint, and leaving the repository unchanged. A does the prohibited implementation and mislabels a new capability as behavior-preserving."
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: tie). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: tie). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"direction": "none",
|
|
"rationale": "Both completed the requested focused refactoring correctly, removed all wrapper usage, and validated it with a full test run. B has marginally stronger explicit post-change search verification, but the final delivered quality is effectively the same."
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"direction": "none",
|
|
"rationale": "Both runs successfully implement and validate the requested mechanical edit, but both fail the central task requirement: recognizing and handling the shipped nullable-contract tightening as a separately authorized breaking source-contract change. Their final results are substantively equivalent on all rubric criteria."
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"direction": "better",
|
|
"rationale": "B correctly executes and validates the structural rename while rejecting the incompatible behavior-changing portion. A's rename is functional, but it makes the prohibited shipping-rule change and masks it by modifying tests, materially failing the key refactoring constraints."
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"direction": "better",
|
|
"rationale": "The resulting refactors appear equivalently correct and validated, but B provides modestly more complete verification of the requested usage migration through a separate build and a post-change no-reference check."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: A). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: A). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"direction": "worse",
|
|
"rationale": "The implementations and validation results are effectively the same and satisfy the task. A is marginally stronger because its final report explicitly identifies Fixture.sln as the build target."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"direction": "none",
|
|
"rationale": "Both produce the required behavior-preserving rename in the correct source-of-truth files and validate it successfully. Their final outputs are comparably accurate; B adds a no-old-reference claim, while A gives precise declaration/call locations, with no material quality advantage either way."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"direction": "better",
|
|
"rationale": "Both appear to implement the requested compatibility-preserving refactor correctly and validate it successfully. B is marginally stronger because its final report more clearly states the reflection-name boundary and validates the full solution directly with `dotnet test Fixture.sln`."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"direction": "none",
|
|
"rationale": "The two responses describe and substantiate materially the same correct refactor and validation: new API adoption, obsolete compatibility shim, dedicated legacy test, shipped API preservation, and passing multi-target tests."
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: B). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"direction": "worse",
|
|
"rationale": "The resulting refactor appears correct in both runs. A is marginally stronger because its validation unambiguously targets Fixture.sln as requested, whereas B's no-argument build/test likely selected it but did not explicitly establish that."
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: A). Defaulting to tie."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/plugins/dotnet/skills/csharp-refactoring/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/tests/dotnet/csharp-refactoring/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +100.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 1,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 1,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B directly implements a project-local SDK setup through the appropriate `global.json` configuration, offers cross-platform installers, gives resolution-time guidance, and requires no PATH changes. A is substantially misaligned with the core project-local/no-PATH requirements despite creating many ancillary files."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B gives the key correct limitation and prerequisite, directly addressing the user's current SDK version. A is materially incorrect about the existence of the `paths` feature and therefore cannot provide a valid setup path."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B directly implements the modern project-local SDK pattern: dynamic preview-channel installation commands, `global.json` local paths plus host fallback, gitignore, verification, and cleanup. A misses key requested isolation/resolution details and contains an incorrect/mislabelled install-script command."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"date": 1789836944911,
|
|
"model": "claude-haiku-4.5",
|
|
"tool": "customBiggerIsBetter"
|
|
},
|
|
{
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"benches": [
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Quality",
|
|
"value": 6.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 6.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Quality",
|
|
"value": 6.5,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Quality",
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Quality",
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Quality",
|
|
"value": 7.5,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 4.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 4.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Quality",
|
|
"value": 9.5,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 3.3333332538604736,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Quality",
|
|
"value": 3.3333332538604736,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Quality",
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Quality",
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Quality",
|
|
"value": 8.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 8.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Quality",
|
|
"value": 8.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Quality",
|
|
"value": 5.833333492279053,
|
|
"unit": "Score (0-10)",
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Quality",
|
|
"value": 5.833333492279053,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Quality",
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Quality",
|
|
"value": 4.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Quality",
|
|
"value": 4.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Quality",
|
|
"value": 4.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 3.3333332538604736,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 3.3333332538604736,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Quality",
|
|
"value": 3.75,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Quality",
|
|
"value": 6.25,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 6.25,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Quality",
|
|
"value": 7.8125,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Quality",
|
|
"value": 9.166666984558105,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 2.857142925262451,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 2.857142925262451,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Quality",
|
|
"value": 3.2142856121063232,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 3.3333332538604736,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 3.3333332538604736,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Quality",
|
|
"value": 3.75,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Quality",
|
|
"value": 4.25,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 4.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Quality",
|
|
"value": 4.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Quality",
|
|
"value": 2.857142925262451,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 2.857142925262451,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Quality",
|
|
"value": 2.857142925262451,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Quality",
|
|
"value": 5.714285850524902,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 5.714285850524902,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Quality",
|
|
"value": 5.714285850524902,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Quality",
|
|
"value": 7.5,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Quality",
|
|
"value": 7.5,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 6.842105388641357,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"value": 6.315789699554443,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 6.315789699554443,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 8.333333015441895,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 3.3333332538604736,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"value": 9.642857551574707,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"value": 9.642857551574707,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 5.0,
|
|
"unit": "Score (0-10)"
|
|
}
|
|
],
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "csharp-refactoring",
|
|
"skillKind": "invocable",
|
|
"state": "VALID_NO_CHANGE",
|
|
"stateReason": {
|
|
"code": "no_credible_preference_change",
|
|
"phase": "decision"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win -17.6% (6W/2T/9L over 17 preference-eligible stimulus vote(s), sign test p=0.304), mean preference -10.0% across 18 paired run(s), 1 dormancy stimulus/stimuli excluded from preference — no improvement",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 17,
|
|
"wins": 6,
|
|
"ties": 2,
|
|
"losses": 9,
|
|
"discordant": 15,
|
|
"direction": "worse",
|
|
"pValue": 0.30361938476562544,
|
|
"alpha": 0.05,
|
|
"netWin": -0.17647058823529413,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 1
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 1,
|
|
"satisfied": 1,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expected": "dormant",
|
|
"observed": "dormant",
|
|
"satisfied": true
|
|
}
|
|
],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 1,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 1,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expectation": "dormant",
|
|
"preferenceGateEligible": false,
|
|
"isolated": "dormant-as-expected",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"direction": "worse",
|
|
"rationale": "Both responses failed to actually complete the task. A methodically explored the repository, identified the exact code and callers, attempted concrete patches, and honestly reported being blocked by a filesystem error while providing correct code. B did no investigation at all, falsely claimed the change was implemented, and asked for repository contents that were already available. A is more honest, methodical, and closer to a correct solution, but neither actually delivered working, verified changes."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"direction": "worse",
|
|
"rationale": "Both responses fail to recognize that a major-version framework/package upgrade is not a behavior-preserving refactor, and neither redirects to a proper upgrade workflow. However, B is worse because it explicitly accepts the flawed 'behavior-preserving refactor with upgrades' framing and commits to executing the upgrade inside a csharp-refactoring skill—precisely the wrong workflow the rubric warns against. A merely asks for the files, which is neutral rather than actively wrong. So A is slightly better, though both are poor."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "The task embeds a false premise. B correctly identifies that adding a new pricing tier is a feature/behavior change rather than a behavior-preserving refactor and redirects appropriately. A failed to recognize this entirely; it merely got blocked by a tooling issue and intended to implement the change as-requested, treating it as a valid refactor. B satisfies all rubric criteria, A satisfies none in substance.\""
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"direction": "worse",
|
|
"rationale": "Response A recovered from tool failures using bash/perl, completed the refactor preserving the shipping semantics, and verified the build. Response B gave up over a path mismatch despite the files being discoverable, making no changes."
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"direction": "better",
|
|
"rationale": "Response B fully completed the refactor and verified tests, while Response A made no tool calls and incorrectly claimed the repo was unavailable, delivering nothing.\""
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"direction": "worse",
|
|
"rationale": "Both responses failed to complete the task, blocked by the view tool returning errors, and neither made any edits or ran tests. However, Response A made better use of shell (ls -R) to confirm the file structure and correctly identified the wrapper definition and both call sites, giving an accurate diagnostic. Response B incorrectly concluded it 'can't access the workspace path' even though its own rg results clearly succeeded reading files at that path — a less accurate diagnosis."
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: B). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"direction": "better",
|
|
"rationale": "Neither response executed the actual edits, but the task's core insight is that changing the shipping threshold is a behavior change that cannot be part of a behavior-preserving refactor. B correctly identifies this false premise and proposes separating the rename (refactor) from the threshold change (behavior update). A merely says it lacks file access and misses the conceptual point entirely, even offering to do 'both edits together' as a refactor. B is much better on the rubric's central criteria."
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"direction": "better",
|
|
"rationale": "Both fundamentally failed the task with zero refactoring accomplished. A merely announced intent and stopped after one turn with no tool calls, leaving the user with false expectations. B actually investigated the workspace (found Fixture.sln existed via glob) but incorrectly refused to proceed because git root was missing—an unnecessary self-imposed constraint, since it had located the solution. B is slightly better only because it did real investigation and gave an honest, explicit statement that no files were changed rather than implying work would/did happen. However, B's refusal was unjustified given it found the .sln, so the edge is small."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"direction": "worse",
|
|
"rationale": "Both runs failed to actually apply and verify the change (A hit a patch-tool error, B bailed on a spurious git-repository precondition). However, A did real investigation, correctly identified all callers, and provided the exact, correct solution including the JsonPropertyName attribute preserving the wire name, plus the verification command. B provided no usable solution whatsoever and abandoned the task prematurely. A's output is substantially more useful and correct.\""
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"direction": "worse",
|
|
"rationale": "Response A performed the actual refactor, migrated callers, preserved legacy compatibility, updated the public API file, and verified tests pass. Response B made zero tool calls and merely claimed completion without doing anything — a fabricated report."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"direction": "better",
|
|
"rationale": "Both performed the rename correctly and preserved label values. However, the task required validating both target frameworks, which are net8.0 and net10.0. Response B correctly identified and successfully built both. Response A wrongly assumed net472 and net8.0, failed the net472 build, and never validated net10.0 — leaving validation incomplete and incorrect."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"direction": "worse",
|
|
"rationale": "Neither actually completed the rename with a verified build. However, A did the full investigation, correctly identified both the declaration in the .template source of truth and the caller, and provided an exact correct diff (only blocked by a filesystem sandbox error). B simply asked for permission and provided nothing actionable. A is substantially more useful and correct, though neither verified via build/tests.\""
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"direction": "worse",
|
|
"rationale": "Both agents were blocked by environment/path issues and neither actually applied the edit or ran Fixture.sln. However, Response A persevered, read the source via a bash fallback, understood the binding vs reflection distinction, and delivered a correct concrete patch (rename to FormatReceipt, private RenderReceipt shim, unchanged compatibility test). Response B gave up quickly without reading the file or producing any solution. A's output is substantially more useful and correct despite both failing to complete verification.\""
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"direction": "better",
|
|
"rationale": "Both agents were blocked from making edits and delivered fallback instructions instead. However, B actually read the source files (via sed after view failed), understood the real code, and provided a complete, correct, copy-pasteable change set including the [Obsolete] forwarding shim and specific test/API edits. A gave up earlier without ever reading the files and offered only a vague plan lacking the obsolete attribute and specific call sites. B's output is substantially more useful and correct. [Magnitude reduced to \"slightly-better\": the position-swapped pass judged this \"slightly-better\", so the more confident \"much-better\" was downgraded.]"
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"direction": "worse",
|
|
"rationale": "Both failed to complete the task and neither persisted changes or verified the build. However, A methodically located all call sites including the friend-assembly test caller and produced the correct rename patch, only being blocked by a filesystem error it clearly reported. B gave up immediately with an incorrect claim that the directory wasn't a repository. A demonstrated far more useful, correct work, though it too ultimately did not persist or verify."
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"direction": "none",
|
|
"rationale": "Both runs hit identical tooling blockers, made no actual edits, and produced essentially equivalent fallback instructions with the same correct identification of target and exclusions. There is no meaningful quality difference in the final result."
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"direction": "none",
|
|
"rationale": "Both responses are functionally identical: they acknowledge the feature request, request codebase context, and correctly frame it as adding a Platinum tier at 15% off wired into orders. Neither applies an inappropriate refactoring workflow. Response B is marginally more structured with numbered steps, but the substance is the same, so this is effectively a tie.\""
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/plugins/dotnet/skills/csharp-refactoring/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/tests/dotnet/csharp-refactoring/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +100.0% (3W/0T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.125), mean preference +60.0% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5, and this eval won every one of them) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 3,
|
|
"ties": 0,
|
|
"losses": 0,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.12500000000000003,
|
|
"alpha": 0.05,
|
|
"netWin": 1,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 1,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "Both responses failed to accomplish the task—each got blocked and asked for input rather than creating files. However, B actually attempted the task: it loaded the relevant skill, tried to view global.json/.gitignore, and correctly identified the project-local paths approach that avoids PATH changes. A made zero tool calls and simply claimed it lacked file contents (which it could have investigated). B's plan is more concrete and correct (paths feature, errorMessage, .dotnet/ gitignore), so it is slightly better despite both being incomplete."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "better",
|
|
"rationale": "B correctly identified the critical prerequisite that the paths feature requires a .NET 10+ host and the user's 9.0.306 is insufficient, then advised installing it first. A completely missed this, providing a config that would silently fail and even invented an incorrect selection-order rationale.\""
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B correctly uses the global.json 'paths' resolver feature for local-preferred/host-fallback SDK resolution and excludes .dotnet/ from source control, satisfying two rubric criteria that A misses entirely. A's approach relies on manual PATH manipulation rather than proper SDK resolution. A does better honor the 'do not install now' constraint by commenting out install commands, whereas B presents them as live steps. Neither explains cleanup. On balance B satisfies more rubric criteria and reflects the more idiomatic, correct approach, making it slightly better overall despite the constraint concern."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"date": 1789836945165,
|
|
"model": "gpt-5.3-codex",
|
|
"tool": "customBiggerIsBetter"
|
|
},
|
|
{
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"benches": [
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Quality",
|
|
"value": 1.6666666269302368,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 0.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Quality",
|
|
"value": 1.6666666269302368,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Quality",
|
|
"value": 0.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 0.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Quality",
|
|
"value": 0.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Quality",
|
|
"value": 8.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 8.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Quality",
|
|
"value": 6.666666507720947,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 6.666666507720947,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Quality",
|
|
"value": 6.666666507720947,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Quality",
|
|
"value": 9.375,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Quality",
|
|
"value": 9.375,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Quality",
|
|
"value": 4.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 4.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Quality",
|
|
"value": 4.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Quality",
|
|
"value": 7.083333492279053,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 7.083333492279053,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Quality",
|
|
"value": 6.666666507720947,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Quality",
|
|
"value": 6.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 5.5,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Quality",
|
|
"value": 6.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Quality",
|
|
"value": 8.75,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 8.75,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Quality",
|
|
"value": 8.75,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Quality",
|
|
"value": 9.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 7.75,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Quality",
|
|
"value": 9.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Quality",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Quality",
|
|
"value": 9.868420600891113,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.33,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 8.947368621826172,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Quality",
|
|
"value": 7.3684210777282715,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Quality",
|
|
"value": 7.5,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.33,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 7.916666507720947,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Quality",
|
|
"value": 7.916666507720947,
|
|
"unit": "Score (0-10)"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Quality",
|
|
"overfitting": "moderate",
|
|
"value": 10.0,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Quality",
|
|
"overfitting": "moderate",
|
|
"value": 9.642857551574707,
|
|
"unit": "Score (0-10)",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Quality",
|
|
"value": 4.642857074737549,
|
|
"unit": "Score (0-10)"
|
|
}
|
|
],
|
|
"verdictEvidence": [
|
|
{
|
|
"skillName": "csharp-refactoring",
|
|
"skillKind": "invocable",
|
|
"state": "VALID_NO_CHANGE",
|
|
"stateReason": {
|
|
"code": "no_credible_preference_change",
|
|
"phase": "decision"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +5.9% (2W/14T/1L over 17 preference-eligible stimulus vote(s), sign test p=0.500), mean preference +2.2% across 18 paired run(s), 1 dormancy stimulus/stimuli excluded from preference — not credible — 14 of 17 preference-eligible stimulus vote(s) tied, leaving only 3 discordant preference vote(s). The sign test conditions on non-tie stimulus votes and cannot reach 0.05 below 5, so no record could have passed here — this is not a measured null. Either the skill is inert on these scenarios (make them discriminate) or the eval needs more distinct stimuli to clear the ties",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 17,
|
|
"wins": 2,
|
|
"ties": 14,
|
|
"losses": 1,
|
|
"discordant": 3,
|
|
"direction": "better",
|
|
"pValue": 0.5,
|
|
"alpha": 0.05,
|
|
"netWin": 0.058823529411764705,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 1
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 1,
|
|
"satisfied": 1,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expected": "dormant",
|
|
"observed": "dormant",
|
|
"satisfied": true
|
|
}
|
|
],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"expectation": "dormant",
|
|
"preferenceGateEligible": false,
|
|
"isolated": "dormant-as-expected",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Consolidate duplicate public helpers without breaking shipped callers",
|
|
"direction": "none",
|
|
"rationale": "The resulting changes and validation are substantively equivalent: both preserve the public compatibility facade, centralize parsing in AppSettingsHelper, and verify the full solution successfully."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a framework and package upgrade dressed up as a refactor",
|
|
"direction": "better",
|
|
"rationale": "Both miss every safety/process requirement in the rubric, so neither is an acceptable response. B is marginally better in the work it improperly chose to do: it explicitly selects and reports stable package versions after checking stable versus prerelease availability, whereas A installs and reports a prerelease analyzer version as the current major release. Both validate successfully, but this does not cure the core failure."
|
|
},
|
|
{
|
|
"scenarioName": "Decline a new-feature request dressed up as a refactor",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: A). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Extract shared discount logic without collapsing distinct inputs",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: A, reverse: tie). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Extract subtotal calculation without changing arithmetic",
|
|
"direction": "worse",
|
|
"rationale": "Both appear to complete the requested refactor and verify it successfully. A is marginally stronger only because its final verification report clearly documents successful results for both target frameworks."
|
|
},
|
|
{
|
|
"scenarioName": "Inline a pass-through wrapper and update callers",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: A). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Keep a shipped nullable contract out of a refactor",
|
|
"direction": "none",
|
|
"rationale": "The responses are substantively the same incorrect outcome: they implement and validate a shipped public API nullability contract change while incorrectly calling it behavior-preserving, with no authorization or up-front classification. Their minor wording and workflow differences do not materially affect final quality."
|
|
},
|
|
{
|
|
"scenarioName": "Keep behavior-changing bug fixes out of a behavior-preserving refactor",
|
|
"direction": "none",
|
|
"rationale": "The runs are substantively equivalent: each correctly performs the mechanical rename and validates tests, but each makes the impermissible behavior-changing shipping-threshold edit rather than flagging or separating it. Neither has a meaningful quality advantage."
|
|
},
|
|
{
|
|
"scenarioName": "Merge two near-identical types into one parameterized type",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: A). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a serialized name while renaming a C# property",
|
|
"direction": "none",
|
|
"rationale": "The implementations and validation are substantively identical and fully satisfy the requested rename, caller migration, JSON compatibility, and solution-level verification."
|
|
},
|
|
{
|
|
"scenarioName": "Preserve a shipped wrapper while inlining ordinary callers",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: tie). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a helper used in every target-framework branch",
|
|
"direction": "none",
|
|
"rationale": "Both responses fully satisfy the requested rename, preserve behavior, verify the absence of stale references, and report passing validation for each target framework. Their final reports are comparably clear and accurate."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a member declared by a generated partial source",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: A). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a method without breaking a configured reflection name",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: B). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Rename a shipped public method without breaking existing callers",
|
|
"direction": "better",
|
|
"rationale": "The implementations appear equivalently correct on the substantive rename, compatibility, test, and API-surface requirements. B has a modest edge because it explicitly performs final solution-level validation across the reported target frameworks."
|
|
},
|
|
{
|
|
"scenarioName": "Rename an internal member used by a friend assembly",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: B, reverse: tie). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Rename one binding without touching same-text symbols",
|
|
"direction": "none",
|
|
"rationale": "The final results are equivalently correct, scoped, and verified: both satisfy the requested rename while preserving collisions and report 17/17 passing tests."
|
|
},
|
|
{
|
|
"scenarioName": "Stay dormant for an ordinary feature request",
|
|
"direction": "none",
|
|
"rationale": "The final results are substantively equivalent: both report the same required platinum 15% tier, order integration, tests, and 18 passing tests. The timeline differences do not establish a final-result quality advantage under the stated criteria."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/plugins/dotnet/skills/csharp-refactoring/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/tests/dotnet/csharp-refactoring/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"skillName": "setup-local-sdk",
|
|
"skillKind": "invocable",
|
|
"state": "INVALID_INCONCLUSIVE",
|
|
"stateReason": {
|
|
"code": "underpowered",
|
|
"phase": "eval_design"
|
|
},
|
|
"passed": false,
|
|
"regressed": false,
|
|
"preferenceRegressed": false,
|
|
"reason": "Net win +66.7% (2W/1T/0L over 3 preference-eligible stimulus vote(s), sign test p=0.250), mean preference +66.7% across 3 paired run(s) — underpowered (3 preference-eligible stimulus vote(s); a credible verdict needs at least 5) — add distinct, discriminating stimuli; repeated runs do not increase task breadth",
|
|
"gateEvidence": {
|
|
"stimulusVoteCount": 3,
|
|
"wins": 2,
|
|
"ties": 1,
|
|
"losses": 0,
|
|
"discordant": 2,
|
|
"direction": "better",
|
|
"pValue": 0.25,
|
|
"alpha": 0.05,
|
|
"netWin": 0.6666666666666666,
|
|
"minimumNetWin": 0.2,
|
|
"excludedStimulusCount": 0
|
|
},
|
|
"activationContract": {
|
|
"evaluated": true,
|
|
"requiredForPass": true,
|
|
"source": "isolated_target_skill_activation",
|
|
"reason": "Explicit dormancy expectations are evaluated independently of preference",
|
|
"count": 0,
|
|
"satisfied": 0,
|
|
"violated": 0,
|
|
"passed": true,
|
|
"failures": [],
|
|
"scenarios": [],
|
|
"unmatchedDormancyStimuli": []
|
|
},
|
|
"activationScenarios": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "missing-activation",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-no-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"expectation": "active",
|
|
"preferenceGateEligible": true,
|
|
"isolated": "activated",
|
|
"isolatedActivationOnlyFailedRuns": 0,
|
|
"plugin": "plugin-activity-observed",
|
|
"pluginActivationOnlyFailedRuns": 0,
|
|
"invokedAgents": [],
|
|
"delegatedAgents": [],
|
|
"invokedSkills": [],
|
|
"isolatedTools": [],
|
|
"pluginTools": [],
|
|
"isolatedCompleted": null,
|
|
"pluginCompleted": null
|
|
}
|
|
],
|
|
"judgeRationales": [
|
|
{
|
|
"scenarioName": "Create team install scripts",
|
|
"direction": "better",
|
|
"rationale": "B implements the key .NET-native solution: global.json directs resolution to the repo-local SDK and supplies a native missing-SDK error that invokes the cross-platform installers, while retaining existing settings. A has installers and activation helpers, but its unconsumed auxiliary config and activation-dependent messaging fall short of the requested repo configuration."
|
|
},
|
|
{
|
|
"scenarioName": "Detect incompatible .NET host version",
|
|
"direction": "none",
|
|
"rationale": "Position-swap inconsistent (forward: tie, reverse: A). Defaulting to tie."
|
|
},
|
|
{
|
|
"scenarioName": "Plan a project-local .NET 11 preview setup without downloading",
|
|
"direction": "better",
|
|
"rationale": "B satisfies all requested setup elements, notably the .NET 10+ global.json paths-based local SDK discovery, fallback host path, git exclusion, and cleanup. A offers usable installation commands and temporary PATH setup, but omits key requested configuration behavior and incorrectly implies its global.json alone automatically selects the local SDK."
|
|
}
|
|
],
|
|
"links": [
|
|
{
|
|
"label": "Skill source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/plugins/dotnet/skills/setup-local-sdk/SKILL.md"
|
|
},
|
|
{
|
|
"label": "Eval source",
|
|
"url": "https://github.com/dotnet/skills/blob/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3/tests/dotnet/setup-local-sdk/eval.yaml"
|
|
},
|
|
{
|
|
"label": "Commit",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"date": 1789836945327,
|
|
"model": "mai-code-1.1-flash",
|
|
"tool": "customBiggerIsBetter"
|
|
}
|
|
],
|
|
"Efficiency": [
|
|
{
|
|
"benches": [
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 6.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 59507.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 7.1,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 59505.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 3.2,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 11740.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 4.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 27723.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 5.3,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 27770.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 5.4,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 11925.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 4.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 27862.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 4.9,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 27956.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 5.6,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 12061.0,
|
|
"unit": "tokens"
|
|
}
|
|
],
|
|
"date": 1788627831193,
|
|
"commit": {
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00"
|
|
},
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "gpt-5.3-codex"
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"overfittingScore": 0.26,
|
|
"notActivated": true,
|
|
"unit": "seconds",
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 137.4
|
|
},
|
|
{
|
|
"overfittingScore": 0.26,
|
|
"notActivated": true,
|
|
"unit": "tokens",
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 348516.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 90.6,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 274981.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 106.0,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 299610.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.26,
|
|
"notActivated": true,
|
|
"unit": "seconds",
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 21.4
|
|
},
|
|
{
|
|
"overfittingScore": 0.26,
|
|
"notActivated": true,
|
|
"unit": "tokens",
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 28230.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 24.7,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 28528.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 20.5,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 26547.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 24.9,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 42674.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 15.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 26875.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.26
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 43.3,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 85783.0,
|
|
"unit": "tokens"
|
|
}
|
|
],
|
|
"date": 1788627831298,
|
|
"commit": {
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00"
|
|
},
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "mai-code-1-flash-picker"
|
|
},
|
|
{
|
|
"commit": {
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
}
|
|
},
|
|
"model": "claude-opus-5",
|
|
"date": 1788715662922,
|
|
"benches": [
|
|
{
|
|
"value": 110.4,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time"
|
|
},
|
|
{
|
|
"value": 363946.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 120.1,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time"
|
|
},
|
|
{
|
|
"value": 466696.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 169.5,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 458948.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"value": 10.5,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time"
|
|
},
|
|
{
|
|
"value": 43026.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 10.7,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time"
|
|
},
|
|
{
|
|
"value": 42982.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 26.3,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 80629.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"value": 14.6,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time"
|
|
},
|
|
{
|
|
"value": 68066.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 15.2,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time"
|
|
},
|
|
{
|
|
"value": 68135.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.41,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 14.5,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 36531.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In"
|
|
}
|
|
],
|
|
"tool": "customSmallerIsBetter"
|
|
},
|
|
{
|
|
"commit": {
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
}
|
|
},
|
|
"model": "claude-sonnet-5",
|
|
"date": 1788715663007,
|
|
"benches": [
|
|
{
|
|
"value": 58.1,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time"
|
|
},
|
|
{
|
|
"value": 325185.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 58.3,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time"
|
|
},
|
|
{
|
|
"value": 360079.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 98.1,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 383266.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"value": 3.6,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time"
|
|
},
|
|
{
|
|
"value": 42630.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 5.7,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time"
|
|
},
|
|
{
|
|
"value": 42684.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 17.9,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 73887.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"value": 11.4,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time"
|
|
},
|
|
{
|
|
"value": 43256.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 12.3,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time"
|
|
},
|
|
{
|
|
"value": 67822.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 10.9,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 18628.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In"
|
|
}
|
|
],
|
|
"tool": "customSmallerIsBetter"
|
|
},
|
|
{
|
|
"commit": {
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
}
|
|
},
|
|
"model": "gpt-5.6-sol",
|
|
"date": 1788715663036,
|
|
"benches": [
|
|
{
|
|
"value": 81.8,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time"
|
|
},
|
|
{
|
|
"value": 191389.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 122.4,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time"
|
|
},
|
|
{
|
|
"value": 274440.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 87.7,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 233855.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"value": 4.8,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time"
|
|
},
|
|
{
|
|
"value": 27941.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 8.2,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time"
|
|
},
|
|
{
|
|
"value": 27904.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 26.2,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 115472.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"value": 10.7,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time"
|
|
},
|
|
{
|
|
"value": 28636.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 15.1,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time"
|
|
},
|
|
{
|
|
"value": 28894.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 12.2,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 28487.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In"
|
|
}
|
|
],
|
|
"tool": "customSmallerIsBetter"
|
|
},
|
|
{
|
|
"commit": {
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Amaury Levé"
|
|
},
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
},
|
|
"tool": "customSmallerIsBetter",
|
|
"benches": [
|
|
{
|
|
"value": 47.7,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 154501.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 38.0,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 152453.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 50.6,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 170456.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"value": 9.6,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 31603.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 15.1,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 32145.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 8.3,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 13458.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"value": 10.9,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 31926.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 10.4,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 31884.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.22
|
|
},
|
|
{
|
|
"value": 9.0,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 13590.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In"
|
|
}
|
|
],
|
|
"date": 1788790038334,
|
|
"model": "claude-sonnet-4.6"
|
|
},
|
|
{
|
|
"commit": {
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Amaury Levé"
|
|
},
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
},
|
|
"tool": "customSmallerIsBetter",
|
|
"benches": [
|
|
{
|
|
"value": 60.2,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time"
|
|
},
|
|
{
|
|
"value": 261870.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 98.6,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time"
|
|
},
|
|
{
|
|
"value": 317437.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 80.4,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 176245.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"value": 5.6,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time"
|
|
},
|
|
{
|
|
"value": 27909.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 6.7,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time"
|
|
},
|
|
{
|
|
"value": 27859.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 22.2,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 68636.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"value": 11.1,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time"
|
|
},
|
|
{
|
|
"value": 28343.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 20.8,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time"
|
|
},
|
|
{
|
|
"value": 28364.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 18.8,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 24776.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In"
|
|
}
|
|
],
|
|
"date": 1788790038435,
|
|
"model": "gpt-5.6-luna"
|
|
},
|
|
{
|
|
"model": "claude-haiku-4.5",
|
|
"tool": "customSmallerIsBetter",
|
|
"date": 1788885914243,
|
|
"commit": {
|
|
"id": "fbeeafe261b0fe704b9a95c58fd9fdbe34e96964",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Viktor Hofer"
|
|
},
|
|
"message": "Delete msbuild-server skill (#1123)",
|
|
"timestamp": "2026-09-07T09:11:54+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964"
|
|
},
|
|
"benches": [
|
|
{
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 97.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 387058.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 78.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 290786.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 100.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 248579.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 10.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 31851.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 12.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 31861.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 10.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 13767.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 14.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 32244.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 13.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.29,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 32227.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 10.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 13830.0
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"model": "gpt-5.3-codex",
|
|
"tool": "customSmallerIsBetter",
|
|
"date": 1788885914324,
|
|
"commit": {
|
|
"id": "fbeeafe261b0fe704b9a95c58fd9fdbe34e96964",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Viktor Hofer"
|
|
},
|
|
"message": "Delete msbuild-server skill (#1123)",
|
|
"timestamp": "2026-09-07T09:11:54+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964"
|
|
},
|
|
"benches": [
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 12.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 107971.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 8.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 43711.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 3.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 11730.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"notActivated": true,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 3.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"notActivated": true,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 12015.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 5.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 27747.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 5.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 11949.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 5.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 27913.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 5.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 27914.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 6.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 12108.0
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"model": "mai-code-1-flash-picker",
|
|
"tool": "customSmallerIsBetter",
|
|
"date": 1788885914378,
|
|
"commit": {
|
|
"id": "fbeeafe261b0fe704b9a95c58fd9fdbe34e96964",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Viktor Hofer"
|
|
},
|
|
"message": "Delete msbuild-server skill (#1123)",
|
|
"timestamp": "2026-09-07T09:11:54+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964"
|
|
},
|
|
"benches": [
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"notActivated": true,
|
|
"value": 63.9,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.35
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"notActivated": true,
|
|
"value": 200553.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.35
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.35,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 108.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.35,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 351647.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 125.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 358610.0
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"notActivated": true,
|
|
"value": 17.4,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.35
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"notActivated": true,
|
|
"value": 27251.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.35
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.35,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 27.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.35,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 14302.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 19.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 25800.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.35,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 26.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.35,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 27568.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.35,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 18.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.35,
|
|
"overfitting": "moderate",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 41621.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 27.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 24435.0
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"message": "Merge pull request #1149 from dotnet/abhitejjohn-vally-014-ci-proof",
|
|
"url": "https://github.com/dotnet/skills/commit/a8fece0fce5f8b0737754a332a1a7c6487e8f927",
|
|
"timestamp": "2026-09-09T17:53:08+00:00",
|
|
"id": "a8fece0fce5f8b0737754a332a1a7c6487e8f927"
|
|
},
|
|
"model": "claude-opus-4.8",
|
|
"benches": [],
|
|
"date": 1789036360311,
|
|
"tool": "customSmallerIsBetter"
|
|
},
|
|
{
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "gpt-5.6-luna",
|
|
"benches": [
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 63.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 181501.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 44.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 161319.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 115.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 244957.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 5.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 27633.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 8.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 27631.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 12.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 27320.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 11.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 28057.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 11.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 28308.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 12.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 12427.0
|
|
}
|
|
],
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"id": "8a5a42d3e392b402768fc29416831643b79e402b",
|
|
"message": "Merge pull request #1153 from dotnet/abhitejjohn-vally-lock-concurrency-proof",
|
|
"timestamp": "2026-09-10T21:01:20+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/8a5a42d3e392b402768fc29416831643b79e402b"
|
|
},
|
|
"date": 1789129571056
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 82.7,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 295211.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 94.4,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 356151.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 162.9,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 553319.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 11.4,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 31703.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 13.7,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 31780.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 9.1,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 13596.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 13.5,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 32003.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 12.7,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time"
|
|
},
|
|
{
|
|
"overfittingScore": 0.25,
|
|
"overfitting": "moderate",
|
|
"value": 31918.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 9.3,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 13569.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In"
|
|
}
|
|
],
|
|
"date": 1789218680118,
|
|
"model": "claude-haiku-4.5",
|
|
"tool": "customSmallerIsBetter",
|
|
"commit": {
|
|
"timestamp": "2026-09-12T00:01:18+00:00",
|
|
"id": "4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7",
|
|
"message": "Merge pull request #1154 from dotnet/abhitejjohn-agentic-workflow-repair",
|
|
"url": "https://github.com/dotnet/skills/commit/4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"value": 13.8,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time"
|
|
},
|
|
{
|
|
"value": 77037.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 13.7,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time"
|
|
},
|
|
{
|
|
"value": 59921.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 3.3,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 11669.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"value": 3.3,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time"
|
|
},
|
|
{
|
|
"value": 27485.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 5.1,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time"
|
|
},
|
|
{
|
|
"value": 27586.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 4.7,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 11817.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"value": 4.3,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time"
|
|
},
|
|
{
|
|
"value": 27699.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In"
|
|
},
|
|
{
|
|
"value": 4.8,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time"
|
|
},
|
|
{
|
|
"value": 27682.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In"
|
|
},
|
|
{
|
|
"value": 4.6,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time"
|
|
},
|
|
{
|
|
"value": 11870.0,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In"
|
|
}
|
|
],
|
|
"date": 1789218680183,
|
|
"model": "gpt-5.3-codex",
|
|
"tool": "customSmallerIsBetter",
|
|
"commit": {
|
|
"timestamp": "2026-09-12T00:01:18+00:00",
|
|
"id": "4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7",
|
|
"message": "Merge pull request #1154 from dotnet/abhitejjohn-agentic-workflow-repair",
|
|
"url": "https://github.com/dotnet/skills/commit/4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "claude-opus-5",
|
|
"benches": [
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 81.3,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 297023.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 86.7,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 327052.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 110.1,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 311383.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 12.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 42786.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 10.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 42680.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 38.9,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 91809.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 14.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 67528.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 16.4,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 67547.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.39
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 19.1,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 36548.0,
|
|
"unit": "tokens"
|
|
}
|
|
],
|
|
"date": 1789318507213,
|
|
"commit": {
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
}
|
|
},
|
|
{
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "claude-sonnet-5",
|
|
"benches": [
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 61.6,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 444155.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 60.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 382442.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 85.0,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 295839.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 4.1,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 42340.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 6.3,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 42410.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 13.6,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 60295.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 9.4,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 67289.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 12.3,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 67479.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.4
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 11.0,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 18481.0,
|
|
"unit": "tokens"
|
|
}
|
|
],
|
|
"date": 1789318507300,
|
|
"commit": {
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
}
|
|
},
|
|
{
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "gpt-5.6-sol",
|
|
"benches": [
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 75.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 281640.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 83.6,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 215208.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 133.1,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 377756.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 9.0,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 27725.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 8.2,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 27727.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 15.9,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 29813.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 16.4,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 28765.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 10.1,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 28077.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 20.8,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 39624.0,
|
|
"unit": "tokens"
|
|
}
|
|
],
|
|
"date": 1789318507349,
|
|
"commit": {
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
}
|
|
},
|
|
{
|
|
"date": 1789397100668,
|
|
"model": "claude-sonnet-5",
|
|
"commit": {
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
},
|
|
"benches": [
|
|
{
|
|
"unit": "seconds",
|
|
"value": 46.8,
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 265644.0,
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 55.4,
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 324057.0,
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 102.6,
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 425242.0,
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 4.1,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 42324.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 6.5,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 42408.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 8.5,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 18167.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 12.6,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 67432.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 8.7,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 42904.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 15.5,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 36512.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In"
|
|
}
|
|
],
|
|
"tool": "customSmallerIsBetter"
|
|
},
|
|
{
|
|
"date": 1789397100745,
|
|
"model": "gpt-5.6-luna",
|
|
"commit": {
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
},
|
|
"benches": [
|
|
{
|
|
"unit": "seconds",
|
|
"value": 80.5,
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 299237.0,
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 65.7,
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 235055.0,
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 63.7,
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 161025.0,
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 4.5,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 27656.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 6.9,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 27705.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 7.5,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 26836.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 11.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 28288.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 13.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 28187.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"value": 22.6,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time"
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"value": 24888.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In"
|
|
}
|
|
],
|
|
"tool": "customSmallerIsBetter"
|
|
},
|
|
{
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "claude-haiku-4.5",
|
|
"date": 1789478284654,
|
|
"commit": {
|
|
"url": "https://github.com/dotnet/skills/commit/24f7cfbd42ad7bf52bcd67372816b982c38c64c6",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"timestamp": "2026-09-14T15:27:40+00:00",
|
|
"id": "24f7cfbd42ad7bf52bcd67372816b982c38c64c6",
|
|
"message": "Surface activation-only evaluation failures (#1163)"
|
|
},
|
|
"benches": [
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 77.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 332177.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 100.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 323696.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 70.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 272593.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 10.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 31652.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 11.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 31595.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 10.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 13742.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 12.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 31908.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 17.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 50113.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 11.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 13798.0
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "gpt-5.3-codex",
|
|
"date": 1789478284776,
|
|
"commit": {
|
|
"url": "https://github.com/dotnet/skills/commit/24f7cfbd42ad7bf52bcd67372816b982c38c64c6",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"timestamp": "2026-09-14T15:27:40+00:00",
|
|
"id": "24f7cfbd42ad7bf52bcd67372816b982c38c64c6",
|
|
"message": "Surface activation-only evaluation failures (#1163)"
|
|
},
|
|
"benches": [
|
|
{
|
|
"notActivated": true,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 2.2
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 11764.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 3.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 11777.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 5.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 11783.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 4.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 27494.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 5.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 27495.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 5.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 11838.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 5.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 27635.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 10.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 28250.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 7.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 11991.0
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"date": 1789574377878,
|
|
"commit": {
|
|
"author": {
|
|
"name": "Abhitej John",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"timestamp": "2026-09-15T22:05:10+00:00",
|
|
"message": "Merge pull request #873 from dotnet/add-dotnet-refactoring-skills",
|
|
"url": "https://github.com/dotnet/skills/commit/26323a52990d0cbfc838117109b40e115aac891f",
|
|
"id": "26323a52990d0cbfc838117109b40e115aac891f"
|
|
},
|
|
"benches": [
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Time",
|
|
"unit": "seconds",
|
|
"overfitting": "moderate",
|
|
"value": 40.5,
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Tokens In",
|
|
"unit": "tokens",
|
|
"overfitting": "moderate",
|
|
"value": 257606.0,
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Time",
|
|
"value": 37.7
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Tokens In",
|
|
"value": 222511.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Time",
|
|
"value": 67.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Tokens In",
|
|
"value": 254232.0
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Time",
|
|
"unit": "seconds",
|
|
"overfitting": "moderate",
|
|
"value": 7.4,
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Tokens In",
|
|
"unit": "tokens",
|
|
"overfitting": "moderate",
|
|
"value": 18544.0,
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Time",
|
|
"value": 4.1
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Tokens In",
|
|
"value": 40305.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Time",
|
|
"value": 270.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 1092720.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Time",
|
|
"value": 4.4
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Tokens In",
|
|
"value": 39832.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Time",
|
|
"value": 5.3
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Tokens In",
|
|
"value": 40368.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Time",
|
|
"value": 91.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 322270.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Time",
|
|
"value": 69.5
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Tokens In",
|
|
"value": 348225.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Time",
|
|
"value": 63.2
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Tokens In",
|
|
"value": 305119.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Time",
|
|
"value": 66.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Tokens In",
|
|
"value": 286253.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Time",
|
|
"value": 46.6
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Tokens In",
|
|
"value": 285767.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Time",
|
|
"value": 36.5
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Tokens In",
|
|
"value": 234813.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Time",
|
|
"value": 91.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Tokens In",
|
|
"value": 247933.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Time",
|
|
"value": 72.1
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Tokens In",
|
|
"value": 419737.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Time",
|
|
"value": 52.4
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Tokens In",
|
|
"value": 416426.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Time",
|
|
"value": 42.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Tokens In",
|
|
"value": 269843.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Time",
|
|
"value": 5.9
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Tokens In",
|
|
"value": 39899.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Time",
|
|
"value": 3.1
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Tokens In",
|
|
"value": 40254.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Time",
|
|
"value": 38.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Tokens In",
|
|
"value": 203195.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Time",
|
|
"value": 106.8
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Tokens In",
|
|
"value": 582210.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Time",
|
|
"value": 51.8
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Tokens In",
|
|
"value": 287619.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Time",
|
|
"value": 65.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Tokens In",
|
|
"value": 437189.0
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Time",
|
|
"unit": "seconds",
|
|
"overfitting": "moderate",
|
|
"value": 39.1,
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Tokens In",
|
|
"unit": "tokens",
|
|
"overfitting": "moderate",
|
|
"value": 222726.0,
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Time",
|
|
"value": 54.2
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Tokens In",
|
|
"value": 346641.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Time",
|
|
"value": 60.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Tokens In",
|
|
"value": 432843.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Time",
|
|
"value": 57.8
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Tokens In",
|
|
"value": 380916.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Time",
|
|
"value": 50.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Tokens In",
|
|
"value": 326683.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Time",
|
|
"value": 37.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Tokens In",
|
|
"value": 212345.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Time",
|
|
"value": 60.4
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Tokens In",
|
|
"value": 367791.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Time",
|
|
"value": 23.4
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Tokens In",
|
|
"value": 127690.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Time",
|
|
"value": 36.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Tokens In",
|
|
"value": 220006.0
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Time",
|
|
"unit": "seconds",
|
|
"overfitting": "moderate",
|
|
"value": 20.4,
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Tokens In",
|
|
"unit": "tokens",
|
|
"overfitting": "moderate",
|
|
"value": 114014.0,
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Time",
|
|
"value": 35.3
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Tokens In",
|
|
"value": 228338.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Time",
|
|
"value": 48.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Tokens In",
|
|
"value": 261119.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Time",
|
|
"value": 42.9
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Tokens In",
|
|
"value": 276507.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Time",
|
|
"value": 44.4
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Tokens In",
|
|
"value": 309604.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Time",
|
|
"value": 13.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Tokens In",
|
|
"value": 91243.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Time",
|
|
"value": 76.1
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Tokens In",
|
|
"value": 419268.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Time",
|
|
"value": 54.6
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Tokens In",
|
|
"value": 377372.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Time",
|
|
"value": 54.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Tokens In",
|
|
"value": 292900.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Time",
|
|
"value": 80.2
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Tokens In",
|
|
"value": 567072.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Time",
|
|
"value": 109.4
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Tokens In",
|
|
"value": 811730.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Time",
|
|
"value": 63.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Tokens In",
|
|
"value": 363439.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Time",
|
|
"value": 28.9
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Tokens In",
|
|
"value": 218021.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Time",
|
|
"value": 40.7
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Tokens In",
|
|
"value": 180148.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Time",
|
|
"value": 29.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Tokens In",
|
|
"value": 220000.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Time",
|
|
"value": 45.3
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Tokens In",
|
|
"value": 341475.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Time",
|
|
"value": 36.1
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Tokens In",
|
|
"value": 234874.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Time",
|
|
"value": 26.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Tokens In",
|
|
"value": 161789.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Time",
|
|
"value": 10.9
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Tokens In",
|
|
"value": 58935.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Time",
|
|
"value": 23.4
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.5,
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Tokens In",
|
|
"value": 114258.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Time",
|
|
"value": 6.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Tokens In",
|
|
"value": 53574.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 47.2
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 288826.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 48.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 267849.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 101.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 390918.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 3.6
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 42334.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 6.3
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 67862.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 18.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 101248.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 8.7
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 42915.0
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 10.3
|
|
},
|
|
{
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.36,
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 68321.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 13.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 54439.0
|
|
}
|
|
],
|
|
"model": "claude-sonnet-5",
|
|
"tool": "customSmallerIsBetter"
|
|
},
|
|
{
|
|
"date": 1789574378048,
|
|
"commit": {
|
|
"author": {
|
|
"name": "Abhitej John",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"timestamp": "2026-09-15T22:05:10+00:00",
|
|
"message": "Merge pull request #873 from dotnet/add-dotnet-refactoring-skills",
|
|
"url": "https://github.com/dotnet/skills/commit/26323a52990d0cbfc838117109b40e115aac891f",
|
|
"id": "26323a52990d0cbfc838117109b40e115aac891f"
|
|
},
|
|
"benches": [
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Time",
|
|
"value": 81.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Tokens In",
|
|
"value": 358118.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Time",
|
|
"value": 80.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Tokens In",
|
|
"value": 417408.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Time",
|
|
"value": 81.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Tokens In",
|
|
"value": 254461.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Time",
|
|
"value": 3.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Tokens In",
|
|
"value": 26124.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Time",
|
|
"value": 4.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Tokens In",
|
|
"value": 26336.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Time",
|
|
"value": 121.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 382184.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Time",
|
|
"value": 3.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Tokens In",
|
|
"value": 26079.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Time",
|
|
"value": 5.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Tokens In",
|
|
"value": 26403.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Time",
|
|
"value": 94.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 232924.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Time",
|
|
"value": 58.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Tokens In",
|
|
"value": 283206.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Time",
|
|
"value": 46.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Tokens In",
|
|
"value": 221152.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Time",
|
|
"value": 55.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Tokens In",
|
|
"value": 177955.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Time",
|
|
"value": 42.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Tokens In",
|
|
"value": 203301.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Time",
|
|
"value": 41.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Tokens In",
|
|
"value": 224627.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Time",
|
|
"value": 57.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Tokens In",
|
|
"value": 173641.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Time",
|
|
"value": 65.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Tokens In",
|
|
"value": 272690.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Time",
|
|
"value": 60.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Tokens In",
|
|
"value": 242953.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Time",
|
|
"value": 53.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Tokens In",
|
|
"value": 301308.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Time",
|
|
"value": 12.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Tokens In",
|
|
"value": 85969.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Time",
|
|
"value": 7.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Tokens In",
|
|
"value": 41304.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Time",
|
|
"value": 73.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Tokens In",
|
|
"value": 253202.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Time",
|
|
"value": 55.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Tokens In",
|
|
"value": 314159.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Time",
|
|
"value": 64.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Tokens In",
|
|
"value": 231199.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Time",
|
|
"value": 101.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Tokens In",
|
|
"value": 280020.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Time",
|
|
"value": 48.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Tokens In",
|
|
"value": 269031.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Time",
|
|
"value": 56.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Tokens In",
|
|
"value": 316536.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Time",
|
|
"value": 77.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Tokens In",
|
|
"value": 262956.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Time",
|
|
"value": 77.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Tokens In",
|
|
"value": 312884.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Time",
|
|
"value": 51.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Tokens In",
|
|
"value": 261227.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Time",
|
|
"value": 70.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Tokens In",
|
|
"value": 211994.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Time",
|
|
"value": 46.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Tokens In",
|
|
"value": 251204.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Time",
|
|
"value": 50.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Tokens In",
|
|
"value": 251103.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Time",
|
|
"value": 82.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Tokens In",
|
|
"value": 225541.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Time",
|
|
"value": 47.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Tokens In",
|
|
"value": 222207.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Time",
|
|
"value": 64.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Tokens In",
|
|
"value": 314123.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Time",
|
|
"value": 76.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Tokens In",
|
|
"value": 279924.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Time",
|
|
"value": 68.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Tokens In",
|
|
"value": 364816.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Time",
|
|
"value": 44.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Tokens In",
|
|
"value": 242964.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Time",
|
|
"value": 41.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Tokens In",
|
|
"value": 149657.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Time",
|
|
"value": 53.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Tokens In",
|
|
"value": 272581.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Time",
|
|
"value": 51.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Tokens In",
|
|
"value": 334111.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Time",
|
|
"value": 51.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Tokens In",
|
|
"value": 180136.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Time",
|
|
"value": 74.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Tokens In",
|
|
"value": 326465.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Time",
|
|
"value": 71.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Tokens In",
|
|
"value": 273211.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Time",
|
|
"value": 55.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Tokens In",
|
|
"value": 240518.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Time",
|
|
"value": 58.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Tokens In",
|
|
"value": 233908.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Time",
|
|
"value": 40.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Tokens In",
|
|
"value": 192604.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Time",
|
|
"value": 46.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Tokens In",
|
|
"value": 200951.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Time",
|
|
"value": 45.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Tokens In",
|
|
"value": 230699.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Time",
|
|
"value": 50.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Tokens In",
|
|
"value": 290091.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Time",
|
|
"value": 41.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Tokens In",
|
|
"value": 179169.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Time",
|
|
"value": 13.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Tokens In",
|
|
"value": 82980.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Time",
|
|
"value": 13.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Tokens In",
|
|
"value": 91154.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Time",
|
|
"value": 6.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Tokens In",
|
|
"value": 47127.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 50.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 233703.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 52.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 261452.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 81.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 297207.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 4.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 27667.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 5.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 28044.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 11.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 27266.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 10.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 28364.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 14.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 28634.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 14.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 36452.0
|
|
}
|
|
],
|
|
"model": "gpt-5.6-luna",
|
|
"tool": "customSmallerIsBetter"
|
|
},
|
|
{
|
|
"benches": [
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 53.5,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 320677.0,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 58.5,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 357525.0,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 55.8,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 288853.0,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 10.6,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 58958.0,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 9.7,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 59207.0,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 90.1,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 328504.0,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 19.8,
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 152992.0,
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 16.9,
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 106264.0,
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 61.4,
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 317112.0,
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 46.9,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 224839.0,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 60.2,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 292895.0,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 73.5,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 224198.0,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 39.9,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 251779.0,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 42.9,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 275979.0,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 94.9,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 198193.0,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 44.0,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 330212.0,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 50.6,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 273279.0,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 38.6,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 219370.0,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 9.2,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 62409.0,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 8.8,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 63062.0,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 25.1,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 156857.0,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 53.1,
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 254926.0,
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 50.5,
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 323138.0,
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 44.3,
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 222916.0,
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 47.4,
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 337969.0,
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 43.7,
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 285713.0,
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 45.1,
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 225614.0,
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 38.4,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 249475.0,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 55.1,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 293117.0,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 36.5,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 153056.0,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 54.6,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 304489.0,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 70.5,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 436906.0,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 37.9,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 156973.0,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 27.0,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Time",
|
|
"unit": "seconds",
|
|
"overfitting": "high",
|
|
"notActivated": true,
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"value": 138118.0,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Tokens In",
|
|
"unit": "tokens",
|
|
"overfitting": "high",
|
|
"notActivated": true,
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 24.8,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 175519.0,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 34.9,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 191094.0,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 35.4,
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 210703.0,
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 42.9,
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 258192.0,
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 33.3,
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 178894.0,
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 40.0,
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 261484.0,
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 50.0,
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 346869.0,
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 47.3,
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 244912.0,
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 77.0,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 479689.0,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 69.5,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 407299.0,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 49.3,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 215510.0,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 42.2,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 257459.0,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 47.9,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 304819.0,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 26.4,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 138778.0,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 47.9,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 309048.0,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 41.8,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 272088.0,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 27.5,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 153550.0,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 14.3,
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 92250.0,
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 7.4,
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high",
|
|
"value": 55128.0,
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 9.3,
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 53568.0,
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 59.7,
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 257559.0,
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 59.4,
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 263516.0,
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 78.9,
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 248390.0,
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 7.6,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 42564.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 11.9,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 68392.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 15.7,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 60108.0,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 16.5,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 43361.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 15.5,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"overfittingScore": 0.3,
|
|
"overfitting": "moderate",
|
|
"value": 43949.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"value": 17.5,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"value": 36554.0,
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"unit": "tokens"
|
|
}
|
|
],
|
|
"commit": {
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
},
|
|
"date": 1789642371907,
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "claude-opus-4.8"
|
|
},
|
|
{
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "claude-sonnet-5",
|
|
"benches": [
|
|
{
|
|
"value": 28.1,
|
|
"unit": "seconds",
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Time"
|
|
},
|
|
{
|
|
"value": 190691.0,
|
|
"unit": "tokens",
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Time",
|
|
"value": 40.5,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Tokens In",
|
|
"value": 265158.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Time",
|
|
"value": 89.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Tokens In",
|
|
"value": 326295.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Time",
|
|
"value": 5.8,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Tokens In",
|
|
"value": 40169.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Time",
|
|
"value": 5.6,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Tokens In",
|
|
"value": 40645.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Time",
|
|
"value": 166.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 955289.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Time",
|
|
"value": 6.2,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Tokens In",
|
|
"value": 40229.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Time",
|
|
"value": 3.9,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Tokens In",
|
|
"value": 40461.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Time",
|
|
"value": 101.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 423465.0
|
|
},
|
|
{
|
|
"value": 41.1,
|
|
"unit": "seconds",
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Time"
|
|
},
|
|
{
|
|
"value": 228167.0,
|
|
"unit": "tokens",
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Time",
|
|
"value": 47.3,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Tokens In",
|
|
"value": 278349.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Time",
|
|
"value": 390.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Tokens In",
|
|
"value": 304857.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Time",
|
|
"value": 35.3,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Tokens In",
|
|
"value": 231123.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Time",
|
|
"value": 46.9,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Tokens In",
|
|
"value": 312531.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Time",
|
|
"value": 387.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Tokens In",
|
|
"value": 294993.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Time",
|
|
"value": 46.7,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Tokens In",
|
|
"value": 374515.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Time",
|
|
"value": 51.7,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Tokens In",
|
|
"value": 335777.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Time",
|
|
"value": 288.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Tokens In",
|
|
"value": 354947.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Time",
|
|
"value": 4.9,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Tokens In",
|
|
"value": 40069.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Time",
|
|
"value": 4.8,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Tokens In",
|
|
"value": 40452.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Time",
|
|
"value": 38.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Tokens In",
|
|
"value": 287162.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Time",
|
|
"value": 47.4,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Tokens In",
|
|
"value": 285871.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Time",
|
|
"value": 47.6,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Tokens In",
|
|
"value": 217472.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Time",
|
|
"value": 230.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Tokens In",
|
|
"value": 443706.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Time",
|
|
"value": 58.9,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Tokens In",
|
|
"value": 497201.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Time",
|
|
"value": 60.6,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Tokens In",
|
|
"value": 443894.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Time",
|
|
"value": 215.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Tokens In",
|
|
"value": 346678.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Time",
|
|
"value": 44.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Tokens In",
|
|
"value": 408842.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Time",
|
|
"value": 48.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Tokens In",
|
|
"value": 411467.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Time",
|
|
"value": 32.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Tokens In",
|
|
"value": 235861.0
|
|
},
|
|
{
|
|
"value": 23.9,
|
|
"unit": "seconds",
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Time"
|
|
},
|
|
{
|
|
"value": 149877.0,
|
|
"unit": "tokens",
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5,
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Tokens In"
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Time",
|
|
"value": 66.1,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Tokens In",
|
|
"value": 471935.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Time",
|
|
"value": 41.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Tokens In",
|
|
"value": 246168.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Time",
|
|
"value": 48.8,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Tokens In",
|
|
"value": 345918.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Time",
|
|
"value": 52.8,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Tokens In",
|
|
"value": 272302.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Time",
|
|
"value": 44.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Tokens In",
|
|
"value": 282506.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Time",
|
|
"value": 33.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Tokens In",
|
|
"value": 254784.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Time",
|
|
"value": 48.6,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Tokens In",
|
|
"value": 313474.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Time",
|
|
"value": 24.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Tokens In",
|
|
"value": 156531.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Time",
|
|
"value": 51.8,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Tokens In",
|
|
"value": 333582.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Time",
|
|
"value": 83.2,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Tokens In",
|
|
"value": 722901.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Time",
|
|
"value": 48.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Tokens In",
|
|
"value": 294211.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Time",
|
|
"value": 65.6,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Tokens In",
|
|
"value": 496368.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Time",
|
|
"value": 78.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Tokens In",
|
|
"value": 618405.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Time",
|
|
"value": 73.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Tokens In",
|
|
"value": 475522.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Time",
|
|
"value": 28.9,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Tokens In",
|
|
"value": 181732.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Time",
|
|
"value": 25.1,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Tokens In",
|
|
"value": 183217.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Time",
|
|
"value": 21.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Tokens In",
|
|
"value": 152097.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Time",
|
|
"value": 27.6,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Tokens In",
|
|
"value": 184400.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Time",
|
|
"value": 23.7,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Tokens In",
|
|
"value": 157808.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Time",
|
|
"value": 32.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Tokens In",
|
|
"value": 226256.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Time",
|
|
"value": 11.1,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Tokens In",
|
|
"value": 55216.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Time",
|
|
"value": 11.4,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Tokens In",
|
|
"value": 74911.0,
|
|
"overfitting": "high",
|
|
"overfittingScore": 0.5
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Time",
|
|
"value": 7.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Tokens In",
|
|
"value": 72100.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 49.4,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 263927.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 58.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 366945.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 128.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 662064.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 4.3,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 42556.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 4.4,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 43204.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 13.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 61422.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 10.5,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 43328.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 10.6,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 68743.0,
|
|
"overfitting": "moderate",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 9.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 18549.0
|
|
}
|
|
],
|
|
"commit": {
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"author": {
|
|
"name": "Abhitej John",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"date": 1789745908368
|
|
},
|
|
{
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "gpt-5.6-luna",
|
|
"benches": [
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Time",
|
|
"value": 44.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Tokens In",
|
|
"value": 277237.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Time",
|
|
"value": 47.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Tokens In",
|
|
"value": 318470.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Time",
|
|
"value": 54.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Tokens In",
|
|
"value": 211884.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Time",
|
|
"value": 4.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Tokens In",
|
|
"value": 26048.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Time",
|
|
"value": 3.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Tokens In",
|
|
"value": 26321.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Time",
|
|
"value": 78.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 292363.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Time",
|
|
"value": 3.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Tokens In",
|
|
"value": 26147.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Time",
|
|
"value": 3.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Tokens In",
|
|
"value": 26381.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Time",
|
|
"value": 79.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 349045.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Time",
|
|
"value": 42.1
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Tokens In",
|
|
"value": 248160.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Time",
|
|
"value": 39.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Tokens In",
|
|
"value": 230164.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Time",
|
|
"value": 54.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Tokens In",
|
|
"value": 174400.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Time",
|
|
"value": 43.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Tokens In",
|
|
"value": 298979.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Time",
|
|
"value": 47.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Tokens In",
|
|
"value": 266437.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Time",
|
|
"value": 54.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Tokens In",
|
|
"value": 142288.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Time",
|
|
"value": 57.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Tokens In",
|
|
"value": 308100.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Time",
|
|
"value": 46.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Tokens In",
|
|
"value": 265075.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Time",
|
|
"value": 51.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Tokens In",
|
|
"value": 227119.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Time",
|
|
"value": 4.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Tokens In",
|
|
"value": 41106.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Time",
|
|
"value": 7.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Tokens In",
|
|
"value": 41737.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Time",
|
|
"value": 216.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Tokens In",
|
|
"value": 317712.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Time",
|
|
"value": 31.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Tokens In",
|
|
"value": 213611.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Time",
|
|
"value": 38.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Tokens In",
|
|
"value": 223960.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Time",
|
|
"value": 37.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Tokens In",
|
|
"value": 199846.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Time",
|
|
"value": 45.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Tokens In",
|
|
"value": 311568.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Time",
|
|
"value": 79.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Tokens In",
|
|
"value": 445569.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Time",
|
|
"value": 70.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Tokens In",
|
|
"value": 327297.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Time",
|
|
"value": 40.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Tokens In",
|
|
"value": 307089.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Time",
|
|
"value": 42.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Tokens In",
|
|
"value": 275796.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Time",
|
|
"value": 63.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Tokens In",
|
|
"value": 237934.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Time",
|
|
"value": 54.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Tokens In",
|
|
"value": 322407.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Time",
|
|
"value": 44.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Tokens In",
|
|
"value": 259359.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Time",
|
|
"value": 30.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Tokens In",
|
|
"value": 168920.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Time",
|
|
"value": 38.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Tokens In",
|
|
"value": 223614.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Time",
|
|
"value": 63.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Tokens In",
|
|
"value": 271245.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Time",
|
|
"value": 26.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Tokens In",
|
|
"value": 184453.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Time",
|
|
"value": 44.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Tokens In",
|
|
"value": 351306.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Time",
|
|
"value": 44.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Tokens In",
|
|
"value": 291659.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Time",
|
|
"value": 29.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Tokens In",
|
|
"value": 194610.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Time",
|
|
"value": 59.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Tokens In",
|
|
"value": 280915.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Time",
|
|
"value": 58.6
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Tokens In",
|
|
"value": 317910.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Time",
|
|
"value": 38.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Tokens In",
|
|
"value": 233178.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Time",
|
|
"value": 75.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Tokens In",
|
|
"value": 540069.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Time",
|
|
"value": 69.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Tokens In",
|
|
"value": 411118.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Time",
|
|
"value": 98.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Tokens In",
|
|
"value": 246691.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Time",
|
|
"value": 41.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Tokens In",
|
|
"value": 252764.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Time",
|
|
"value": 50.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Tokens In",
|
|
"value": 202752.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Time",
|
|
"value": 28.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Tokens In",
|
|
"value": 165056.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Time",
|
|
"value": 33.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Tokens In",
|
|
"value": 256651.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Time",
|
|
"value": 34.7
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Tokens In",
|
|
"value": 215514.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Time",
|
|
"value": 39.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Tokens In",
|
|
"value": 267841.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Time",
|
|
"value": 8.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Tokens In",
|
|
"value": 70400.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Time",
|
|
"value": 11.8
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Tokens In",
|
|
"value": 79407.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Time",
|
|
"value": 4.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Tokens In",
|
|
"value": 35598.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 33.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 195788.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 36.9
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 287397.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 65.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 268684.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 5.5
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 27818.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 4.0
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 28183.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 10.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 27699.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 7.3
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 28070.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 7.2
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 28786.0
|
|
},
|
|
{
|
|
"unit": "seconds",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 10.4
|
|
},
|
|
{
|
|
"unit": "tokens",
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 57743.0
|
|
}
|
|
],
|
|
"commit": {
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"author": {
|
|
"name": "Abhitej John",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"date": 1789745908562
|
|
},
|
|
{
|
|
"date": 1789836944911,
|
|
"benches": [
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Time",
|
|
"value": 71.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Tokens In",
|
|
"value": 463392.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 70.7,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 426345.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Time",
|
|
"value": 77.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Tokens In",
|
|
"value": 464740.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 27.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 126911.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 28.9,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 142547.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Time",
|
|
"value": 104.2,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 429334.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 31.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 162393.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 8.9,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 29920.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Time",
|
|
"value": 96.2,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 370098.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 64.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 373834.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 65.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 251111.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Time",
|
|
"value": 66.4,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Tokens In",
|
|
"value": 210092.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 60.9,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 341563.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 59.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 349638.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Time",
|
|
"value": 56.5,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Tokens In",
|
|
"value": 186482.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Time",
|
|
"value": 62.6,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Tokens In",
|
|
"value": 342594.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 75.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 560293.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Time",
|
|
"value": 58.2,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Tokens In",
|
|
"value": 371241.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 81.3,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 500342.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 29.7,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 135123.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Time",
|
|
"value": 208.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Tokens In",
|
|
"value": 1001730.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 57.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 295695.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 72.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 527876.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Time",
|
|
"value": 68.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Tokens In",
|
|
"value": 267320.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 94.6,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 484479.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 65.6,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 378358.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Time",
|
|
"value": 73.6,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Tokens In",
|
|
"value": 342516.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Time",
|
|
"value": 51.4,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Tokens In",
|
|
"value": 330379.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 78.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 477236.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Time",
|
|
"value": 69.9,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Tokens In",
|
|
"value": 458605.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Time",
|
|
"value": 64.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Tokens In",
|
|
"value": 306229.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51,
|
|
"overfitting": "high"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 91.4,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 421805.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Time",
|
|
"value": 66.2,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Tokens In",
|
|
"value": 329045.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 68.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 357203.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 62.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 396645.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Time",
|
|
"value": 60.2,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Tokens In",
|
|
"value": 331505.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 61.4,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 325462.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 73.6,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 492716.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Time",
|
|
"value": 53.8,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Tokens In",
|
|
"value": 210520.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 70.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 349020.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 87.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 491416.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Time",
|
|
"value": 75.9,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Tokens In",
|
|
"value": 307445.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 128.7,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 712675.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 100.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 535596.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Time",
|
|
"value": 93.3,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Tokens In",
|
|
"value": 466114.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 74.1,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 466699.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 70.3,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 392380.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Time",
|
|
"value": 61.8,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Tokens In",
|
|
"value": 467522.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 56.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 354251.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 61.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 428338.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Time",
|
|
"value": 43.5,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Tokens In",
|
|
"value": 236672.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Time",
|
|
"overfitting": "high",
|
|
"value": 23.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Tokens In",
|
|
"overfitting": "high",
|
|
"value": 167245.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Time",
|
|
"overfitting": "high",
|
|
"value": 13.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Tokens In",
|
|
"overfitting": "high",
|
|
"value": 99350.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.51
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Time",
|
|
"value": 4.6,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Tokens In",
|
|
"value": 39643.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 71.0,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 299687.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 83.6,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 322221.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 185.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 461864.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 10.3,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 31645.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 10.9,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 32135.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 9.8,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 13789.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 15.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 32152.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 16.9,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 32749.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 9.3,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 13776.0,
|
|
"unit": "tokens"
|
|
}
|
|
],
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "claude-haiku-4.5"
|
|
},
|
|
{
|
|
"date": 1789836945165,
|
|
"benches": [
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Time",
|
|
"value": 2.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Tokens In",
|
|
"value": 11895.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 7.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 79377.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Time",
|
|
"value": 11.0,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Tokens In",
|
|
"value": 106265.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Time",
|
|
"value": 1.4,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Tokens In",
|
|
"value": 11800.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 2.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 26198.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Time",
|
|
"value": 3.0,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 11635.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 2.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 25962.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 2.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 26229.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Time",
|
|
"value": 9.1,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 74179.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 7.9,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 98339.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 1.6,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 11968.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Time",
|
|
"value": 25.9,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Tokens In",
|
|
"value": 115472.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Time",
|
|
"value": 33.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Tokens In",
|
|
"value": 158651.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 1.7,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 11912.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Time",
|
|
"value": 3.2,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Tokens In",
|
|
"value": 11635.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Time",
|
|
"value": 6.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Tokens In",
|
|
"value": 73390.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 4.4,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 40684.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Time",
|
|
"value": 9.3,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Tokens In",
|
|
"value": 74264.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Time",
|
|
"value": 1.6,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Tokens In",
|
|
"value": 11861.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 2.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 11983.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Time",
|
|
"value": 2.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Tokens In",
|
|
"value": 11668.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Time",
|
|
"value": 1.9,
|
|
"unit": "seconds",
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Tokens In",
|
|
"value": 11873.0,
|
|
"unit": "tokens",
|
|
"notActivated": true
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Time",
|
|
"value": 3.1,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Tokens In",
|
|
"value": 26262.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Time",
|
|
"value": 2.2,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Tokens In",
|
|
"value": 11650.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Time",
|
|
"value": 3.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Tokens In",
|
|
"value": 40341.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Time",
|
|
"value": 20.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Tokens In",
|
|
"value": 163689.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Time",
|
|
"value": 2.4,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Tokens In",
|
|
"value": 11608.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 5.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 40273.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 6.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 55233.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Time",
|
|
"value": 15.1,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Tokens In",
|
|
"value": 138373.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Time",
|
|
"value": 2.1,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Tokens In",
|
|
"value": 11863.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 1.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 11997.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Time",
|
|
"value": 44.6,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Tokens In",
|
|
"value": 173747.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Time",
|
|
"value": 20.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Tokens In",
|
|
"value": 77997.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 10.7,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 88574.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Time",
|
|
"value": 11.5,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Tokens In",
|
|
"value": 102403.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 3.1,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 25925.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 1.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 11926.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Time",
|
|
"value": 10.3,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Tokens In",
|
|
"value": 116992.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 8.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 84585.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 1.4,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 11959.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Time",
|
|
"value": 14.5,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Tokens In",
|
|
"value": 129817.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Time",
|
|
"value": 14.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Tokens In",
|
|
"value": 126276.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 2.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 11949.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Time",
|
|
"value": 8.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Tokens In",
|
|
"value": 65336.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Time",
|
|
"value": 1.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Tokens In",
|
|
"value": 11823.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 1.9,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 11981.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Time",
|
|
"value": 16.1,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Tokens In",
|
|
"value": 147514.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Time",
|
|
"value": 12.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Tokens In",
|
|
"value": 106340.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 1.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 11958.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.2
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Time",
|
|
"value": 12.1,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Tokens In",
|
|
"value": 118107.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Time",
|
|
"value": 1.9,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Tokens In",
|
|
"value": 11822.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Time",
|
|
"value": 1.8,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Plugin Tokens In",
|
|
"value": 11948.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Time",
|
|
"value": 1.4,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Tokens In",
|
|
"value": 11603.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 5.3,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 59139.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"value": 2.3,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"value": 12100.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 2.2,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 11729.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 2.8,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 27497.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"value": 2.8,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"value": 27905.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 2.9,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 11799.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"value": 4.5,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"value": 27803.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"value": 4.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"value": 28097.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 3.3,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 11988.0,
|
|
"unit": "tokens"
|
|
}
|
|
],
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "gpt-5.3-codex"
|
|
},
|
|
{
|
|
"date": 1789836945327,
|
|
"benches": [
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Time",
|
|
"value": 41.7,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Skilled Tokens In",
|
|
"value": 238792.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 41.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 178962.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Time",
|
|
"value": 55.6,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Consolidate duplicate public helpers without breaking shipped callers - Vanilla Tokens In",
|
|
"value": 238566.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Time",
|
|
"value": 73.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Skilled Tokens In",
|
|
"value": 308424.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 50.9,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 297907.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Time",
|
|
"value": 91.2,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a framework and package upgrade dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 226198.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Time",
|
|
"value": 49.1,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Skilled Tokens In",
|
|
"value": 194139.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 38.3,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 219364.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Time",
|
|
"value": 47.8,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Decline a new-feature request dressed up as a refactor - Vanilla Tokens In",
|
|
"value": 167053.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Time",
|
|
"value": 31.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Skilled Tokens In",
|
|
"value": 194520.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 32.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 170194.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Time",
|
|
"value": 43.2,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract shared discount logic without collapsing distinct inputs - Vanilla Tokens In",
|
|
"value": 168862.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Time",
|
|
"value": 32.9,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Skilled Tokens In",
|
|
"value": 176223.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 28.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 172820.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Time",
|
|
"value": 43.0,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Extract subtotal calculation without changing arithmetic - Vanilla Tokens In",
|
|
"value": 132258.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Time",
|
|
"value": 37.9,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Skilled Tokens In",
|
|
"value": 225953.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 33.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 250695.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Time",
|
|
"value": 41.3,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Inline a pass-through wrapper and update callers - Vanilla Tokens In",
|
|
"value": 189743.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Time",
|
|
"value": 41.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Skilled Tokens In",
|
|
"value": 214419.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 43.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 288271.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Time",
|
|
"value": 53.6,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep a shipped nullable contract out of a refactor - Vanilla Tokens In",
|
|
"value": 237914.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Time",
|
|
"value": 42.1,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Skilled Tokens In",
|
|
"value": 219218.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 38.3,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 290267.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Time",
|
|
"value": 34.8,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Keep behavior-changing bug fixes out of a behavior-preserving refactor - Vanilla Tokens In",
|
|
"value": 152729.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Time",
|
|
"value": 55.9,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Skilled Tokens In",
|
|
"value": 291729.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 73.4,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 338232.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Time",
|
|
"value": 65.9,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Merge two near-identical types into one parameterized type - Vanilla Tokens In",
|
|
"value": 278199.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Time",
|
|
"value": 30.6,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Skilled Tokens In",
|
|
"value": 143834.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 40.3,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 183527.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Time",
|
|
"value": 33.6,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a serialized name while renaming a C# property - Vanilla Tokens In",
|
|
"value": 167407.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Time",
|
|
"value": 35.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Skilled Tokens In",
|
|
"value": 166050.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 50.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 246618.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Time",
|
|
"value": 46.5,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Preserve a shipped wrapper while inlining ordinary callers - Vanilla Tokens In",
|
|
"value": 182940.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Time",
|
|
"value": 45.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Skilled Tokens In",
|
|
"value": 264679.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 49.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 204326.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Time",
|
|
"value": 38.5,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a helper used in every target-framework branch - Vanilla Tokens In",
|
|
"value": 178494.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Time",
|
|
"value": 32.3,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Skilled Tokens In",
|
|
"value": 145646.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 26.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 133900.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Time",
|
|
"value": 37.6,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a member declared by a generated partial source - Vanilla Tokens In",
|
|
"value": 178549.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Time",
|
|
"value": 41.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Skilled Tokens In",
|
|
"value": 266702.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 34.7,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 187636.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Time",
|
|
"value": 40.3,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a method without breaking a configured reflection name - Vanilla Tokens In",
|
|
"value": 182566.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Time",
|
|
"value": 66.4,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Skilled Tokens In",
|
|
"value": 370913.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 42.6,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 223543.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Time",
|
|
"value": 49.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename a shipped public method without breaking existing callers - Vanilla Tokens In",
|
|
"value": 266975.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Time",
|
|
"value": 32.1,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Skilled Tokens In",
|
|
"value": 175373.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 31.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 196993.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Time",
|
|
"value": 36.9,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename an internal member used by a friend assembly - Vanilla Tokens In",
|
|
"value": 179802.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Time",
|
|
"value": 35.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Skilled Tokens In",
|
|
"value": 177171.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 42.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 228181.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Time",
|
|
"value": 35.4,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Rename one binding without touching same-text symbols - Vanilla Tokens In",
|
|
"value": 227851.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 112.3,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 636874.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.49
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Time",
|
|
"value": 63.4,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "csharp-refactoring/Stay dormant for an ordinary feature request - Vanilla Tokens In",
|
|
"value": 504461.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Time",
|
|
"value": 34.7,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.33,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "setup-local-sdk/Create team install scripts - Skilled Tokens In",
|
|
"value": 158547.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.33,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 42.5,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 293276.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Time",
|
|
"value": 63.9,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Create team install scripts - Vanilla Tokens In",
|
|
"value": 343749.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Time",
|
|
"value": 7.0,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.33,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"notActivated": true,
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Skilled Tokens In",
|
|
"value": 25678.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.33,
|
|
"overfitting": "moderate"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 8.2,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 26797.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Time",
|
|
"value": 8.4,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Detect incompatible .NET host version - Vanilla Tokens In",
|
|
"value": 26090.0,
|
|
"unit": "tokens"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Time",
|
|
"overfitting": "moderate",
|
|
"value": 10.8,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Skilled Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 27238.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Time",
|
|
"overfitting": "moderate",
|
|
"value": 5.6,
|
|
"unit": "seconds",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Plugin Tokens In",
|
|
"overfitting": "moderate",
|
|
"value": 26001.0,
|
|
"unit": "tokens",
|
|
"overfittingScore": 0.33
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Time",
|
|
"value": 7.7,
|
|
"unit": "seconds"
|
|
},
|
|
{
|
|
"name": "setup-local-sdk/Plan a project-local .NET 11 preview setup without downloading - Vanilla Tokens In",
|
|
"value": 12040.0,
|
|
"unit": "tokens"
|
|
}
|
|
],
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"tool": "customSmallerIsBetter",
|
|
"model": "mai-code-1.1-flash"
|
|
}
|
|
],
|
|
"SkillValue": [
|
|
{
|
|
"judgeModel": "claude-opus-4.8",
|
|
"date": 1788627831193,
|
|
"commit": {
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00"
|
|
},
|
|
"skills": [
|
|
{
|
|
"timedOut": false,
|
|
"pairedN": 3,
|
|
"activationFired": 3,
|
|
"treatment": {
|
|
"timeMs": 5186.666666666667,
|
|
"tokensOut": 256.3333333333333,
|
|
"cacheRead": 25600.0,
|
|
"cacheWrite": 0.0,
|
|
"tokens": 38364.0,
|
|
"n": 3,
|
|
"tokensIn": 38107.666666666664
|
|
},
|
|
"baseAvailable": 3,
|
|
"baseline": {
|
|
"timeMs": 4741.666666666667,
|
|
"tokensOut": 215.0,
|
|
"cacheRead": 3072.0,
|
|
"cacheWrite": 0.0,
|
|
"tokens": 11908.666666666666,
|
|
"n": 3,
|
|
"tokensIn": 11693.666666666666
|
|
},
|
|
"activationExpected": 3,
|
|
"hasPassData": true,
|
|
"treatmentFail": 1,
|
|
"baselineFail": 3,
|
|
"passTotal": 3,
|
|
"treatAvailable": 3,
|
|
"skill": "setup-local-sdk"
|
|
}
|
|
],
|
|
"model": "gpt-5.3-codex"
|
|
},
|
|
{
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"date": 1788627831298,
|
|
"commit": {
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00"
|
|
},
|
|
"skills": [
|
|
{
|
|
"timedOut": false,
|
|
"pairedN": 3,
|
|
"activationFired": 1,
|
|
"treatment": {
|
|
"timeMs": 61245.0,
|
|
"tokensOut": 7065.333333333333,
|
|
"cacheRead": 113834.66666666667,
|
|
"cacheWrite": 0.0,
|
|
"tokens": 139806.66666666666,
|
|
"n": 3,
|
|
"tokensIn": 132741.33333333334
|
|
},
|
|
"baseAvailable": 3,
|
|
"baseline": {
|
|
"timeMs": 56620.333333333336,
|
|
"tokensOut": 6347.0,
|
|
"cacheRead": 114261.33333333333,
|
|
"cacheWrite": 0.0,
|
|
"tokens": 137313.33333333334,
|
|
"n": 3,
|
|
"tokensIn": 130966.33333333333
|
|
},
|
|
"activationExpected": 3,
|
|
"hasPassData": true,
|
|
"treatmentFail": 0,
|
|
"baselineFail": 3,
|
|
"passTotal": 3,
|
|
"treatAvailable": 3,
|
|
"skill": "setup-local-sdk"
|
|
}
|
|
],
|
|
"model": "mai-code-1-flash-picker"
|
|
},
|
|
{
|
|
"commit": {
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
}
|
|
},
|
|
"skills": [
|
|
{
|
|
"skill": "setup-local-sdk",
|
|
"activationExpected": 3,
|
|
"baseline": {
|
|
"timeMs": 70121.0,
|
|
"tokens": 192036.0,
|
|
"cacheRead": 161114.33333333334,
|
|
"n": 3,
|
|
"tokensOut": 5988.0,
|
|
"cacheWrite": 24918.333333333332,
|
|
"tokensIn": 186048.0
|
|
},
|
|
"activationFired": 3,
|
|
"timedOut": false,
|
|
"baseAvailable": 3,
|
|
"baselineFail": 3,
|
|
"treatmentFail": 0,
|
|
"pairedN": 3,
|
|
"hasPassData": true,
|
|
"treatment": {
|
|
"timeMs": 45194.0,
|
|
"tokens": 158346.0,
|
|
"cacheRead": 138155.0,
|
|
"n": 3,
|
|
"tokensOut": 3862.6666666666665,
|
|
"cacheWrite": 16316.333333333334,
|
|
"tokensIn": 154483.33333333334
|
|
},
|
|
"treatAvailable": 3,
|
|
"passTotal": 3
|
|
}
|
|
],
|
|
"date": 1788715662922,
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"model": "claude-opus-5"
|
|
},
|
|
{
|
|
"commit": {
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
}
|
|
},
|
|
"skills": [
|
|
{
|
|
"skill": "setup-local-sdk",
|
|
"activationExpected": 3,
|
|
"baseline": {
|
|
"timeMs": 42301.0,
|
|
"tokens": 158593.66666666666,
|
|
"cacheRead": 133496.33333333334,
|
|
"n": 3,
|
|
"tokensOut": 4092.6666666666665,
|
|
"cacheWrite": 20990.0,
|
|
"tokensIn": 154501.0
|
|
},
|
|
"activationFired": 3,
|
|
"timedOut": false,
|
|
"baseAvailable": 3,
|
|
"baselineFail": 3,
|
|
"treatmentFail": 0,
|
|
"pairedN": 3,
|
|
"hasPassData": true,
|
|
"treatment": {
|
|
"timeMs": 24372.666666666668,
|
|
"tokens": 137023.66666666666,
|
|
"cacheRead": 119047.66666666667,
|
|
"n": 3,
|
|
"tokensOut": 2514.3333333333335,
|
|
"cacheWrite": 15451.0,
|
|
"tokensIn": 134509.33333333334
|
|
},
|
|
"treatAvailable": 3,
|
|
"passTotal": 3
|
|
}
|
|
],
|
|
"date": 1788715663007,
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"model": "claude-sonnet-5"
|
|
},
|
|
{
|
|
"commit": {
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
}
|
|
},
|
|
"skills": [
|
|
{
|
|
"skill": "setup-local-sdk",
|
|
"activationExpected": 3,
|
|
"baseline": {
|
|
"timeMs": 42038.0,
|
|
"tokens": 125938.0,
|
|
"cacheRead": 75434.66666666667,
|
|
"n": 3,
|
|
"tokensOut": 3795.6666666666665,
|
|
"cacheWrite": 0.0,
|
|
"tokensIn": 122142.33333333333
|
|
},
|
|
"activationFired": 3,
|
|
"timedOut": false,
|
|
"baseAvailable": 3,
|
|
"baselineFail": 3,
|
|
"treatmentFail": 2,
|
|
"pairedN": 3,
|
|
"hasPassData": true,
|
|
"treatment": {
|
|
"timeMs": 32443.666666666668,
|
|
"tokens": 82655.33333333333,
|
|
"cacheRead": 64940.666666666664,
|
|
"n": 3,
|
|
"tokensOut": 2687.0,
|
|
"cacheWrite": 7801.0,
|
|
"tokensIn": 79968.33333333333
|
|
},
|
|
"treatAvailable": 3,
|
|
"passTotal": 3
|
|
}
|
|
],
|
|
"date": 1788715663036,
|
|
"judgeModel": "claude-opus-4.8",
|
|
"model": "gpt-5.6-sol"
|
|
},
|
|
{
|
|
"commit": {
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Amaury Levé"
|
|
},
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
},
|
|
"skills": [
|
|
{
|
|
"baseline": {
|
|
"timeMs": 22648.333333333332,
|
|
"tokensOut": 1427.3333333333333,
|
|
"cacheRead": 49922.666666666664,
|
|
"cacheWrite": 14478.333333333334,
|
|
"n": 3,
|
|
"tokensIn": 64407.333333333336,
|
|
"tokens": 65834.66666666667
|
|
},
|
|
"treatAvailable": 3,
|
|
"passTotal": 3,
|
|
"hasPassData": true,
|
|
"activationExpected": 3,
|
|
"pairedN": 3,
|
|
"baseAvailable": 3,
|
|
"treatmentFail": 1,
|
|
"activationFired": 3,
|
|
"timedOut": false,
|
|
"treatment": {
|
|
"timeMs": 22742.333333333332,
|
|
"tokensOut": 1633.0,
|
|
"cacheRead": 60146.333333333336,
|
|
"cacheWrite": 10889.333333333334,
|
|
"n": 3,
|
|
"tokensIn": 71043.66666666667,
|
|
"tokens": 72676.66666666667
|
|
},
|
|
"baselineFail": 3,
|
|
"skill": "setup-local-sdk"
|
|
}
|
|
],
|
|
"date": 1788790038334,
|
|
"model": "claude-sonnet-4.6",
|
|
"judgeModel": "gpt-5.6-terra"
|
|
},
|
|
{
|
|
"commit": {
|
|
"message": "Improve non-passing dotnet-test scenarios (#1114)",
|
|
"id": "ac8f41264bdd557e58924a3110eea8e0917dcf4d",
|
|
"timestamp": "2026-09-04T10:21:25+00:00",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Amaury Levé"
|
|
},
|
|
"url": "https://github.com/dotnet/skills/commit/ac8f41264bdd557e58924a3110eea8e0917dcf4d"
|
|
},
|
|
"skills": [
|
|
{
|
|
"baseline": {
|
|
"timeMs": 40482.666666666664,
|
|
"tokensOut": 3117.0,
|
|
"cacheRead": 69927.0,
|
|
"cacheWrite": 16824.666666666668,
|
|
"n": 3,
|
|
"tokensIn": 86768.66666666667,
|
|
"tokens": 89885.66666666667
|
|
},
|
|
"treatAvailable": 3,
|
|
"passTotal": 3,
|
|
"hasPassData": true,
|
|
"activationExpected": 3,
|
|
"pairedN": 3,
|
|
"baseAvailable": 3,
|
|
"treatmentFail": 1,
|
|
"activationFired": 3,
|
|
"timedOut": false,
|
|
"treatment": {
|
|
"timeMs": 25647.0,
|
|
"tokensOut": 2221.0,
|
|
"cacheRead": 78847.66666666667,
|
|
"cacheWrite": 24954.0,
|
|
"n": 3,
|
|
"tokensIn": 103819.66666666667,
|
|
"tokens": 106040.66666666667
|
|
},
|
|
"baselineFail": 3,
|
|
"skill": "setup-local-sdk"
|
|
}
|
|
],
|
|
"date": 1788790038435,
|
|
"model": "gpt-5.6-luna",
|
|
"judgeModel": "claude-opus-4.8"
|
|
},
|
|
{
|
|
"model": "claude-haiku-4.5",
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"skills": [
|
|
{
|
|
"activationExpected": 3,
|
|
"pairedN": 3,
|
|
"baseAvailable": 3,
|
|
"treatment": {
|
|
"cacheRead": 133215.33333333334,
|
|
"tokensOut": 3360.3333333333335,
|
|
"tokensIn": 147024.0,
|
|
"cacheWrite": 13761.666666666666,
|
|
"timeMs": 40791.666666666664,
|
|
"n": 3,
|
|
"tokens": 150384.33333333334
|
|
},
|
|
"timedOut": false,
|
|
"baseline": {
|
|
"cacheRead": 79659.33333333333,
|
|
"tokensOut": 3769.0,
|
|
"tokensIn": 88289.66666666667,
|
|
"cacheWrite": 8595.0,
|
|
"timeMs": 40420.666666666664,
|
|
"n": 3,
|
|
"tokens": 92058.66666666667
|
|
},
|
|
"treatmentFail": 1,
|
|
"skill": "setup-local-sdk",
|
|
"passTotal": 3,
|
|
"baselineFail": 3,
|
|
"treatAvailable": 3,
|
|
"activationFired": 3,
|
|
"hasPassData": true
|
|
}
|
|
],
|
|
"date": 1788885914243,
|
|
"commit": {
|
|
"id": "fbeeafe261b0fe704b9a95c58fd9fdbe34e96964",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Viktor Hofer"
|
|
},
|
|
"message": "Delete msbuild-server skill (#1123)",
|
|
"timestamp": "2026-09-07T09:11:54+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964"
|
|
}
|
|
},
|
|
{
|
|
"model": "gpt-5.3-codex",
|
|
"judgeModel": "claude-opus-4.8",
|
|
"skills": [
|
|
{
|
|
"activationExpected": 3,
|
|
"pairedN": 3,
|
|
"baseAvailable": 3,
|
|
"treatment": {
|
|
"cacheRead": 35498.666666666664,
|
|
"tokensOut": 325.6666666666667,
|
|
"tokensIn": 48974.0,
|
|
"cacheWrite": 0.0,
|
|
"timeMs": 7075.333333333333,
|
|
"n": 3,
|
|
"tokens": 49299.666666666664
|
|
},
|
|
"timedOut": false,
|
|
"baseline": {
|
|
"cacheRead": 0.0,
|
|
"tokensOut": 236.0,
|
|
"tokensIn": 11693.0,
|
|
"cacheWrite": 0.0,
|
|
"timeMs": 4956.333333333333,
|
|
"n": 3,
|
|
"tokens": 11929.0
|
|
},
|
|
"treatmentFail": 2,
|
|
"skill": "setup-local-sdk",
|
|
"passTotal": 3,
|
|
"baselineFail": 3,
|
|
"treatAvailable": 3,
|
|
"activationFired": 2,
|
|
"hasPassData": true
|
|
}
|
|
],
|
|
"date": 1788885914324,
|
|
"commit": {
|
|
"id": "fbeeafe261b0fe704b9a95c58fd9fdbe34e96964",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Viktor Hofer"
|
|
},
|
|
"message": "Delete msbuild-server skill (#1123)",
|
|
"timestamp": "2026-09-07T09:11:54+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964"
|
|
}
|
|
},
|
|
{
|
|
"model": "mai-code-1-flash-picker",
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"skills": [
|
|
{
|
|
"activationExpected": 3,
|
|
"pairedN": 3,
|
|
"baseAvailable": 3,
|
|
"treatment": {
|
|
"cacheRead": 66688.0,
|
|
"tokensOut": 4614.0,
|
|
"tokensIn": 80510.0,
|
|
"cacheWrite": 0.0,
|
|
"timeMs": 35986.0,
|
|
"n": 3,
|
|
"tokens": 85124.0
|
|
},
|
|
"timedOut": false,
|
|
"baseline": {
|
|
"cacheRead": 111616.0,
|
|
"tokensOut": 6421.333333333333,
|
|
"tokensIn": 129860.33333333333,
|
|
"cacheWrite": 0.0,
|
|
"timeMs": 57310.666666666664,
|
|
"n": 3,
|
|
"tokens": 136281.66666666666
|
|
},
|
|
"treatmentFail": 1,
|
|
"skill": "setup-local-sdk",
|
|
"passTotal": 3,
|
|
"baselineFail": 3,
|
|
"treatAvailable": 3,
|
|
"activationFired": 1,
|
|
"hasPassData": true
|
|
}
|
|
],
|
|
"date": 1788885914378,
|
|
"commit": {
|
|
"id": "fbeeafe261b0fe704b9a95c58fd9fdbe34e96964",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Viktor Hofer"
|
|
},
|
|
"message": "Delete msbuild-server skill (#1123)",
|
|
"timestamp": "2026-09-07T09:11:54+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/fbeeafe261b0fe704b9a95c58fd9fdbe34e96964"
|
|
}
|
|
},
|
|
{
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"message": "Merge pull request #1149 from dotnet/abhitejjohn-vally-014-ci-proof",
|
|
"url": "https://github.com/dotnet/skills/commit/a8fece0fce5f8b0737754a332a1a7c6487e8f927",
|
|
"timestamp": "2026-09-09T17:53:08+00:00",
|
|
"id": "a8fece0fce5f8b0737754a332a1a7c6487e8f927"
|
|
},
|
|
"model": "claude-opus-4.8",
|
|
"skills": [
|
|
{
|
|
"treatmentFail": 0,
|
|
"skill": "setup-local-sdk",
|
|
"hasPassData": false,
|
|
"baseAvailable": 0,
|
|
"baselineFail": 0,
|
|
"activationFired": 0,
|
|
"timedOut": false,
|
|
"treatAvailable": 0,
|
|
"passTotal": 0,
|
|
"baseline": {
|
|
"cacheWrite": null,
|
|
"timeMs": null,
|
|
"n": 0,
|
|
"tokens": null,
|
|
"cacheRead": null,
|
|
"tokensOut": null,
|
|
"tokensIn": null
|
|
},
|
|
"treatment": {
|
|
"cacheWrite": null,
|
|
"timeMs": null,
|
|
"n": 0,
|
|
"tokens": null,
|
|
"cacheRead": null,
|
|
"tokensOut": null,
|
|
"tokensIn": null
|
|
},
|
|
"pairedN": 0,
|
|
"activationExpected": 0
|
|
}
|
|
],
|
|
"date": 1789036360311,
|
|
"judgeModel": "gpt-5.6-terra"
|
|
},
|
|
{
|
|
"model": "gpt-5.6-luna",
|
|
"skills": [
|
|
{
|
|
"treatAvailable": 3,
|
|
"pairedN": 3,
|
|
"skill": "setup-local-sdk",
|
|
"baseAvailable": 3,
|
|
"baselineFail": 3,
|
|
"activationExpected": 3,
|
|
"treatment": {
|
|
"cacheWrite": 17708.0,
|
|
"tokensIn": 77198.33333333333,
|
|
"timeMs": 26731.666666666668,
|
|
"tokens": 79063.66666666667,
|
|
"tokensOut": 1865.3333333333333,
|
|
"cacheRead": 59476.333333333336,
|
|
"n": 3
|
|
},
|
|
"timedOut": false,
|
|
"hasPassData": true,
|
|
"treatmentFail": 1,
|
|
"baseline": {
|
|
"cacheWrite": 16435.333333333332,
|
|
"tokensIn": 91363.33333333333,
|
|
"timeMs": 46655.333333333336,
|
|
"tokens": 94901.33333333333,
|
|
"tokensOut": 3538.0,
|
|
"cacheRead": 74911.0,
|
|
"n": 3
|
|
},
|
|
"activationFired": 3,
|
|
"passTotal": 3
|
|
}
|
|
],
|
|
"judgeModel": "claude-opus-4.8",
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"id": "8a5a42d3e392b402768fc29416831643b79e402b",
|
|
"message": "Merge pull request #1153 from dotnet/abhitejjohn-vally-lock-concurrency-proof",
|
|
"timestamp": "2026-09-10T21:01:20+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/8a5a42d3e392b402768fc29416831643b79e402b"
|
|
},
|
|
"date": 1789129571056
|
|
},
|
|
{
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"skills": [
|
|
{
|
|
"baselineFail": 3,
|
|
"pairedN": 3,
|
|
"baseAvailable": 3,
|
|
"activationFired": 3,
|
|
"skill": "setup-local-sdk",
|
|
"treatment": {
|
|
"cacheRead": 103423.0,
|
|
"tokensIn": 116431.66666666667,
|
|
"tokens": 119639.0,
|
|
"timeMs": 35838.333333333336,
|
|
"cacheWrite": 12968.333333333334,
|
|
"n": 3,
|
|
"tokensOut": 3207.3333333333335
|
|
},
|
|
"activationExpected": 3,
|
|
"timedOut": false,
|
|
"treatAvailable": 3,
|
|
"treatmentFail": 1,
|
|
"passTotal": 3,
|
|
"hasPassData": true,
|
|
"baseline": {
|
|
"cacheRead": 176412.0,
|
|
"tokensIn": 187892.33333333334,
|
|
"tokens": 193494.66666666666,
|
|
"timeMs": 60436.0,
|
|
"cacheWrite": 11428.666666666666,
|
|
"n": 3,
|
|
"tokensOut": 5602.333333333333
|
|
}
|
|
}
|
|
],
|
|
"date": 1789218680118,
|
|
"model": "claude-haiku-4.5",
|
|
"commit": {
|
|
"timestamp": "2026-09-12T00:01:18+00:00",
|
|
"id": "4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7",
|
|
"message": "Merge pull request #1154 from dotnet/abhitejjohn-agentic-workflow-repair",
|
|
"url": "https://github.com/dotnet/skills/commit/4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"judgeModel": "claude-opus-4.8",
|
|
"skills": [
|
|
{
|
|
"baselineFail": 3,
|
|
"pairedN": 3,
|
|
"baseAvailable": 3,
|
|
"activationFired": 3,
|
|
"skill": "setup-local-sdk",
|
|
"treatment": {
|
|
"cacheRead": 30549.333333333332,
|
|
"tokensIn": 43265.666666666664,
|
|
"tokens": 44073.666666666664,
|
|
"timeMs": 7108.333333333333,
|
|
"cacheWrite": 0.0,
|
|
"n": 3,
|
|
"tokensOut": 808.0
|
|
},
|
|
"activationExpected": 3,
|
|
"timedOut": false,
|
|
"treatAvailable": 3,
|
|
"treatmentFail": 1,
|
|
"passTotal": 3,
|
|
"hasPassData": true,
|
|
"baseline": {
|
|
"cacheRead": 3072.0,
|
|
"tokensIn": 11575.666666666666,
|
|
"tokens": 11785.333333333334,
|
|
"timeMs": 4202.0,
|
|
"cacheWrite": 0.0,
|
|
"n": 3,
|
|
"tokensOut": 209.66666666666666
|
|
}
|
|
}
|
|
],
|
|
"date": 1789218680183,
|
|
"model": "gpt-5.3-codex",
|
|
"commit": {
|
|
"timestamp": "2026-09-12T00:01:18+00:00",
|
|
"id": "4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7",
|
|
"message": "Merge pull request #1154 from dotnet/abhitejjohn-agentic-workflow-repair",
|
|
"url": "https://github.com/dotnet/skills/commit/4c72b17fa2c2aaa307d8f1e337ec75cab45ac5c7",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"model": "claude-opus-5",
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"skills": [
|
|
{
|
|
"baseAvailable": 3,
|
|
"treatmentFail": 1,
|
|
"pairedN": 3,
|
|
"hasPassData": true,
|
|
"skill": "setup-local-sdk",
|
|
"treatment": {
|
|
"tokensIn": 132595.0,
|
|
"cacheRead": 116834.0,
|
|
"timeMs": 35912.0,
|
|
"tokens": 135779.0,
|
|
"n": 3,
|
|
"tokensOut": 3184.0,
|
|
"cacheWrite": 15750.333333333334
|
|
},
|
|
"activationExpected": 3,
|
|
"baseline": {
|
|
"tokensIn": 142195.66666666666,
|
|
"cacheRead": 120896.66666666667,
|
|
"timeMs": 56018.0,
|
|
"tokens": 146580.0,
|
|
"n": 3,
|
|
"tokensOut": 4384.333333333333,
|
|
"cacheWrite": 21285.666666666668
|
|
},
|
|
"baselineFail": 3,
|
|
"timedOut": false,
|
|
"passTotal": 3,
|
|
"treatAvailable": 3,
|
|
"activationFired": 3
|
|
}
|
|
],
|
|
"date": 1789318507213,
|
|
"commit": {
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
}
|
|
},
|
|
{
|
|
"model": "claude-sonnet-5",
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"skills": [
|
|
{
|
|
"baseAvailable": 3,
|
|
"treatmentFail": 0,
|
|
"pairedN": 3,
|
|
"hasPassData": true,
|
|
"skill": "setup-local-sdk",
|
|
"treatment": {
|
|
"tokensIn": 182071.66666666666,
|
|
"cacheRead": 165823.66666666666,
|
|
"timeMs": 25060.666666666668,
|
|
"tokens": 184594.66666666666,
|
|
"n": 3,
|
|
"tokensOut": 2523.0,
|
|
"cacheWrite": 16234.0
|
|
},
|
|
"activationExpected": 3,
|
|
"baseline": {
|
|
"tokensIn": 121368.66666666667,
|
|
"cacheRead": 99273.33333333333,
|
|
"timeMs": 36501.0,
|
|
"tokens": 124871.66666666667,
|
|
"n": 3,
|
|
"tokensOut": 3503.0,
|
|
"cacheWrite": 22084.0
|
|
},
|
|
"baselineFail": 3,
|
|
"timedOut": false,
|
|
"passTotal": 3,
|
|
"treatAvailable": 3,
|
|
"activationFired": 3
|
|
}
|
|
],
|
|
"date": 1789318507300,
|
|
"commit": {
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
}
|
|
},
|
|
{
|
|
"model": "gpt-5.6-sol",
|
|
"judgeModel": "claude-opus-4.8",
|
|
"skills": [
|
|
{
|
|
"baseAvailable": 3,
|
|
"treatmentFail": 2,
|
|
"pairedN": 3,
|
|
"hasPassData": true,
|
|
"skill": "setup-local-sdk",
|
|
"treatment": {
|
|
"tokensIn": 110275.33333333333,
|
|
"cacheRead": 80554.66666666667,
|
|
"timeMs": 33719.666666666664,
|
|
"tokens": 112710.0,
|
|
"n": 3,
|
|
"tokensOut": 2434.6666666666665,
|
|
"cacheWrite": 0.0
|
|
},
|
|
"activationExpected": 3,
|
|
"baseline": {
|
|
"tokensIn": 143995.66666666666,
|
|
"cacheRead": 117760.0,
|
|
"timeMs": 56606.0,
|
|
"tokens": 149064.33333333334,
|
|
"n": 3,
|
|
"tokensOut": 5068.666666666667,
|
|
"cacheWrite": 0.0
|
|
},
|
|
"baselineFail": 3,
|
|
"timedOut": false,
|
|
"passTotal": 3,
|
|
"treatAvailable": 3,
|
|
"activationFired": 3
|
|
}
|
|
],
|
|
"date": 1789318507349,
|
|
"commit": {
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3"
|
|
}
|
|
},
|
|
{
|
|
"date": 1789397100668,
|
|
"model": "claude-sonnet-5",
|
|
"commit": {
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
},
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"skills": [
|
|
{
|
|
"hasPassData": true,
|
|
"baselineFail": 3,
|
|
"activationFired": 3,
|
|
"treatment": {
|
|
"tokensOut": 2167.0,
|
|
"n": 3,
|
|
"timeMs": 21154.666666666668,
|
|
"tokens": 125133.33333333333,
|
|
"cacheWrite": 15129.333333333334,
|
|
"cacheRead": 107827.0,
|
|
"tokensIn": 122966.33333333333
|
|
},
|
|
"activationExpected": 3,
|
|
"baseline": {
|
|
"tokensOut": 3976.3333333333335,
|
|
"n": 3,
|
|
"timeMs": 42188.666666666664,
|
|
"tokens": 159973.66666666666,
|
|
"cacheWrite": 10081.0,
|
|
"cacheRead": 145902.33333333334,
|
|
"tokensIn": 155997.33333333334
|
|
},
|
|
"pairedN": 3,
|
|
"baseAvailable": 3,
|
|
"treatmentFail": 0,
|
|
"timedOut": false,
|
|
"skill": "setup-local-sdk",
|
|
"treatAvailable": 3,
|
|
"passTotal": 3
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"date": 1789397100745,
|
|
"model": "gpt-5.6-luna",
|
|
"commit": {
|
|
"url": "https://github.com/dotnet/skills/commit/4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"timestamp": "2026-09-12T07:43:41+00:00",
|
|
"id": "4ecd7d9c76fa458807684771c2bfc7acf1e00ad3",
|
|
"message": "Merge pull request #1134 from dotnet/bot/weekly-version-sync",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
},
|
|
"judgeModel": "claude-opus-4.8",
|
|
"skills": [
|
|
{
|
|
"hasPassData": true,
|
|
"baselineFail": 3,
|
|
"activationFired": 3,
|
|
"treatment": {
|
|
"tokensOut": 2784.0,
|
|
"n": 3,
|
|
"timeMs": 31980.666666666668,
|
|
"tokens": 118393.66666666667,
|
|
"cacheWrite": 19543.333333333332,
|
|
"cacheRead": 96048.33333333333,
|
|
"tokensIn": 115609.66666666667
|
|
},
|
|
"activationExpected": 3,
|
|
"baseline": {
|
|
"tokensOut": 3140.3333333333335,
|
|
"n": 3,
|
|
"timeMs": 31274.0,
|
|
"tokens": 70916.33333333333,
|
|
"cacheWrite": 15623.0,
|
|
"cacheRead": 52139.0,
|
|
"tokensIn": 67776.0
|
|
},
|
|
"pairedN": 3,
|
|
"baseAvailable": 3,
|
|
"treatmentFail": 2,
|
|
"timedOut": false,
|
|
"skill": "setup-local-sdk",
|
|
"treatAvailable": 3,
|
|
"passTotal": 3
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"model": "claude-haiku-4.5",
|
|
"date": 1789478284654,
|
|
"commit": {
|
|
"url": "https://github.com/dotnet/skills/commit/24f7cfbd42ad7bf52bcd67372816b982c38c64c6",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"timestamp": "2026-09-14T15:27:40+00:00",
|
|
"id": "24f7cfbd42ad7bf52bcd67372816b982c38c64c6",
|
|
"message": "Surface activation-only evaluation failures (#1163)"
|
|
},
|
|
"skills": [
|
|
{
|
|
"pairedN": 3,
|
|
"treatment": {
|
|
"tokens": 131912.33333333334,
|
|
"cacheWrite": 13105.666666666666,
|
|
"n": 3,
|
|
"timeMs": 33259.0,
|
|
"tokensIn": 129016.66666666667,
|
|
"cacheRead": 115867.66666666667,
|
|
"tokensOut": 2895.6666666666665
|
|
},
|
|
"baseAvailable": 3,
|
|
"baselineFail": 3,
|
|
"baseline": {
|
|
"tokens": 100044.33333333333,
|
|
"cacheWrite": 7749.0,
|
|
"n": 3,
|
|
"timeMs": 30542.666666666668,
|
|
"tokensIn": 97345.66666666667,
|
|
"cacheRead": 89561.66666666667,
|
|
"tokensOut": 2698.6666666666665
|
|
},
|
|
"activationExpected": 3,
|
|
"activationFired": 3,
|
|
"treatAvailable": 3,
|
|
"skill": "setup-local-sdk",
|
|
"treatmentFail": 0,
|
|
"timedOut": false,
|
|
"hasPassData": true,
|
|
"passTotal": 3
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"judgeModel": "claude-opus-4.8",
|
|
"model": "gpt-5.3-codex",
|
|
"date": 1789478284776,
|
|
"commit": {
|
|
"url": "https://github.com/dotnet/skills/commit/24f7cfbd42ad7bf52bcd67372816b982c38c64c6",
|
|
"author": {
|
|
"name": "Amaury Levé",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"timestamp": "2026-09-14T15:27:40+00:00",
|
|
"id": "24f7cfbd42ad7bf52bcd67372816b982c38c64c6",
|
|
"message": "Surface activation-only evaluation failures (#1163)"
|
|
},
|
|
"skills": [
|
|
{
|
|
"pairedN": 3,
|
|
"treatment": {
|
|
"tokens": 22297.666666666668,
|
|
"cacheWrite": 0.0,
|
|
"n": 3,
|
|
"timeMs": 4082.6666666666665,
|
|
"tokensIn": 22103.0,
|
|
"cacheRead": 11776.0,
|
|
"tokensOut": 194.66666666666666
|
|
},
|
|
"baseAvailable": 3,
|
|
"baselineFail": 3,
|
|
"baseline": {
|
|
"tokens": 11870.666666666666,
|
|
"cacheWrite": 0.0,
|
|
"n": 3,
|
|
"timeMs": 6048.666666666667,
|
|
"tokensIn": 11576.666666666666,
|
|
"cacheRead": 3072.0,
|
|
"tokensOut": 294.0
|
|
},
|
|
"activationExpected": 3,
|
|
"activationFired": 2,
|
|
"treatAvailable": 3,
|
|
"skill": "setup-local-sdk",
|
|
"treatmentFail": 1,
|
|
"timedOut": false,
|
|
"hasPassData": true,
|
|
"passTotal": 3
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"date": 1789574377878,
|
|
"skills": [
|
|
{
|
|
"activationExpected": 17,
|
|
"skill": "csharp-refactoring",
|
|
"baseline": {
|
|
"timeMs": 61756.055555555555,
|
|
"tokensIn": 297852.27777777775,
|
|
"tokens": 301271.8333333333,
|
|
"cacheRead": 282785.3333333333,
|
|
"n": 18,
|
|
"cacheWrite": 15040.833333333334,
|
|
"tokensOut": 3419.5555555555557
|
|
},
|
|
"treatAvailable": 18,
|
|
"treatmentFail": 3,
|
|
"passTotal": 18,
|
|
"hasPassData": true,
|
|
"timedOut": false,
|
|
"pairedN": 18,
|
|
"treatment": {
|
|
"timeMs": 45292.38888888889,
|
|
"tokensIn": 272856.27777777775,
|
|
"tokens": 275474.72222222225,
|
|
"cacheRead": 257696.16666666666,
|
|
"n": 18,
|
|
"cacheWrite": 15137.333333333334,
|
|
"tokensOut": 2618.4444444444443
|
|
},
|
|
"baselineFail": 6,
|
|
"baseAvailable": 18,
|
|
"activationFired": 13
|
|
},
|
|
{
|
|
"activationExpected": 3,
|
|
"skill": "setup-local-sdk",
|
|
"baseline": {
|
|
"timeMs": 44435.333333333336,
|
|
"tokensIn": 177830.33333333334,
|
|
"tokens": 182201.66666666666,
|
|
"cacheRead": 166891.66666666666,
|
|
"n": 3,
|
|
"cacheWrite": 10922.0,
|
|
"tokensOut": 4371.333333333333
|
|
},
|
|
"treatAvailable": 3,
|
|
"treatmentFail": 0,
|
|
"passTotal": 3,
|
|
"hasPassData": true,
|
|
"timedOut": false,
|
|
"pairedN": 3,
|
|
"treatment": {
|
|
"timeMs": 19817.333333333332,
|
|
"tokensIn": 122421.0,
|
|
"tokens": 124691.66666666667,
|
|
"cacheRead": 107152.33333333333,
|
|
"n": 3,
|
|
"cacheWrite": 15258.666666666666,
|
|
"tokensOut": 2270.6666666666665
|
|
},
|
|
"baselineFail": 2,
|
|
"baseAvailable": 3,
|
|
"activationFired": 3
|
|
}
|
|
],
|
|
"commit": {
|
|
"author": {
|
|
"name": "Abhitej John",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"timestamp": "2026-09-15T22:05:10+00:00",
|
|
"message": "Merge pull request #873 from dotnet/add-dotnet-refactoring-skills",
|
|
"url": "https://github.com/dotnet/skills/commit/26323a52990d0cbfc838117109b40e115aac891f",
|
|
"id": "26323a52990d0cbfc838117109b40e115aac891f"
|
|
},
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"model": "claude-sonnet-5"
|
|
},
|
|
{
|
|
"date": 1789574378048,
|
|
"skills": [
|
|
{
|
|
"activationExpected": 17,
|
|
"skill": "csharp-refactoring",
|
|
"baseline": {
|
|
"timeMs": 65942.66666666667,
|
|
"tokensIn": 221158.66666666666,
|
|
"tokens": 224092.66666666666,
|
|
"cacheRead": 200428.83333333334,
|
|
"n": 18,
|
|
"cacheWrite": 20688.833333333332,
|
|
"tokensOut": 2934.0
|
|
},
|
|
"treatAvailable": 18,
|
|
"treatmentFail": 2,
|
|
"passTotal": 18,
|
|
"hasPassData": true,
|
|
"timedOut": false,
|
|
"pairedN": 18,
|
|
"treatment": {
|
|
"timeMs": 47625.444444444445,
|
|
"tokensIn": 227193.27777777778,
|
|
"tokens": 229801.16666666666,
|
|
"cacheRead": 205548.27777777778,
|
|
"n": 18,
|
|
"cacheWrite": 21607.333333333332,
|
|
"tokensOut": 2607.8888888888887
|
|
},
|
|
"baselineFail": 6,
|
|
"baseAvailable": 18,
|
|
"activationFired": 17
|
|
},
|
|
{
|
|
"activationExpected": 3,
|
|
"skill": "setup-local-sdk",
|
|
"baseline": {
|
|
"timeMs": 35750.666666666664,
|
|
"tokensIn": 116958.33333333333,
|
|
"tokens": 120308.33333333333,
|
|
"cacheRead": 100432.0,
|
|
"n": 3,
|
|
"cacheWrite": 16504.333333333332,
|
|
"tokensOut": 3350.0
|
|
},
|
|
"treatAvailable": 3,
|
|
"treatmentFail": 1,
|
|
"passTotal": 3,
|
|
"hasPassData": true,
|
|
"timedOut": false,
|
|
"pairedN": 3,
|
|
"treatment": {
|
|
"timeMs": 21982.333333333332,
|
|
"tokensIn": 94844.0,
|
|
"tokens": 96578.0,
|
|
"cacheRead": 77305.0,
|
|
"n": 3,
|
|
"cacheWrite": 17522.0,
|
|
"tokensOut": 1734.0
|
|
},
|
|
"baselineFail": 2,
|
|
"baseAvailable": 3,
|
|
"activationFired": 3
|
|
}
|
|
],
|
|
"commit": {
|
|
"author": {
|
|
"name": "Abhitej John",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"timestamp": "2026-09-15T22:05:10+00:00",
|
|
"message": "Merge pull request #873 from dotnet/add-dotnet-refactoring-skills",
|
|
"url": "https://github.com/dotnet/skills/commit/26323a52990d0cbfc838117109b40e115aac891f",
|
|
"id": "26323a52990d0cbfc838117109b40e115aac891f"
|
|
},
|
|
"judgeModel": "claude-opus-4.8",
|
|
"model": "gpt-5.6-luna"
|
|
},
|
|
{
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"commit": {
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
}
|
|
},
|
|
"date": 1789642371907,
|
|
"skills": [
|
|
{
|
|
"baselineFail": 7,
|
|
"passTotal": 18,
|
|
"hasPassData": true,
|
|
"activationExpected": 17,
|
|
"baseAvailable": 18,
|
|
"treatment": {
|
|
"cacheRead": 220930.94444444444,
|
|
"timeMs": 38950.27777777778,
|
|
"tokensIn": 236481.77777777778,
|
|
"n": 18,
|
|
"tokensOut": 2266.8888888888887,
|
|
"cacheWrite": 15530.944444444445,
|
|
"tokens": 238748.66666666666
|
|
},
|
|
"baseline": {
|
|
"cacheRead": 186496.11111111112,
|
|
"timeMs": 46182.22222222222,
|
|
"tokensIn": 201631.83333333334,
|
|
"n": 18,
|
|
"tokensOut": 2143.277777777778,
|
|
"cacheWrite": 15115.944444444445,
|
|
"tokens": 203775.11111111112
|
|
},
|
|
"treatAvailable": 18,
|
|
"skill": "csharp-refactoring",
|
|
"activationFired": 16,
|
|
"timedOut": false,
|
|
"pairedN": 18,
|
|
"treatmentFail": 2
|
|
},
|
|
{
|
|
"baselineFail": 3,
|
|
"passTotal": 3,
|
|
"hasPassData": true,
|
|
"activationExpected": 3,
|
|
"baseAvailable": 3,
|
|
"treatment": {
|
|
"cacheRead": 97297.0,
|
|
"timeMs": 27962.666666666668,
|
|
"tokensIn": 112267.33333333333,
|
|
"n": 3,
|
|
"tokensOut": 2227.3333333333335,
|
|
"cacheWrite": 14961.0,
|
|
"tokens": 114494.66666666667
|
|
},
|
|
"baseline": {
|
|
"cacheRead": 102323.66666666667,
|
|
"timeMs": 37375.0,
|
|
"tokensIn": 112304.0,
|
|
"n": 3,
|
|
"tokensOut": 2713.3333333333335,
|
|
"cacheWrite": 9969.0,
|
|
"tokens": 115017.33333333333
|
|
},
|
|
"treatAvailable": 3,
|
|
"skill": "setup-local-sdk",
|
|
"activationFired": 3,
|
|
"timedOut": false,
|
|
"pairedN": 3,
|
|
"treatmentFail": 0
|
|
}
|
|
],
|
|
"model": "claude-opus-4.8"
|
|
},
|
|
{
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"model": "claude-sonnet-5",
|
|
"skills": [
|
|
{
|
|
"hasPassData": true,
|
|
"activationFired": 14,
|
|
"baselineFail": 7,
|
|
"baseAvailable": 18,
|
|
"timedOut": false,
|
|
"baseline": {
|
|
"cacheRead": 303270.22222222225,
|
|
"cacheWrite": 19923.777777777777,
|
|
"tokensIn": 323222.77777777775,
|
|
"n": 18,
|
|
"tokens": 326591.3333333333,
|
|
"tokensOut": 3368.5555555555557,
|
|
"timeMs": 124015.11111111111
|
|
},
|
|
"treatmentFail": 1,
|
|
"treatment": {
|
|
"cacheRead": 224242.38888888888,
|
|
"cacheWrite": 14519.444444444445,
|
|
"tokensIn": 238782.27777777778,
|
|
"n": 18,
|
|
"tokens": 241041.88888888888,
|
|
"tokensOut": 2259.6111111111113,
|
|
"timeMs": 33841.5
|
|
},
|
|
"activationExpected": 17,
|
|
"pairedN": 18,
|
|
"treatAvailable": 18,
|
|
"skill": "csharp-refactoring",
|
|
"passTotal": 18
|
|
},
|
|
{
|
|
"hasPassData": true,
|
|
"activationFired": 3,
|
|
"baselineFail": 3,
|
|
"baseAvailable": 3,
|
|
"timedOut": false,
|
|
"baseline": {
|
|
"cacheRead": 229878.0,
|
|
"cacheWrite": 12444.333333333334,
|
|
"tokensIn": 242342.33333333334,
|
|
"n": 3,
|
|
"tokens": 247345.0,
|
|
"tokensOut": 5002.666666666667,
|
|
"timeMs": 50392.0
|
|
},
|
|
"treatmentFail": 0,
|
|
"treatment": {
|
|
"cacheRead": 98968.0,
|
|
"cacheWrite": 15267.333333333334,
|
|
"tokensIn": 114244.66666666667,
|
|
"n": 3,
|
|
"tokens": 116603.66666666667,
|
|
"tokensOut": 2359.0,
|
|
"timeMs": 21417.333333333332
|
|
},
|
|
"activationExpected": 3,
|
|
"pairedN": 3,
|
|
"treatAvailable": 3,
|
|
"skill": "setup-local-sdk",
|
|
"passTotal": 3
|
|
}
|
|
],
|
|
"commit": {
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"author": {
|
|
"name": "Abhitej John",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"date": 1789745908368
|
|
},
|
|
{
|
|
"judgeModel": "claude-opus-4.8",
|
|
"model": "gpt-5.6-luna",
|
|
"skills": [
|
|
{
|
|
"hasPassData": true,
|
|
"activationFired": 17,
|
|
"baselineFail": 7,
|
|
"baseAvailable": 18,
|
|
"timedOut": false,
|
|
"baseline": {
|
|
"cacheRead": 196276.94444444444,
|
|
"cacheWrite": 21703.222222222223,
|
|
"tokensIn": 218019.66666666666,
|
|
"n": 18,
|
|
"tokens": 220901.94444444444,
|
|
"tokensOut": 2882.277777777778,
|
|
"timeMs": 58749.77777777778
|
|
},
|
|
"treatmentFail": 2,
|
|
"treatment": {
|
|
"cacheRead": 217509.05555555556,
|
|
"cacheWrite": 21790.833333333332,
|
|
"tokensIn": 239338.72222222222,
|
|
"n": 18,
|
|
"tokens": 242009.5,
|
|
"tokensOut": 2670.777777777778,
|
|
"timeMs": 37449.11111111111
|
|
},
|
|
"activationExpected": 17,
|
|
"pairedN": 18,
|
|
"treatAvailable": 18,
|
|
"skill": "csharp-refactoring",
|
|
"passTotal": 18
|
|
},
|
|
{
|
|
"hasPassData": true,
|
|
"activationFired": 3,
|
|
"baselineFail": 2,
|
|
"baseAvailable": 3,
|
|
"timedOut": false,
|
|
"baseline": {
|
|
"cacheRead": 96216.0,
|
|
"cacheWrite": 18698.333333333332,
|
|
"tokensIn": 114937.33333333333,
|
|
"n": 3,
|
|
"tokens": 118042.0,
|
|
"tokensOut": 3104.6666666666665,
|
|
"timeMs": 28702.666666666668
|
|
},
|
|
"treatmentFail": 2,
|
|
"treatment": {
|
|
"cacheRead": 64611.333333333336,
|
|
"cacheWrite": 17473.666666666668,
|
|
"tokensIn": 82100.0,
|
|
"n": 3,
|
|
"tokens": 83892.0,
|
|
"tokensOut": 1792.0,
|
|
"timeMs": 15365.333333333334
|
|
},
|
|
"activationExpected": 3,
|
|
"pairedN": 3,
|
|
"treatAvailable": 3,
|
|
"skill": "setup-local-sdk",
|
|
"passTotal": 3
|
|
}
|
|
],
|
|
"commit": {
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"author": {
|
|
"name": "Abhitej John",
|
|
"username": "AbhitejJohn"
|
|
},
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"date": 1789745908562
|
|
},
|
|
{
|
|
"skills": [
|
|
{
|
|
"treatment": {
|
|
"cacheRead": 335826.0,
|
|
"tokensOut": 4269.277777777777,
|
|
"timeMs": 63859.166666666664,
|
|
"tokensIn": 354639.94444444444,
|
|
"n": 18,
|
|
"tokens": 358909.22222222225,
|
|
"cacheWrite": 18709.88888888889
|
|
},
|
|
"treatAvailable": 18,
|
|
"skill": "csharp-refactoring",
|
|
"activationExpected": 17,
|
|
"baselineFail": 9,
|
|
"treatmentFail": 5,
|
|
"hasPassData": true,
|
|
"baseline": {
|
|
"cacheRead": 337804.6666666667,
|
|
"tokensOut": 4831.222222222223,
|
|
"timeMs": 74406.05555555556,
|
|
"tokensIn": 355759.0,
|
|
"n": 18,
|
|
"tokens": 360590.22222222225,
|
|
"cacheWrite": 17849.166666666668
|
|
},
|
|
"activationFired": 13,
|
|
"timedOut": false,
|
|
"baseAvailable": 18,
|
|
"passTotal": 18,
|
|
"pairedN": 18
|
|
},
|
|
{
|
|
"treatment": {
|
|
"cacheRead": 105593.66666666667,
|
|
"tokensOut": 2925.0,
|
|
"timeMs": 32344.0,
|
|
"tokensIn": 118236.33333333333,
|
|
"n": 3,
|
|
"tokens": 121161.33333333333,
|
|
"cacheWrite": 12601.666666666666
|
|
},
|
|
"treatAvailable": 3,
|
|
"skill": "setup-local-sdk",
|
|
"activationExpected": 3,
|
|
"baselineFail": 3,
|
|
"treatmentFail": 1,
|
|
"hasPassData": true,
|
|
"baseline": {
|
|
"cacheRead": 142610.0,
|
|
"tokensOut": 6820.333333333333,
|
|
"timeMs": 68268.0,
|
|
"tokensIn": 156322.66666666666,
|
|
"n": 3,
|
|
"tokens": 163143.0,
|
|
"cacheWrite": 13670.0
|
|
},
|
|
"activationFired": 3,
|
|
"timedOut": false,
|
|
"baseAvailable": 3,
|
|
"passTotal": 3,
|
|
"pairedN": 3
|
|
}
|
|
],
|
|
"date": 1789836944911,
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"model": "claude-haiku-4.5"
|
|
},
|
|
{
|
|
"skills": [
|
|
{
|
|
"treatment": {
|
|
"cacheRead": 39281.77777777778,
|
|
"tokensOut": 351.05555555555554,
|
|
"timeMs": 7262.611111111111,
|
|
"tokensIn": 51927.61111111111,
|
|
"n": 18,
|
|
"tokens": 52278.666666666664,
|
|
"cacheWrite": 0.0
|
|
},
|
|
"treatAvailable": 18,
|
|
"skill": "csharp-refactoring",
|
|
"activationExpected": 17,
|
|
"baselineFail": 15,
|
|
"treatmentFail": 14,
|
|
"hasPassData": true,
|
|
"baseline": {
|
|
"cacheRead": 64369.77777777778,
|
|
"tokensOut": 619.7222222222222,
|
|
"timeMs": 11284.777777777777,
|
|
"tokensIn": 78950.72222222222,
|
|
"n": 18,
|
|
"tokens": 79570.44444444444,
|
|
"cacheWrite": 0.0
|
|
},
|
|
"activationFired": 6,
|
|
"timedOut": false,
|
|
"baseAvailable": 18,
|
|
"passTotal": 18,
|
|
"pairedN": 18
|
|
},
|
|
{
|
|
"treatment": {
|
|
"cacheRead": 25258.666666666668,
|
|
"tokensOut": 324.3333333333333,
|
|
"timeMs": 4201.333333333333,
|
|
"tokensIn": 37822.0,
|
|
"n": 3,
|
|
"tokens": 38146.333333333336,
|
|
"cacheWrite": 0.0
|
|
},
|
|
"treatAvailable": 3,
|
|
"skill": "setup-local-sdk",
|
|
"activationExpected": 3,
|
|
"baselineFail": 3,
|
|
"treatmentFail": 2,
|
|
"hasPassData": true,
|
|
"baseline": {
|
|
"cacheRead": 4608.0,
|
|
"tokensOut": 262.3333333333333,
|
|
"timeMs": 2778.6666666666665,
|
|
"tokensIn": 11576.333333333334,
|
|
"n": 3,
|
|
"tokens": 11838.666666666666,
|
|
"cacheWrite": 0.0
|
|
},
|
|
"activationFired": 3,
|
|
"timedOut": false,
|
|
"baseAvailable": 3,
|
|
"passTotal": 3,
|
|
"pairedN": 3
|
|
}
|
|
],
|
|
"date": 1789836945165,
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"judgeModel": "claude-opus-4.8",
|
|
"model": "gpt-5.3-codex"
|
|
},
|
|
{
|
|
"skills": [
|
|
{
|
|
"treatment": {
|
|
"cacheRead": 208682.66666666666,
|
|
"tokensOut": 4181.055555555556,
|
|
"timeMs": 46471.555555555555,
|
|
"tokensIn": 240855.55555555556,
|
|
"n": 18,
|
|
"tokens": 245036.61111111112,
|
|
"cacheWrite": 0.0
|
|
},
|
|
"treatAvailable": 18,
|
|
"skill": "csharp-refactoring",
|
|
"activationExpected": 17,
|
|
"baselineFail": 8,
|
|
"treatmentFail": 9,
|
|
"hasPassData": true,
|
|
"baseline": {
|
|
"cacheRead": 181184.0,
|
|
"tokensOut": 4141.388888888889,
|
|
"timeMs": 47683.27777777778,
|
|
"tokensIn": 210334.55555555556,
|
|
"n": 18,
|
|
"tokens": 214475.94444444444,
|
|
"cacheWrite": 0.0
|
|
},
|
|
"activationFired": 0,
|
|
"timedOut": false,
|
|
"baseAvailable": 18,
|
|
"passTotal": 18,
|
|
"pairedN": 18
|
|
},
|
|
{
|
|
"treatment": {
|
|
"cacheRead": 49920.0,
|
|
"tokensOut": 4085.6666666666665,
|
|
"timeMs": 17467.333333333332,
|
|
"tokensIn": 66402.0,
|
|
"n": 3,
|
|
"tokens": 70487.66666666667,
|
|
"cacheWrite": 0.0
|
|
},
|
|
"treatAvailable": 3,
|
|
"skill": "setup-local-sdk",
|
|
"activationExpected": 3,
|
|
"baselineFail": 3,
|
|
"treatmentFail": 1,
|
|
"hasPassData": true,
|
|
"baseline": {
|
|
"cacheRead": 105557.33333333333,
|
|
"tokensOut": 6352.333333333333,
|
|
"timeMs": 26656.333333333332,
|
|
"tokensIn": 120940.66666666667,
|
|
"n": 3,
|
|
"tokens": 127293.0,
|
|
"cacheWrite": 0.0
|
|
},
|
|
"activationFired": 1,
|
|
"timedOut": false,
|
|
"baseAvailable": 3,
|
|
"passTotal": 3,
|
|
"pairedN": 3
|
|
}
|
|
],
|
|
"date": 1789836945327,
|
|
"commit": {
|
|
"author": {
|
|
"username": "AbhitejJohn",
|
|
"name": "Abhitej John"
|
|
},
|
|
"message": "Merge pull request #1184 from dotnet/dependabot/npm_and_yarn/eng/evaluation-tools/microsoft/vally-cli-0.16.0",
|
|
"url": "https://github.com/dotnet/skills/commit/8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3",
|
|
"timestamp": "2026-09-17T03:09:52+00:00",
|
|
"id": "8bbfe7a4d1c5c0cd42cd04e38031779c75f2dda3"
|
|
},
|
|
"judgeModel": "gpt-5.6-terra",
|
|
"model": "mai-code-1.1-flash"
|
|
}
|
|
]
|
|
},
|
|
"lastUpdate": 1789836945327,
|
|
"schemaVersion": 2
|
|
}
|