mirror of
https://github.com/dotnet/skills.git
synced 2026-09-20 09:49:54 +08:00
Viktor Hofer
3fff1f2573
* Skill-validator: perf, cancellation, and reliability improvements Performance: - Cache skill-validator-dist.tar.gz in CI (evaluation.yml and skill-check.yml) to skip .NET SDK setup, build, and tar on cache hit. Evaluate and comment-on-pr jobs restore from cache with artifact download fallback. - Replace SemaphoreSlim with Lock in SessionDatabase for lighter-weight synchronous locking. Cancellation: - Thread CancellationToken from System.CommandLine entry point through the entire evaluate call chain: Run -> EvaluateTarget -> EvaluateSkill/EvaluateAgent -> ExecuteScenario/ExecuteAgentScenario -> ExecuteRun/ExecuteAgentRun, plus EvaluateSkillNoise -> ExecuteNoiseTest. - Add CancellationToken parameter to AgentRunner.RunAgent, Judge.JudgeRun, PairwiseJudge.Judge, and OverfittingJudge.Analyze, forwarding to RetryHelper and LlmSession.SendAsync respectively. - Pass CancellationToken to all ConcurrencyLimiter.RunAsync call sites so semaphore waits are cancellable. - Re-throw OperationCanceledException at the top-level skill evaluation loop so Ctrl+C / CI cancellation propagates instead of being swallowed as a skill failure. Reliability: - Smart timeline truncation in Judge and PairwiseJudge: error events (session.error, runner.error) are always preserved during truncation regardless of position, with a summary marker showing event type breakdown. - Add jitter (±25%) to RetryHelper exponential backoff to prevent thundering herd on rate-limit recovery. - Add CancellationToken parameter to ConcurrencyLimiter.RunAsync<T>. - Add ConcurrencyLimiter tests (6 tests) and RetryHelper jitter tests (2 tests). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address PR review feedback - RetryHelper: re-clamp delay after jitter so it never exceeds the remaining budget or MaxSingleDelayMs. - Judge/PairwiseJudge: handle edge case where error count alone exceeds MaxTimelineEvents by trimming errors with a summary instead of unconditionally appending all of them. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Add high-variance detection and refactor FailureKind to enum - Add Statistics.CoefficientOfVariation() for measuring score consistency - Flag scenarios with CV > 0.5 as high-variance in ScenarioComparison - Show console warning and markdown footnote for high-variance scenarios - Include high-variance scenario names in Comparator verdict reason - Refactor FailureKind from magic strings to strongly-typed enum with JsonStringEnumMemberName attributes for backwards-compatible JSON output - Add 6 new CV unit tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Viktor Hofer <vihofer@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1112 lines
43 KiB
C#
1112 lines
43 KiB
C#
using System.Text.Json;
|
|
using SkillValidator.Evaluate;
|
|
using SkillValidator.Shared;
|
|
|
|
namespace SkillValidator.Tests;
|
|
|
|
public class OverfittingJudgeTests
|
|
{
|
|
// --- Score computation tests ---
|
|
|
|
[Fact]
|
|
public void ComputeScore_AllOutcomeBroad_ReturnsZero()
|
|
{
|
|
var rubric = new List<RubricOverfitAssessment>
|
|
{
|
|
new("sc1", "criterion1", "outcome", 0.9, "Good outcome test"),
|
|
new("sc1", "criterion2", "outcome", 0.8, "Another outcome test"),
|
|
};
|
|
var assertions = new List<AssertionOverfitAssessment>
|
|
{
|
|
new("sc1", "file_exists: *.binlog", "broad", 0.9, "Checks file existence"),
|
|
};
|
|
|
|
var score = OverfittingJudge.ComputeOverfittingScore(rubric, assertions);
|
|
Assert.Equal(0.0, score);
|
|
}
|
|
|
|
[Fact]
|
|
public void ComputeScore_AllVocabularyNarrow_ReturnsHigh()
|
|
{
|
|
var rubric = new List<RubricOverfitAssessment>
|
|
{
|
|
new("sc1", "Uses --clreventlevel flag", "vocabulary", 0.9, "Tests exact flag name"),
|
|
new("sc1", "Checked GC Heap Size > 500MB", "vocabulary", 0.8, "Tests exact counter"),
|
|
};
|
|
var assertions = new List<AssertionOverfitAssessment>
|
|
{
|
|
new("sc1", "output_matches: (-bl:\\{\\{\\}\\})", "narrow", 0.95, "Tests specific escaping"),
|
|
};
|
|
|
|
var score = OverfittingJudge.ComputeOverfittingScore(rubric, assertions);
|
|
// rubricAvg = (1.0*0.9 + 1.0*0.8) / 2 = 0.85
|
|
// assertionAvg = (1.0*0.95) / 1 = 0.95
|
|
// combined = 0.7*0.85 + 0.3*0.95 = 0.595 + 0.285 = 0.88
|
|
Assert.True(score > 0.5, $"Expected high score, got {score}");
|
|
}
|
|
|
|
[Fact]
|
|
public void ComputeScore_MixedClassifications_ReturnsMedium()
|
|
{
|
|
var rubric = new List<RubricOverfitAssessment>
|
|
{
|
|
new("sc1", "Identified root cause", "outcome", 0.9, "Good"),
|
|
new("sc1", "Built twice to check", "technique", 0.7, "Diagnostic step"),
|
|
new("sc2", "Used specific label", "vocabulary", 0.8, "Exact wording"),
|
|
};
|
|
var assertions = new List<AssertionOverfitAssessment>
|
|
{
|
|
new("sc1", "file_exists: *.binlog", "broad", 0.9, "Outcome check"),
|
|
new("sc2", "output_matches: specific-pattern", "narrow", 0.85, "Narrow match"),
|
|
};
|
|
|
|
var score = OverfittingJudge.ComputeOverfittingScore(rubric, assertions);
|
|
// rubricAvg = (0 + 0.5*0.7 + 1.0*0.8) / 3 = (0 + 0.35 + 0.8) / 3 = 0.3833
|
|
// assertionAvg = (0 + 1.0*0.85) / 2 = 0.425
|
|
// combined = 0.7*0.3833 + 0.3*0.425 = 0.26833 + 0.1275 = 0.3958
|
|
Assert.True(score > 0.2 && score < 0.5, $"Expected moderate score, got {score}");
|
|
}
|
|
|
|
[Fact]
|
|
public void ComputeScore_EmptyInputs_ReturnsZero()
|
|
{
|
|
var score = OverfittingJudge.ComputeOverfittingScore(
|
|
new List<RubricOverfitAssessment>(),
|
|
new List<AssertionOverfitAssessment>());
|
|
Assert.Equal(0.0, score);
|
|
}
|
|
|
|
[Fact]
|
|
public void ComputeScore_TechniqueOnly_ReturnsMedium()
|
|
{
|
|
var rubric = new List<RubricOverfitAssessment>
|
|
{
|
|
new("sc1", "Ran dotnet-counters monitor", "technique", 1.0, "Specific tool"),
|
|
};
|
|
var assertions = new List<AssertionOverfitAssessment>();
|
|
|
|
var score = OverfittingJudge.ComputeOverfittingScore(rubric, assertions);
|
|
// rubricAvg = 0.5 * 1.0 / 1 = 0.5
|
|
// assertionAvg = 0 (no assertions)
|
|
// combined = 0.7 * 0.5 + 0.3 * 0 = 0.35
|
|
Assert.Equal(0.35, score, 2);
|
|
}
|
|
|
|
// --- JSON response parsing tests ---
|
|
|
|
private static readonly string ValidOverfittingJson = JsonSerializer.Serialize(new
|
|
{
|
|
rubric_assessments = new[]
|
|
{
|
|
new
|
|
{
|
|
scenario = "generate-unique-binlog",
|
|
criterion = "Used -bl:{} for unique binlog names",
|
|
classification = "outcome",
|
|
confidence = 0.85,
|
|
reasoning = "This is genuinely the only way to do it"
|
|
},
|
|
new
|
|
{
|
|
scenario = "generate-unique-binlog",
|
|
criterion = "PowerShell escaping {{}}",
|
|
classification = "vocabulary",
|
|
confidence = 0.9,
|
|
reasoning = "Tests shell escaping LLMs already know"
|
|
}
|
|
},
|
|
assertion_assessments = new[]
|
|
{
|
|
new
|
|
{
|
|
scenario = "generate-unique-binlog",
|
|
assertion_summary = "file_exists: *.binlog",
|
|
classification = "broad",
|
|
confidence = 0.95,
|
|
reasoning = "Tests the outcome"
|
|
},
|
|
new
|
|
{
|
|
scenario = "generate-unique-binlog",
|
|
assertion_summary = "output_matches: (-bl:\\{\\{\\}\\})",
|
|
classification = "narrow",
|
|
confidence = 0.9,
|
|
reasoning = "Tests specific syntax"
|
|
}
|
|
},
|
|
cross_scenario_issues = new[] { "Repetitive testing of shell escaping across scenarios" },
|
|
overall_overfitting_score = 0.45,
|
|
overall_reasoning = "The eval has moderate overfitting due to vocabulary testing."
|
|
});
|
|
|
|
[Fact]
|
|
public void ParseResponse_ValidJson_ParsesCorrectly()
|
|
{
|
|
var result = OverfittingJudge.ParseOverfittingResponse(ValidOverfittingJson);
|
|
|
|
Assert.Equal(2, result.RubricAssessments.Count);
|
|
Assert.Equal(2, result.AssertionAssessments.Count);
|
|
Assert.Single(result.CrossScenarioIssues);
|
|
Assert.NotEmpty(result.OverallReasoning);
|
|
}
|
|
|
|
[Fact]
|
|
public void ParseResponse_ValidJson_ComputesBlendedScore()
|
|
{
|
|
var result = OverfittingJudge.ParseOverfittingResponse(ValidOverfittingJson);
|
|
|
|
// Computed: rubricAvg = (0*0.85 + 1.0*0.9) / 2 = 0.45
|
|
// assertionAvg = (0*0.95 + 1.0*0.9) / 2 = 0.45
|
|
// computed = 0.7*0.45 + 0.3*0.45 = 0.45
|
|
// LLM overall = 0.45
|
|
// Final = 0.6*0.45 + 0.4*0.45 = 0.45
|
|
Assert.True(result.Score >= 0.0 && result.Score <= 1.0);
|
|
}
|
|
|
|
[Fact]
|
|
public void ParseResponse_InCodeBlock_ParsesCorrectly()
|
|
{
|
|
var content = "```json\n" + ValidOverfittingJson + "\n```";
|
|
var result = OverfittingJudge.ParseOverfittingResponse(content);
|
|
|
|
Assert.Equal(2, result.RubricAssessments.Count);
|
|
Assert.Equal(2, result.AssertionAssessments.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public void ParseResponse_WithSurroundingText_ParsesCorrectly()
|
|
{
|
|
var content = "Here is my analysis:\n\n" + ValidOverfittingJson + "\n\nThat concludes the assessment.";
|
|
var result = OverfittingJudge.ParseOverfittingResponse(content);
|
|
|
|
Assert.Equal(2, result.RubricAssessments.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public void ParseResponse_NoJson_Throws()
|
|
{
|
|
Assert.Throws<InvalidOperationException>(() =>
|
|
OverfittingJudge.ParseOverfittingResponse("No JSON here at all"));
|
|
}
|
|
|
|
// --- Severity mapping tests ---
|
|
|
|
[Theory]
|
|
[InlineData(0.0, OverfittingSeverity.Low)]
|
|
[InlineData(0.10, OverfittingSeverity.Low)]
|
|
[InlineData(0.19, OverfittingSeverity.Low)]
|
|
[InlineData(0.20, OverfittingSeverity.Moderate)]
|
|
[InlineData(0.35, OverfittingSeverity.Moderate)]
|
|
[InlineData(0.49, OverfittingSeverity.Moderate)]
|
|
[InlineData(0.50, OverfittingSeverity.High)]
|
|
[InlineData(0.75, OverfittingSeverity.High)]
|
|
[InlineData(1.0, OverfittingSeverity.High)]
|
|
public void SeverityMapping_CorrectThresholds(double score, OverfittingSeverity expected)
|
|
{
|
|
// Build a response where both computed and LLM overall equal the target score
|
|
// With no rubric/assertions, computed = 0, final = 0.4 * clamp(llmScore, 0, 1)
|
|
// To get exact score: we need computed portion + LLM portion
|
|
// Use rubric items to produce the exact computed score we want
|
|
var rubric = new List<object>();
|
|
if (score > 0)
|
|
{
|
|
// One vocabulary item with confidence = score (produces computedScore = 0.7*score)
|
|
// And set LLM overall to score (produces 0.4*score)
|
|
// final = 0.6*(0.7*score) + 0.4*score = 0.42*score + 0.4*score = 0.82*score
|
|
// That's not right either. Just test the severity at the boundary.
|
|
}
|
|
|
|
var json = JsonSerializer.Serialize(new
|
|
{
|
|
rubric_assessments = Array.Empty<object>(),
|
|
assertion_assessments = Array.Empty<object>(),
|
|
cross_scenario_issues = Array.Empty<string>(),
|
|
overall_overfitting_score = Math.Min(score / 0.4, 1.0),
|
|
overall_reasoning = "test"
|
|
});
|
|
|
|
var result = OverfittingJudge.ParseOverfittingResponse(json);
|
|
// final = 0.6 * 0 + 0.4 * min(score/0.4, 1.0)
|
|
// For score <= 0.4: final = score
|
|
// For score > 0.4: final = 0.4 (clamped)
|
|
// This test verifies the score is valid and severity mapping holds for achievable scores
|
|
Assert.True(result.Score >= 0.0 && result.Score <= 1.0);
|
|
if (score <= 0.4)
|
|
{
|
|
Assert.Equal(expected, result.Severity);
|
|
}
|
|
}
|
|
|
|
// --- OverfittingResult serialization ---
|
|
|
|
[Fact]
|
|
public void OverfittingResult_SerializesToJson_WithStringSeverity()
|
|
{
|
|
var result = new OverfittingResult(
|
|
0.55,
|
|
OverfittingSeverity.High,
|
|
new List<RubricOverfitAssessment>
|
|
{
|
|
new("sc1", "test criterion", "vocabulary", 0.9, "reason")
|
|
},
|
|
new List<AssertionOverfitAssessment>
|
|
{
|
|
new("sc1", "output_matches: pattern", "narrow", 0.85, "reason")
|
|
},
|
|
new List<PromptOverfitAssessment>(),
|
|
new List<string> { "cross-scenario issue" },
|
|
"Overall reasoning"
|
|
);
|
|
|
|
var json = JsonSerializer.Serialize(result, new JsonSerializerOptions
|
|
{
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
|
});
|
|
|
|
Assert.Contains("\"severity\":\"High\"", json);
|
|
Assert.Contains("\"score\":0.55", json);
|
|
}
|
|
|
|
// --- Prompt building tests ---
|
|
|
|
[Fact]
|
|
public void BuildSystemPrompt_ContainsKeyElements()
|
|
{
|
|
var prompt = OverfittingJudge.BuildSystemPrompt();
|
|
|
|
Assert.Contains("DOMAIN EXPERT TEST", prompt);
|
|
Assert.Contains("LLM KNOWLEDGE TEST", prompt);
|
|
Assert.Contains("outcome", prompt);
|
|
Assert.Contains("technique", prompt);
|
|
Assert.Contains("vocabulary", prompt);
|
|
Assert.Contains("broad", prompt);
|
|
Assert.Contains("narrow", prompt);
|
|
Assert.Contains("Few-shot examples", prompt);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task BuildUserPrompt_IncludesSkillAndEvalContent()
|
|
{
|
|
var skill = new SkillInfo(
|
|
Name: "test-skill",
|
|
Description: "A test skill",
|
|
Path: "/skills/test-skill",
|
|
SkillMdPath: "/skills/test-skill/SKILL.md",
|
|
SkillMdContent: "# Test Skill\nThis teaches something.");
|
|
var evalSkill = new EvalSkillInfo(
|
|
Skill: skill,
|
|
EvalPath: null,
|
|
EvalConfig: new EvalConfig(new List<EvalScenario>
|
|
{
|
|
new("scenario1", "Do something",
|
|
Rubric: new List<string> { "Did the thing correctly" })
|
|
}));
|
|
|
|
var prompt = await OverfittingJudge.BuildUserPromptAsync(evalSkill);
|
|
|
|
Assert.Contains("SKILL_CONTENT_START", prompt);
|
|
Assert.Contains("SKILL_CONTENT_END", prompt);
|
|
Assert.Contains("EVAL_CONTENT_START", prompt);
|
|
Assert.Contains("EVAL_CONTENT_END", prompt);
|
|
Assert.Contains("test-skill", prompt);
|
|
Assert.Contains("Test Skill", prompt);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task BuildUserPrompt_TruncatesLargeSkillContent()
|
|
{
|
|
var largeContent = new string('x', 50_000);
|
|
var skill = new SkillInfo(
|
|
Name: "large-skill",
|
|
Description: "A large skill",
|
|
Path: "/skills/large-skill",
|
|
SkillMdPath: "/skills/large-skill/SKILL.md",
|
|
SkillMdContent: largeContent);
|
|
var evalSkill = new EvalSkillInfo(
|
|
Skill: skill,
|
|
EvalPath: null,
|
|
EvalConfig: new EvalConfig(new List<EvalScenario>()));
|
|
|
|
var prompt = await OverfittingJudge.BuildUserPromptAsync(evalSkill);
|
|
|
|
Assert.Contains("TRUNCATED", prompt);
|
|
Assert.Contains("large-skill/SKILL.md", prompt);
|
|
}
|
|
|
|
// --- Markdown table integration ---
|
|
|
|
[Fact]
|
|
public void MarkdownTable_IncludesOverfitColumn()
|
|
{
|
|
var verdicts = new List<SkillVerdict>
|
|
{
|
|
new()
|
|
{
|
|
SkillName = "test-skill",
|
|
SkillPath = "/test",
|
|
Passed = true,
|
|
Scenarios = new List<ScenarioComparison>
|
|
{
|
|
new()
|
|
{
|
|
ScenarioName = "sc1",
|
|
Baseline = new RunResult(
|
|
new RunMetrics { AgentOutput = "baseline" },
|
|
new JudgeResult(new List<RubricScore>(), 3.5, "OK")),
|
|
SkilledIsolated = new RunResult(
|
|
new RunMetrics { AgentOutput = "skilled" },
|
|
new JudgeResult(new List<RubricScore>(), 4.5, "Good")),
|
|
ImprovementScore = 0.25,
|
|
Breakdown = new MetricBreakdown(0, 0, 0, 0, 0, 0, 0),
|
|
}
|
|
},
|
|
OverallImprovementScore = 0.25,
|
|
Reason = "Pass",
|
|
OverfittingResult = new OverfittingResult(
|
|
0.38,
|
|
OverfittingSeverity.Moderate,
|
|
new List<RubricOverfitAssessment>(),
|
|
new List<AssertionOverfitAssessment>(),
|
|
new List<PromptOverfitAssessment>(),
|
|
new List<string>(),
|
|
"Moderate overfitting detected"
|
|
)
|
|
}
|
|
};
|
|
|
|
var md = Reporter.GenerateMarkdownSummary(verdicts);
|
|
|
|
Assert.Contains("Overfit", md);
|
|
Assert.DoesNotContain("Notes", md);
|
|
Assert.Contains("Quality", md);
|
|
Assert.DoesNotContain("Baseline", md);
|
|
Assert.Contains("\U0001f7e2", md); // 🟢 green circle for improvement
|
|
Assert.Contains("🟡 0.38", md);
|
|
}
|
|
|
|
[Fact]
|
|
public void MarkdownTable_ShowsDashWhenNoOverfitting()
|
|
{
|
|
var verdicts = new List<SkillVerdict>
|
|
{
|
|
new()
|
|
{
|
|
SkillName = "test-skill",
|
|
SkillPath = "/test",
|
|
Passed = true,
|
|
Scenarios = new List<ScenarioComparison>
|
|
{
|
|
new()
|
|
{
|
|
ScenarioName = "sc1",
|
|
Baseline = new RunResult(
|
|
new RunMetrics { AgentOutput = "baseline" },
|
|
new JudgeResult(new List<RubricScore>(), 3.5, "OK")),
|
|
SkilledIsolated = new RunResult(
|
|
new RunMetrics { AgentOutput = "skilled" },
|
|
new JudgeResult(new List<RubricScore>(), 4.5, "Good")),
|
|
ImprovementScore = 0.25,
|
|
Breakdown = new MetricBreakdown(0, 0, 0, 0, 0, 0, 0),
|
|
}
|
|
},
|
|
OverallImprovementScore = 0.25,
|
|
Reason = "Pass",
|
|
}
|
|
};
|
|
|
|
var md = Reporter.GenerateMarkdownSummary(verdicts);
|
|
|
|
Assert.Contains("Overfit", md);
|
|
Assert.DoesNotContain("Notes", md);
|
|
Assert.Contains("Quality", md);
|
|
Assert.Contains("\u2192", md); // → arrow in quality cell
|
|
Assert.Contains("| \u2014 |", md); // — dash in Overfit column when no result
|
|
}
|
|
|
|
[Fact]
|
|
public void MarkdownTable_ShowsFootnoteWhenVerdictDisagreesWithQuality()
|
|
{
|
|
// Quality improved (+1.0) but composite is negative due to token/time overhead.
|
|
// ImprovementScore derived from breakdown * DefaultWeights:
|
|
// -5.0*0.05 + -3.0*0.025 + 0*0.15 + -3.0*0.025 + 0.4*0.40 + 0.2*0.30 + 0*0.05 = -0.18
|
|
var verdicts = new List<SkillVerdict>
|
|
{
|
|
new()
|
|
{
|
|
SkillName = "test-skill",
|
|
SkillPath = "/test",
|
|
Passed = false,
|
|
Scenarios = new List<ScenarioComparison>
|
|
{
|
|
new()
|
|
{
|
|
ScenarioName = "overhead-scenario",
|
|
Baseline = new RunResult(
|
|
new RunMetrics { AgentOutput = "baseline", TokenEstimate = 1000, ToolCallCount = 5, WallTimeMs = 2100 },
|
|
new JudgeResult(new List<RubricScore>(), 3.5, "OK")),
|
|
SkilledIsolated = new RunResult(
|
|
new RunMetrics { AgentOutput = "skilled", TokenEstimate = 8000, ToolCallCount = 15, WallTimeMs = 8500 },
|
|
new JudgeResult(new List<RubricScore>(), 4.5, "Good")),
|
|
ImprovementScore = -0.18,
|
|
Breakdown = new MetricBreakdown(
|
|
TokenReduction: -5.0,
|
|
ToolCallReduction: -3.0,
|
|
TaskCompletionImprovement: 0,
|
|
TimeReduction: -3.0,
|
|
QualityImprovement: 0.4,
|
|
OverallJudgmentImprovement: 0.2,
|
|
ErrorReduction: 0),
|
|
}
|
|
},
|
|
OverallImprovementScore = -0.18,
|
|
Reason = "Below threshold",
|
|
}
|
|
};
|
|
|
|
var md = Reporter.GenerateMarkdownSummary(verdicts);
|
|
|
|
Assert.Contains("[1]", md);
|
|
Assert.Contains("(Isolated) Quality improved but weighted score is", md);
|
|
Assert.Contains("due to:", md);
|
|
// Raw metrics should appear in footnote
|
|
Assert.Contains("tokens (1000", md);
|
|
Assert.Contains("8000)", md);
|
|
Assert.Contains("tool calls (5", md);
|
|
Assert.Contains("time (2.1s", md);
|
|
}
|
|
|
|
[Fact]
|
|
public void MarkdownTable_NoFootnoteWhenVerdictMatchesQuality()
|
|
{
|
|
// Quality improved and composite is positive — no footnote needed.
|
|
// ImprovementScore derived from breakdown * DefaultWeights:
|
|
// 0*0.05 + 0*0.025 + 0*0.15 + 0*0.025 + 0.5*0.40 + 0.5*0.30 + 0*0.05 = 0.35
|
|
var verdicts = new List<SkillVerdict>
|
|
{
|
|
new()
|
|
{
|
|
SkillName = "test-skill",
|
|
SkillPath = "/test",
|
|
Passed = true,
|
|
Scenarios = new List<ScenarioComparison>
|
|
{
|
|
new()
|
|
{
|
|
ScenarioName = "clean-pass",
|
|
Baseline = new RunResult(
|
|
new RunMetrics { AgentOutput = "baseline" },
|
|
new JudgeResult(new List<RubricScore>(), 3.0, "OK")),
|
|
SkilledIsolated = new RunResult(
|
|
new RunMetrics { AgentOutput = "skilled" },
|
|
new JudgeResult(new List<RubricScore>(), 4.5, "Good")),
|
|
ImprovementScore = 0.35,
|
|
Breakdown = new MetricBreakdown(
|
|
TokenReduction: 0,
|
|
ToolCallReduction: 0,
|
|
TaskCompletionImprovement: 0,
|
|
TimeReduction: 0,
|
|
QualityImprovement: 0.5,
|
|
OverallJudgmentImprovement: 0.5,
|
|
ErrorReduction: 0),
|
|
}
|
|
},
|
|
OverallImprovementScore = 0.35,
|
|
Reason = "Pass",
|
|
}
|
|
};
|
|
|
|
var md = Reporter.GenerateMarkdownSummary(verdicts);
|
|
|
|
Assert.DoesNotContain("[1]", md);
|
|
Assert.DoesNotContain("weighted score", md);
|
|
}
|
|
|
|
[Fact]
|
|
public void MarkdownTable_ShowsFootnoteWhenQualityDroppedButCompositePositive()
|
|
{
|
|
// Quality dropped (-1.0) but composite is positive due to efficiency gains.
|
|
// ImprovementScore derived from breakdown * DefaultWeights:
|
|
// 2.0*0.05 + 0*0.025 + 1.0*0.15 + 0*0.025 + -0.2*0.40 + -0.1*0.30 + 0*0.05 = 0.14
|
|
var verdicts = new List<SkillVerdict>
|
|
{
|
|
new()
|
|
{
|
|
SkillName = "test-skill",
|
|
SkillPath = "/test",
|
|
Passed = true,
|
|
Scenarios = new List<ScenarioComparison>
|
|
{
|
|
new()
|
|
{
|
|
ScenarioName = "efficiency-offset",
|
|
Baseline = new RunResult(
|
|
new RunMetrics { AgentOutput = "baseline", TokenEstimate = 5000, TaskCompleted = false },
|
|
new JudgeResult(new List<RubricScore>(), 4.0, "Good")),
|
|
SkilledIsolated = new RunResult(
|
|
new RunMetrics { AgentOutput = "skilled", TokenEstimate = 2000, TaskCompleted = true },
|
|
new JudgeResult(new List<RubricScore>(), 3.0, "OK")),
|
|
ImprovementScore = 0.14,
|
|
Breakdown = new MetricBreakdown(
|
|
TokenReduction: 2.0,
|
|
ToolCallReduction: 0,
|
|
TaskCompletionImprovement: 1.0,
|
|
TimeReduction: 0,
|
|
QualityImprovement: -0.2,
|
|
OverallJudgmentImprovement: -0.1,
|
|
ErrorReduction: 0),
|
|
}
|
|
},
|
|
OverallImprovementScore = 0.14,
|
|
Reason = "Pass",
|
|
}
|
|
};
|
|
|
|
var md = Reporter.GenerateMarkdownSummary(verdicts);
|
|
|
|
Assert.Contains("[1]", md);
|
|
Assert.Contains("(Isolated) Quality dropped but weighted score is", md);
|
|
Assert.Contains("due to:", md);
|
|
// Raw metrics should appear in footnote
|
|
Assert.Contains("completion", md);
|
|
Assert.Contains("tokens (5000", md);
|
|
Assert.Contains("2000)", md);
|
|
}
|
|
|
|
[Fact]
|
|
public void MarkdownTable_ShowsFootnoteWhenQualityUnchangedButVerdictNegative()
|
|
{
|
|
// Quality scores are identical between baseline and skill runs, but verdict is negative.
|
|
// A footnote should explain what efficiency metrics caused the negative score.
|
|
// ImprovementScore derived from breakdown * DefaultWeights:
|
|
// -2.0*0.05 + 0 + 0 + 0 + 0 + 0 + 0 = -0.10
|
|
var verdicts = new List<SkillVerdict>
|
|
{
|
|
new()
|
|
{
|
|
SkillName = "test-skill",
|
|
SkillPath = "/test",
|
|
Passed = false,
|
|
Scenarios = new List<ScenarioComparison>
|
|
{
|
|
new()
|
|
{
|
|
ScenarioName = "quality-unchanged",
|
|
Baseline = new RunResult(
|
|
new RunMetrics { AgentOutput = "baseline", TokenEstimate = 1000, ToolCallCount = 5 },
|
|
new JudgeResult(new List<RubricScore>(), 3.5, "OK")),
|
|
SkilledIsolated = new RunResult(
|
|
new RunMetrics { AgentOutput = "skilled", TokenEstimate = 3000, ToolCallCount = 5 },
|
|
new JudgeResult(new List<RubricScore>(), 3.5, "OK")),
|
|
ImprovementScore = -0.10,
|
|
Breakdown = new MetricBreakdown(
|
|
TokenReduction: -2.0,
|
|
ToolCallReduction: 0,
|
|
TaskCompletionImprovement: 0,
|
|
TimeReduction: 0,
|
|
QualityImprovement: 0,
|
|
OverallJudgmentImprovement: 0,
|
|
ErrorReduction: 0),
|
|
}
|
|
},
|
|
OverallImprovementScore = -0.10,
|
|
Reason = "Below threshold",
|
|
}
|
|
};
|
|
|
|
var md = Reporter.GenerateMarkdownSummary(verdicts);
|
|
|
|
Assert.Contains("[1]", md);
|
|
Assert.Contains("(Isolated) Quality unchanged but weighted score is", md);
|
|
Assert.Contains("tokens (1000", md);
|
|
}
|
|
|
|
[Fact]
|
|
public void MarkdownSummary_ShowsErrorsSectionForPreEvalFailures()
|
|
{
|
|
var verdicts = new List<SkillVerdict>
|
|
{
|
|
new()
|
|
{
|
|
SkillName = "broken-skill",
|
|
SkillPath = "/test",
|
|
Passed = false,
|
|
Scenarios = [],
|
|
OverallImprovementScore = 0,
|
|
Reason = "Skill description is 1,370 characters — maximum is 1,024.",
|
|
FailureKind = FailureKind.SpecConformanceFailure,
|
|
}
|
|
};
|
|
|
|
var md = Reporter.GenerateMarkdownSummary(verdicts);
|
|
|
|
Assert.Contains("### ❌ Skill validation errors", md);
|
|
Assert.Contains("- `broken-skill: Skill description is 1,370 characters", md);
|
|
}
|
|
|
|
[Fact]
|
|
public void MarkdownSummary_OmitsErrorsSectionWhenAllVerdictsPassed()
|
|
{
|
|
var verdicts = new List<SkillVerdict>
|
|
{
|
|
new()
|
|
{
|
|
SkillName = "good-skill",
|
|
SkillPath = "/test",
|
|
Passed = true,
|
|
Scenarios = new List<ScenarioComparison>
|
|
{
|
|
new()
|
|
{
|
|
ScenarioName = "test-scenario",
|
|
Baseline = new RunResult(
|
|
new RunMetrics { AgentOutput = "baseline" },
|
|
new JudgeResult(new List<RubricScore>(), 3.5, "OK")),
|
|
SkilledIsolated = new RunResult(
|
|
new RunMetrics { AgentOutput = "skilled" },
|
|
new JudgeResult(new List<RubricScore>(), 4.5, "Good")),
|
|
ImprovementScore = 0.25,
|
|
Breakdown = new MetricBreakdown(0, 0, 0, 0, 0, 0, 0),
|
|
}
|
|
},
|
|
OverallImprovementScore = 0.25,
|
|
Reason = "Pass",
|
|
}
|
|
};
|
|
|
|
var md = Reporter.GenerateMarkdownSummary(verdicts);
|
|
|
|
Assert.DoesNotContain("### ❌ Skill validation errors", md);
|
|
}
|
|
|
|
[Fact]
|
|
public void MarkdownSummary_OmitsErrorsSectionForFailuresWithScenarios()
|
|
{
|
|
var verdicts = new List<SkillVerdict>
|
|
{
|
|
new()
|
|
{
|
|
SkillName = "threshold-fail",
|
|
SkillPath = "/test",
|
|
Passed = false,
|
|
Scenarios = new List<ScenarioComparison>
|
|
{
|
|
new()
|
|
{
|
|
ScenarioName = "test-scenario",
|
|
Baseline = new RunResult(
|
|
new RunMetrics { AgentOutput = "baseline" },
|
|
new JudgeResult(new List<RubricScore>(), 4.0, "OK")),
|
|
SkilledIsolated = new RunResult(
|
|
new RunMetrics { AgentOutput = "skilled" },
|
|
new JudgeResult(new List<RubricScore>(), 3.0, "Worse")),
|
|
ImprovementScore = -0.25,
|
|
Breakdown = new MetricBreakdown(0, 0, 0, 0, 0, 0, 0),
|
|
}
|
|
},
|
|
OverallImprovementScore = -0.25,
|
|
Reason = "Regression detected",
|
|
FailureKind = FailureKind.CompletionRegression,
|
|
}
|
|
};
|
|
|
|
var md = Reporter.GenerateMarkdownSummary(verdicts);
|
|
|
|
Assert.DoesNotContain("### ❌ Skill validation errors", md);
|
|
}
|
|
|
|
[Fact]
|
|
public void MarkdownTable_SkillsLoaded_NoDuplicateWhenIsolatedAndPluginIdentical()
|
|
{
|
|
var activation = new SkillActivationInfo(
|
|
Activated: true,
|
|
DetectedSkills: new List<string> { "my-skill" },
|
|
ExtraTools: new List<string> { "report_intent", "skill" },
|
|
SkillEventCount: 1);
|
|
|
|
var verdicts = new List<SkillVerdict>
|
|
{
|
|
new()
|
|
{
|
|
SkillName = "my-skill",
|
|
SkillPath = "/test",
|
|
Passed = true,
|
|
Scenarios = new List<ScenarioComparison>
|
|
{
|
|
new()
|
|
{
|
|
ScenarioName = "sc1",
|
|
Baseline = new RunResult(
|
|
new RunMetrics { AgentOutput = "baseline" },
|
|
new JudgeResult(new List<RubricScore>(), 3.5, "OK")),
|
|
SkilledIsolated = new RunResult(
|
|
new RunMetrics { AgentOutput = "skilled" },
|
|
new JudgeResult(new List<RubricScore>(), 4.5, "Good")),
|
|
SkilledPlugin = new RunResult(
|
|
new RunMetrics { AgentOutput = "skilled-plugin" },
|
|
new JudgeResult(new List<RubricScore>(), 4.5, "Good")),
|
|
ImprovementScore = 0.25,
|
|
Breakdown = new MetricBreakdown(0, 0, 0, 0, 0, 0, 0),
|
|
SkillActivationIsolated = activation,
|
|
SkillActivationPlugin = activation,
|
|
}
|
|
},
|
|
OverallImprovementScore = 0.25,
|
|
Reason = "Pass",
|
|
}
|
|
};
|
|
|
|
var md = Reporter.GenerateMarkdownSummary(verdicts);
|
|
|
|
// Skills loaded cell should contain the activation info exactly once (no " / " separator)
|
|
Assert.DoesNotContain("my-skill; tools: report_intent, skill / ✅ my-skill", md);
|
|
Assert.Contains("✅ my-skill; tools: report_intent, skill", md);
|
|
}
|
|
|
|
[Fact]
|
|
public void MarkdownTable_SkillsLoaded_ShowsBothWhenIsolatedAndPluginDiffer()
|
|
{
|
|
var isolatedActivation = new SkillActivationInfo(
|
|
Activated: true,
|
|
DetectedSkills: new List<string> { "my-skill" },
|
|
ExtraTools: new List<string> { "report_intent" },
|
|
SkillEventCount: 1);
|
|
var pluginActivation = new SkillActivationInfo(
|
|
Activated: true,
|
|
DetectedSkills: new List<string> { "my-skill" },
|
|
ExtraTools: new List<string> { "report_intent", "skill" },
|
|
SkillEventCount: 1);
|
|
|
|
var verdicts = new List<SkillVerdict>
|
|
{
|
|
new()
|
|
{
|
|
SkillName = "my-skill",
|
|
SkillPath = "/test",
|
|
Passed = true,
|
|
Scenarios = new List<ScenarioComparison>
|
|
{
|
|
new()
|
|
{
|
|
ScenarioName = "sc1",
|
|
Baseline = new RunResult(
|
|
new RunMetrics { AgentOutput = "baseline" },
|
|
new JudgeResult(new List<RubricScore>(), 3.5, "OK")),
|
|
SkilledIsolated = new RunResult(
|
|
new RunMetrics { AgentOutput = "skilled" },
|
|
new JudgeResult(new List<RubricScore>(), 4.5, "Good")),
|
|
SkilledPlugin = new RunResult(
|
|
new RunMetrics { AgentOutput = "skilled-plugin" },
|
|
new JudgeResult(new List<RubricScore>(), 4.5, "Good")),
|
|
ImprovementScore = 0.25,
|
|
Breakdown = new MetricBreakdown(0, 0, 0, 0, 0, 0, 0),
|
|
SkillActivationIsolated = isolatedActivation,
|
|
SkillActivationPlugin = pluginActivation,
|
|
}
|
|
},
|
|
OverallImprovementScore = 0.25,
|
|
Reason = "Pass",
|
|
}
|
|
};
|
|
|
|
var md = Reporter.GenerateMarkdownSummary(verdicts);
|
|
|
|
// Both activation entries should appear separated by " / "
|
|
Assert.Contains("✅ my-skill; tools: report_intent / ✅ my-skill; tools: report_intent, skill", md);
|
|
}
|
|
|
|
// --- Prompt overfitting detection tests ---
|
|
|
|
[Fact]
|
|
public void DetectPromptOverfitting_ExplicitSkillName_Detected()
|
|
{
|
|
var skill = new SkillInfo(
|
|
Name: "migrate-dotnet10-to-dotnet11",
|
|
Description: "Migration skill",
|
|
Path: "/skills/migrate-dotnet10-to-dotnet11",
|
|
SkillMdPath: "/skills/migrate-dotnet10-to-dotnet11/SKILL.md",
|
|
SkillMdContent: "# Migration Skill");
|
|
var evalSkill = new EvalSkillInfo(
|
|
Skill: skill,
|
|
EvalPath: null,
|
|
EvalConfig: new EvalConfig(new List<EvalScenario>
|
|
{
|
|
new("scenario1",
|
|
"Use the migrate-dotnet10-to-dotnet11 skill to help me migrate my .NET 10 console app to .NET 11."),
|
|
}));
|
|
|
|
var assessments = OverfittingJudge.DetectPromptOverfitting(evalSkill);
|
|
|
|
Assert.Single(assessments);
|
|
Assert.Equal("explicit_skill_reference", assessments[0].Issue);
|
|
Assert.Equal(1.0, assessments[0].Confidence);
|
|
Assert.Contains("migrate-dotnet10-to-dotnet11", assessments[0].Reasoning);
|
|
}
|
|
|
|
[Fact]
|
|
public void DetectPromptOverfitting_MultipleScenarios_AllDetected()
|
|
{
|
|
var skill = new SkillInfo(
|
|
Name: "migrate-dotnet10-to-dotnet11",
|
|
Description: "Migration skill",
|
|
Path: "/skills/migrate-dotnet10-to-dotnet11",
|
|
SkillMdPath: "/skills/migrate-dotnet10-to-dotnet11/SKILL.md",
|
|
SkillMdContent: "# Migration Skill");
|
|
var evalSkill = new EvalSkillInfo(
|
|
Skill: skill,
|
|
EvalPath: null,
|
|
EvalConfig: new EvalConfig(new List<EvalScenario>
|
|
{
|
|
new("scenario1",
|
|
"Use the migrate-dotnet10-to-dotnet11 skill to help me with compression changes."),
|
|
new("scenario2",
|
|
"Use the migrate-dotnet10-to-dotnet11 skill to help me with C# 15 compiler changes."),
|
|
new("scenario3",
|
|
"Use the migrate-dotnet10-to-dotnet11 skill to help me with EF Core Cosmos DB."),
|
|
}));
|
|
|
|
var assessments = OverfittingJudge.DetectPromptOverfitting(evalSkill);
|
|
|
|
Assert.Equal(3, assessments.Count);
|
|
Assert.All(assessments, a => Assert.Equal("explicit_skill_reference", a.Issue));
|
|
}
|
|
|
|
[Fact]
|
|
public void DetectPromptOverfitting_UseSkillPhrase_Detected()
|
|
{
|
|
var skill = new SkillInfo(
|
|
Name: "dotnet-pinvoke",
|
|
Description: "P/Invoke skill",
|
|
Path: "/skills/dotnet-pinvoke",
|
|
SkillMdPath: "/skills/dotnet-pinvoke/SKILL.md",
|
|
SkillMdContent: "# P/Invoke Skill");
|
|
var evalSkill = new EvalSkillInfo(
|
|
Skill: skill,
|
|
EvalPath: null,
|
|
EvalConfig: new EvalConfig(new List<EvalScenario>
|
|
{
|
|
new("scenario1",
|
|
"Use the pinvoke skill to help me create a P/Invoke binding."),
|
|
}));
|
|
|
|
var assessments = OverfittingJudge.DetectPromptOverfitting(evalSkill);
|
|
|
|
Assert.Single(assessments);
|
|
Assert.Equal("skill_instruction", assessments[0].Issue);
|
|
Assert.Equal(0.9, assessments[0].Confidence);
|
|
}
|
|
|
|
[Fact]
|
|
public void DetectPromptOverfitting_NeutralPrompt_NothingDetected()
|
|
{
|
|
var skill = new SkillInfo(
|
|
Name: "migrate-dotnet10-to-dotnet11",
|
|
Description: "Migration skill",
|
|
Path: "/skills/migrate-dotnet10-to-dotnet11",
|
|
SkillMdPath: "/skills/migrate-dotnet10-to-dotnet11/SKILL.md",
|
|
SkillMdContent: "# Migration Skill");
|
|
var evalSkill = new EvalSkillInfo(
|
|
Skill: skill,
|
|
EvalPath: null,
|
|
EvalConfig: new EvalConfig(new List<EvalScenario>
|
|
{
|
|
new("scenario1",
|
|
"I need to migrate my .NET 10 console app to .NET 11. What breaks?"),
|
|
}));
|
|
|
|
var assessments = OverfittingJudge.DetectPromptOverfitting(evalSkill);
|
|
|
|
Assert.Empty(assessments);
|
|
}
|
|
|
|
[Fact]
|
|
public void DetectPromptOverfitting_CaseInsensitive_Detected()
|
|
{
|
|
var skill = new SkillInfo(
|
|
Name: "Migrate-Dotnet10-To-Dotnet11",
|
|
Description: "Migration skill",
|
|
Path: "/skills/migrate-dotnet10-to-dotnet11",
|
|
SkillMdPath: "/skills/migrate-dotnet10-to-dotnet11/SKILL.md",
|
|
SkillMdContent: "# Migration Skill");
|
|
var evalSkill = new EvalSkillInfo(
|
|
Skill: skill,
|
|
EvalPath: null,
|
|
EvalConfig: new EvalConfig(new List<EvalScenario>
|
|
{
|
|
new("scenario1",
|
|
"Use the migrate-dotnet10-to-dotnet11 skill to help me."),
|
|
}));
|
|
|
|
var assessments = OverfittingJudge.DetectPromptOverfitting(evalSkill);
|
|
|
|
Assert.Single(assessments);
|
|
Assert.Equal("explicit_skill_reference", assessments[0].Issue);
|
|
}
|
|
|
|
[Fact]
|
|
public void DetectPromptOverfitting_NoEvalConfig_ReturnsEmpty()
|
|
{
|
|
var skill = new SkillInfo(
|
|
Name: "test-skill",
|
|
Description: "Test",
|
|
Path: "/skills/test-skill",
|
|
SkillMdPath: "/skills/test-skill/SKILL.md",
|
|
SkillMdContent: "# Test");
|
|
var evalSkill = new EvalSkillInfo(
|
|
Skill: skill,
|
|
EvalPath: null,
|
|
EvalConfig: null);
|
|
|
|
var assessments = OverfittingJudge.DetectPromptOverfitting(evalSkill);
|
|
|
|
Assert.Empty(assessments);
|
|
}
|
|
|
|
// --- Score computation with prompt assessments ---
|
|
|
|
[Fact]
|
|
public void ComputeScore_WithPromptIssues_BoostsScore()
|
|
{
|
|
var rubric = new List<RubricOverfitAssessment>
|
|
{
|
|
new("sc1", "Explains GZipStream changes", "outcome", 0.9, "Good outcome test"),
|
|
};
|
|
var assertions = new List<AssertionOverfitAssessment>
|
|
{
|
|
new("sc1", "output_matches: GZipStream", "broad", 0.9, "Broad test"),
|
|
};
|
|
var prompts = new List<PromptOverfitAssessment>
|
|
{
|
|
new("sc1", "explicit_skill_reference", 1.0, "Prompt names the skill"),
|
|
};
|
|
|
|
var scoreWithPrompts = OverfittingJudge.ComputeOverfittingScore(rubric, assertions, prompts);
|
|
var scoreWithout = OverfittingJudge.ComputeOverfittingScore(rubric, assertions);
|
|
|
|
// With prompt issues: 0.4*1.0 + 0.4*0.0 + 0.2*0.0 = 0.4
|
|
// Without: 0.7*0.0 + 0.3*0.0 = 0.0
|
|
Assert.True(scoreWithPrompts > scoreWithout,
|
|
$"Score with prompts ({scoreWithPrompts}) should exceed score without ({scoreWithout})");
|
|
Assert.True(scoreWithPrompts >= 0.4,
|
|
$"Score with prompt issues should be at least 0.4, got {scoreWithPrompts}");
|
|
}
|
|
|
|
[Fact]
|
|
public void ComputeScore_AllScenariosExplicitRef_HighScore()
|
|
{
|
|
var rubric = new List<RubricOverfitAssessment>
|
|
{
|
|
new("sc1", "criterion1", "outcome", 0.9, "Good"),
|
|
new("sc2", "criterion2", "outcome", 0.8, "Good"),
|
|
};
|
|
var assertions = new List<AssertionOverfitAssessment>
|
|
{
|
|
new("sc1", "output_matches: pattern", "broad", 0.9, "Broad"),
|
|
};
|
|
var prompts = new List<PromptOverfitAssessment>
|
|
{
|
|
new("sc1", "explicit_skill_reference", 1.0, "Names skill"),
|
|
new("sc2", "explicit_skill_reference", 1.0, "Names skill"),
|
|
};
|
|
|
|
var score = OverfittingJudge.ComputeOverfittingScore(rubric, assertions, prompts);
|
|
|
|
// 0.4*1.0 + 0.4*0.0 + 0.2*0.0 = 0.4
|
|
Assert.Equal(0.4, score, 2);
|
|
}
|
|
|
|
[Fact]
|
|
public void ComputeScore_PromptIssuesWithVocabulary_CompoundsHigh()
|
|
{
|
|
var rubric = new List<RubricOverfitAssessment>
|
|
{
|
|
new("sc1", "Uses specific term", "vocabulary", 0.9, "Vocabulary test"),
|
|
};
|
|
var assertions = new List<AssertionOverfitAssessment>
|
|
{
|
|
new("sc1", "output_matches: specific-pattern", "narrow", 0.85, "Narrow match"),
|
|
};
|
|
var prompts = new List<PromptOverfitAssessment>
|
|
{
|
|
new("sc1", "explicit_skill_reference", 1.0, "Names skill"),
|
|
};
|
|
|
|
var score = OverfittingJudge.ComputeOverfittingScore(rubric, assertions, prompts);
|
|
|
|
// 0.4*1.0 + 0.4*(1.0*0.9) + 0.2*(1.0*0.85) = 0.4 + 0.36 + 0.17 = 0.93
|
|
Assert.True(score > 0.8, $"Combined prompt+vocabulary should produce high score, got {score}");
|
|
}
|
|
|
|
// --- ParseResponse with prompt assessments ---
|
|
|
|
[Fact]
|
|
public void ParseResponse_WithPromptAssessments_ParsesCorrectly()
|
|
{
|
|
var json = JsonSerializer.Serialize(new
|
|
{
|
|
rubric_assessments = new[]
|
|
{
|
|
new { scenario = "sc1", criterion = "test", classification = "outcome", confidence = 0.9, reasoning = "ok" }
|
|
},
|
|
assertion_assessments = Array.Empty<object>(),
|
|
prompt_assessments = new[]
|
|
{
|
|
new { scenario = "sc1", issue = "explicit_skill_reference", confidence = 1.0, reasoning = "Prompt names the skill" }
|
|
},
|
|
cross_scenario_issues = Array.Empty<string>(),
|
|
overall_overfitting_score = 0.5,
|
|
overall_reasoning = "Prompt overfitting detected"
|
|
});
|
|
|
|
var result = OverfittingJudge.ParseOverfittingResponse(json);
|
|
|
|
Assert.Single(result.PromptAssessments);
|
|
Assert.Equal("explicit_skill_reference", result.PromptAssessments[0].Issue);
|
|
Assert.Equal(1.0, result.PromptAssessments[0].Confidence);
|
|
}
|
|
|
|
[Fact]
|
|
public void ParseResponse_DeterministicPromptsMergedWithLlm()
|
|
{
|
|
var json = JsonSerializer.Serialize(new
|
|
{
|
|
rubric_assessments = Array.Empty<object>(),
|
|
assertion_assessments = Array.Empty<object>(),
|
|
prompt_assessments = new[]
|
|
{
|
|
new { scenario = "sc1", issue = "explicit_skill_reference", confidence = 0.8, reasoning = "LLM detected" },
|
|
new { scenario = "sc2", issue = "explicit_skill_reference", confidence = 0.9, reasoning = "LLM detected" }
|
|
},
|
|
cross_scenario_issues = Array.Empty<string>(),
|
|
overall_overfitting_score = 0.5,
|
|
overall_reasoning = "test"
|
|
});
|
|
|
|
var deterministicAssessments = new List<PromptOverfitAssessment>
|
|
{
|
|
new("sc1", "explicit_skill_reference", 1.0, "Deterministic: prompt contains skill name")
|
|
};
|
|
|
|
var result = OverfittingJudge.ParseOverfittingResponse(json, deterministicAssessments);
|
|
|
|
// sc1: deterministic wins (same scenario+issue already covered), sc2: LLM addition
|
|
Assert.Equal(2, result.PromptAssessments.Count);
|
|
var sc1 = result.PromptAssessments.First(p => p.Scenario == "sc1");
|
|
Assert.Equal(1.0, sc1.Confidence); // deterministic confidence
|
|
Assert.Contains("Deterministic", sc1.Reasoning);
|
|
}
|
|
|
|
// --- BuildSystemPrompt includes prompt classification ---
|
|
|
|
[Fact]
|
|
public void BuildSystemPrompt_ContainsPromptClassification()
|
|
{
|
|
var prompt = OverfittingJudge.BuildSystemPrompt();
|
|
|
|
Assert.Contains("prompt_assessments", prompt);
|
|
Assert.Contains("explicit_skill_reference", prompt);
|
|
Assert.Contains("skill_instruction", prompt);
|
|
Assert.Contains("neutral", prompt);
|
|
Assert.Contains("Scenario prompt classifications", prompt);
|
|
}
|
|
}
|