mirror of
https://github.com/wshobson/agents.git
synced 2026-09-14 18:04:20 +08:00
Seth Hobson
31fdae14d6
* fix(plugin-eval): populate model_usage from judge and Monte Carlo layers (#660) * fix(plugin-eval): attribute judge usage to SDK model, scope it per-call Addresses CodeRabbit + Codex review on #668: - query_llm now keys usage_sink by the SDK-reported model (collect_sdk_output(messages).model), falling back to the requested model only when the stream reported none. Keeps judge attribution consistent with the Monte Carlo layer, which already aggregates by SDK-reported model, and stops routing/fallback substitutions from misattributing tokens to the wrong model. - JudgeAnalyzer no longer accumulates model_usage as instance state. analyze_skill() now creates a local dict and threads it through the four assess_* calls, so a reused analyzer (or concurrent analyze_skill calls) can no longer leak or mix token counts between runs. - Parameterized a bare `dict` annotation introduced by this feature in test_judge.py's _result() helper. - Added tests: SDK-reported model differing from the requested model, and repeated analyze_skill() calls on one analyzer not leaking usage.
83 lines
3.3 KiB
Python
83 lines
3.3 KiB
Python
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from plugin_eval.engine import EvalEngine
|
|
from plugin_eval.models import Depth, EvalConfig, LayerResult, PluginEvalResult
|
|
|
|
|
|
class TestEvalEngine:
|
|
def test_quick_eval_skill(self, sample_skill_dir: Path):
|
|
config = EvalConfig(depth=Depth.QUICK)
|
|
engine = EvalEngine(config)
|
|
result = engine.evaluate_skill(sample_skill_dir)
|
|
assert isinstance(result, PluginEvalResult)
|
|
assert len(result.layers) == 1
|
|
assert result.layers[0].layer == "static"
|
|
assert result.composite is not None
|
|
assert result.composite.confidence_label == "Estimated"
|
|
|
|
def test_quick_eval_plugin(self, sample_plugin_dir: Path):
|
|
config = EvalConfig(depth=Depth.QUICK)
|
|
engine = EvalEngine(config)
|
|
result = engine.evaluate_plugin(sample_plugin_dir)
|
|
assert isinstance(result, PluginEvalResult)
|
|
assert result.composite.score > 0
|
|
|
|
def test_composite_score_within_bounds(self, sample_skill_dir: Path):
|
|
config = EvalConfig(depth=Depth.QUICK)
|
|
engine = EvalEngine(config)
|
|
result = engine.evaluate_skill(sample_skill_dir)
|
|
assert 0 <= result.composite.score <= 100
|
|
|
|
def test_layer_blend_renormalization(self):
|
|
"""When only L1 is available, L1 weights should renormalize to 1.0."""
|
|
engine = EvalEngine(EvalConfig(depth=Depth.QUICK))
|
|
blended = engine._blend_layer_scores(
|
|
static_scores={"triggering_accuracy": 0.9, "orchestration_fitness": 0.8},
|
|
judge_scores=None,
|
|
mc_scores=None,
|
|
)
|
|
assert blended["triggering_accuracy"] > 0
|
|
assert blended["orchestration_fitness"] > 0
|
|
|
|
def test_quick_eval_skill_has_empty_model_usage(self, sample_skill_dir: Path):
|
|
"""Static-only (quick) runs never touch the SDK, so model_usage stays empty."""
|
|
config = EvalConfig(depth=Depth.QUICK)
|
|
engine = EvalEngine(config)
|
|
result = engine.evaluate_skill(sample_skill_dir)
|
|
assert result.model_usage == {}
|
|
|
|
|
|
class TestMergeModelUsage:
|
|
"""EvalEngine._merge_model_usage sums per-model tokens across layers."""
|
|
|
|
def test_merges_disjoint_models_across_layers(self):
|
|
layers = [
|
|
LayerResult(layer="static", score=0.9),
|
|
LayerResult(
|
|
layer="judge", score=0.8, metadata={"model_usage": {"claude-haiku-4-5": 10}}
|
|
),
|
|
LayerResult(
|
|
layer="monte_carlo", score=0.7, metadata={"model_usage": {"claude-sonnet-5": 500}}
|
|
),
|
|
]
|
|
merged = EvalEngine._merge_model_usage(layers)
|
|
assert merged == {"claude-haiku-4-5": 10, "claude-sonnet-5": 500}
|
|
|
|
def test_sums_the_same_model_name_across_layers(self):
|
|
layers = [
|
|
LayerResult(
|
|
layer="judge", score=0.8, metadata={"model_usage": {"claude-sonnet-5": 300}}
|
|
),
|
|
LayerResult(
|
|
layer="monte_carlo", score=0.7, metadata={"model_usage": {"claude-sonnet-5": 500}}
|
|
),
|
|
]
|
|
merged = EvalEngine._merge_model_usage(layers)
|
|
assert merged == {"claude-sonnet-5": 800}
|
|
|
|
def test_static_only_layers_merge_to_empty(self):
|
|
layers = [LayerResult(layer="static", score=0.9)]
|
|
assert EvalEngine._merge_model_usage(layers) == {}
|