mirror of
https://github.com/conorluddy/ios-simulator-skill.git
synced 2026-09-14 20:07:12 +08:00
Conor Luddy
00afc436d6
Closes #82. Three coverage gaps from the PR #78 review, now closed. The crashed-worker recovery path had no test coverage. Adds: - test_mark_crashed_sets_status — direct call after create - test_mark_crashed_after_claim_overrides_running — overrides running state - test_mark_crashed_silent_on_missing_session — no-op on dead session ID - test_load_summary_returns_none_when_missing — crashed session has no summary.json The 50-event fixture in tests/fixtures/sample_session.py only ever produced CRITICAL events (all duration ≥ 600 ms). L1 hit ~36% of its 200-token budget; L2 only ~8% of 2000. Token-budget tests trivially passed. Hardened make_events(): - 12 clusters spanning all 4 Severity tiers (MINOR/WARN/CRITICAL/FROZEN) - One hot cluster at 18 events (near-max count); others spread across cold tail - Two processes (MyApp + Helper) → process_distribution populates - First-cluster events packed tightly → detect_temporal_bursts populates - 7s gap inserted mid-window → detect_quiet_periods populates - Long symbol names → per-line formatter stress After hardening: L0=28/30 (93%), L1=91/200 (46%), L2=298/2000 (15%). L1/L2 are naturally sparse at default top_n=3; the real win is that all four L2 aggregate branches (Severity / Bursts / Quiet periods / Processes) now actually render. Adds qualitative assertions in test_token_budgets.py: - test_fixture_covers_all_severity_tiers — guard against fixture regression - test_fixture_has_at_least_ten_clusters - test_l2_renders_all_aggregate_branches — every L2 branch must populate - test_l1_floor_is_meaningful — floor at 60 tokens - test_l2_floor_renders_full_summary — floor at 200 tokens diff_sessions() produces 5 verdict strings; only 3 were tested. format_diff was only rendered for 'no change' and version mismatch — the new/resolved/drift render paths were never asserted. Adds 6 new in tests/test_diff.py: - test_diff_verdict_regression_new_minor - test_diff_verdict_improvement_resolved - test_diff_verdict_drift_mixed_signs (the bidirectional drift case) - test_format_diff_renders_new_block - test_format_diff_renders_resolved_block - test_format_diff_renders_drift_block Small helper _summary_with_clusters() builds SessionSummary instances with controlled per-cluster max_duration_ms — keeps verdict tests isolated from the event pipeline. test_diff_detects_resolved_cluster filtered by an old symbol name from the soft fixture ([ImageDecoder decode:]). Updated to use the new equivalent. - ruff check: All checks passed - black --check (scripts): clean - python -m pytest tests/: 103 passed (was 88) - pytest tests/: 103 passed
118 lines
3.5 KiB
Python
118 lines
3.5 KiB
Python
"""Contract: --stop output ≤ 200 tokens, --get-details ≤ 2000 tokens (char/4)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from common.hang_pipeline import (
|
|
compress_to_budget,
|
|
estimate_tokens,
|
|
format_cluster_detail,
|
|
format_l0,
|
|
format_l1,
|
|
format_l2,
|
|
)
|
|
from tests.fixtures.sample_session import (
|
|
assert_token_budget,
|
|
make_events,
|
|
make_summary,
|
|
)
|
|
|
|
|
|
def test_l0_is_one_line_under_30_tokens():
|
|
summary = make_summary()
|
|
out = format_l0(summary)
|
|
assert "\n" not in out
|
|
assert_token_budget(out, 30)
|
|
|
|
|
|
def test_l1_default_under_200_tokens():
|
|
"""Issue #77 contract: default --stop output ≤ 200 tokens on 30s/50-event session."""
|
|
summary = make_summary()
|
|
out = format_l1(summary)
|
|
assert_token_budget(out, 200)
|
|
|
|
|
|
def test_l2_full_under_2000_tokens():
|
|
"""Issue #77 contract: --get-details (without --cluster) ≤ 2000 tokens."""
|
|
summary = make_summary()
|
|
out = format_l2(summary)
|
|
assert_token_budget(out, 2000)
|
|
|
|
|
|
def test_l3_cluster_detail_under_2000_tokens():
|
|
summary = make_summary()
|
|
cluster = summary.clusters[0]
|
|
events = [e for e in make_events() if e.fingerprint == cluster.fingerprint]
|
|
out = format_cluster_detail(cluster, events)
|
|
assert_token_budget(out, 2000)
|
|
|
|
|
|
def test_compress_to_budget_honours_tight_limits():
|
|
summary = make_summary()
|
|
# Tight budget should fall back to L0 (~20 tokens).
|
|
out = compress_to_budget(summary, max_tokens=30)
|
|
assert estimate_tokens(out) <= 30
|
|
assert "\n" not in out
|
|
|
|
|
|
def test_compress_to_budget_uses_l1_at_default_window():
|
|
summary = make_summary()
|
|
out = compress_to_budget(summary, max_tokens=150)
|
|
assert estimate_tokens(out) <= 150
|
|
|
|
|
|
def test_compress_to_budget_uses_l2_when_room():
|
|
summary = make_summary()
|
|
out = compress_to_budget(summary, max_tokens=1500)
|
|
assert estimate_tokens(out) <= 1500
|
|
# L2 reports a "Severity:" histogram line; L1 does not.
|
|
assert "Severity:" in out
|
|
|
|
|
|
def test_compress_with_no_budget_returns_l1():
|
|
summary = make_summary()
|
|
out = compress_to_budget(summary, max_tokens=None)
|
|
assert "Drill:" in out
|
|
|
|
|
|
# === fixture-stress assertions (#82) ===
|
|
|
|
|
|
def test_fixture_covers_all_severity_tiers():
|
|
"""The contract tests are only meaningful if the fixture actually exercises
|
|
every severity band. Guard against a future fixture regression."""
|
|
summary = make_summary()
|
|
severities_present = {c.severity.value for c in summary.clusters}
|
|
assert severities_present == {"minor", "warn", "critical", "frozen"}
|
|
|
|
|
|
def test_fixture_has_at_least_ten_clusters():
|
|
summary = make_summary()
|
|
assert len(summary.clusters) >= 10
|
|
|
|
|
|
def test_l2_renders_all_aggregate_branches():
|
|
"""L2 formatter has four optional branches (severity histogram, bursts,
|
|
quiet periods, process distribution). Fixture must populate all four so
|
|
none are silently skipped."""
|
|
summary = make_summary()
|
|
out = format_l2(summary)
|
|
assert "Severity:" in out
|
|
assert "Bursts:" in out
|
|
assert "Quiet periods:" in out
|
|
assert "Processes:" in out
|
|
|
|
|
|
def test_l1_floor_is_meaningful():
|
|
"""L1 should be far enough above zero that the top-N lines actually render."""
|
|
summary = make_summary()
|
|
out = format_l1(summary)
|
|
# Header (~30 tokens) + 3 cluster lines + drill hint should land well above 60.
|
|
assert estimate_tokens(out) >= 60
|
|
|
|
|
|
def test_l2_floor_renders_full_summary():
|
|
"""L2 must include header + every cluster + every aggregate branch — easily 200+ tokens."""
|
|
summary = make_summary()
|
|
out = format_l2(summary)
|
|
assert estimate_tokens(out) >= 200
|