mirror of
https://github.com/conorluddy/ios-simulator-skill.git
synced 2026-09-14 20:07:12 +08:00
Conor Luddy
00afc436d6
Closes #82. Three coverage gaps from the PR #78 review, now closed. The crashed-worker recovery path had no test coverage. Adds: - test_mark_crashed_sets_status — direct call after create - test_mark_crashed_after_claim_overrides_running — overrides running state - test_mark_crashed_silent_on_missing_session — no-op on dead session ID - test_load_summary_returns_none_when_missing — crashed session has no summary.json The 50-event fixture in tests/fixtures/sample_session.py only ever produced CRITICAL events (all duration ≥ 600 ms). L1 hit ~36% of its 200-token budget; L2 only ~8% of 2000. Token-budget tests trivially passed. Hardened make_events(): - 12 clusters spanning all 4 Severity tiers (MINOR/WARN/CRITICAL/FROZEN) - One hot cluster at 18 events (near-max count); others spread across cold tail - Two processes (MyApp + Helper) → process_distribution populates - First-cluster events packed tightly → detect_temporal_bursts populates - 7s gap inserted mid-window → detect_quiet_periods populates - Long symbol names → per-line formatter stress After hardening: L0=28/30 (93%), L1=91/200 (46%), L2=298/2000 (15%). L1/L2 are naturally sparse at default top_n=3; the real win is that all four L2 aggregate branches (Severity / Bursts / Quiet periods / Processes) now actually render. Adds qualitative assertions in test_token_budgets.py: - test_fixture_covers_all_severity_tiers — guard against fixture regression - test_fixture_has_at_least_ten_clusters - test_l2_renders_all_aggregate_branches — every L2 branch must populate - test_l1_floor_is_meaningful — floor at 60 tokens - test_l2_floor_renders_full_summary — floor at 200 tokens diff_sessions() produces 5 verdict strings; only 3 were tested. format_diff was only rendered for 'no change' and version mismatch — the new/resolved/drift render paths were never asserted. Adds 6 new in tests/test_diff.py: - test_diff_verdict_regression_new_minor - test_diff_verdict_improvement_resolved - test_diff_verdict_drift_mixed_signs (the bidirectional drift case) - test_format_diff_renders_new_block - test_format_diff_renders_resolved_block - test_format_diff_renders_drift_block Small helper _summary_with_clusters() builds SessionSummary instances with controlled per-cluster max_duration_ms — keeps verdict tests isolated from the event pipeline. test_diff_detects_resolved_cluster filtered by an old symbol name from the soft fixture ([ImageDecoder decode:]). Updated to use the new equivalent. - ruff check: All checks passed - black --check (scripts): clean - python -m pytest tests/: 103 passed (was 88) - pytest tests/: 103 passed
278 lines
9.1 KiB
Python
278 lines
9.1 KiB
Python
"""Diff intelligence: regressions, improvements, drift, version mismatch."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from common.hang_pipeline import (
|
|
SummaryBuilder,
|
|
bucket_severity,
|
|
compute_fingerprint,
|
|
diff_sessions,
|
|
format_diff,
|
|
)
|
|
from tests.fixtures.sample_session import make_events, make_summary
|
|
|
|
|
|
# === diff_sessions ===
|
|
|
|
|
|
def test_diff_no_change_when_identical():
|
|
a = make_summary(session_id="hang-a")
|
|
b = make_summary(session_id="hang-b")
|
|
result = diff_sessions(a, b)
|
|
assert result["verdict"] == "no change"
|
|
assert not result["new_clusters"]
|
|
assert not result["resolved_clusters"]
|
|
|
|
|
|
def test_diff_detects_new_critical_cluster():
|
|
a = make_summary(session_id="hang-a")
|
|
b_events = make_events()
|
|
# Add a new high-severity event with a fresh fingerprint.
|
|
from common.hang_pipeline import NormalisedEvent
|
|
|
|
b_events.append(
|
|
NormalisedEvent(
|
|
delta_ms=29_000,
|
|
process="MyApp",
|
|
pid=42,
|
|
duration_ms=1800,
|
|
severity=bucket_severity(1800),
|
|
symbol="[BrandNew explode:]",
|
|
message_prefix="brand-new hang",
|
|
fingerprint=compute_fingerprint("[BrandNew explode:]", "brand-new hang"),
|
|
)
|
|
)
|
|
b = SummaryBuilder(
|
|
session_id="hang-b",
|
|
started_at="2026-05-22T14:35:00",
|
|
duration_ms=30_000,
|
|
).build(b_events)
|
|
result = diff_sessions(a, b)
|
|
assert any(
|
|
"[BrandNew explode:]" == c["symbol_or_prefix"] for c in result["new_clusters"]
|
|
)
|
|
assert "regression" in result["verdict"]
|
|
assert "critical" in result["verdict"]
|
|
|
|
|
|
def test_diff_detects_resolved_cluster():
|
|
a = make_summary(session_id="hang-a")
|
|
# B has fewer fingerprints than A — at least one resolved.
|
|
dropped_symbol = "[ImageDecoderForLargeRasterAssets decodeWithOptions:]"
|
|
b_events = [e for e in make_events() if e.symbol != dropped_symbol]
|
|
b = SummaryBuilder(
|
|
session_id="hang-b",
|
|
started_at="2026-05-22T14:35:00",
|
|
duration_ms=30_000,
|
|
).build(b_events)
|
|
result = diff_sessions(a, b)
|
|
assert any(
|
|
c["symbol_or_prefix"] == dropped_symbol for c in result["resolved_clusters"]
|
|
)
|
|
|
|
|
|
def test_diff_flags_drift_when_max_duration_grows():
|
|
a_events = make_events()
|
|
a = SummaryBuilder(
|
|
session_id="hang-a", started_at="t", duration_ms=30_000
|
|
).build(a_events)
|
|
# B with the same fingerprints but 40% longer durations.
|
|
from common.hang_pipeline import NormalisedEvent
|
|
|
|
b_events = [
|
|
NormalisedEvent(
|
|
delta_ms=e.delta_ms,
|
|
process=e.process,
|
|
pid=e.pid,
|
|
duration_ms=e.duration_ms * 1.4,
|
|
severity=bucket_severity(e.duration_ms * 1.4),
|
|
symbol=e.symbol,
|
|
message_prefix=e.message_prefix,
|
|
fingerprint=e.fingerprint,
|
|
)
|
|
for e in a_events
|
|
]
|
|
b = SummaryBuilder(
|
|
session_id="hang-b", started_at="t", duration_ms=30_000
|
|
).build(b_events)
|
|
result = diff_sessions(a, b)
|
|
assert len(result["drift"]) > 0
|
|
# 40% bump should flag as worsened, not improved.
|
|
assert all(d["delta_pct"] > 0 for d in result["drift"])
|
|
|
|
|
|
def test_diff_version_mismatch_skips_structural():
|
|
a = make_summary(session_id="hang-a")
|
|
b = make_summary(session_id="hang-b")
|
|
b.fingerprint_version = 99
|
|
result = diff_sessions(a, b)
|
|
assert result["version_mismatch"] is True
|
|
assert "skipped" in result["verdict"]
|
|
# No structural keys when mismatched.
|
|
assert "new_clusters" not in result or not result.get("new_clusters")
|
|
|
|
|
|
# === format_diff ===
|
|
|
|
|
|
def test_format_diff_no_change():
|
|
a = make_summary(session_id="hang-a")
|
|
b = make_summary(session_id="hang-b")
|
|
out = format_diff(diff_sessions(a, b))
|
|
assert "no change" in out
|
|
assert "hang-a" in out
|
|
assert "hang-b" in out
|
|
|
|
|
|
def test_format_diff_version_mismatch_carries_warning():
|
|
a = make_summary(session_id="hang-a")
|
|
b = make_summary(session_id="hang-b")
|
|
b.fingerprint_version = 99
|
|
out = format_diff(diff_sessions(a, b))
|
|
assert "mismatch" in out.lower()
|
|
|
|
|
|
# === shared helper for SessionSummary fabrication ===
|
|
|
|
|
|
def _summary_with_clusters(session_id: str, clusters_spec: list[tuple[str, float, str]]):
|
|
"""Build a SessionSummary from (fingerprint, max_dur, severity) tuples."""
|
|
from common.hang_pipeline import Cluster, NormalisedEvent, SessionSummary, Severity
|
|
|
|
clusters = [
|
|
Cluster(
|
|
fingerprint=fp,
|
|
count=1,
|
|
max_duration_ms=max_dur,
|
|
total_duration_ms=max_dur,
|
|
first_delta_ms=0,
|
|
severity=Severity(sev),
|
|
symbol_or_prefix=fp,
|
|
sample_event=NormalisedEvent(
|
|
delta_ms=0,
|
|
process="p",
|
|
pid=1,
|
|
duration_ms=max_dur,
|
|
severity=Severity(sev),
|
|
symbol=fp,
|
|
message_prefix=fp,
|
|
fingerprint=fp,
|
|
),
|
|
)
|
|
for fp, max_dur, sev in clusters_spec
|
|
]
|
|
return SessionSummary(
|
|
session_id=session_id,
|
|
started_at="t",
|
|
duration_ms=1000,
|
|
event_count=len(clusters),
|
|
dropped_below_threshold=0,
|
|
matched_lines=len(clusters),
|
|
total_lines=len(clusters),
|
|
clusters=clusters,
|
|
aggregates={},
|
|
)
|
|
|
|
|
|
# === zero-duration drift handling (#79) ===
|
|
|
|
|
|
def test_diff_zero_to_zero_counts_as_stable():
|
|
a = _summary_with_clusters("hang-a", [("fp:silent", 0.0, "minor")])
|
|
b = _summary_with_clusters("hang-b", [("fp:silent", 0.0, "minor")])
|
|
result = diff_sessions(a, b)
|
|
assert result["stable_count"] == 1
|
|
assert not result["drift"]
|
|
|
|
|
|
def test_diff_zero_to_nonzero_is_drift_with_inf_delta():
|
|
a = _summary_with_clusters("hang-a", [("fp:newsignal", 0.0, "minor")])
|
|
b = _summary_with_clusters("hang-b", [("fp:newsignal", 800.0, "critical")])
|
|
result = diff_sessions(a, b)
|
|
assert len(result["drift"]) == 1
|
|
assert result["drift"][0]["delta_pct"] == float("inf")
|
|
assert result["drift"][0]["max_duration_ms_a"] == 0.0
|
|
assert result["drift"][0]["max_duration_ms_b"] == 800.0
|
|
|
|
|
|
def test_diff_nonzero_to_zero_is_drift_with_minus_one_hundred():
|
|
a = _summary_with_clusters("hang-a", [("fp:fixed", 800.0, "critical")])
|
|
b = _summary_with_clusters("hang-b", [("fp:fixed", 0.0, "minor")])
|
|
result = diff_sessions(a, b)
|
|
assert len(result["drift"]) == 1
|
|
assert result["drift"][0]["delta_pct"] == -100.0
|
|
|
|
|
|
def test_diff_nonzero_unchanged_still_uses_threshold():
|
|
# Regression guard — the new zero-handling branches must not break the standard path.
|
|
a = _summary_with_clusters("hang-a", [("fp:steady", 500.0, "critical")])
|
|
b = _summary_with_clusters("hang-b", [("fp:steady", 505.0, "critical")]) # 1% drift
|
|
result = diff_sessions(a, b)
|
|
assert result["stable_count"] == 1
|
|
assert not result["drift"]
|
|
|
|
|
|
def test_format_diff_renders_inf_delta_as_new():
|
|
a = _summary_with_clusters("hang-a", [("fp:newsignal", 0.0, "minor")])
|
|
b = _summary_with_clusters("hang-b", [("fp:newsignal", 800.0, "critical")])
|
|
out = format_diff(diff_sessions(a, b))
|
|
assert "new" in out
|
|
assert "inf" not in out.lower()
|
|
|
|
|
|
# === verdict coverage (#82): the 3 verdicts not exercised elsewhere ===
|
|
|
|
|
|
def test_diff_verdict_regression_new_minor():
|
|
# A is empty, B has one new MINOR cluster — verdict should NOT be "new critical".
|
|
a = _summary_with_clusters("hang-a", [])
|
|
b = _summary_with_clusters("hang-b", [("fp:newminor", 200.0, "minor")])
|
|
result = diff_sessions(a, b)
|
|
assert result["verdict"] == "regression: 1 new minor"
|
|
|
|
|
|
def test_diff_verdict_improvement_resolved():
|
|
a = _summary_with_clusters("hang-a", [("fp:wasthere", 500.0, "critical")])
|
|
b = _summary_with_clusters("hang-b", [])
|
|
result = diff_sessions(a, b)
|
|
assert result["verdict"] == "improvement: 1 resolved"
|
|
|
|
|
|
def test_diff_verdict_drift_mixed_signs():
|
|
# Shared fingerprints with one worsening + one improving in B.
|
|
a = _summary_with_clusters("hang-a", [("fp:up", 500.0, "critical"), ("fp:down", 800.0, "critical")])
|
|
b = _summary_with_clusters("hang-b", [("fp:up", 800.0, "critical"), ("fp:down", 500.0, "critical")])
|
|
result = diff_sessions(a, b)
|
|
assert result["verdict"] == "drift: 1 worsened, 1 improved"
|
|
|
|
|
|
# === format_diff render coverage (#82) ===
|
|
|
|
|
|
def test_format_diff_renders_new_block():
|
|
a = _summary_with_clusters("hang-a", [])
|
|
b = _summary_with_clusters("hang-b", [("fp:fresh", 1500.0, "critical")])
|
|
out = format_diff(diff_sessions(a, b))
|
|
assert "New (" in out
|
|
assert "fp:fresh" in out
|
|
assert "1500" in out
|
|
|
|
|
|
def test_format_diff_renders_resolved_block():
|
|
a = _summary_with_clusters("hang-a", [("fp:gone", 700.0, "warn")])
|
|
b = _summary_with_clusters("hang-b", [])
|
|
out = format_diff(diff_sessions(a, b))
|
|
assert "Resolved (" in out
|
|
assert "fp:gone" in out
|
|
|
|
|
|
def test_format_diff_renders_drift_block():
|
|
# 60% bump triggers drift entry.
|
|
a = _summary_with_clusters("hang-a", [("fp:slow", 500.0, "critical")])
|
|
b = _summary_with_clusters("hang-b", [("fp:slow", 800.0, "critical")])
|
|
out = format_diff(diff_sessions(a, b))
|
|
assert "Drift (" in out
|
|
assert "fp:slow" in out
|
|
assert "500" in out
|
|
assert "800" in out
|