mirror of
https://github.com/AgriciDaniel/claude-blog.git
synced 2026-09-19 03:32:21 +08:00
Daniel
0f74346a65
Sixth-round hostile audit caught 20 file:line defects v1.8.4 missed, including 4 HIGH-severity items: CLAUDE.md was stale at v1.7.0 (never bumped through any 1.8.x), docs/ARCHITECTURE.md was stale at v1.7.0, scripts/lint_prose.py (the v1.8.4 CI linter) shipped with ZERO unit tests, and the v1.8.4 OUTERMOST nonce-authority instruction had no regression test. v1.8.5 closes the HIGH + MEDIUM findings AND adds two more coherence regression tests so documentation drift cannot recur silently. Honest framing: the asymptote is ~92-95/100. Each round catches fewer items at lower severity. v1.8.5 lifts the calibrated score back toward ~92/100; a 7th audit will surface additional LOW/INFO items. The infrastructure layer in v1.8.4 (lint_prose + version- coherence) is now itself tested in v1.8.5. HIGH findings closed - 6TH-AUDIT-001 CLAUDE.md updated v1.7.0 -> v1.8.5 baseline - 6TH-AUDIT-002 docs/ARCHITECTURE.md updated 27 -> 30 sub-skills - 6TH-AUDIT-005 tests/test_lint_prose.py (new): 18 behavioral tests - 6TH-AUDIT-008 OUTERMOST regression in test_security_v1_8_0.py + behavioral test in test_load_untrusted_root.py MEDIUM findings closed - 6TH-AUDIT-003/004 28 vs 29 command count reconciled (README:195 and README:111 arithmetic fixed) - 6TH-AUDIT-010 tests/test_version_coherence.py (new): 2 tests replicating the v1.8.4 CI YAML heredoc as pytest assertions - 6TH-AUDIT-011 `claude plugin validate` step added to CI as best-effort (continue-on-error when CLI not on runner) LOW findings closed - 6TH-AUDIT-006 lint_prose.py nested-fence handling: track opening delimiter so ``` inside ~~~ doesn't close outer - 6TH-AUDIT-007 lint_prose.py double-backtick mask: pre-mask ``code`` spans first to avoid empty-span match leak - 6TH-AUDIT-019 tests/test_command_coherence.py (new): asserts SKILL.md table and COMMANDS.md table declare same command set README updates - Quality Gates section enumerates the 6 CI gates (pytest, plugin validate, stale-path, prose hygiene, version coherence, command coherence) Tests - 130 pass + 0 skips (v1.8.4 was 106 + 0 skips). 24 new tests added. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
70 lines
2.4 KiB
Python
70 lines
2.4 KiB
Python
"""Command-count coherence regression test.
|
|
|
|
Asserts that the orchestrator (skills/blog/SKILL.md) and the command
|
|
reference (docs/COMMANDS.md) declare the same set of `/blog X` commands.
|
|
|
|
Added v1.8.5 (6TH-AUDIT-019): the v1.8.4 audit found `README:195` said
|
|
"28 user-facing commands" while `docs/COMMANDS.md:3` said "29 ... slash
|
|
commands" and `skills/blog/SKILL.md` had 29 rows. This test catches the
|
|
class-of-defect on every PR.
|
|
|
|
Stdlib + pytest only.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
# Pattern for orchestrator table rows: `| \`/blog X ...\` |`
|
|
_SKILL_ROW = re.compile(r"^\|\s*`/blog\s+([\w-]+)", re.MULTILINE)
|
|
# Pattern for COMMANDS.md table rows: `| \`X <args>\` | blog-X | ...`
|
|
_COMMANDS_ROW = re.compile(r"^\|\s*`(\w[\w-]*)\b", re.MULTILINE)
|
|
|
|
|
|
def _extract_skill_commands() -> set[str]:
|
|
text = (ROOT / "skills" / "blog" / "SKILL.md").read_text(encoding="utf-8")
|
|
return set(_SKILL_ROW.findall(text))
|
|
|
|
|
|
def _extract_commands_md_commands() -> set[str]:
|
|
text = (ROOT / "docs" / "COMMANDS.md").read_text(encoding="utf-8")
|
|
# The overview table sits between the "## Command Overview" heading
|
|
# and the next "---" separator. Match table rows in that block.
|
|
block = re.search(
|
|
r"## Command Overview.*?^---",
|
|
text, re.DOTALL | re.MULTILINE,
|
|
)
|
|
assert block is not None, "Command Overview section missing from COMMANDS.md"
|
|
cmds = set(_COMMANDS_ROW.findall(block.group(0)))
|
|
cmds.discard("Command") # the table header column name
|
|
return cmds
|
|
|
|
|
|
def test_orchestrator_and_commands_md_declare_same_commands() -> None:
|
|
skill_cmds = _extract_skill_commands()
|
|
commands_cmds = _extract_commands_md_commands()
|
|
assert skill_cmds == commands_cmds, (
|
|
f"command sets disagree:\n"
|
|
f" in SKILL.md but not COMMANDS.md: {skill_cmds - commands_cmds}\n"
|
|
f" in COMMANDS.md but not SKILL.md: {commands_cmds - skill_cmds}"
|
|
)
|
|
|
|
|
|
def test_orchestrator_has_at_least_28_commands() -> None:
|
|
"""Sanity floor: regression guard against a future refactor that
|
|
accidentally deletes commands from the SKILL.md table."""
|
|
skill_cmds = _extract_skill_commands()
|
|
assert len(skill_cmds) >= 28, (
|
|
f"orchestrator commands dropped below floor: {len(skill_cmds)}, "
|
|
f"current set = {sorted(skill_cmds)}"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v"])
|