mirror of
https://github.com/boshu2/agentops.git
synced 2026-09-14 15:08:13 +08:00
Bo
c516dd55a9
## What changed - preserve canonical trigger text while compacting generated Codex descriptions - harden portable skill validation for duplicate YAML keys, symlink containment, and current optional-field rules - regenerate all owned Codex projections and hashes - add focused negative and generator regression coverage ## Verification - portable conformance: 52/52 - portable Bats: 8/8 - generator acceptance: 18/18 - `scripts/regen-all.sh --check` - Bash syntax, blocking ShellCheck, and `git diff --check` The interrupted all-skills remediation snapshot is deliberately excluded: it remains preserved on `codex/all-skills-pass-20260816` and is not merge-safe.
304 lines
12 KiB
Bash
Executable File
304 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# validate-codex-api-conformance.sh — Check generated codex skills against Codex API contract.
|
|
# Exit 0 = pass, exit 1 = failures found.
|
|
# Contract: docs/contracts/codex-skill-api.md
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
SKILLS_ROOT="${CODEX_SKILLS_ROOT:-$REPO_ROOT/skills-codex}"
|
|
|
|
# Cross-runtime skills legitimately reference non-Codex runtimes/paths (cc-hooks
|
|
# documents ~/.claude/settings.json). Shared exemption list with codex-sync and
|
|
# the other Codex gates.
|
|
CROSS_RUNTIME_FILE="$REPO_ROOT/scripts/lint/codex-cross-runtime-skills.txt"
|
|
is_cross_runtime() {
|
|
[[ -f "$CROSS_RUNTIME_FILE" ]] || return 1
|
|
grep -vE '^[[:space:]]*#|^[[:space:]]*$' "$CROSS_RUNTIME_FILE" | grep -qxF "$1"
|
|
}
|
|
|
|
# parity_only twins are GENERATED by codex-sync and verified by its byte-exact
|
|
# drift gate; conformance is re-checked only on BESPOKE (hand-authored) twins.
|
|
BESPOKE_SKILLS="$(python3 -c "import json; d=json.load(open('$REPO_ROOT/skills-codex-overrides/catalog.json')); print(chr(10).join(e['name'] for e in d.get('skills',[]) if e.get('treatment')=='bespoke'))" 2>/dev/null || true)"
|
|
is_bespoke() { grep -qxF "$1" <<<"$BESPOKE_SKILLS"; }
|
|
|
|
failures=0
|
|
warnings=0
|
|
|
|
if [[ ! -d "$SKILLS_ROOT" ]]; then
|
|
echo "Error: skills-codex directory not found: $SKILLS_ROOT" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# --- Check 1: Portable Agent Skills package contract ---
|
|
# The checked-in Codex tree is the portable release projection. Validate every
|
|
# generated and bespoke package here; generator parity alone cannot establish
|
|
# portable conformance. This intentionally enforces the normative specification
|
|
# rather than the narrower behavior of any one reference implementation.
|
|
echo "=== Check 1: Portable Agent Skills contract ==="
|
|
portable_output=""
|
|
if ! portable_output="$(python3 - "$SKILLS_ROOT" "$REPO_ROOT" 2>&1 <<'PY'
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
from yaml.constructor import ConstructorError
|
|
from yaml.resolver import BaseResolver
|
|
|
|
skills_root = Path(sys.argv[1]).resolve()
|
|
repo_root = Path(sys.argv[2]).resolve()
|
|
try:
|
|
skills_root.relative_to(repo_root)
|
|
link_root = repo_root
|
|
except ValueError:
|
|
# A detached catalog bundle is its own containment boundary. Cross-skill
|
|
# links may resolve to sibling packages inside that catalog.
|
|
link_root = skills_root
|
|
allowed = {"name", "description", "license", "compatibility", "metadata", "allowed-tools"}
|
|
name_re = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
|
link_re = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)")
|
|
errors: list[tuple[str, str]] = []
|
|
checked = 0
|
|
|
|
|
|
class UniqueKeyLoader(yaml.SafeLoader):
|
|
pass
|
|
|
|
|
|
def construct_unique_mapping(loader: UniqueKeyLoader, node: yaml.Node, deep: bool = False) -> dict:
|
|
mapping: dict = {}
|
|
for key_node, value_node in node.value:
|
|
key = loader.construct_object(key_node, deep=deep)
|
|
try:
|
|
duplicate = key in mapping
|
|
except TypeError as exc:
|
|
raise ConstructorError(
|
|
"while constructing a mapping",
|
|
node.start_mark,
|
|
"found an unhashable mapping key",
|
|
key_node.start_mark,
|
|
) from exc
|
|
if duplicate:
|
|
raise ConstructorError(
|
|
"while constructing a mapping",
|
|
node.start_mark,
|
|
f"found duplicate key {key!r}",
|
|
key_node.start_mark,
|
|
)
|
|
mapping[key] = loader.construct_object(value_node, deep=deep)
|
|
return mapping
|
|
|
|
|
|
UniqueKeyLoader.add_constructor(
|
|
BaseResolver.DEFAULT_MAPPING_TAG,
|
|
construct_unique_mapping,
|
|
)
|
|
|
|
|
|
def fail(skill: str, message: str) -> None:
|
|
errors.append((skill, message))
|
|
|
|
|
|
for skill_dir in sorted(path for path in skills_root.iterdir() if path.is_dir()):
|
|
skill = skill_dir.name
|
|
if skill_dir.is_symlink():
|
|
fail(skill, "skill package directory must not be a symlink")
|
|
continue
|
|
skill_md = skill_dir / "SKILL.md"
|
|
if not skill_md.is_file() or skill_md.is_symlink():
|
|
fail(skill, "missing regular SKILL.md")
|
|
continue
|
|
checked += 1
|
|
try:
|
|
text = skill_md.read_text(encoding="utf-8")
|
|
except (OSError, UnicodeError) as exc:
|
|
fail(skill, f"SKILL.md is not loadable UTF-8: {exc}")
|
|
continue
|
|
if not text.startswith("---\n"):
|
|
fail(skill, "SKILL.md must start with YAML frontmatter")
|
|
continue
|
|
marker = text.find("\n---\n", 4)
|
|
if marker < 0:
|
|
fail(skill, "SKILL.md frontmatter is not closed")
|
|
continue
|
|
raw_frontmatter = text[4:marker]
|
|
body = text[marker + 5 :]
|
|
try:
|
|
metadata = yaml.load(raw_frontmatter, Loader=UniqueKeyLoader)
|
|
except yaml.YAMLError as exc:
|
|
fail(skill, f"invalid YAML frontmatter: {exc}")
|
|
continue
|
|
if not isinstance(metadata, dict):
|
|
fail(skill, "frontmatter must be a mapping")
|
|
continue
|
|
unexpected = sorted(set(metadata) - allowed)
|
|
if unexpected:
|
|
fail(skill, f"host-only frontmatter fields: {', '.join(unexpected)}")
|
|
name = metadata.get("name")
|
|
if not isinstance(name, str) or not name_re.fullmatch(name) or len(name) > 64:
|
|
fail(skill, "name must be 1-64 lowercase ASCII letters/digits/hyphens without edge or repeated hyphens")
|
|
elif name != skill:
|
|
fail(skill, f"name {name!r} does not match directory {skill!r}")
|
|
description = metadata.get("description")
|
|
if not isinstance(description, str) or not description.strip() or len(description) > 1024:
|
|
fail(skill, "description must be a nonempty string of at most 1024 characters")
|
|
if "license" in metadata and (
|
|
not isinstance(metadata["license"], str) or not metadata["license"].strip()
|
|
):
|
|
fail(skill, "license must be a nonempty string")
|
|
if "compatibility" in metadata:
|
|
compatibility = metadata["compatibility"]
|
|
if not isinstance(compatibility, str) or not compatibility.strip() or len(compatibility) > 500:
|
|
fail(skill, "compatibility must be a nonempty string of at most 500 characters")
|
|
if "metadata" in metadata:
|
|
extension = metadata["metadata"]
|
|
if not isinstance(extension, dict) or any(
|
|
not isinstance(key, str) or not isinstance(value, str)
|
|
for key, value in (extension.items() if isinstance(extension, dict) else ())
|
|
):
|
|
fail(skill, "metadata must map strings to strings")
|
|
if "allowed-tools" in metadata:
|
|
tools = metadata["allowed-tools"]
|
|
if not isinstance(tools, str) or not tools.strip() or "," in tools or "\t" in tools or "\n" in tools:
|
|
fail(skill, "allowed-tools must be a nonempty space-separated string without comma delimiters")
|
|
if not body.strip():
|
|
fail(skill, "SKILL.md body must be nonempty")
|
|
|
|
for resource in sorted(skill_dir.rglob("*")):
|
|
if resource.is_symlink():
|
|
fail(skill, f"resource must not be a symlink: {resource.relative_to(skill_dir)}")
|
|
|
|
# Validate actual Markdown resource links after removing code, where text
|
|
# such as errors.AsType[T](err) is not a link. External URLs and anchors are
|
|
# valid but do not identify bundled resources.
|
|
for markdown in sorted(skill_dir.rglob("*.md")):
|
|
if markdown.is_symlink():
|
|
continue
|
|
try:
|
|
markdown_text = markdown.read_text(encoding="utf-8")
|
|
except (OSError, UnicodeError) as exc:
|
|
fail(skill, f"resource is not loadable UTF-8: {markdown.relative_to(skill_dir)}: {exc}")
|
|
continue
|
|
markdown_text = re.sub(r"```.*?```", "", markdown_text, flags=re.S)
|
|
markdown_text = re.sub(r"`[^`\n]*`", "", markdown_text)
|
|
for match in link_re.finditer(markdown_text):
|
|
raw_target = match.group(1).strip()
|
|
target = raw_target.split(maxsplit=1)[0].strip("<>")
|
|
if not target or target.startswith("#") or re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target):
|
|
continue
|
|
if target.startswith(("/", "~")) or re.match(r"^[A-Za-z]:[\\/]", target):
|
|
fail(skill, f"resource link must be relative: {target}")
|
|
continue
|
|
path_part = target.split("#", 1)[0].split("?", 1)[0]
|
|
resolved = (markdown.parent / path_part).resolve()
|
|
try:
|
|
resolved.relative_to(link_root)
|
|
except ValueError:
|
|
fail(skill, f"resource link escapes catalog: {target}")
|
|
continue
|
|
if not resolved.exists():
|
|
fail(skill, f"resource link does not resolve: {markdown.relative_to(skill_dir)} -> {target}")
|
|
|
|
if checked == 0:
|
|
fail("<root>", "no skill packages found")
|
|
|
|
for skill, message in errors:
|
|
print(f" FAIL [{skill}] {message}")
|
|
if errors:
|
|
raise SystemExit(1)
|
|
print(f" PASS [portable] {checked} package(s)")
|
|
PY
|
|
)"; then
|
|
printf '%s\n' "$portable_output"
|
|
portable_failures="$(printf '%s\n' "$portable_output" | grep -c '^ FAIL \[' || true)"
|
|
if [[ "$portable_failures" -eq 0 ]]; then
|
|
echo " FAIL [portable-validator] validator did not complete"
|
|
portable_failures=1
|
|
fi
|
|
failures=$((failures + portable_failures))
|
|
else
|
|
printf '%s\n' "$portable_output"
|
|
fi
|
|
|
|
# --- Check 2: No Claude-only primitive names ---
|
|
echo "=== Check 2: Claude primitive references ==="
|
|
# All Claude-only primitives — none have working Codex equivalents
|
|
# (todo_write/update_plan empirically verified as unavailable via a headless codex run)
|
|
CLAUDE_PRIMITIVES='TaskCreate|TaskList|TaskUpdate|TaskGet|TaskStop|TeamCreate|TeamDelete|SendMessage|EnterPlanMode|ExitPlanMode|EnterWorktree|task-create|task-list|task-update|task-get|task-stop|team-create|team-delete|send-message|enter-plan-mode|exit-plan-mode|enter-worktree|todo_write|update_plan'
|
|
|
|
while IFS= read -r skill_md; do
|
|
skill_name="$(basename "$(dirname "$skill_md")")"
|
|
is_bespoke "$skill_name" || continue # parity twins are generator/drift-verified
|
|
|
|
# Search body (after frontmatter) for Claude primitives
|
|
body=$(awk 'BEGIN{skip=0} NR==1 && /^---$/{skip=1; next} skip && /^---$/{skip=0; next} !skip{print}' "$skill_md")
|
|
matches=$(echo "$body" | grep -onE "\b($CLAUDE_PRIMITIVES)\b" 2>/dev/null || true)
|
|
if [[ -n "$matches" ]]; then
|
|
count=$(echo "$matches" | wc -l | tr -d ' ')
|
|
echo " FAIL [$skill_name] $count Claude primitive reference(s)"
|
|
failures=$((failures + 1))
|
|
fi
|
|
done < <(find "$SKILLS_ROOT" -mindepth 2 -maxdepth 2 -name 'SKILL.md' -type f | sort)
|
|
|
|
# --- Check 3: No Claude-specific paths ---
|
|
echo "=== Check 3: Claude-specific paths ==="
|
|
while IFS= read -r skill_md; do
|
|
skill_name="$(basename "$(dirname "$skill_md")")"
|
|
is_bespoke "$skill_name" || continue # parity twins are generator/drift-verified
|
|
|
|
# Cross-runtime skills may reference ~/.claude accurately (cc-hooks documents
|
|
# the Claude Code hook config path); skip this check for them.
|
|
if is_cross_runtime "$skill_name"; then
|
|
continue
|
|
fi
|
|
# shellcheck disable=SC2088 # intentional literal match of the documented path
|
|
matches=$(grep -n '~/\.claude/' "$skill_md" 2>/dev/null || true)
|
|
if [[ -n "$matches" ]]; then
|
|
count=$(echo "$matches" | wc -l | tr -d ' ')
|
|
echo " FAIL [$skill_name] $count ~/.claude/ path reference(s)"
|
|
failures=$((failures + 1))
|
|
fi
|
|
done < <(find "$SKILLS_ROOT" -mindepth 2 -maxdepth 2 -name 'SKILL.md' -type f | sort)
|
|
|
|
# --- Check 4: agents/openai.yaml validity (if present) ---
|
|
echo "=== Check 4: agents/openai.yaml validity ==="
|
|
while IFS= read -r yaml_file; do
|
|
skill_name="$(basename "$(dirname "$(dirname "$yaml_file")")")"
|
|
# Basic YAML syntax check
|
|
if ! python3 -c "import yaml; yaml.safe_load(open('$yaml_file'))" 2>/dev/null; then
|
|
echo " FAIL [$skill_name] Invalid YAML: $yaml_file"
|
|
failures=$((failures + 1))
|
|
fi
|
|
done < <(find "$SKILLS_ROOT" -path '*/agents/openai.yaml' -type f 2>/dev/null | sort)
|
|
|
|
# --- Check 5: No Skill() tool invocations ---
|
|
echo "=== Check 5: Skill() tool invocations ==="
|
|
while IFS= read -r skill_md; do
|
|
skill_name="$(basename "$(dirname "$skill_md")")"
|
|
is_bespoke "$skill_name" || continue # parity twins are generator/drift-verified
|
|
|
|
matches=$(grep -n 'Skill(skill=' "$skill_md" 2>/dev/null || true)
|
|
if [[ -n "$matches" ]]; then
|
|
count=$(echo "$matches" | wc -l | tr -d ' ')
|
|
echo " FAIL [$skill_name] $count Skill() tool invocation(s) (use \$skill syntax)"
|
|
failures=$((failures + 1))
|
|
fi
|
|
done < <(find "$SKILLS_ROOT" -mindepth 2 -maxdepth 2 -name 'SKILL.md' -type f | sort)
|
|
|
|
# --- Summary ---
|
|
echo ""
|
|
echo "=== Summary ==="
|
|
echo "Failures: $failures"
|
|
echo "Warnings: $warnings"
|
|
|
|
if [[ $failures -gt 0 ]]; then
|
|
echo ""
|
|
echo "Codex API conformance check FAILED with $failures failure(s)."
|
|
exit 1
|
|
else
|
|
echo ""
|
|
echo "Codex API conformance check passed."
|
|
exit 0
|
|
fi
|