mirror of
https://github.com/getcargohq/cargo-skills.git
synced 2026-09-14 19:17:15 +08:00
57254032cf
* Record which skills a session actually loaded Routing is the thing this bundle turns on, and it has been argued rather than measured. evals/routing.jsonl now scores 100% on the real-model tier — a ceiling effect, not a victory: every prompt in it was written by someone looking at the descriptions they were grading, and several were added to pin behavior that had just been implemented. A suite that always passes has stopped discovering anything. Prompts nobody wrote to pass have to come from real sessions. - hooks/skill-loads.sh reads a Claude Code transcript and emits a marker naming the skills invoked and the recipes/playbooks/guides opened: `[cargo-skills: cargo-gtm | docs: build-tam,waterfall]`. The session hooks append it to the row's summary, so it is queryable today over workspace_management.sessions with no schema change. - Wired into both session-checkpoint.sh and session-end.sh. Recomputed from the whole transcript each time rather than accumulated, so a session that dies abruptly still carries it. Emits nothing at all for sessions that never touched Cargo. - evals/README.md documents the three questions the data answers (which skills never load, which sessions loaded two before settling, which load with no doc read after) and how each turns into an eval case. - Self-test wired into skills-lint, because a regression here corrupts telemetry silently instead of failing loudly. Two bugs the fixtures passed and real transcripts caught: - **Prose mentions counted as reads.** The doc regex matched a bare path anywhere in the transcript, and a skill's own catalog table names every recipe and playbook — so one cargo-gtm load reported all 43 playbooks as opened. Now anchored to a `"file_path":` tool input. Fixture 7 is that exact case. - **Dangling prefix** when docs were read without a Skill call, giving `[cargo-skills: | docs: …]`. Both halves are now independently optional. Also: the doc list caps at 12, and says `+N more` rather than looking complete. PRIVACY: skill and document names only, all of them already public in this repository. Never a prompt, file contents, an argument, a record, or anything a user typed. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * Fix Bugbot issues: preserve marker in sync finalize, retain skill load order 1. End hook drops skill marker (high severity): The sync finalize now includes the skill marker in the placeholder summary, so it persists even when refine() fails, times out, or is skipped. The marker is fast (no LLM call, just grep/sed). 2. Skill order lost to sorting (medium severity): Replaced 'sort -u' with 'awk !seen[$0]++' to preserve load order. The first skill in a multi-skill session is the misroute candidate, so order matters for harvest question 2. Added test case 10 to verify load order preservation. Co-authored-by: Auree Aubert <aureeaubert@users.noreply.github.com> --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: Auree Aubert <aureeaubert@users.noreply.github.com>
223 lines
9.8 KiB
Bash
Executable File
223 lines
9.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Cargo session-end hook: summarize the transcript and finalize the session row.
|
|
#
|
|
# SINGLE SOURCE OF TRUTH for both delivery channels — runs as the PLUGIN copy
|
|
# (default) or as the STANDALONE copy the installer's fallback channel downloads
|
|
# to ~/.claude/hooks/. Behavior is identical in both; only deference differs:
|
|
# the plugin copy exits when a standalone copy exists, so the row is never
|
|
# double-finalized. Mode auto-detects from the script's location and can be
|
|
# forced with a first argument: plugin | standalone.
|
|
#
|
|
# Failures never block a session and fall back to the "Session ended."
|
|
# placeholder, but every step logs to $CARGO_SESSION_LOG (default
|
|
# ~/.claude/cargo-session.log) so a stuck placeholder row can be diagnosed
|
|
# instead of failing silently.
|
|
set -u
|
|
|
|
# Re-entrancy guard. The refinement step below summarizes the transcript by
|
|
# shelling out to `claude -p`, and that subprocess is itself a full Claude Code
|
|
# session: when it exits it fires SessionEnd, which runs this hook again, which
|
|
# spawns another summarizer. The chain only ends when some child's transcript
|
|
# happens to be missing — observed in the wild as 264 headless sessions against
|
|
# 8 real ones (~16.5M tokens) on one machine in a day.
|
|
#
|
|
# The marker is exported across the `claude -p` call and propagates to the
|
|
# child's own hooks through the environment. Every Cargo lifecycle hook checks
|
|
# it and exits immediately, so a summarizer child registers no session row,
|
|
# refreshes no CLI, and spawns no second summarizer.
|
|
if [ "${CARGO_SESSION_SUMMARIZER:-}" = "1" ]; then
|
|
exit 0
|
|
fi
|
|
|
|
SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
|
MODE="${1:-}"
|
|
if [ -z "$MODE" ]; then
|
|
if [ "$SCRIPT_DIR" = "$HOME/.claude/hooks" ]; then MODE="standalone"; else MODE="plugin"; fi
|
|
fi
|
|
# A standalone copy owns the lifecycle only when it is REGISTERED in
|
|
# ~/.claude/settings.json — a leftover file nothing invokes must not suppress
|
|
# the plugin copy. When registration cannot be verified (no jq, or no
|
|
# settings.json), default to NOT owned: the plugin copy runs. That is the safe
|
|
# direction — the session upsert is idempotent, so a duplicate with an active
|
|
# standalone is harmless, whereas a wrong defer silently skips CLI pinning and
|
|
# session logging with no signal.
|
|
standalone_owns() {
|
|
s="$HOME/.claude/hooks/$1"
|
|
[ -x "$s" ] || return 1
|
|
command -v jq >/dev/null 2>&1 || return 1
|
|
[ -f "$HOME/.claude/settings.json" ] || return 1
|
|
jq -e --arg cmd "$s" \
|
|
'[.hooks[]?[]? | .hooks[]? | select(.command | contains($cmd))] | length > 0' \
|
|
"$HOME/.claude/settings.json" >/dev/null 2>&1
|
|
}
|
|
if [ "$MODE" = "plugin" ] && standalone_owns "session-end.sh"; then
|
|
exit 0
|
|
fi
|
|
|
|
LOG="${CARGO_SESSION_LOG:-$HOME/.claude/cargo-session.log}"
|
|
mkdir -p "$(dirname "$LOG")" 2>/dev/null || true
|
|
log() { printf '[%s] session-end(%s): %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$MODE" "$*" >>"$LOG" 2>/dev/null || true; }
|
|
|
|
INPUT="$(cat 2>/dev/null || true)"
|
|
SESSION_ID="$(printf '%s' "$INPUT" | jq -r '.session_id // "unknown"' 2>/dev/null || echo "unknown")"
|
|
TRANSCRIPT_PATH="$(printf '%s' "$INPUT" | jq -r '.transcript_path // ""' 2>/dev/null || echo "")"
|
|
|
|
# Resolve the claude binary. Hooks frequently run with a minimal PATH, and
|
|
# `claude` is a Node script whose shebang also needs `node` on PATH — so a bare
|
|
# `command -v claude` misses it whenever the binary lives in a Node/version-
|
|
# manager bin dir (nvm, volta, asdf, /opt/node*, npm global prefix). Widen PATH
|
|
# to those dirs first, then resolve.
|
|
add_path() {
|
|
[ -n "${1:-}" ] && [ -d "$1" ] || return 0
|
|
case ":$PATH:" in
|
|
*":$1:"*) ;;
|
|
*) PATH="$1:$PATH" ;;
|
|
esac
|
|
}
|
|
|
|
# Node's own bin dir is where npm-global CLIs (including claude) usually land.
|
|
if command -v node >/dev/null 2>&1; then
|
|
add_path "$(dirname "$(command -v node)")"
|
|
fi
|
|
if command -v npm >/dev/null 2>&1; then
|
|
add_path "$(npm prefix -g 2>/dev/null)/bin"
|
|
fi
|
|
add_path "$HOME/.claude/local"
|
|
add_path "$HOME/.local/bin"
|
|
add_path "${VOLTA_HOME:-$HOME/.volta}/bin"
|
|
add_path "$HOME/.asdf/shims"
|
|
add_path "/usr/local/bin"
|
|
add_path "/opt/homebrew/bin"
|
|
for d in /opt/node*/bin "${NVM_DIR:-$HOME/.nvm}"/versions/node/*/bin; do
|
|
add_path "$d"
|
|
done
|
|
export PATH
|
|
|
|
CLAUDE_BIN="$(command -v claude 2>/dev/null || true)"
|
|
|
|
# 1) Synchronous, fast finalize so the session row is always closed with
|
|
# --finished even if the async refinement below never completes. The hook MUST
|
|
# return quickly — a blocking `claude -p` here is aborted during session
|
|
# teardown, which Claude Code reports as "Hook cancelled".
|
|
#
|
|
# Include the skill marker in the placeholder summary so it persists even when
|
|
# refine() fails, times out, or is skipped. The marker is fast: no LLM call,
|
|
# just grep/sed on the transcript.
|
|
PLACEHOLDER_SUMMARY="Session ended."
|
|
if [ -n "$TRANSCRIPT_PATH" ] && [ -f "$TRANSCRIPT_PATH" ] && [ -x "$SCRIPT_DIR/skill-loads.sh" ]; then
|
|
PLACEHOLDER_MARKER="$(bash "$SCRIPT_DIR/skill-loads.sh" "$TRANSCRIPT_PATH" 2>/dev/null || true)"
|
|
[ -n "$PLACEHOLDER_MARKER" ] && PLACEHOLDER_SUMMARY="$PLACEHOLDER_SUMMARY $PLACEHOLDER_MARKER"
|
|
fi
|
|
|
|
if command -v cargo-ai >/dev/null 2>&1; then
|
|
if cargo-ai workspaceManagement session upsert \
|
|
--session-id "$SESSION_ID" \
|
|
--title "Claude Code session ${SESSION_ID}" \
|
|
--summary "$PLACEHOLDER_SUMMARY" \
|
|
--finished >>"$LOG" 2>&1; then
|
|
log "finalized $SESSION_ID with placeholder summary"
|
|
else
|
|
log "session upsert failed for $SESSION_ID"
|
|
fi
|
|
else
|
|
log "cargo-ai not found; cannot finalize $SESSION_ID"
|
|
fi
|
|
|
|
# 2) Async refinement: summarize the transcript with claude and re-upsert a
|
|
# better title/summary. Fully detached (setsid/nohup + closed fds) so it
|
|
# survives the parent session exiting and never blocks — or is cancelled
|
|
# with — the hook.
|
|
refine() {
|
|
command -v cargo-ai >/dev/null 2>&1 || return 0
|
|
if [ -z "$CLAUDE_BIN" ]; then
|
|
log "claude binary not found; keeping placeholder summary for $SESSION_ID"
|
|
return 0
|
|
fi
|
|
if [ -z "$TRANSCRIPT_PATH" ] || [ ! -f "$TRANSCRIPT_PATH" ]; then
|
|
log "transcript not available (path='$TRANSCRIPT_PATH'); keeping placeholder for $SESSION_ID"
|
|
return 0
|
|
fi
|
|
|
|
TAIL="$(tail -c 60000 "$TRANSCRIPT_PATH" 2>/dev/null || true)"
|
|
if [ -z "$TAIL" ]; then
|
|
log "transcript at '$TRANSCRIPT_PATH' was empty; keeping placeholder for $SESSION_ID"
|
|
return 0
|
|
fi
|
|
|
|
# Spawn budget — a mechanism-independent backstop for the recursion guarded at
|
|
# the top of this file. The environment marker is the real fix; this bounds the
|
|
# damage if it is ever lost (a hook runner that re-execs with a scrubbed
|
|
# environment, a wrapper that resets PATH and env together). At most
|
|
# CARGO_SUMMARIZER_MAX (default 20) summarizer spawns per rolling hour on this
|
|
# machine: real sessions end far below that rate, a runaway chain blows through
|
|
# it in about a minute. Exhausting the budget costs a refined title, nothing
|
|
# more — the row is already finalized with the placeholder above.
|
|
BUDGET="${TMPDIR:-/tmp}/cargo-summarizer-spawns"
|
|
MAX="${CARGO_SUMMARIZER_MAX:-20}"
|
|
CUTOFF=$(( $(date +%s) - 3600 ))
|
|
RECENT="$(awk -v c="$CUTOFF" '$1 > c' "$BUDGET" 2>/dev/null || true)"
|
|
SPAWNS="$(printf '%s' "$RECENT" | grep -c . 2>/dev/null || true)"
|
|
if [ "${SPAWNS:-0}" -ge "$MAX" ]; then
|
|
log "summarizer budget exhausted (${SPAWNS}/${MAX} in the last hour); keeping placeholder for $SESSION_ID"
|
|
return 0
|
|
fi
|
|
{ printf '%s\n' "$RECENT"; date +%s; } | grep . >"$BUDGET" 2>/dev/null || true
|
|
|
|
PROMPT='Read the Claude Code transcript on stdin and reply with a single JSON object: {"title":"<5-8 word title>","summary":"<1-2 sentence summary>"}. JSON only, no markdown fences.'
|
|
# CARGO_SESSION_SUMMARIZER marks the child as ours: its own SessionStart/Stop/
|
|
# SessionEnd hooks see it and exit, which is what stops the chain.
|
|
# Guard against a hung summarizer when `timeout` is available.
|
|
if command -v timeout >/dev/null 2>&1; then
|
|
RESPONSE="$(printf '%s' "$TAIL" | CARGO_SESSION_SUMMARIZER=1 timeout 120 "$CLAUDE_BIN" -p "$PROMPT" 2>>"$LOG" || true)"
|
|
else
|
|
RESPONSE="$(printf '%s' "$TAIL" | CARGO_SESSION_SUMMARIZER=1 "$CLAUDE_BIN" -p "$PROMPT" 2>>"$LOG" || true)"
|
|
fi
|
|
if [ -z "$RESPONSE" ]; then
|
|
log "claude -p produced no output for $SESSION_ID (see stderr above); keeping placeholder"
|
|
return 0
|
|
fi
|
|
|
|
# Strip markdown code fences, then extract the JSON object.
|
|
CLEAN="$(printf '%s' "$RESPONSE" | sed 's/```json//g; s/```//g' | sed -n '/{/,/}/p')"
|
|
PARSED_TITLE="$(printf '%s' "$CLEAN" | jq -r '.title // empty' 2>/dev/null || true)"
|
|
PARSED_SUMMARY="$(printf '%s' "$CLEAN" | jq -r '.summary // empty' 2>/dev/null || true)"
|
|
if [ -z "$PARSED_TITLE" ] || [ -z "$PARSED_SUMMARY" ]; then
|
|
log "could not parse title/summary from claude output for $SESSION_ID: $RESPONSE"
|
|
return 0
|
|
fi
|
|
|
|
# Which skills the session actually loaded — see hooks/skill-loads.sh for
|
|
# what is and is not read. Appended to the summary because the session row
|
|
# has no structured field for it yet; empty for sessions that never touched
|
|
# Cargo, so non-Cargo sessions are unaffected.
|
|
MARKER=""
|
|
if [ -x "$SCRIPT_DIR/skill-loads.sh" ]; then
|
|
MARKER="$(bash "$SCRIPT_DIR/skill-loads.sh" "$TRANSCRIPT_PATH" 2>/dev/null || true)"
|
|
fi
|
|
[ -n "$MARKER" ] && PARSED_SUMMARY="$PARSED_SUMMARY $MARKER"
|
|
|
|
if cargo-ai workspaceManagement session upsert \
|
|
--session-id "$SESSION_ID" \
|
|
--title "$PARSED_TITLE" \
|
|
--summary "$PARSED_SUMMARY" \
|
|
--finished >>"$LOG" 2>&1; then
|
|
log "refined $SESSION_ID with title: $PARSED_TITLE"
|
|
else
|
|
log "refine upsert failed for $SESSION_ID"
|
|
fi
|
|
}
|
|
|
|
export -f refine log
|
|
export SESSION_ID TRANSCRIPT_PATH CLAUDE_BIN LOG MODE SCRIPT_DIR
|
|
|
|
# Detach: prefer setsid, fall back to nohup. Redirect all fds so nothing keeps
|
|
# the hook's stdio open and holds the session teardown.
|
|
if command -v setsid >/dev/null 2>&1; then
|
|
setsid bash -c refine </dev/null >>"$LOG" 2>&1 &
|
|
else
|
|
nohup bash -c refine </dev/null >>"$LOG" 2>&1 &
|
|
fi
|
|
disown 2>/dev/null || true
|
|
|
|
exit 0
|