mirror of
https://github.com/rtk-ai/rtk.git
synced 2026-09-19 07:33:17 +08:00
Florian BRUNIAUX
a005bb15c0
* feat(cargo): aggregate test output into single line (#83) Problem: `cargo test` shows 24+ summary lines even when all pass. An LLM only needs to know IF something failed, not 24x "ok". Before (24 lines): ``` ✓ test result: ok. 2 passed; 0 failed; ... ✓ test result: ok. 0 passed; 0 failed; ... ... (x24) ``` After (1 line): ``` ✓ cargo test: 137 passed (24 suites, 1.45s) ``` Changes: - Add AggregatedTestResult struct with regex parsing - Merge multiple test summaries when all pass - Format: "N passed, M ignored, P filtered out (X suites, Ys)" - Fallback to original behavior if parsing fails - Failures still show full details (no aggregation) Tests: 6 new + 1 modified, covering all cases: - Multi-suite aggregation - Single suite (singular "suite") - Zero tests - With ignored/filtered out - Failures → no aggregation (detail preserved) - Regex fallback Closes #83 Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com> * feat: add Python and Go language support Implements comprehensive support for Python and Go development tooling with 70-90% token reduction across all commands. Python commands (3): - rtk ruff: Linter/formatter with JSON (check) and text (format) parsing (80%+) - rtk pytest: Test runner with state machine text parser (90%+) - rtk pip: Package manager with auto-detect uv (70-85%) Go commands (4): - rtk go test: NDJSON streaming parser for interleaved test events (90%+) - rtk go build: Text filter showing errors only (80%) - rtk go vet: Text filter for issues (75%) - rtk golangci-lint: JSON parser grouped by rule (85%) Architecture: - Standalone Python commands (mirror lint/prettier pattern) - Go sub-enum (mirror git/cargo pattern) - 5 new modules: ruff_cmd, pytest_cmd, pip_cmd, go_cmd, golangci_cmd - Hook integration in rtk-rewrite.sh for transparent rewrites - Comprehensive tests (47 new tests, all passing) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com> * feat(benchmark): add Python and Go commands Add benchmark sections for Python (ruff, pytest, pip) and Go (go test/build/vet, golangci-lint) to validate >80% token savings in CI pipeline. Sections conditionally execute based on project markers (pyproject.toml, go.mod) and tool availability. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
387 lines
12 KiB
Bash
Executable File
387 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
set -e
|
|
|
|
RTK="./target/release/rtk"
|
|
BENCH_DIR="./scripts/benchmark"
|
|
|
|
# Mode local : générer les fichiers debug
|
|
if [ -z "$CI" ]; then
|
|
rm -rf "$BENCH_DIR"
|
|
mkdir -p "$BENCH_DIR/unix" "$BENCH_DIR/rtk" "$BENCH_DIR/diff"
|
|
fi
|
|
|
|
# Nom de fichier safe
|
|
safe_name() {
|
|
echo "$1" | tr ' /' '_-' | tr -cd 'a-zA-Z0-9_-'
|
|
}
|
|
|
|
# Fonction pour compter les tokens (~4 chars = 1 token)
|
|
count_tokens() {
|
|
local input="$1"
|
|
local len=${#input}
|
|
echo $(( (len + 3) / 4 ))
|
|
}
|
|
|
|
# Compteurs globaux
|
|
TOTAL_UNIX=0
|
|
TOTAL_RTK=0
|
|
TOTAL_TESTS=0
|
|
GOOD_TESTS=0
|
|
FAIL_TESTS=0
|
|
SKIP_TESTS=0
|
|
|
|
# Fonction de benchmark — une ligne par test
|
|
bench() {
|
|
local name="$1"
|
|
local unix_cmd="$2"
|
|
local rtk_cmd="$3"
|
|
|
|
unix_out=$(eval "$unix_cmd" 2>/dev/null || true)
|
|
rtk_out=$(eval "$rtk_cmd" 2>/dev/null || true)
|
|
|
|
unix_tokens=$(count_tokens "$unix_out")
|
|
rtk_tokens=$(count_tokens "$rtk_out")
|
|
|
|
TOTAL_TESTS=$((TOTAL_TESTS + 1))
|
|
|
|
local icon=""
|
|
local tag=""
|
|
|
|
if [ -z "$rtk_out" ]; then
|
|
icon="❌"
|
|
tag="FAIL"
|
|
FAIL_TESTS=$((FAIL_TESTS + 1))
|
|
TOTAL_UNIX=$((TOTAL_UNIX + unix_tokens))
|
|
TOTAL_RTK=$((TOTAL_RTK + unix_tokens))
|
|
elif [ "$rtk_tokens" -ge "$unix_tokens" ] && [ "$unix_tokens" -gt 0 ]; then
|
|
icon="⚠️"
|
|
tag="SKIP"
|
|
SKIP_TESTS=$((SKIP_TESTS + 1))
|
|
TOTAL_UNIX=$((TOTAL_UNIX + unix_tokens))
|
|
TOTAL_RTK=$((TOTAL_RTK + unix_tokens))
|
|
else
|
|
icon="✅"
|
|
tag="GOOD"
|
|
GOOD_TESTS=$((GOOD_TESTS + 1))
|
|
TOTAL_UNIX=$((TOTAL_UNIX + unix_tokens))
|
|
TOTAL_RTK=$((TOTAL_RTK + rtk_tokens))
|
|
fi
|
|
|
|
if [ "$tag" = "FAIL" ]; then
|
|
printf "%s %-24s │ %-40s │ %-40s │ %6d → %6s (--)\n" \
|
|
"$icon" "$name" "$unix_cmd" "$rtk_cmd" "$unix_tokens" "-"
|
|
else
|
|
if [ "$unix_tokens" -gt 0 ]; then
|
|
local pct=$(( (unix_tokens - rtk_tokens) * 100 / unix_tokens ))
|
|
else
|
|
local pct=0
|
|
fi
|
|
printf "%s %-24s │ %-40s │ %-40s │ %6d → %6d (%+d%%)\n" \
|
|
"$icon" "$name" "$unix_cmd" "$rtk_cmd" "$unix_tokens" "$rtk_tokens" "$pct"
|
|
fi
|
|
|
|
# Fichiers debug en local uniquement
|
|
if [ -z "$CI" ]; then
|
|
local filename=$(safe_name "$name")
|
|
local prefix="GOOD"
|
|
[ "$tag" = "FAIL" ] && prefix="FAIL"
|
|
[ "$tag" = "SKIP" ] && prefix="BAD"
|
|
|
|
local ts=$(date "+%d/%m/%Y %H:%M:%S")
|
|
|
|
printf "# %s\n> %s\n\n\`\`\`bash\n$ %s\n\`\`\`\n\n\`\`\`\n%s\n\`\`\`\n" \
|
|
"$name" "$ts" "$unix_cmd" "$unix_out" > "$BENCH_DIR/unix/${filename}.md"
|
|
|
|
printf "# %s\n> %s\n\n\`\`\`bash\n$ %s\n\`\`\`\n\n\`\`\`\n%s\n\`\`\`\n" \
|
|
"$name" "$ts" "$rtk_cmd" "$rtk_out" > "$BENCH_DIR/rtk/${filename}.md"
|
|
|
|
{
|
|
echo "# Diff: $name"
|
|
echo "> $ts"
|
|
echo ""
|
|
echo "| Metric | Unix | RTK |"
|
|
echo "|--------|------|-----|"
|
|
echo "| Tokens | $unix_tokens | $rtk_tokens |"
|
|
echo ""
|
|
echo "## Unix"
|
|
echo "\`\`\`"
|
|
echo "$unix_out"
|
|
echo "\`\`\`"
|
|
echo ""
|
|
echo "## RTK"
|
|
echo "\`\`\`"
|
|
echo "$rtk_out"
|
|
echo "\`\`\`"
|
|
} > "$BENCH_DIR/diff/${prefix}-${filename}.md"
|
|
fi
|
|
}
|
|
|
|
# Section header
|
|
section() {
|
|
echo ""
|
|
echo "── $1 ──"
|
|
}
|
|
|
|
# ═══════════════════════════════════════════
|
|
echo "RTK Benchmark"
|
|
echo "═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════"
|
|
printf " %-24s │ %-40s │ %-40s │ %s\n" "TEST" "SHELL" "RTK" "TOKENS"
|
|
echo "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────"
|
|
|
|
# ===================
|
|
# ls
|
|
# ===================
|
|
section "ls"
|
|
bench "ls" "ls -la" "$RTK ls"
|
|
bench "ls src/" "ls -la src/" "$RTK ls src/"
|
|
bench "ls -l src/" "ls -l src/" "$RTK ls -l src/"
|
|
bench "ls -la src/" "ls -la src/" "$RTK ls -la src/"
|
|
bench "ls -lh src/" "ls -lh src/" "$RTK ls -lh src/"
|
|
bench "ls src/ -l" "ls -l src/" "$RTK ls src/ -l"
|
|
bench "ls -a" "ls -la" "$RTK ls -a"
|
|
bench "ls multi" "ls -la src/ scripts/" "$RTK ls src/ scripts/"
|
|
|
|
# ===================
|
|
# read
|
|
# ===================
|
|
section "read"
|
|
bench "read" "cat src/main.rs" "$RTK read src/main.rs"
|
|
bench "read -l minimal" "cat src/main.rs" "$RTK read src/main.rs -l minimal"
|
|
bench "read -l aggressive" "cat src/main.rs" "$RTK read src/main.rs -l aggressive"
|
|
bench "read -n" "cat -n src/main.rs" "$RTK read src/main.rs -n"
|
|
|
|
# ===================
|
|
# find
|
|
# ===================
|
|
section "find"
|
|
bench "find *" "find . -type f" "$RTK find '*'"
|
|
bench "find *.rs" "find . -name '*.rs' -type f" "$RTK find '*.rs'"
|
|
bench "find --max 10" "find . -not -path './target/*' -not -path './.git/*' -type f | head -10" "$RTK find '*' --max 10"
|
|
bench "find --max 100" "find . -not -path './target/*' -not -path './.git/*' -type f | head -100" "$RTK find '*' --max 100"
|
|
|
|
# ===================
|
|
# git
|
|
# ===================
|
|
section "git"
|
|
bench "git status" "git status" "$RTK git status"
|
|
bench "git log -n 10" "git log -10" "$RTK git log -n 10"
|
|
bench "git log -n 5" "git log -5" "$RTK git log -n 5"
|
|
bench "git diff" "git diff HEAD~1 2>/dev/null || echo ''" "$RTK git diff"
|
|
|
|
# ===================
|
|
# grep
|
|
# ===================
|
|
section "grep"
|
|
bench "grep fn" "grep -rn 'fn ' src/ || true" "$RTK grep 'fn ' src/"
|
|
bench "grep struct" "grep -rn 'struct ' src/ || true" "$RTK grep 'struct ' src/"
|
|
bench "grep -l 40" "grep -rn 'fn ' src/ || true" "$RTK grep 'fn ' src/ -l 40"
|
|
bench "grep --max 20" "grep -rn 'fn ' src/ | head -20 || true" "$RTK grep 'fn ' src/ --max 20"
|
|
bench "grep -c" "grep -ron 'fn ' src/ || true" "$RTK grep 'fn ' src/ -c"
|
|
|
|
# ===================
|
|
# json
|
|
# ===================
|
|
section "json"
|
|
cat > /tmp/rtk_bench.json << 'JSONEOF'
|
|
{
|
|
"name": "rtk",
|
|
"version": "0.2.1",
|
|
"config": {
|
|
"debug": false,
|
|
"max_depth": 10,
|
|
"filters": ["node_modules", "target", ".git"]
|
|
},
|
|
"dependencies": {
|
|
"serde": "1.0",
|
|
"clap": "4.0",
|
|
"anyhow": "1.0"
|
|
}
|
|
}
|
|
JSONEOF
|
|
bench "json" "cat /tmp/rtk_bench.json" "$RTK json /tmp/rtk_bench.json"
|
|
bench "json -d 2" "cat /tmp/rtk_bench.json" "$RTK json /tmp/rtk_bench.json -d 2"
|
|
rm -f /tmp/rtk_bench.json
|
|
|
|
# ===================
|
|
# deps
|
|
# ===================
|
|
section "deps"
|
|
bench "deps" "cat Cargo.toml" "$RTK deps"
|
|
|
|
# ===================
|
|
# env
|
|
# ===================
|
|
section "env"
|
|
bench "env" "env" "$RTK env"
|
|
bench "env -f PATH" "env | grep PATH" "$RTK env -f PATH"
|
|
bench "env --show-all" "env" "$RTK env --show-all"
|
|
|
|
# ===================
|
|
# err
|
|
# ===================
|
|
section "err"
|
|
bench "err cargo build" "cargo build 2>&1 || true" "$RTK err cargo build"
|
|
|
|
# ===================
|
|
# test
|
|
# ===================
|
|
section "test"
|
|
bench "test cargo test" "cargo test 2>&1 || true" "$RTK test cargo test"
|
|
|
|
# ===================
|
|
# log
|
|
# ===================
|
|
section "log"
|
|
LOG_FILE="/tmp/rtk_bench_sample.log"
|
|
cat > "$LOG_FILE" << 'LOGEOF'
|
|
2024-01-15 10:00:01 INFO Application started
|
|
2024-01-15 10:00:02 INFO Loading configuration
|
|
2024-01-15 10:00:03 ERROR Connection failed: timeout
|
|
2024-01-15 10:00:04 ERROR Connection failed: timeout
|
|
2024-01-15 10:00:05 ERROR Connection failed: timeout
|
|
2024-01-15 10:00:06 ERROR Connection failed: timeout
|
|
2024-01-15 10:00:07 ERROR Connection failed: timeout
|
|
2024-01-15 10:00:08 WARN Retrying connection
|
|
2024-01-15 10:00:09 INFO Connection established
|
|
2024-01-15 10:00:10 INFO Processing request
|
|
2024-01-15 10:00:11 INFO Processing request
|
|
2024-01-15 10:00:12 INFO Processing request
|
|
2024-01-15 10:00:13 INFO Request completed
|
|
LOGEOF
|
|
bench "log" "cat $LOG_FILE" "$RTK log $LOG_FILE"
|
|
rm -f "$LOG_FILE"
|
|
|
|
# ===================
|
|
# summary
|
|
# ===================
|
|
section "summary"
|
|
bench "summary cargo --help" "cargo --help" "$RTK summary cargo --help"
|
|
bench "summary rustc --help" "rustc --help 2>/dev/null || echo 'rustc not found'" "$RTK summary rustc --help"
|
|
|
|
# ===================
|
|
# Modern JavaScript Stack (skip si pas de package.json)
|
|
# ===================
|
|
if [ -f "package.json" ]; then
|
|
section "modern JS stack"
|
|
|
|
if command -v tsc &> /dev/null || [ -f "node_modules/.bin/tsc" ]; then
|
|
bench "tsc" "tsc --noEmit 2>&1 || true" "$RTK tsc --noEmit"
|
|
fi
|
|
|
|
if command -v prettier &> /dev/null || [ -f "node_modules/.bin/prettier" ]; then
|
|
bench "prettier --check" "prettier --check . 2>&1 || true" "$RTK prettier --check ."
|
|
fi
|
|
|
|
if command -v eslint &> /dev/null || [ -f "node_modules/.bin/eslint" ]; then
|
|
bench "lint" "eslint . 2>&1 || true" "$RTK lint ."
|
|
fi
|
|
|
|
if [ -f "next.config.js" ] || [ -f "next.config.mjs" ] || [ -f "next.config.ts" ]; then
|
|
if command -v next &> /dev/null || [ -f "node_modules/.bin/next" ]; then
|
|
bench "next build" "next build 2>&1 || true" "$RTK next build"
|
|
fi
|
|
fi
|
|
|
|
if [ -f "playwright.config.ts" ] || [ -f "playwright.config.js" ]; then
|
|
if command -v playwright &> /dev/null || [ -f "node_modules/.bin/playwright" ]; then
|
|
bench "playwright test" "playwright test 2>&1 || true" "$RTK playwright test"
|
|
fi
|
|
fi
|
|
|
|
if [ -f "prisma/schema.prisma" ]; then
|
|
if command -v prisma &> /dev/null || [ -f "node_modules/.bin/prisma" ]; then
|
|
bench "prisma generate" "prisma generate 2>&1 || true" "$RTK prisma generate"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ===================
|
|
# docker (skip si pas dispo)
|
|
# ===================
|
|
if command -v docker &> /dev/null; then
|
|
section "docker"
|
|
bench "docker ps" "docker ps 2>/dev/null || true" "$RTK docker ps"
|
|
bench "docker images" "docker images 2>/dev/null || true" "$RTK docker images"
|
|
fi
|
|
|
|
# ===================
|
|
# kubectl (skip si pas dispo)
|
|
# ===================
|
|
if command -v kubectl &> /dev/null; then
|
|
section "kubectl"
|
|
bench "kubectl pods" "kubectl get pods 2>/dev/null || true" "$RTK kubectl pods"
|
|
bench "kubectl services" "kubectl get services 2>/dev/null || true" "$RTK kubectl services"
|
|
fi
|
|
|
|
# ===================
|
|
# Python (skip si pas de projet Python)
|
|
# ===================
|
|
if [ -f "pyproject.toml" ] || [ -f "requirements.txt" ] || [ -f "setup.py" ]; then
|
|
section "Python stack"
|
|
|
|
if command -v ruff &> /dev/null; then
|
|
bench "ruff check" "ruff check . 2>&1 || true" "$RTK ruff check ."
|
|
bench "ruff format --check" "ruff format --check . 2>&1 || true" "$RTK ruff format --check ."
|
|
fi
|
|
|
|
if command -v pytest &> /dev/null; then
|
|
bench "pytest" "pytest --tb=short -q 2>&1 || true" "$RTK pytest"
|
|
fi
|
|
|
|
if command -v pip &> /dev/null; then
|
|
bench "pip list" "pip list 2>&1 || true" "$RTK pip list"
|
|
bench "pip outdated" "pip list --outdated 2>&1 || true" "$RTK pip outdated"
|
|
fi
|
|
fi
|
|
|
|
# ===================
|
|
# Go (skip si pas de go.mod)
|
|
# ===================
|
|
if [ -f "go.mod" ]; then
|
|
section "Go stack"
|
|
|
|
if command -v go &> /dev/null; then
|
|
bench "go test" "go test ./... 2>&1 || true" "$RTK go test ./..."
|
|
bench "go build" "go build ./... 2>&1 || true" "$RTK go build ./..."
|
|
bench "go vet" "go vet ./... 2>&1 || true" "$RTK go vet ./..."
|
|
fi
|
|
|
|
if command -v golangci-lint &> /dev/null; then
|
|
bench "golangci-lint" "golangci-lint run 2>&1 || true" "$RTK golangci-lint run"
|
|
fi
|
|
fi
|
|
|
|
# ===================
|
|
# Résumé global
|
|
# ===================
|
|
echo ""
|
|
echo "═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════"
|
|
|
|
if [ "$TOTAL_TESTS" -gt 0 ]; then
|
|
GOOD_PCT=$((GOOD_TESTS * 100 / TOTAL_TESTS))
|
|
if [ "$TOTAL_UNIX" -gt 0 ]; then
|
|
TOTAL_SAVED=$((TOTAL_UNIX - TOTAL_RTK))
|
|
TOTAL_SAVE_PCT=$((TOTAL_SAVED * 100 / TOTAL_UNIX))
|
|
else
|
|
TOTAL_SAVED=0
|
|
TOTAL_SAVE_PCT=0
|
|
fi
|
|
|
|
echo ""
|
|
echo " ✅ $GOOD_TESTS good ⚠️ $SKIP_TESTS skip ❌ $FAIL_TESTS fail $GOOD_TESTS/$TOTAL_TESTS ($GOOD_PCT%)"
|
|
echo " Tokens: $TOTAL_UNIX → $TOTAL_RTK (-$TOTAL_SAVE_PCT%)"
|
|
echo ""
|
|
|
|
# Fichiers debug en local
|
|
if [ -z "$CI" ]; then
|
|
echo " Debug: $BENCH_DIR/{unix,rtk,diff}/"
|
|
fi
|
|
echo ""
|
|
|
|
# Exit code non-zero si moins de 80% good
|
|
if [ "$GOOD_PCT" -lt 80 ]; then
|
|
echo " BENCHMARK FAILED: $GOOD_PCT% good (minimum 80%)"
|
|
exit 1
|
|
fi
|
|
fi
|