mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Dragan Spiridonov
294 lines
12 KiB
Bash
294 lines
12 KiB
Bash
#!/bin/bash
|
|
#
|
|
# QCSD Production Telemetry Collection Script
|
|
# Collects DORA-approximation metrics from GitHub API for npm package releases.
|
|
#
|
|
# Usage: ./scripts/collect-production-telemetry.sh [OPTIONS]
|
|
#
|
|
# Options:
|
|
# --release-id <version> Release version (default: latest tag)
|
|
# --lookback <days> Days of history to analyze (default: 30)
|
|
# --output <path> Output directory (default: docs/telemetry/production)
|
|
# --trigger-type <type> Trigger context: post-deploy|scheduled|manual (default: manual)
|
|
#
|
|
# Environment:
|
|
# GITHUB_REPOSITORY Owner/repo (default: auto-detect from git remote)
|
|
# GH_TOKEN / GITHUB_TOKEN GitHub authentication (gh CLI must be authenticated)
|
|
#
|
|
# Maintainer-invoked diagnostic only. This script deliberately has no GitHub
|
|
# Actions trigger: its GitHub-derived values are delivery-process proxies, not
|
|
# production observability. See docs/telemetry/README.md before using it.
|
|
# Individual API failures produce null fields, not script crash.
|
|
|
|
# Note: no set -e — individual commands use || fallbacks for error tolerance
|
|
|
|
# ─── Colors ───────────────────────────────────────────────────────────────────
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m'
|
|
|
|
# ─── Defaults ─────────────────────────────────────────────────────────────────
|
|
RELEASE_ID="auto"
|
|
LOOKBACK_DAYS=30
|
|
OUTPUT_DIR="docs/telemetry/production"
|
|
TRIGGER_TYPE="manual"
|
|
REPO="${GITHUB_REPOSITORY:-""}"
|
|
|
|
# ─── Parse Arguments ──────────────────────────────────────────────────────────
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--release-id) RELEASE_ID="$2"; shift 2 ;;
|
|
--lookback) LOOKBACK_DAYS="$2"; shift 2 ;;
|
|
--output) OUTPUT_DIR="$2"; shift 2 ;;
|
|
--trigger-type) TRIGGER_TYPE="$2"; shift 2 ;;
|
|
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
# ─── Auto-detect repo ────────────────────────────────────────────────────────
|
|
if [ -z "$REPO" ]; then
|
|
REPO=$(gh repo view --json nameWithOwner -q '.nameWithOwner' 2>/dev/null || echo "")
|
|
if [ -z "$REPO" ]; then
|
|
echo -e "${RED}ERROR: Cannot detect repository. Set GITHUB_REPOSITORY or run from a git repo with gh auth.${NC}"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# ─── Auto-detect release ID ──────────────────────────────────────────────────
|
|
if [ "$RELEASE_ID" = "auto" ]; then
|
|
RELEASE_ID=$(gh release view --repo "$REPO" --json tagName -q '.tagName' 2>/dev/null || echo "unknown")
|
|
fi
|
|
|
|
# ─── Calculate date boundaries ────────────────────────────────────────────────
|
|
# macOS and Linux compatible date calculation
|
|
if date -v-1d > /dev/null 2>&1; then
|
|
SINCE_DATE=$(date -u -v-${LOOKBACK_DAYS}d +%Y-%m-%dT%H:%M:%SZ)
|
|
else
|
|
SINCE_DATE=$(date -u -d "${LOOKBACK_DAYS} days ago" +%Y-%m-%dT%H:%M:%SZ)
|
|
fi
|
|
COLLECTION_TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
DATE_STAMP=$(date -u +%Y%m%d)
|
|
|
|
echo "========================================"
|
|
echo -e "${BLUE}QCSD Production Telemetry Collection${NC}"
|
|
echo "========================================"
|
|
echo "Repository: $REPO"
|
|
echo "Release: $RELEASE_ID"
|
|
echo "Lookback: ${LOOKBACK_DAYS} days (since $SINCE_DATE)"
|
|
echo "Output: $OUTPUT_DIR"
|
|
echo "Trigger: $TRIGGER_TYPE"
|
|
echo "========================================"
|
|
echo ""
|
|
|
|
# ─── Helper: safe API call ────────────────────────────────────────────────────
|
|
safe_api() {
|
|
local result
|
|
if result=$("$@" 2>/dev/null); then
|
|
echo "$result"
|
|
else
|
|
echo "null"
|
|
fi
|
|
}
|
|
|
|
# ─── 1. Deployment Frequency ─────────────────────────────────────────────────
|
|
echo -e "${BLUE}[1/5]${NC} Collecting deployment frequency..."
|
|
|
|
RELEASE_DATA=$(safe_api gh api "repos/${REPO}/releases?per_page=100" --paginate --jq "[.[] | select(.published_at >= \"${SINCE_DATE}\") | {tag: .tag_name, published: .published_at, created: .created_at}]")
|
|
|
|
if [ "$RELEASE_DATA" != "null" ] && [ -n "$RELEASE_DATA" ]; then
|
|
RELEASE_COUNT=$(echo "$RELEASE_DATA" | jq 'length')
|
|
FREQ_PER_WEEK=$(echo "scale=2; $RELEASE_COUNT * 7 / $LOOKBACK_DAYS" | bc 2>/dev/null || echo "null")
|
|
echo -e " ${GREEN}Found $RELEASE_COUNT releases in ${LOOKBACK_DAYS}d ($FREQ_PER_WEEK/week)${NC}"
|
|
else
|
|
RELEASE_COUNT=0
|
|
FREQ_PER_WEEK="null"
|
|
RELEASE_DATA="[]"
|
|
echo -e " ${YELLOW}No release data available${NC}"
|
|
fi
|
|
|
|
# ─── 2. Lead Time for Changes ────────────────────────────────────────────────
|
|
echo -e "${BLUE}[2/5]${NC} Computing lead time..."
|
|
|
|
if [ "$RELEASE_DATA" != "[]" ] && [ "$RELEASE_DATA" != "null" ]; then
|
|
LEAD_TIMES=$(echo "$RELEASE_DATA" | jq '[.[] | {
|
|
tag: .tag,
|
|
created: .created,
|
|
published: .published,
|
|
lead_hours: ((((.published | fromdateiso8601) - (.created | fromdateiso8601)) / 3600) | round)
|
|
}]')
|
|
MEDIAN_LEAD=$(echo "$LEAD_TIMES" | jq '[.[].lead_hours] | sort | if length > 0 then .[length/2 | floor] else null end')
|
|
echo -e " ${GREEN}Median lead time: ${MEDIAN_LEAD}h across $RELEASE_COUNT releases${NC}"
|
|
else
|
|
LEAD_TIMES="[]"
|
|
MEDIAN_LEAD="null"
|
|
echo -e " ${YELLOW}No lead time data (no releases)${NC}"
|
|
fi
|
|
|
|
# ─── 3. Change Failure Rate ──────────────────────────────────────────────────
|
|
echo -e "${BLUE}[3/5]${NC} Computing change failure rate..."
|
|
|
|
CFR_DATA=$(safe_api gh api "repos/${REPO}/actions/workflows/npm-publish.yml/runs?per_page=100" --jq "{
|
|
total: [.workflow_runs[] | select(.created_at >= \"${SINCE_DATE}\")] | length,
|
|
failed: [.workflow_runs[] | select(.created_at >= \"${SINCE_DATE}\" and .conclusion == \"failure\")] | length
|
|
}")
|
|
|
|
if [ "$CFR_DATA" != "null" ] && [ -n "$CFR_DATA" ]; then
|
|
CFR_TOTAL=$(echo "$CFR_DATA" | jq '.total')
|
|
CFR_FAILED=$(echo "$CFR_DATA" | jq '.failed')
|
|
if [ "$CFR_TOTAL" -gt 0 ] 2>/dev/null; then
|
|
CFR_RATE=$(echo "scale=1; $CFR_FAILED * 100 / $CFR_TOTAL" | bc 2>/dev/null || echo "null")
|
|
else
|
|
CFR_RATE=0
|
|
fi
|
|
echo -e " ${GREEN}${CFR_FAILED}/${CFR_TOTAL} failed (${CFR_RATE}%)${NC}"
|
|
else
|
|
CFR_TOTAL=0
|
|
CFR_FAILED=0
|
|
CFR_RATE="null"
|
|
echo -e " ${YELLOW}No workflow run data available${NC}"
|
|
fi
|
|
|
|
# ─── 4. MTTR Approximation ───────────────────────────────────────────────────
|
|
echo -e "${BLUE}[4/5]${NC} Computing MTTR approximation (bug issue lifecycle)..."
|
|
|
|
MTTR_DATA=$(safe_api gh api "repos/${REPO}/issues?labels=bug&state=closed&since=${SINCE_DATE}&per_page=100" --paginate --jq '[.[] | select(.pull_request == null) | {
|
|
number: .number,
|
|
created: .created_at,
|
|
closed: .closed_at,
|
|
hours: ((((.closed_at | fromdateiso8601) - (.created_at | fromdateiso8601)) / 3600) | round)
|
|
}]')
|
|
|
|
if [ "$MTTR_DATA" != "null" ] && [ -n "$MTTR_DATA" ]; then
|
|
BUGS_CLOSED=$(echo "$MTTR_DATA" | jq 'length')
|
|
MTTR_MEDIAN=$(echo "$MTTR_DATA" | jq '[.[].hours] | sort | if length > 0 then .[length/2 | floor] else null end')
|
|
echo -e " ${GREEN}${BUGS_CLOSED} bugs closed, median ${MTTR_MEDIAN}h${NC}"
|
|
else
|
|
BUGS_CLOSED=0
|
|
MTTR_MEDIAN="null"
|
|
MTTR_DATA="[]"
|
|
echo -e " ${YELLOW}No closed bug data available${NC}"
|
|
fi
|
|
|
|
# ─── 5. Open Issues Snapshot ─────────────────────────────────────────────────
|
|
echo -e "${BLUE}[5/5]${NC} Collecting open issues snapshot..."
|
|
|
|
OPEN_BUGS=$(safe_api gh issue list --repo "${REPO}" --label bug --state open --json number --jq 'length')
|
|
OPEN_BUGS=${OPEN_BUGS:-0}
|
|
echo -e " ${GREEN}${OPEN_BUGS} open bugs${NC}"
|
|
|
|
# ─── Build JSON Output ───────────────────────────────────────────────────────
|
|
echo ""
|
|
echo -e "${BLUE}Building telemetry JSON...${NC}"
|
|
|
|
# Get current release context
|
|
CURRENT_RELEASE=$(safe_api gh release view "${RELEASE_ID}" --repo "${REPO}" --json tagName,publishedAt,createdAt --jq '{tag: .tagName, publishedAt: .publishedAt, createdAt: .createdAt}')
|
|
if [ "$CURRENT_RELEASE" = "null" ]; then
|
|
CURRENT_RELEASE='{"tag": "'"$RELEASE_ID"'", "publishedAt": null, "createdAt": null}'
|
|
fi
|
|
|
|
# Recent releases list (last 5)
|
|
RECENT_RELEASES=$(safe_api gh api "repos/${REPO}/releases?per_page=5" --jq '[.[] | {tag: .tag_name, publishedAt: .published_at}]')
|
|
if [ "$RECENT_RELEASES" = "null" ]; then
|
|
RECENT_RELEASES="[]"
|
|
fi
|
|
|
|
# Assemble final JSON
|
|
OUTPUT_JSON=$(jq -n \
|
|
--arg ts "$COLLECTION_TS" \
|
|
--arg rid "$RELEASE_ID" \
|
|
--argjson lookback "$LOOKBACK_DAYS" \
|
|
--arg trigger "$TRIGGER_TYPE" \
|
|
--arg repo "$REPO" \
|
|
--argjson relCount "$RELEASE_COUNT" \
|
|
--argjson freqWeek "${FREQ_PER_WEEK:-null}" \
|
|
--argjson leadTimes "$LEAD_TIMES" \
|
|
--argjson medianLead "${MEDIAN_LEAD:-null}" \
|
|
--argjson cfrTotal "$CFR_TOTAL" \
|
|
--argjson cfrFailed "$CFR_FAILED" \
|
|
--argjson cfrRate "${CFR_RATE:-null}" \
|
|
--argjson bugsClosed "$BUGS_CLOSED" \
|
|
--argjson mttrMedian "${MTTR_MEDIAN:-null}" \
|
|
--argjson mttrData "$MTTR_DATA" \
|
|
--argjson openBugs "${OPEN_BUGS:-0}" \
|
|
--argjson currentRelease "$CURRENT_RELEASE" \
|
|
--argjson recentReleases "$RECENT_RELEASES" \
|
|
'{
|
|
collectionTimestamp: $ts,
|
|
releaseId: $rid,
|
|
lookbackDays: $lookback,
|
|
source: "github-api",
|
|
triggerType: $trigger,
|
|
repository: $repo,
|
|
dora: {
|
|
deploymentFrequency: {
|
|
value: $freqWeek,
|
|
unit: "per_week",
|
|
rawCount: $relCount,
|
|
period: "\($lookback)d"
|
|
},
|
|
leadTime: {
|
|
value: $medianLead,
|
|
unit: "hours",
|
|
measurements: $leadTimes
|
|
},
|
|
changeFailureRate: {
|
|
value: $cfrRate,
|
|
totalRuns: $cfrTotal,
|
|
failedRuns: $cfrFailed,
|
|
period: "\($lookback)d"
|
|
},
|
|
mttr: {
|
|
value: $mttrMedian,
|
|
unit: "hours",
|
|
bugsClosed: $bugsClosed,
|
|
medianHours: $mttrMedian,
|
|
details: $mttrData,
|
|
period: "\($lookback)d"
|
|
}
|
|
},
|
|
releaseContext: {
|
|
currentRelease: $currentRelease,
|
|
recentReleases: $recentReleases
|
|
},
|
|
issues: {
|
|
openBugs: $openBugs,
|
|
closedBugsInPeriod: $bugsClosed
|
|
},
|
|
limitations: [
|
|
"DORA metrics are approximated from GitHub API, not from APM/observability tooling",
|
|
"Lead time measures tag-to-publish, not commit-to-production",
|
|
"Change failure rate uses workflow failures as proxy for production incidents",
|
|
"MTTR uses bug issue lifecycle as proxy for incident recovery time"
|
|
]
|
|
}')
|
|
|
|
# ─── Write Output ─────────────────────────────────────────────────────────────
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
OUTFILE="${OUTPUT_DIR}/telemetry-${RELEASE_ID}-${DATE_STAMP}.json"
|
|
LATEST="${OUTPUT_DIR}/latest.json"
|
|
|
|
echo "$OUTPUT_JSON" > "$OUTFILE"
|
|
cp "$OUTFILE" "$LATEST"
|
|
|
|
echo ""
|
|
echo "========================================"
|
|
echo -e "${GREEN}Telemetry collection complete${NC}"
|
|
echo "========================================"
|
|
echo "Output: $OUTFILE"
|
|
echo "Latest: $LATEST"
|
|
echo ""
|
|
|
|
# ─── Summary ──────────────────────────────────────────────────────────────────
|
|
echo -e "${BLUE}DORA Summary:${NC}"
|
|
echo " Deployment Frequency: ${FREQ_PER_WEEK:-?}/week ($RELEASE_COUNT releases in ${LOOKBACK_DAYS}d)"
|
|
echo " Lead Time: ${MEDIAN_LEAD:-?}h median"
|
|
echo " Change Failure Rate: ${CFR_RATE:-?}% ($CFR_FAILED/$CFR_TOTAL)"
|
|
echo " MTTR: ${MTTR_MEDIAN:-?}h median ($BUGS_CLOSED bugs)"
|
|
echo " Open Bugs: $OPEN_BUGS"
|
|
echo ""
|
|
echo -e "${BLUE}Invoke production swarm:${NC}"
|
|
echo " /qcsd-production-swarm TELEMETRY_DATA=$LATEST RELEASE_ID=$RELEASE_ID"
|