mirror of
https://github.com/CopilotKit/CopilotKit.git
synced 2026-09-14 16:26:20 +08:00
Jordan Ritter
6e66ea626b
Use jq to build JSON payloads safely and payload-file-path to avoid inline multiline content. Limits error context to 3 lines, strips ANSI codes, and caps field length at 200 chars. - drift-detection: split payload build from post, sanitize playwright output - starter-smoke: replace toJSON(format(...)) double-encoding with jq - showcase_deploy: replace 300-char inline ternary with readable shell conditional
268 lines
10 KiB
YAML
268 lines
10 KiB
YAML
name: "Showcase: Drift Detection"
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "0 */6 * * *" # L1-L3 every 6 hours (0/6/12/18 UTC)
|
|
- cron: "0 0 * * *" # L4 daily at midnight UTC
|
|
- cron: "0 9 * * 1" # Weekly version drift (Monday 9am UTC) — unchanged
|
|
workflow_dispatch:
|
|
inputs:
|
|
run_version_check:
|
|
description: "Run version drift check"
|
|
type: boolean
|
|
default: false
|
|
|
|
jobs:
|
|
drift-detection:
|
|
name: E2E Smoke Suite
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
# Only run on schedule (excluding weekly version-drift slot) or manual dispatch
|
|
if: (github.event_name == 'schedule' && github.event.schedule != '0 9 * * 1') || github.event_name == 'workflow_dispatch'
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
with:
|
|
lfs: false
|
|
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 20
|
|
|
|
- name: Install dependencies
|
|
working-directory: showcase/tests
|
|
run: npm ci
|
|
|
|
- name: Install Playwright
|
|
working-directory: showcase/tests
|
|
run: npx playwright install --with-deps chromium
|
|
|
|
- name: Run L1-L3 tests
|
|
id: l1_l3
|
|
working-directory: showcase/tests
|
|
env:
|
|
CI: true
|
|
run: |
|
|
npx playwright test integration-smoke --grep "@health|@agent|@chat" --reporter=github 2>&1 | tee /tmp/e2e-output.log
|
|
exit ${PIPESTATUS[0]}
|
|
|
|
- name: Run L4 tests (daily only)
|
|
id: l4
|
|
if: github.event.schedule == '0 0 * * *' || github.event_name == 'workflow_dispatch'
|
|
working-directory: showcase/tests
|
|
env:
|
|
CI: true
|
|
run: |
|
|
npx playwright test integration-smoke --grep "@tools" --reporter=github 2>&1 | tee -a /tmp/e2e-output.log
|
|
exit ${PIPESTATUS[0]}
|
|
|
|
- name: Extract failure summary
|
|
id: failures
|
|
if: failure()
|
|
run: |
|
|
# Extract failed test names and error messages from playwright output
|
|
DETAILS=$(grep -E "^\s+\d+\)|Error:" /tmp/e2e-output.log 2>/dev/null | head -15 | sed 's/"/\\"/g' || echo "See CI logs for details")
|
|
DETAILS="${DETAILS:0:1200}"
|
|
|
|
{
|
|
echo "details<<EOFEOF"
|
|
echo "$DETAILS"
|
|
echo "EOFEOF"
|
|
} >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Build Slack payload
|
|
if: failure()
|
|
run: |
|
|
SUMMARY=$(echo "${{ steps.failures.outputs.details }}" | head -3 | sed 's/\x1b\[[0-9;]*m//g' | cut -c1-200)
|
|
URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
jq -n --arg text ":x: *Showcase E2E suite failed*\n<$URL|View run>\n\`\`\`$SUMMARY\`\`\`" \
|
|
'{text: $text}' > /tmp/slack-payload.json
|
|
|
|
- name: Post failure to Slack
|
|
if: failure()
|
|
uses: slackapi/slack-github-action@v2.1.0
|
|
with:
|
|
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
|
webhook-type: incoming-webhook
|
|
payload-file-path: /tmp/slack-payload.json
|
|
|
|
- name: Create issue on failure
|
|
if: failure()
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const title = '[Drift] Showcase E2E suite failing';
|
|
const body = `## E2E Drift Detection Alert\n\n**Run**: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}\n**Schedule**: ${context.payload.schedule || 'manual'}\n\nThe centralized E2E smoke suite is failing. Please investigate.`;
|
|
|
|
const { data: issues } = await github.rest.issues.listForRepo({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
labels: 'showcase-drift',
|
|
state: 'open',
|
|
});
|
|
|
|
const existing = issues.find(i => i.title === title);
|
|
if (!existing) {
|
|
await github.rest.issues.create({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
title,
|
|
body,
|
|
labels: ['showcase-drift'],
|
|
});
|
|
}
|
|
|
|
version-drift:
|
|
name: Version Drift Report
|
|
runs-on: ubuntu-latest
|
|
# Only on weekly Monday schedule or manual trigger
|
|
if: github.event.schedule == '0 9 * * 1' || github.event.inputs.run_version_check == 'true'
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 20
|
|
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Check Python package drift
|
|
id: python_drift
|
|
run: |
|
|
report=""
|
|
for pkg_dir in showcase/packages/*/; do
|
|
req_file="$pkg_dir/requirements.txt"
|
|
[ -f "$req_file" ] || continue
|
|
slug=$(basename "$pkg_dir")
|
|
echo "=== $slug ==="
|
|
while IFS= read -r line; do
|
|
# Skip empty lines and comments
|
|
[[ -z "$line" || "$line" =~ ^# ]] && continue
|
|
# Extract package name and pinned version
|
|
if [[ "$line" =~ ^([a-zA-Z0-9_-]+)\[?[a-z]*\]?==([0-9.]+) ]]; then
|
|
pkg="${BASH_REMATCH[1]}"
|
|
pinned="${BASH_REMATCH[2]}"
|
|
latest=$(pip index versions "$pkg" 2>/dev/null | head -1 | grep -oP '\([\d.]+\)' | tr -d '()' || echo "unknown")
|
|
if [ "$latest" != "unknown" ] && [ "$latest" != "$pinned" ]; then
|
|
echo " $pkg: pinned=$pinned latest=$latest"
|
|
report+="| $slug | $pkg | $pinned | $latest |\n"
|
|
fi
|
|
fi
|
|
done < "$req_file"
|
|
done
|
|
echo "report<<EOF" >> $GITHUB_OUTPUT
|
|
echo -e "$report" >> $GITHUB_OUTPUT
|
|
echo "EOF" >> $GITHUB_OUTPUT
|
|
|
|
- name: Check npm package drift
|
|
id: npm_drift
|
|
run: |
|
|
tmpfile=$(mktemp)
|
|
for pkg_dir in showcase/packages/*/; do
|
|
pkg_json="$pkg_dir/package.json"
|
|
[ -f "$pkg_json" ] || continue
|
|
slug=$(basename "$pkg_dir")
|
|
echo "=== $slug ==="
|
|
# Check key deps: @copilotkit/*, @mastra/*, @ag-ui/*
|
|
node -e "
|
|
const pkg = require('./$pkg_json');
|
|
const deps = {...(pkg.dependencies || {})};
|
|
const interesting = Object.entries(deps).filter(([k]) =>
|
|
k.startsWith('@copilotkit/') || k.startsWith('@copilotkitnext/') ||
|
|
k.startsWith('@mastra/') || k.startsWith('@ag-ui/') ||
|
|
k === 'langchain' || k === 'langgraph'
|
|
);
|
|
interesting.forEach(([name, version]) => {
|
|
console.log(name + '=' + version);
|
|
});
|
|
" 2>/dev/null | while IFS='=' read -r name version; do
|
|
if [[ "$version" =~ ^[0-9] ]]; then
|
|
latest=$(npm view "$name" version 2>/dev/null || echo "unknown")
|
|
if [ "$latest" != "unknown" ] && [ "$latest" != "$version" ]; then
|
|
echo " $name: pinned=$version latest=$latest"
|
|
echo "| $slug | $name | $version | $latest |" >> "$tmpfile"
|
|
fi
|
|
fi
|
|
done
|
|
done
|
|
report=$(cat "$tmpfile" 2>/dev/null || echo "")
|
|
rm -f "$tmpfile"
|
|
echo "report<<EOF" >> $GITHUB_OUTPUT
|
|
echo -e "$report" >> $GITHUB_OUTPUT
|
|
echo "EOF" >> $GITHUB_OUTPUT
|
|
|
|
- name: Create drift report issue
|
|
id: drift_issue
|
|
if: steps.python_drift.outputs.report != '' || steps.npm_drift.outputs.report != ''
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const pythonDrift = `${{ steps.python_drift.outputs.report }}`.trim();
|
|
const npmDrift = `${{ steps.npm_drift.outputs.report }}`.trim();
|
|
|
|
if (!pythonDrift && !npmDrift) return;
|
|
|
|
const title = `[Drift] Showcase packages have outdated pinned versions`;
|
|
const body = `## Weekly Version Drift Report
|
|
|
|
Showcase packages have pinned versions that differ from the latest releases.
|
|
Review each and update if the new version is compatible.
|
|
|
|
${pythonDrift ? `### Python Packages\n| Package | Dep | Pinned | Latest |\n|---------|-----|--------|--------|\n${pythonDrift}` : ''}
|
|
|
|
${npmDrift ? `### npm Packages\n| Package | Dep | Pinned | Latest |\n|---------|-----|--------|--------|\n${npmDrift}` : ''}
|
|
|
|
**Action**: For each outdated dep, check the Dojo example for the correct version.
|
|
Update \`requirements.txt\` / \`package.json\`, rebuild, and verify demos still work.
|
|
`;
|
|
|
|
const { data: issues } = await github.rest.issues.listForRepo({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
labels: 'showcase-drift,version-drift',
|
|
state: 'open',
|
|
});
|
|
|
|
let issueUrl;
|
|
if (issues.length === 0) {
|
|
const { data: created } = await github.rest.issues.create({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
title,
|
|
body,
|
|
labels: ['showcase-drift', 'version-drift'],
|
|
});
|
|
issueUrl = created.html_url;
|
|
} else {
|
|
await github.rest.issues.update({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
issue_number: issues[0].number,
|
|
body,
|
|
});
|
|
issueUrl = issues[0].html_url;
|
|
}
|
|
core.setOutput('issue_url', issueUrl);
|
|
|
|
- name: Notify Slack (version drift)
|
|
if: steps.python_drift.outputs.report != '' || steps.npm_drift.outputs.report != ''
|
|
uses: slackapi/slack-github-action@v2.1.0
|
|
with:
|
|
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
|
webhook-type: incoming-webhook
|
|
payload: |
|
|
{ "text": ":warning: *Version drift*: dependency updates available | <${{ steps.drift_issue.outputs.issue_url }}|View issue>" }
|
|
|
|
- name: Notify Slack (version drift failure)
|
|
if: failure()
|
|
uses: slackapi/slack-github-action@v2.1.0
|
|
with:
|
|
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
|
webhook-type: incoming-webhook
|
|
payload: |
|
|
{ "text": ":x: *Version drift check*: failed | <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" }
|