mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Dragan Spiridonov
d8a04b22d2
Adds a new `security-audit` job to optimized-ci.yml that runs in parallel with the journey shards and fails the build on any high/critical CVE in production deps. devDep CVEs are reported as warnings — they don't ship to users but are visible so we can fix them before they migrate. Also adds the same `npm audit --audit-level=high --omit=dev` step to the publish workflow's build job as a defense-in-depth gate: blocks `npm publish` if a new prod-tree CVE has landed since the last main-branch CI run. Would have caught the 6 high `@typescript-eslint`/`minimatch` advisories that Tier B just cleared the moment they were published, rather than when someone happened to run `npm audit`. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
409 lines
15 KiB
YAML
409 lines
15 KiB
YAML
name: Optimized CI
|
|
|
|
on:
|
|
push:
|
|
branches: [main, develop]
|
|
pull_request:
|
|
branches: [main, develop]
|
|
workflow_dispatch:
|
|
|
|
# On PRs: cancel stale runs when a new push arrives (old run is obsolete).
|
|
# On main: never cancel — every push must run to completion for accurate CI signal.
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
permissions:
|
|
contents: read
|
|
actions: write
|
|
pull-requests: write
|
|
checks: write
|
|
|
|
# ─── Shared exclusions for journey test shards ───
|
|
# The original test:journeys:ci command excludes these directories:
|
|
# browser, browser-integration, mcp, sync, learning, domains, llm, n8n, *.e2e.test.ts
|
|
# By targeting specific subdirectories per shard, most exclusions are implicit.
|
|
# Only the "remaining" shard needs explicit excludes.
|
|
|
|
jobs:
|
|
# ─── Security audit (fail fast on high/critical CVEs) ───
|
|
# Cheap (~30s), independent of the test shards. Runs in parallel with them
|
|
# so a known CVE blocks the build without waiting for journey tests. Scoped
|
|
# to production deps (--omit=dev) because devDep CVEs do not ship to users;
|
|
# those are visible in the second step as a warning, not a failure.
|
|
security-audit:
|
|
name: Security Audit
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci --ignore-scripts
|
|
- name: Block on high+critical CVEs in production deps
|
|
run: npm audit --audit-level=high --omit=dev
|
|
- name: Report devDep CVEs (non-blocking)
|
|
if: always()
|
|
run: npm audit --audit-level=high || echo "::warning::devDep CVEs found — fix before they migrate into prod"
|
|
|
|
# ─── Journey Tests: 4 parallel shards ───
|
|
# Previously a single 25-min job that timed out. Now 4 shards of ~5-8 min each.
|
|
|
|
journey-governance:
|
|
name: Journey — Governance
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run governance integration tests
|
|
run: timeout 480 npx vitest run tests/integration/governance/ --reporter=verbose
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=1024'
|
|
|
|
journey-ruvector-coordination:
|
|
name: Journey — RuVector + Coordination
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run ruvector + coordination integration tests
|
|
run: bash scripts/ci-vitest-run.sh tests/integration/ruvector/ tests/integration/coordination/ --reporter=verbose
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=1024'
|
|
|
|
journey-root:
|
|
name: Journey — Root-level
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run root-level integration tests
|
|
run: |
|
|
# Run only test files directly in tests/integration/ (not in subdirectories)
|
|
FILES=$(find tests/integration -maxdepth 1 -name '*.test.ts' ! -name '*.e2e.test.ts' | sort)
|
|
if [ -n "$FILES" ]; then
|
|
bash scripts/ci-vitest-run.sh $FILES --reporter=verbose
|
|
else
|
|
echo "No root-level integration tests found"
|
|
fi
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=1024'
|
|
|
|
journey-remaining:
|
|
name: Journey — Remaining Subdirs
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run remaining integration subdirectory tests
|
|
run: |
|
|
# Everything in tests/integration subdirs EXCEPT the ones covered by
|
|
# other shards (governance, ruvector, coordination) and the original
|
|
# exclusions (browser, mcp, sync, learning, domains, llm, n8n).
|
|
bash scripts/ci-vitest-run.sh \
|
|
tests/integration/adapters/ \
|
|
tests/integration/cli/ \
|
|
tests/integration/code-intelligence/ \
|
|
tests/integration/feedback/ \
|
|
tests/integration/migration/ \
|
|
tests/integration/opencode/ \
|
|
tests/integration/parsers/ \
|
|
tests/integration/planning/ \
|
|
tests/integration/protocols/ \
|
|
tests/integration/requirements-validation/ \
|
|
tests/integration/rl-suite/ \
|
|
tests/integration/rvf/ \
|
|
tests/integration/security/ \
|
|
tests/integration/templates/ \
|
|
tests/integration/test-execution/ \
|
|
tests/integration/test-generation/ \
|
|
tests/integration/validation/ \
|
|
--reporter=verbose
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=1024'
|
|
|
|
# ─── Contract + Code Intelligence: separate parallel jobs ───
|
|
|
|
contract-tests:
|
|
name: Contract Tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run contract tests
|
|
run: |
|
|
if [ -d "tests/unit/domains/contract-testing" ] && [ "$(ls -A tests/unit/domains/contract-testing/*.test.ts 2>/dev/null)" ]; then
|
|
timeout 480 npm run test:contracts
|
|
else
|
|
echo "No contract tests found"
|
|
fi
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=512'
|
|
|
|
code-intelligence-tests:
|
|
name: Code Intelligence Tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run code intelligence tests
|
|
run: timeout 480 npm run test:code-intelligence
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=768'
|
|
SKIP_INTEGRATION_TESTS: 'true'
|
|
|
|
# ─── Infrastructure tests (unchanged, already parallel) ───
|
|
|
|
infrastructure-tests:
|
|
name: Infrastructure Tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run Infrastructure Tests
|
|
run: |
|
|
if [ -d "tests/infrastructure" ] && [ "$(ls -A tests/infrastructure/*.test.ts 2>/dev/null)" ]; then
|
|
timeout 480 npm run test:infrastructure
|
|
else
|
|
echo "No infrastructure tests found"
|
|
fi
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=768'
|
|
- name: Run Regression Tests
|
|
run: |
|
|
if [ -d "tests/regression" ] && [ "$(find tests/regression -name '*.test.ts' 2>/dev/null | head -1)" ]; then
|
|
timeout 480 npm run test:regression
|
|
else
|
|
echo "No regression tests found"
|
|
fi
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=512'
|
|
|
|
# ─── Postgres integration tests (unchanged) ───
|
|
|
|
integration-tests:
|
|
name: Postgres Integration Tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
services:
|
|
postgres:
|
|
image: postgres:16-alpine
|
|
env:
|
|
POSTGRES_DB: aqe_test
|
|
POSTGRES_USER: aqe_test
|
|
POSTGRES_PASSWORD: aqe_test
|
|
ports:
|
|
- 15432:5432
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run Postgres integration tests
|
|
run: timeout 480 npm run test:integration:pg
|
|
env:
|
|
POSTGRES_URL: postgresql://aqe_test:aqe_test@localhost:15432/aqe_test
|
|
NODE_OPTIONS: '--max-old-space-size=1024'
|
|
|
|
# ─── Performance gates (after journey shards complete) ───
|
|
|
|
performance-gates:
|
|
name: Performance Gates
|
|
runs-on: ubuntu-latest
|
|
needs: [journey-governance, journey-ruvector-coordination, journey-root, journey-remaining, contract-tests, code-intelligence-tests]
|
|
if: success()
|
|
timeout-minutes: 10
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run Performance Gates
|
|
id: perf-gates
|
|
run: |
|
|
timeout 480 npm run performance:gate > perf-report.md 2>&1 || PERF_EXIT=$?
|
|
echo "PERF_EXIT_CODE=${PERF_EXIT:-0}" >> $GITHUB_OUTPUT
|
|
cat perf-report.md
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: performance-report
|
|
path: perf-report.md
|
|
retention-days: 30
|
|
- name: Check performance gate results
|
|
run: |
|
|
EXIT_CODE="${{ steps.perf-gates.outputs.PERF_EXIT_CODE }}"
|
|
if [ "$EXIT_CODE" -eq 0 ]; then
|
|
echo "All performance gates passed"
|
|
elif [ "$EXIT_CODE" -eq 2 ]; then
|
|
echo "Performance gates passed with warnings"
|
|
else
|
|
echo "Performance gates failed (exit code: $EXIT_CODE)"
|
|
if [ "$EXIT_CODE" -eq 1 ]; then
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# ─── Coverage analysis (PRs only, after journey shards) ───
|
|
|
|
coverage:
|
|
name: Coverage Analysis
|
|
runs-on: ubuntu-latest
|
|
needs: [journey-governance, journey-ruvector-coordination, journey-root, journey-remaining]
|
|
timeout-minutes: 10
|
|
if: success() && github.event_name == 'pull_request'
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run tests with coverage (excluding browser E2E)
|
|
run: bash scripts/ci-vitest-run.sh --coverage --reporter=verbose
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=1024'
|
|
CI: 'true'
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: coverage-report
|
|
path: coverage/
|
|
retention-days: 7
|
|
- name: Check coverage thresholds
|
|
run: |
|
|
if [ -f "coverage/coverage-summary.json" ]; then
|
|
COVERAGE=$(cat coverage/coverage-summary.json | jq '.total.lines.pct')
|
|
echo "Line coverage: ${COVERAGE}%"
|
|
if (( $(echo "$COVERAGE < 80" | bc -l) )); then
|
|
echo "Coverage below 80% threshold"
|
|
else
|
|
echo "Coverage meets threshold"
|
|
fi
|
|
fi
|
|
|
|
# ─── Test dashboard (summary) ───
|
|
|
|
dashboard:
|
|
name: Test Dashboard
|
|
runs-on: ubuntu-latest
|
|
needs: [journey-governance, journey-ruvector-coordination, journey-root, journey-remaining, contract-tests, code-intelligence-tests, infrastructure-tests]
|
|
if: always()
|
|
timeout-minutes: 2
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0' }
|
|
- name: Generate Dashboard
|
|
run: node scripts/test-dashboard.cjs
|
|
- name: Generate Migration Metrics
|
|
run: |
|
|
echo "# CI Test Metrics" > ci-metrics.md
|
|
echo "" >> ci-metrics.md
|
|
echo "**Date**: $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> ci-metrics.md
|
|
echo "**Commit**: ${{ github.sha }}" >> ci-metrics.md
|
|
echo "" >> ci-metrics.md
|
|
|
|
total_files=$(find tests -name "*.test.ts" 2>/dev/null | wc -l || echo "0")
|
|
total_lines=$(find tests -name "*.test.ts" -exec wc -l {} + 2>/dev/null | tail -1 | awk '{print $1}' || echo "0")
|
|
large_files=$(find tests -name "*.test.ts" -exec wc -l {} \; 2>/dev/null | awk '$1 > 600' | wc -l || echo "0")
|
|
skipped=$(grep -r "describe.skip\|it.skip\|test.skip" tests --include="*.test.ts" 2>/dev/null | wc -l || echo "0")
|
|
|
|
echo "## Current State" >> ci-metrics.md
|
|
echo "- Total test files: $total_files (target: 50)" >> ci-metrics.md
|
|
echo "- Total lines: $total_lines (target: 40,000)" >> ci-metrics.md
|
|
echo "- Files > 600 lines: $large_files (target: 0)" >> ci-metrics.md
|
|
echo "- Skipped tests: $skipped (target: 0)" >> ci-metrics.md
|
|
echo "" >> ci-metrics.md
|
|
|
|
baseline_files=426
|
|
baseline_lines=208253
|
|
files_reduced=$((baseline_files - total_files))
|
|
lines_reduced=$((baseline_lines - total_lines))
|
|
|
|
echo "## Progress from Baseline" >> ci-metrics.md
|
|
echo "- Files reduced: $files_reduced (-$((files_reduced * 100 / baseline_files))%)" >> ci-metrics.md
|
|
echo "- Lines reduced: $lines_reduced (-$((lines_reduced * 100 / baseline_lines))%)" >> ci-metrics.md
|
|
|
|
cat ci-metrics.md
|
|
- uses: actions/upload-artifact@v4
|
|
with:
|
|
name: ci-metrics
|
|
path: ci-metrics.md
|
|
retention-days: 30
|
|
- name: Comment on PR
|
|
if: github.event_name == 'pull_request'
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const fs = require('fs');
|
|
|
|
const metrics = fs.readFileSync('ci-metrics.md', 'utf8');
|
|
|
|
const body = `## Test Suite Metrics
|
|
|
|
${metrics}
|
|
|
|
---
|
|
*Generated by Optimized CI*`;
|
|
|
|
const { data: comments } = await github.rest.issues.listComments({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
issue_number: context.issue.number,
|
|
});
|
|
|
|
const botComment = comments.find(c =>
|
|
c.user?.type === 'Bot' && c.body?.includes('Test Suite Metrics')
|
|
);
|
|
|
|
if (botComment) {
|
|
await github.rest.issues.updateComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
comment_id: botComment.id,
|
|
body
|
|
});
|
|
} else {
|
|
await github.rest.issues.createComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
issue_number: context.issue.number,
|
|
body
|
|
});
|
|
}
|