mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Dragan Spiridonov
8eb060b6e2
The #528 and #535 integration tests (tests/integration/mcp, gated on describe.skipIf(!BUILT)) never executed in CI — a false green: - the mcp-integration-tests job ran `npm ci` but no `npm run build`, so the dist bundles were absent and both tests silently skipped; - the step was `continue-on-error: true`, so even a real failure passed; - neither test was in the release gate's test:integration:fast list. Fixes: - mcp-tools-test.yml: add `npm run build` before test:mcp:integration and drop continue-on-error so the job is a real gate (the exit-124 hang tolerance stays for the known vitest-hang flake). - both tests: skip only for LOCAL unbuilt checkouts (`!BUILT && !process.env.CI`) — in CI a missing build now fails loudly instead of skipping. Verified: CI=1 with the bundle removed fails the suite instead of reporting green. - package.json: add both tests to test:integration:fast so they also run on the release gate (npm-publish.yml), which builds first. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
271 lines
8.8 KiB
YAML
271 lines
8.8 KiB
YAML
name: MCP Tools Testing
|
|
permissions:
|
|
contents: read
|
|
actions: write
|
|
issues: write
|
|
pull-requests: write
|
|
checks: write
|
|
|
|
on:
|
|
push:
|
|
branches: [main, testing-with-qe]
|
|
paths:
|
|
- 'src/mcp/**'
|
|
- 'tests/mcp/**'
|
|
- 'tests/integration/mcp/**'
|
|
- 'package.json'
|
|
- 'package-lock.json'
|
|
pull_request:
|
|
branches: [main]
|
|
paths:
|
|
- 'src/mcp/**'
|
|
- 'tests/mcp/**'
|
|
- 'tests/integration/mcp/**'
|
|
- 'package.json'
|
|
|
|
# On PRs: cancel stale runs when a new push arrives (old run is obsolete).
|
|
# On main: never cancel — every push must run to completion for accurate CI signal.
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
jobs:
|
|
# Split MCP unit tests into parallel shards by subdirectory.
|
|
# Previously ran as one sequential job hitting the 30-min timeout.
|
|
# Now each shard runs in ~8-12 min instead of ~25 min total.
|
|
mcp-unit-handlers:
|
|
name: MCP Units — Handlers
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run handler tests
|
|
run: bash scripts/ci-vitest-run.sh tests/unit/mcp/handlers/ --fileParallelism=false --reporter=verbose
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=1024'
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: mcp-unit-handlers-results
|
|
path: junit.xml
|
|
retention-days: 14
|
|
|
|
mcp-unit-security:
|
|
name: MCP Units — Security
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run security tests
|
|
run: timeout 480 npx vitest run tests/unit/mcp/security/ --fileParallelism=false --reporter=verbose
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=1024'
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: mcp-unit-security-results
|
|
path: junit.xml
|
|
retention-days: 14
|
|
|
|
mcp-unit-tools-transport:
|
|
name: MCP Units — Tools + Transport + Metrics
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run tools, transport, metrics tests
|
|
run: >-
|
|
timeout 480 npx vitest run
|
|
tests/unit/mcp/tools/
|
|
tests/unit/mcp/transport/
|
|
tests/unit/mcp/metrics/
|
|
--fileParallelism=false
|
|
--reporter=verbose
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=1024'
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: mcp-unit-tools-transport-results
|
|
path: junit.xml
|
|
retention-days: 14
|
|
|
|
mcp-unit-root:
|
|
name: MCP Units — Root Files
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
- name: Run root-level MCP tests
|
|
run: >-
|
|
bash scripts/ci-vitest-run.sh
|
|
tests/unit/mcp/mcp-server.test.ts
|
|
tests/unit/mcp/mcp-performance-benchmark.test.ts
|
|
tests/unit/mcp/tool-registry-security.test.ts
|
|
tests/unit/mcp/tool-scoping.test.ts
|
|
tests/unit/mcp/wrapped-domain-handlers.test.ts
|
|
--fileParallelism=false
|
|
--reporter=verbose
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=1024'
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: mcp-unit-root-results
|
|
path: junit.xml
|
|
retention-days: 14
|
|
|
|
mcp-integration-tests:
|
|
name: MCP Integration Tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 20
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
# C3: integration tests under tests/integration/mcp drive the BUILT
|
|
# bundles (dist/cli + dist/mcp). Without this build step they were
|
|
# silently skipping (describe.skipIf(!BUILT)) — a false green.
|
|
- run: npm run build
|
|
|
|
- name: Run MCP integration tests
|
|
run: |
|
|
timeout 480 npm run test:mcp:integration; EXIT=$?
|
|
if [ $EXIT -eq 124 ] && [ -f junit.xml ]; then
|
|
FAILURES=$(grep -c '<failure' junit.xml 2>/dev/null || echo "0")
|
|
if [ "$FAILURES" = "0" ]; then
|
|
echo "::warning::Vitest hung after tests passed (exit 124). Treating as success."
|
|
exit 0
|
|
fi
|
|
fi
|
|
exit $EXIT
|
|
env:
|
|
NODE_OPTIONS: '--max-old-space-size=1024'
|
|
# C3: was `continue-on-error: true`, which let real failures pass as
|
|
# green. Removed so this job is an actual gate. The exit-124 hang
|
|
# tolerance above still absorbs the known vitest-hang flake.
|
|
|
|
- name: Generate test report
|
|
uses: dorny/test-reporter@v1
|
|
if: always() && hashFiles('junit.xml') != ''
|
|
continue-on-error: true
|
|
with:
|
|
name: MCP Integration Test Results
|
|
path: 'junit.xml'
|
|
reporter: jest-junit
|
|
fail-on-error: false
|
|
fail-on-empty: false
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: mcp-integration-test-results
|
|
path: |
|
|
test-results/
|
|
junit.xml
|
|
retention-days: 30
|
|
|
|
mcp-validation:
|
|
name: Validate All MCP Tools
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with: { node-version: '24.13.0', cache: 'npm' }
|
|
- run: npm ci
|
|
- run: npm run build
|
|
|
|
- name: Validate MCP tools
|
|
run: timeout 480 npm run mcp:validate
|
|
continue-on-error: true
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: mcp-validation-report
|
|
path: reports/mcp-validation-*.json
|
|
retention-days: 30
|
|
|
|
- name: Generate MCP report
|
|
run: timeout 480 npm run mcp:report
|
|
if: always()
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: mcp-report
|
|
path: reports/mcp-report-*.md
|
|
retention-days: 30
|
|
|
|
mcp-summary:
|
|
name: MCP Test Summary
|
|
runs-on: ubuntu-latest
|
|
needs: [mcp-unit-handlers, mcp-unit-security, mcp-unit-tools-transport, mcp-unit-root, mcp-integration-tests, mcp-validation]
|
|
if: always()
|
|
|
|
steps:
|
|
- name: Download all artifacts
|
|
uses: actions/download-artifact@v4
|
|
|
|
- name: Create summary comment
|
|
uses: actions/github-script@v7
|
|
if: github.event_name == 'pull_request'
|
|
with:
|
|
script: |
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
let summary = '## MCP Tools Test Summary\n\n';
|
|
|
|
// Check validation report
|
|
try {
|
|
const validationFiles = fs.readdirSync('mcp-validation-report');
|
|
if (validationFiles.length > 0) {
|
|
const reportPath = path.join('mcp-validation-report', validationFiles[0]);
|
|
const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
|
|
summary += `### Validation Results\n`;
|
|
summary += `- Total Tools: ${report.totalTools}\n`;
|
|
summary += `- Valid Tools: ${report.validTools}\n`;
|
|
summary += `- Invalid Tools: ${report.invalidTools}\n`;
|
|
summary += `- Coverage: ${report.coverage}%\n\n`;
|
|
}
|
|
} catch (err) {
|
|
summary += '### Validation Results\n- Validation report not found\n\n';
|
|
}
|
|
|
|
summary += '### Unit Test Results\n';
|
|
summary += '- Handlers: ${{ needs.mcp-unit-handlers.result }}\n';
|
|
summary += '- Security: ${{ needs.mcp-unit-security.result }}\n';
|
|
summary += '- Tools/Transport: ${{ needs.mcp-unit-tools-transport.result }}\n';
|
|
summary += '- Root: ${{ needs.mcp-unit-root.result }}\n\n';
|
|
summary += '### Other Results\n';
|
|
summary += '- Integration Tests: ${{ needs.mcp-integration-tests.result }}\n';
|
|
summary += '- Validation: ${{ needs.mcp-validation.result }}\n';
|
|
|
|
github.rest.issues.createComment({
|
|
issue_number: context.issue.number,
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
body: summary
|
|
});
|