mcp-use__mcp-use/.github/workflows/conformance.yml

name: MCP Conformance Tests
on:
  workflow_dispatch:
    inputs:
      python_only:
        description: 'Run only Python conformance tests'
        type: boolean
        default: false
      typescript_only:
        description: 'Run only TypeScript conformance tests'
        type: boolean
        default: false
  push:
    branches:
      - main
      - canary
    paths:
      - 'libraries/python/mcp_use/server/**'
      - 'libraries/python/mcp_use/client/**'
      - 'libraries/python/pyproject.toml'
      - 'libraries/typescript/packages/server/src/**'
      - 'libraries/typescript/packages/server/package.json'
      - 'libraries/typescript/packages/client/src/**'
      - 'libraries/typescript/packages/client/package.json'
      - 'libraries/python/tests/integration/servers_for_testing/conformance_server.py'
      - 'libraries/python/tests/integration/clients_for_testing/conformance_client.py'
      - 'libraries/typescript/packages/server/examples/conformance/**'
      - 'libraries/typescript/packages/client/examples/conformance/**'
      - '.github/workflows/conformance.yml'
  pull_request:
    branches:
      - main
      - canary
    paths:
      - 'libraries/python/mcp_use/server/**'
      - 'libraries/python/mcp_use/client/**'
      - 'libraries/python/pyproject.toml'
      - 'libraries/typescript/packages/server/src/**'
      - 'libraries/typescript/packages/server/package.json'
      - 'libraries/typescript/packages/client/src/**'
      - 'libraries/typescript/packages/client/package.json'
      - 'libraries/python/tests/integration/servers_for_testing/conformance_server.py'
      - 'libraries/python/tests/integration/clients_for_testing/conformance_client.py'
      - 'libraries/typescript/packages/server/examples/conformance/**'
      - 'libraries/typescript/packages/client/examples/conformance/**'
      - '.github/workflows/conformance.yml'

permissions:
  contents: read
jobs:
  # Detect which languages had changes so we only run the relevant tests.
  # On workflow_dispatch the manual inputs take precedence instead.
  detect-changes:
    runs-on: ubuntu-latest
    outputs:
      python: ${{ steps.compute.outputs.python }}
      typescript: ${{ steps.compute.outputs.typescript }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: filter
        if: github.event_name != 'workflow_dispatch'
        with:
          filters: |
            python:
              - 'libraries/python/**'
              - '.github/workflows/conformance.yml'
            typescript:
              - 'libraries/typescript/**'
              - '.github/workflows/conformance.yml'
      - name: Compute outputs
        id: compute
        env:
          EVENT_NAME: ${{ github.event_name }}
          INPUT_TYPESCRIPT_ONLY: ${{ github.event.inputs.typescript_only }}
          INPUT_PYTHON_ONLY: ${{ github.event.inputs.python_only }}
          FILTER_PYTHON: ${{ steps.filter.outputs.python }}
          FILTER_TYPESCRIPT: ${{ steps.filter.outputs.typescript }}
        run: |
          if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
            # Manual dispatch: use the explicit inputs
            if [ "$INPUT_TYPESCRIPT_ONLY" = "true" ]; then
              echo "python=false" >> $GITHUB_OUTPUT
            else
              echo "python=true" >> $GITHUB_OUTPUT
            fi
            if [ "$INPUT_PYTHON_ONLY" = "true" ]; then
              echo "typescript=false" >> $GITHUB_OUTPUT
            else
              echo "typescript=true" >> $GITHUB_OUTPUT
            fi
          else
            # Push / PR: use path detection
            echo "python=$FILTER_PYTHON" >> $GITHUB_OUTPUT
            echo "typescript=$FILTER_TYPESCRIPT" >> $GITHUB_OUTPUT
          fi

  python-conformance-test:
    # mcp-conformance-action@v1 is pinned here to
    # @modelcontextprotocol/conformance 0.1.16. Keep Python on that stable
    # runner until its v2 coverage is migrated deliberately.
    name: Python Conformance Tests (v1)
    needs: detect-changes
    if: needs.detect-changes.outputs.python == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 20

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python
        if: needs.detect-changes.outputs.python == 'true'
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install uv
        if: needs.detect-changes.outputs.python == 'true'
        uses: astral-sh/setup-uv@v4
        with:
          version: "latest"

      - name: Set up Node.js for the v1 conformance runner
        uses: actions/setup-node@v4
        with:
          node-version: '22.23.1'

      - name: Prepare Python
        working-directory: libraries/python
        run: |
          uv venv
          source .venv/bin/activate
          uv pip install -e ".[dev]"

      - name: Verify Python conformance fixtures
        working-directory: libraries/python
        run: |
          source .venv/bin/activate
          uv pip check
          python -c "import mcp_use; import tests.integration.servers_for_testing.conformance_server"

      - name: Run Python conformance tests
        uses: mcp-use/mcp-conformance-action@v1
        env:
          MCP_USE_ANONYMIZED_TELEMETRY: false
        with:
          mode: test
          test-type: both
          conformance-version: '0.1.16'
          servers: '[{"name":"python","start-command":"source .venv/bin/activate && python tests/integration/servers_for_testing/conformance_server.py --transport streamable-http --port 8000","url":"http://127.0.0.1:8000/mcp","working-directory":"libraries/python"}]'
          clients: '[{"name":"python-client","command":".venv/bin/python tests/integration/clients_for_testing/conformance_client.py","working-directory":"libraries/python"}]'
          badge-gist-id: ${{ secrets.CONFORMANCE_GIST_ID }}
          badge-gist-token: ${{ secrets.GIST_SECRET }}

  typescript-v2-conformance-test:
    name: TypeScript Conformance Tests (v2; legacy + modern modes)
    needs: detect-changes
    if: needs.detect-changes.outputs.typescript == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '22.23.1'

      - name: Install pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 10

      - name: Prepare TypeScript conformance examples
        working-directory: libraries/typescript
        run: |
          pnpm install --frozen-lockfile
          pnpm build

      - name: Run TypeScript conformance tests
        env:
          # Pin the v2 runner independently from the v1 action used above.
          CONFORMANCE_RUNNER: '@modelcontextprotocol/conformance@0.2.0-alpha.10'
          # Dated versions intentionally exclude off-timeline extensions such as tasks.
          SPEC_VERSIONS: '2025-11-25 2026-07-28'
          LEGACY_SERVER_EXPECTED_FAILURES: 'libraries/typescript/packages/server/examples/conformance/expected-failures.yml'
        run: |
          set +e
          exit_code=0
          results_dir=conformance-results/typescript
          mkdir -p "$results_dir"

          (
            cd libraries/typescript/packages/server/examples/conformance
            PORT=3000 node ../../dist/bin.js dev --no-open
          ) > "$results_dir/server.log" 2>&1 &
          server_pid=$!
          trap 'kill "$server_pid" 2>/dev/null; wait "$server_pid" 2>/dev/null' EXIT

          server_ready=false
          for attempt in {1..60}; do
            if curl --silent --show-error --output /dev/null http://127.0.0.1:3000/mcp; then
              server_ready=true
              break
            fi
            if ! kill -0 "$server_pid" 2>/dev/null; then
              echo 'TypeScript conformance server exited before it became ready.'
              cat "$results_dir/server.log"
              break
            fi
            sleep 1
          done

          if [ "$server_ready" != true ]; then
            echo 'TypeScript conformance server did not become ready within 60 seconds.'
            exit_code=1
          fi

          for spec_version in $SPEC_VERSIONS; do
            version_results_dir="$results_dir/$spec_version"
            mkdir -p "$version_results_dir"

            if [ "$server_ready" = true ]; then
              expected_failure_args=()
              if [ "$spec_version" = '2025-11-25' ]; then
                expected_failure_args=(--expected-failures "$LEGACY_SERVER_EXPECTED_FAILURES")
              fi

              pnpm dlx "$CONFORMANCE_RUNNER" server \
                --url http://127.0.0.1:3000/mcp \
                --suite all \
                --spec-version "$spec_version" \
                --output-dir "$version_results_dir/server" \
                "${expected_failure_args[@]}" || exit_code=1

              if [ "$spec_version" = '2025-11-25' ]; then
                printf '%s\n' \
                  '### TypeScript legacy server exclusions' \
                  '' \
                  '- `tools-call-sampling` — stateless legacy HTTP has no client session for sampling.' \
                  '- `tools-call-elicitation` — stateless legacy HTTP has no client session for elicitation.' \
                  '- `elicitation-sep1034-defaults` — stateless legacy HTTP has no client session for elicitation.' \
                  '- `elicitation-sep1330-enums` — stateless legacy HTTP has no client session for elicitation.' \
                  '- `server-sse-polling` — stateless legacy HTTP has no session or SSE resumability.' \
                  '- `server-sse-multiple-streams` — stateless legacy HTTP has no session for multiple SSE streams.' \
                  '- `resources-subscribe` — stateless legacy HTTP does not retain resource subscriptions.' \
                  '- `resources-unsubscribe` — stateless legacy HTTP does not retain resource subscriptions.' \
                  >> "$GITHUB_STEP_SUMMARY"
              fi
            fi

            pnpm dlx "$CONFORMANCE_RUNNER" client \
              --command 'cd libraries/typescript/packages/client/examples/conformance && ./run-conformance-client-node.sh' \
              --suite all \
              --spec-version "$spec_version" \
              --output-dir "$version_results_dir/client-node" || exit_code=1

            pnpm dlx "$CONFORMANCE_RUNNER" client \
              --command 'cd libraries/typescript/packages/client/examples/conformance && ./run-conformance-client-browser.sh' \
              --suite all \
              --spec-version "$spec_version" \
              --output-dir "$version_results_dir/client-browser" || exit_code=1

            pnpm dlx "$CONFORMANCE_RUNNER" client \
              --command 'cd libraries/typescript/packages/client/examples/conformance && ./run-conformance-client-react.sh' \
              --suite all \
              --spec-version "$spec_version" \
              --output-dir "$version_results_dir/client-react" || exit_code=1
          done

          exit "$exit_code"

      - name: Summarize TypeScript conformance scores
        if: always()
        working-directory: .
        env:
          CONFORMANCE_RUNNER: '@modelcontextprotocol/conformance@0.2.0-alpha.10'
          SPEC_VERSIONS: '2025-11-25 2026-07-28'
          LEGACY_SERVER_EXPECTED_FAILURES: 'libraries/typescript/packages/server/examples/conformance/expected-failures.yml'
        run: |
          node libraries/typescript/scripts/summarize-conformance.mjs \
            conformance-results/typescript \
            --runner "$CONFORMANCE_RUNNER" \
            --spec-versions "$SPEC_VERSIONS" \
            --expected-failures "$LEGACY_SERVER_EXPECTED_FAILURES" \
            --output conformance-results/typescript/summary.md

      - name: Upload TypeScript v2 conformance results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: typescript-v2-conformance-results
          path: conformance-results/typescript
          if-no-files-found: warn
          retention-days: 14