fix: report ErrSearchTimeout when results never hydrate; raise integration selector timeout to 15s

- breaker: stop counting client cancellations/deadlines (incl. bare
  rate-limiter wait errors) and circuit-open as engine failures
- rate limiting: cache limiters in SearchEngineOptions and rawEngine so
  pacing applies on raw/library paths; pool wrapper delegates
- /extract SSRF guard: public-IP-only policy with dial-time IP pinning,
  redirect re-validation, rendered-mode preflight, http/https allow-list,
  ErrTargetNotAllowed -> HTTP 400, extract.allow_private_networks
  escape hatch (default off)
- browser: no rod Must* on the request path; CLI parses block_resources
  without panicking

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,76 @@
+name: Bug report
+description: Report a reproducible OpenSERP bug.
+title: "bug: "
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to report a bug. Reproducible reports are the fastest to fix.
+  - type: textarea
+    id: summary
+    attributes:
+      label: What happened?
+      description: Describe the unexpected behavior and what you expected instead.
+      placeholder: "Google search returns empty results for ..."
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: How do we reproduce it?
+      description: Include the exact command, request, config, and minimal input.
+      placeholder: |
+        1. Run `openserp serve ...`
+        2. Request `curl "..."`
+        3. See ...
+    validations:
+      required: true
+  - type: dropdown
+    id: mode
+    attributes:
+      label: Retrieval mode
+      options:
+        - browser
+        - raw
+        - both
+        - not sure
+    validations:
+      required: true
+  - type: dropdown
+    id: engine
+    attributes:
+      label: Engine
+      options:
+        - google
+        - yandex
+        - baidu
+        - bing
+        - duckduckgo
+        - ecosia
+        - megasearch
+        - not engine-specific
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: OpenSERP version
+      description: Paste `openserp --version`, the Docker tag, or the commit SHA.
+      placeholder: "0.8.0 / v0.7.15 / commit SHA"
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Environment
+      description: OS, Go version, Docker/browser version, proxy/captcha setup if relevant.
+      placeholder: "Windows 11, Go 1.24, Docker, Chromium ..."
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs or response body
+      description: Paste relevant logs, stack traces, or API responses. Remove secrets and proxy credentials.
+      render: shell

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Contributing guide
+    url: https://github.com/karust/openserp/blob/main/docs/CONTRIBUTING.md
+    about: Start here before opening a pull request.
+  - name: Add a new engine guide
+    url: https://github.com/karust/openserp/blob/main/docs/ADDING_AN_ENGINE.md
+    about: Implementation checklist for new search engines.

.github/ISSUE_TEMPLATE/engine_request.yml

@@ -0,0 +1,47 @@
+name: Engine request
+description: Request support for a new search engine or engine vertical.
+title: "engine: "
+labels: ["enhancement"]
+body:
+  - type: input
+    id: engine
+    attributes:
+      label: Engine name
+      placeholder: "Brave Search"
+    validations:
+      required: true
+  - type: input
+    id: search_url
+    attributes:
+      label: Public search URL
+      placeholder: "https://search.brave.com/search?q=openserp"
+    validations:
+      required: true
+  - type: dropdown
+    id: vertical
+    attributes:
+      label: Result type
+      options:
+        - web
+        - images
+        - web and images
+        - other
+    validations:
+      required: true
+  - type: textarea
+    id: selectors
+    attributes:
+      label: Known stable selectors or sample HTML
+      description: If you inspected the page, mention stable data attributes or attach a sanitized fixture.
+  - type: textarea
+    id: notes
+    attributes:
+      label: Notes
+      description: Region/language quirks, captcha behavior, pagination, or API constraints.
+  - type: checkboxes
+    id: contribution
+    attributes:
+      label: Contribution
+      options:
+        - label: I can help test this engine.
+        - label: I am willing to implement this engine.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,47 @@
+name: Feature request
+description: Suggest an OpenSERP improvement that is not engine-specific.
+title: "feat: "
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem
+      description: What user problem does this solve?
+      placeholder: "It is hard to ..."
+    validations:
+      required: true
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposed solution
+      description: Describe the behavior, API, CLI, or documentation change you want.
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives considered
+      description: What workarounds or smaller versions have you considered?
+  - type: dropdown
+    id: area
+    attributes:
+      label: Area
+      options:
+        - CLI
+        - API
+        - docs
+        - parser
+        - proxy
+        - captcha
+        - resilience
+        - tests
+        - other
+    validations:
+      required: true
+  - type: checkboxes
+    id: contribution
+    attributes:
+      label: Contribution
+      options:
+        - label: I am willing to open a PR for this.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,20 @@
+## Summary
+
+- 
+
+## Why
+
+- 
+
+## Testing
+
+- [ ] `make test`
+- [ ] `make lint`
+- [ ] `make test-integration` (only for browser, proxy, captcha, or live-engine changes)
+
+## Checklist
+
+- [ ] I linked the related issue or explained why there is none.
+- [ ] I updated docs or examples for changed user-facing behavior.
+- [ ] I kept unit tests deterministic and free of browser/network dependencies.
+- [ ] I removed secrets, proxy credentials, and private logs from examples.

.github/workflows/ci.yml

@@ -42,3 +42,6 @@ jobs:
 
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v6
+        with:
+          version: v1.64.8
+          args: --config .golangci.yml

.github/workflows/docker.yml

@@ -0,0 +1,118 @@
+name: Docker
+
+on:
+  push:
+    branches: [main]
+    tags: ["v*.*.*"]
+  pull_request:
+    branches: [main]
+  # Manual republish (e.g. base-image refresh without a version bump).
+  # Must be dispatched FROM A TAG REF: metadata-action only derives semver
+  # image tags from the git tag, so a branch dispatch is rejected below.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  # Fail fast with a clear message instead of silently pushing a bare
+  # `latest` (or nothing) when someone dispatches from a branch.
+  guard-dispatch:
+    if: github.event_name == 'workflow_dispatch' && !startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          echo "::error::Manual runs must be dispatched from a v*.*.* tag ref (use the 'Run workflow' ref selector). Branch refs produce no semver image tags."
+          exit 1
+
+  # Catch tag/code drift before anything is published (root.go has drifted
+  # from tags before).
+  verify-version:
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check tag matches cmd/root.go version
+        run: |
+          TAG_VERSION="${GITHUB_REF_NAME#v}"
+          CODE_VERSION=$(sed -nE 's/^[[:space:]]*version[[:space:]]*=[[:space:]]*"([0-9]+\.[0-9]+\.[0-9]+)".*/\1/p' cmd/root.go)
+          echo "tag=v${TAG_VERSION} code=${CODE_VERSION}"
+          if [ "$TAG_VERSION" != "$CODE_VERSION" ]; then
+            echo "::error::Git tag v${TAG_VERSION} does not match version \"${CODE_VERSION}\" in cmd/root.go. Bump cmd/root.go before tagging."
+            exit 1
+          fi
+
+  # PR / main validation: build only, single platform (multi-arch under QEMU
+  # roughly doubles CI time for no extra signal; tag builds cover arm64).
+  build:
+    if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build (no push)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64
+          push: false
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  # Publish on version tags (and manual dispatch from a tag ref).
+  # No fork-secrets concern: this never runs for pull_request events.
+  publish:
+    if: startsWith(github.ref, 'refs/tags/v')
+    needs: verify-version
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # `latest` is applied automatically on semver tags (latest=auto).
+      # Caveat: it follows the most recently pushed tag, not the highest
+      # version — we only tag forward, never backport.
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: karust/openserp
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      # The pinned chromedp/headless-shell digest is a multi-arch OCI index
+      # (amd64 + arm64/v8), and the builder stage cross-compiles via
+      # $BUILDPLATFORM, so only the small final-stage apt layer runs under QEMU.
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/release.yml

@@ -0,0 +1,100 @@
+name: Release
+
+# Publishes binary archives to the GitHub release for a version tag.
+# Docker images are published separately by docker.yml on the same tag.
+
+on:
+  push:
+    tags: ["v*.*.*"]
+  # Manual re-run (e.g. after a workflow fix) — must be dispatched from a tag ref.
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  binaries:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Guard non-tag dispatch
+        if: github.event_name == 'workflow_dispatch' && !startsWith(github.ref, 'refs/tags/v')
+        run: |
+          echo "::error::Manual runs must be dispatched from a v*.*.* tag ref (use the 'Run workflow' ref selector)."
+          exit 1
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Full history + tags, needed to diff against the previous release tag
+          fetch-depth: 0
+
+      - name: Check tag matches cmd/root.go version
+        run: |
+          TAG_VERSION="${GITHUB_REF_NAME#v}"
+          CODE_VERSION=$(sed -nE 's/^[[:space:]]*version[[:space:]]*=[[:space:]]*"([0-9]+\.[0-9]+\.[0-9]+)".*/\1/p' cmd/root.go)
+          echo "tag=v${TAG_VERSION} code=${CODE_VERSION}"
+          if [ "$TAG_VERSION" != "$CODE_VERSION" ]; then
+            echo "::error::Git tag v${TAG_VERSION} does not match version \"${CODE_VERSION}\" in cmd/root.go. Bump cmd/root.go before tagging."
+            exit 1
+          fi
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.24"
+          cache: true
+
+      - name: Run unit tests
+        run: go test -count=1 ./...
+
+      - name: Build release binaries
+        run: |
+          VERSION="${GITHUB_REF_NAME#v}"
+          mkdir -p dist
+          for GOOS in darwin linux windows; do
+            for GOARCH in amd64 386 arm64; do
+              if [ "$GOOS" = "darwin" ] && [ "$GOARCH" = "386" ]; then
+                continue
+              fi
+              BIN="openserp"
+              if [ "$GOOS" = "windows" ]; then
+                BIN="openserp.exe"
+              fi
+              rm -f "$BIN"
+              echo "Building ${GOOS}/${GOARCH}..."
+              CGO_ENABLED=0 GOOS="$GOOS" GOARCH="$GOARCH" \
+                go build -trimpath -ldflags="-s -w" -o "$BIN" .
+              tar --create --gzip --file="dist/openserp-${GOOS}-${GOARCH}-${VERSION}.tgz" "$BIN"
+              rm -f "$BIN"
+            done
+          done
+          ls -lh dist/
+
+      # Most changes land directly on main without PRs, and GitHub's
+      # auto-generated notes only list merged PRs — so build the changelog
+      # from commit subjects since the previous tag. Auto-notes stay enabled
+      # below for the compare link and occasional contributor PR credits.
+      - name: Generate commit changelog
+        run: |
+          PREV_TAG=$(git describe --tags --abbrev=0 "${GITHUB_REF_NAME}^" 2>/dev/null || true)
+          {
+            echo "## Changes"
+            echo ""
+            if [ -n "$PREV_TAG" ]; then
+              echo "Commits since ${PREV_TAG}:"
+              echo ""
+              git log --no-merges --pretty='- %s' "${PREV_TAG}..${GITHUB_REF_NAME}"
+            else
+              git log --no-merges --pretty='- %s' "${GITHUB_REF_NAME}"
+            fi
+          } > release_notes.md
+          cat release_notes.md
+
+      # Creates the release if it doesn't exist, or attaches the archives to
+      # an existing one (e.g. a release drafted manually in the GitHub UI).
+      - name: Create GitHub release and upload binaries
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*.tgz
+          body_path: release_notes.md
+          generate_release_notes: true

.gitignore

@@ -38,3 +38,4 @@ sanitize_html_fixtures.py
 core/testdata/*
 .tmp-gocache
 .tmpcache/
+google/data/geotargets-2026-05-28.csv

.golangci.yml

@@ -0,0 +1,7 @@
+run:
+  tests: true
+  timeout: 5m
+
+issues:
+  max-issues-per-linter: 0
+  max-same-issues: 0

Makefile

@@ -0,0 +1,25 @@
+GO ?= go
+BINARY ?= openserp
+PKGS ?= ./...
+GOFILES := $(shell git ls-files '*.go')
+
+.PHONY: build test test-integration lint run fmt
+
+build:
+	$(GO) build -o $(BINARY) .
+
+test:
+	$(GO) test -race -count=1 $(PKGS)
+
+test-integration:
+	OPENSERP_INTEGRATION_TESTS=1 $(GO) test -race -count=1 -timeout=120s -tags=integration $(PKGS)
+
+lint:
+	$(GO) vet $(PKGS)
+	golangci-lint run --config .golangci.yml
+
+run:
+	$(GO) run . serve
+
+fmt:
+	gofmt -w $(GOFILES)

README.md

@@ -1,19 +1,24 @@
-# OpenSERP (Search Engine Results)
+![OpenSERP](./logo.svg)
 
-![OpenSERP](/logo.svg)
+# OpenSERP (Search Engine Results)
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/karust/openserp)](https://goreportcard.com/report/github.com/karust/openserp)
 [![Go Reference](https://pkg.go.dev/badge/github/karust/openserp?style=for-the-badge)](https://pkg.go.dev/github.com/karust/openserp)
-[![release](https://img.shields.io/github/release/karust/openserp)](https://github.com/karust/openserp/releases)
-[![Docker Pulls](https://img.shields.io/docker/v/karust/openserp)](https://hub.docker.com/repository/docker/karust/openserp)
+[![release](https://img.shields.io/github/v/release/karust/openserp)](https://github.com/karust/openserp/releases)
+[![Docker Pulls](https://img.shields.io/docker/v/karust/openserp)](https://hub.docker.com/r/karust/openserp)
 [![CI](https://github.com/karust/openserp/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/karust/openserp/actions/workflows/ci.yml)
 
-**OpenSERP** is an API and CLI for accessing search engine results from **Google, Yandex, Baidu, Bing, and DuckDuckGo**.
-A developer-friendly alternative to paid SERP API services!
+**OpenSERP** is a free, open-source API and CLI for accessing normalized search engine results from **Google, Yandex, Baidu, Bing, DuckDuckGo, and Ecosia**.
+
+Run it locally, self-host it, or use the optional hosted API when you do not want to manage infrastructure.
 
 **Official website:** [openserp.org](https://openserp.org)
 
-> 💡 OpenSerp is free and open-source. Only links listed in this repository and on the official website are associated with the project.
+**Feedback:** [GitHub Issues](https://github.com/karust/openserp/issues) or [feedback@openserp.org](mailto:feedback@openserp.org)
+
+**Latest updates, usage examples**: [Telegram](https://t.me/+RJEKspw3mUlhZDMy)
+
+> 💡 OpenSERP is free and open-source. Only links listed in this repository and on the official website are associated with the project.
 
 ## Features
 
@@ -21,39 +26,73 @@ A developer-friendly alternative to paid SERP API services!
 - 🌐 **Megasearch** - cross-engine aggregation with deduplication
 - 🖼 **Images** - image search is also available
 - 🎯 **Advanced filters** - language, date range, file type, and site queries
-- 🌍 **Configurable** - proxy, cache, and resilient mode support
+- ✨ **SERP features** - AI summaries, answer boxes, people-also-ask, and related searches in a response
+- 📄 **URL extraction** - turn target pages into clean markdown/text for grounding and automation
+- 🌍 **Configurable** - proxy, cache, and resilient mode
 - 🐳 **Docker-ready** - local and container deployment
-- 📝 **Data Formats** - JSON, Markdown, Text, NdJSON response format types are supported
+- 📝 **Data Formats** - JSON, Markdown, Text, NdJSON response formats
 
-## Quick Start⚡️
+## ⚡ Quick Start
 
 ### Docker
 
-```bash
+```sh
 # Run the API server via prebuilt image
-docker run -p 127.0.0.1:7000:7000 -it karust/openserp serve -a 0.0.0.0 -p 7000
+docker run --rm -p 127.0.0.1:7000:7000 karust/openserp:latest serve -a 0.0.0.0 -p 7000
 
-# Or use docker-compose
-docker compose up --build
+# Or use docker-compose (pulls the prebuilt image)
+docker compose up
 ```
 
 ### From Source
 
-```bash
+```sh
 git clone https://github.com/karust/openserp.git
 cd openserp
 go build -o openserp .
 ./openserp serve
 ```
 
+## Deployment Options
+
+- **Self-hosted (this repo)** - free, MIT-licensed, with full control over runtime, proxies, cache, and scaling.
+- **[Hosted API](https://openserp.org/cloud)** - optional managed version from the project maintainers, with the same API shape.
+
+The hosted API helps fund continued development of the open-source project. Same endpoints, same response schema, and client code can migrate either direction.
+
 ## API Docs
 
+Once the server is running, the interactive docs are available locally:
+
 - Swagger UI: `http://127.0.0.1:7000/docs`
 - OpenAPI YAML: `http://127.0.0.1:7000/openapi.yaml`
 
+To browse the spec without running the server, see [docs/openapi.yaml](./docs/openapi.yaml). For a higher-level overview of how OpenSERP works internally, see the [architecture docs](https://openserp.org/docs/architecture/).
+
+## SDKs & Examples
+
+Official client packages. Each works against your self-hosted server (set `baseUrl`) or the [hosted API](https://openserp.org/cloud) (set `apiKey`):
+
+| Type                        | Package                                                                                      | Install                         |
+| --------------------------- | -------------------------------------------------------------------------------------------- | ------------------------------- |
+| JavaScript / TypeScript SDK | [`@openserp/sdk`](https://www.npmjs.com/package/@openserp/sdk)                               | `npm install @openserp/sdk`     |
+| Python SDK                  | [`openserp`](https://pypi.org/project/openserp/)                                             | `pip install openserp`          |
+| MCP server (AI agents)      | [`@openserp/mcp`](https://www.npmjs.com/package/@openserp/mcp)                               | `npx @openserp/mcp`             |
+| n8n community node          | [`@openserp/n8n-nodes-openserp`](https://www.npmjs.com/package/@openserp/n8n-nodes-openserp) | Install via n8n community nodes |
+
+See [**examples**](./examples) for small JavaScript and Python use cases covering search, AI grounding, SEO, content extraction, and image search.
+
+```js
+import { OpenSERP } from "@openserp/sdk";
+
+// Use your self-hosted server
+const client = new OpenSERP({ baseUrl: "http://localhost:7000" });
+const { results } = await client.search({ engine: "google", text: "openserp", limit: 5 });
+```
+
 ## Search Endpoints
 
-Available engine names: `google`, `yandex`, `baidu`, `bing`, `duckduckgo`.
+Available engine names: `google`, `yandex`, `baidu`, `bing`, `duckduckgo`, `ecosia`.
 
 Dedicated engine endpoints:
 
@@ -73,11 +112,17 @@ Megasearch:
 # Search all configured engines
 curl "http://127.0.0.1:7000/mega/search?text=golang&limit=10"
 
-# Search selected engines
-curl "http://127.0.0.1:7000/mega/search?text=golang&engines=duckduckgo,bing&limit=15"
+# Fast mode: only one fastest engine is queried
+curl "http://127.0.0.1:7000/mega/search?text=golang&mode=fast&engines=google,bing,yandex"
+
+# Any mode: sequential fallback in provided order (default order if none provided)
+curl "http://127.0.0.1:7000/mega/search?text=golang&mode=any&engines=google,yandex,bing"
+
+# Balanced mode (default): parallel all engines with aggregation controls
+curl "http://127.0.0.1:7000/mega/search?text=golang&mode=balanced&dedupe=true&merge=true"
 
 # Advanced filtering
-curl "http://127.0.0.1:7000/mega/search?text=golang&engines=google,bing&limit=20&date=20250101..20251231&lang=EN"
+curl "http://127.0.0.1:7000/mega/search?text=golang&engines=google,bing&limit=20&date=20250101..20251231&lang=EN&region=US"
 
 # Image megasearch
 curl "http://127.0.0.1:7000/mega/image?text=golang+logo&limit=20"
@@ -89,30 +134,50 @@ List engines:
 curl "http://127.0.0.1:7000/mega/engines"
 ```
 
+URL extraction:
+
+```bash
+# Extract one URL as JSON
+curl "http://127.0.0.1:7000/extract?url=https://example.com&mode=auto"
+
+# Return clean page markdown
+curl "http://127.0.0.1:7000/extract?url=https://example.com&format=markdown"
+
+# Embed extracted content under the top search results
+curl "http://127.0.0.1:7000/google/search?text=llm+observability&extract=true&extract_top=2&format=markdown"
+```
+
 ## 🔍 Query Parameters
 
 Common parameters:
 
-| Parameter | Description                | Example                              |
-| --------- | -------------------------- | ------------------------------------ |
-| `text`    | Search query               | `golang programming`                 |
-| `lang`    | Language code              | `EN`, `DE`, `RU`, `ES`               |
-| `date`    | Date range                 | `20250101..20251231`                 |
-| `file`    | File extension             | `pdf`, `doc`, `xls`                  |
-| `site`    | Site-specific search       | `github.com`                         |
-| `limit`   | Number of results, max 100 | `10`, `25`, `50`                     |
-| `start`   | Pagination offset          | `0`, `10`, `20`                      |
-| `format`  | Output format              | `json`, `markdown`, `text`, `ndjson` |
+| Parameter      | Description                                                                                                                          | Example                              |
+| -------------- | ------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------ |
+| `text`         | Search query                                                                                                                         | `golang programming`                 |
+| `lang`         | Language code                                                                                                                        | `EN`, `DE`, `RU`, `ES`               |
+| `region`       | Market/location hint. Countries/locales work across engines; Google also accepts city names via `uule`; Yandex accepts numeric `lr`. | `DE`, `en-GB`, `Berlin`, `213`       |
+| `date`         | Date range                                                                                                                           | `20250101..20251231`                 |
+| `file`         | File extension                                                                                                                       | `pdf`, `doc`, `xls`                  |
+| `site`         | Site-specific search                                                                                                                 | `github.com`                         |
+| `limit`        | Number of organic results, max 100. When omitted or `<=10`, only the first SERP page is parsed.                                      | `25`, `50`                           |
+| `start`        | Pagination offset                                                                                                                    | `0`, `10`, `20`                      |
+| `format`       | Output format                                                                                                                        | `json`, `markdown`, `text`, `ndjson` |
+| `extract`      | Fetch and embed target-page content for top web results                                                                              | `true`                               |
+| `extract_top`  | Number of top web results to extract, clamped to 1-5                                                                                 | `3`                                  |
+| `extract_mode` | Extraction strategy: raw HTTP first, raw only, or browser-rendered                                                                   | `auto`, `fast`, `rendered`           |
 
 Engine-specific parameters:
 
-| Parameter | Supported engines | Notes                                                                  |
-| --------- | ----------------- | ---------------------------------------------------------------------- |
-| `filter`  | `google`          | Duplicate filter: `true` hides similar results, `false` includes them. |
-| `answers` | `google`          | Include Google answer boxes in output.                                 |
+| Parameter  | Supported engines | Notes                                                                  |
+| ---------- | ----------------- | ---------------------------------------------------------------------- |
+| `filter`   | `google`          | Duplicate filter: `true` hides similar results, `false` includes them. |
+| `features` | browser `Search`  | Populate `serp_features[]` from the live page. Defaults to `true`.     |
 
 ## Search Response Example
 
+<details>
+<summary>Search response example</summary>
+
 ```json
 {
   "query": {
@@ -124,7 +189,7 @@ Engine-specific parameters:
     "requested_at": "2026-04-25T22:27:52Z",
     "took_ms": 6410,
     "engines_failed": [],
-    "version": "1.0"
+    "version": "2.1"
   },
   "results": [
     {
@@ -137,22 +202,14 @@ Engine-specific parameters:
       "snippet": "Official Go documentation, tutorials, references, and release notes.",
       "domain": "go.dev",
       "favicon": "https://go.dev/favicon.ico",
-      "is_ad": false,
       "position": {
-        "absolute": 1,
-        "page": 1,
-        "on_page": 1
+        "absolute": 1
       },
       "engine": "google",
       "domain_info": {
         "tld": "dev",
         "sld": "go",
-        "is_gov": false,
-        "is_edu": false,
-        "is_social": false
-      },
-      "classification": {
-        "content_type": "webpage"
+        "category": ""
       }
     }
   ],
@@ -164,9 +221,14 @@ Engine-specific parameters:
 }
 ```
 
+</details>
+
 ## Mega Response Notes
 
-`/mega/search` returns the same envelope plus `clusters`. Results are deduplicated by normalized URL; clusters keep the per-engine occurrences:
+`/mega/search` returns the same envelope plus `clusters`. Results are deduplicated by normalized URL; clusters keep the per-engine occurrences.
+
+<details>
+<summary>Cluster example</summary>
 
 ```json
 {
@@ -184,8 +246,13 @@ Engine-specific parameters:
 }
 ```
 
+</details>
+
 ## Image Response Example
 
+<details>
+<summary>Image result example</summary>
+
 ```json
 {
   "id": "i_a1b2c3d4e5f6a1b2",
@@ -206,6 +273,8 @@ Engine-specific parameters:
 }
 ```
 
+</details>
+
 ## Error Responses
 
 `400 Bad Request`:
@@ -242,6 +311,8 @@ Simple global proxy:
 
 Advanced proxy configuration is available in [config.yaml](./config.yaml). You can enable tagged proxy pools and per-request override via `X-Use-Proxy: <tag>` or `X-Use-Proxy: direct`.
 
+A [managed API](https://openserp.org/cloud) is also available for teams that do not want to operate infrastructure.
+
 ## Health & Stats
 
 ```bash
@@ -257,8 +328,14 @@ curl "http://127.0.0.1:7000/stats/cb"
 
 This project is licensed under the MIT License. See [LICENSE](LICENSE).
 
-## 🤝 Contributing
+## Contributing
 
 Contributions are welcome. See [docs/CONTRIBUTING.md](./docs/CONTRIBUTING.md).
 
-###### _"OpenSERP" is the name of this open-source project. Use of the name in a way that implies affiliation, endorsement, or official status is not permitted._
+## Feedback & Updates
+
+- [GitHub Issues](https://github.com/karust/openserp/issues) - bugs, feature ideas, and reproducible issues.
+- [Telegram channel](https://t.me/openserp_cloud) - OpenSERP news, release notes, and project updates. Direct messages are open for quick feedback and hosted API questions.
+- [feedback@openserp.org](mailto:feedback@openserp.org) - private notes, longer feedback, or anything that does not fit GitHub Issues.
+
+###### _"OpenSERP" is the name of this open-source project. The official website is [openserp.org](https://openserp.org). Resources not linked on this page are not affiliated with the project._

baidu/all_test.go

@@ -35,7 +35,7 @@ func TestBuildURL(t *testing.T) {
 				if got := params.Get("gpc"); got != "stf=1388534400,1672531200|stftype=2" {
 					t.Fatalf("unexpected gpc value: %q", got)
 				}
-				if got := params.Get("rn"); got != "10" {
+				if got := params.Get("rn"); got != "" {
 					t.Fatalf("unexpected rn value: %q", got)
 				}
 				if got := params.Get("pn"); got != "" {
@@ -140,6 +140,23 @@ func TestBuildImageURL(t *testing.T) {
 				}
 			},
 		},
+		{
+			name: "small limit omits result count param",
+			query: core.Query{
+				Text:  "golang",
+				Limit: 10,
+			},
+			pageNum: 2,
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("rn"); got != "" {
+					t.Fatalf("rn should be omitted when Limit<=10, got %q", got)
+				}
+				if got := params.Get("pn"); got != "60" {
+					t.Fatalf("unexpected pn value: %q", got)
+				}
+			},
+		},
 		{
 			name:    "empty fields return error",
 			query:   core.Query{},

baidu/captcha_selector_test.go

@@ -17,17 +17,17 @@ func TestBaiduPageTypeSelectors(t *testing.T) {
 		selector string
 		wantHit  bool
 	}{
-		{"search_captcha.html", sel.Captcha, true},
-		{"search_captcha.html", sel.Timeout, true},
-		{"search_captcha.html", sel.Results, false},
+		{"search_captcha.html", Selectors.Captcha, true},
+		{"search_captcha.html", Selectors.Timeout, true},
+		{"search_captcha.html", Selectors.Results, false},
 
-		{"search_results.html", sel.Results, true},
-		{"search_results.html", sel.Captcha, false},
-		{"search_results.html", sel.Timeout, false},
+		{"search_results.html", Selectors.Results, true},
+		{"search_results.html", Selectors.Captcha, false},
+		{"search_results.html", Selectors.Timeout, false},
 
-		{"search_no_results.html", sel.Captcha, false},
-		{"search_no_results.html", sel.Timeout, false},
-		{"search_no_results.html", sel.Results, false},
+		{"search_no_results.html", Selectors.Captcha, false},
+		{"search_no_results.html", Selectors.Timeout, false},
+		{"search_no_results.html", Selectors.Results, false},
 	}
 
 	for _, tt := range tests {

baidu/features.go

@@ -0,0 +1,40 @@
+package baidu
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"github.com/karust/openserp/core"
+)
+
+func extractBaiduFeatures(doc *goquery.Document) []core.SerpFeature {
+	features := core.ExtractSerpFeaturesBySelectors(doc, []core.SerpFeatureSelector{
+		{
+			Type:          core.ResultTypeAISummary,
+			Title:         "AI summary",
+			Container:     []string{"div[tpl='app/chat-input']", "div[tpl='ai_chat']", "div[tpl*='ai']", ".op-ai-answer", ".cosc-result", ".ai-answer"},
+			TitleSelector: []string{".c-title", "h2", "h3"},
+			TextSelector:  []string{".cosc-answer", ".op_ai_answer_content", ".ai-answer-content", ".c-abstract"},
+			LinkSelector:  []string{"a[href^='http']"},
+			Position:      1,
+			Confidence:    0.7,
+		},
+		{
+			Type:          core.ResultTypeAnswerBox,
+			Title:         "Answer",
+			Container:     []string{".op_exactqa_s_answer", ".op_dict_content", ".op_weather4_twoicon", "div[tpl='calculator']", "div[tpl='app/calc']"},
+			TitleSelector: []string{".c-title", "h2", "h3"},
+			TextSelector:  []string{".op_exactqa_s_answer", ".op_dict_content", ".op_weather4_twoicon", ".op_new_val_screen_result", ".c-abstract"},
+			LinkSelector:  []string{"a[href^='http']"},
+			Position:      1,
+			Confidence:    0.75,
+		},
+		{
+			Type:         core.ResultTypeRelatedSearches,
+			Title:        "Related searches",
+			Container:    []string{"div[tpl='app/rs']", "#rs_new", "#rs", ".opr-recommends-merge-content", ".c-recommend"},
+			ItemSelector: []string{"a"},
+			LinkSelector: []string{"a[href^='http']", "a"},
+			Confidence:   0.75,
+		},
+	})
+	return core.DeduplicateSerpFeatures(features)
+}

baidu/parse_html.go

@@ -0,0 +1,197 @@
+package baidu
+
+import (
+	"io"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/karust/openserp/core"
+)
+
+// ParseHTML parses a Baidu SERP HTML document and returns search results.
+// No network I/O.
+func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(r)
+	if err != nil {
+		return nil, err
+	}
+	return parseBaiduDocument(doc), nil
+}
+
+func parseBaiduDocument(doc *goquery.Document) []core.SearchResult {
+	features := extractBaiduFeatures(doc)
+	// Match all result-card variants in one pass so DOM order is preserved and
+	// every card type (organic www_index, baike/encyclopedia, op cards) is
+	// collected. Selecting one variant at a time and returning on the first hit
+	// dropped baike and other result-op cards that interleave with organic rows.
+	results := parseBaiduSelection(doc.Find(baiduResultSelector()))
+	return core.AttachFeaturesToFirstResult(results, features)
+}
+
+func baiduResultSelectors() []string {
+	selectors := make([]string, 0, len(Selectors.ResultsAlt)+1)
+	selectors = append(selectors, Selectors.Results)
+	selectors = append(selectors, Selectors.ResultsAlt...)
+	return selectors
+}
+
+func baiduResultSelector() string {
+	return strings.Join(baiduResultSelectors(), ", ")
+}
+
+func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult {
+	var results []core.SearchResult
+	rank := 1
+	adRank := 1
+	absoluteRank := 1
+
+	sel.Each(func(_ int, item *goquery.Selection) {
+		isAd := baiduSelectionHasAdMarker(item)
+
+		// h3-first: organic results always carry a heading; this filters out
+		// non-result blocks that may share the wrapper class.
+		titleTag := item.Find("h3").First()
+		var (
+			title   string
+			linkTag *goquery.Selection
+		)
+		if titleTag.Length() > 0 {
+			title = strings.TrimSpace(titleTag.Text())
+			if child := titleTag.Find("a[href]").First(); child.Length() > 0 {
+				linkTag = child
+			} else if closest := titleTag.Closest("a[href]"); closest.Length() > 0 {
+				linkTag = closest
+			}
+		}
+		if linkTag == nil || linkTag.Length() == 0 {
+			first := item.Find(Selectors.Link).First()
+			if first.Length() == 0 {
+				return
+			}
+			linkTag = first
+		}
+		if title == "" {
+			title = strings.TrimSpace(linkTag.Text())
+		}
+		if title == "" {
+			return
+		}
+
+		href, exists := linkTag.Attr("href")
+		if !exists {
+			return
+		}
+		href = strings.TrimSpace(href)
+		if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") {
+			return
+		}
+		// Organic Baidu results link out through an absolute redirect
+		// (http://www.baidu.com/link?url=...). Op cards like "People also search"
+		// (tpl=recommend_list) instead carry relative on-site search links
+		// (/s?wd=...); treat those as related-search modules, not organic rows.
+		if strings.HasPrefix(href, "/") {
+			return
+		}
+		// Baidu result cards carry the canonical destination in the mu= attribute
+		// (e.g. baike.baidu.com, britannica.com), while the visible link is an
+		// opaque www.baidu.com/link?url= redirect. Prefer mu= so callers get the
+		// real URL, which also enables domain-based classification (encyclopedia,
+		// news, etc.) downstream.
+		if mu := canonicalBaiduURL(item); mu != "" {
+			href = mu
+		}
+
+		desc := ""
+		if descTag := item.Find(Selectors.Desc).First(); descTag.Length() > 0 {
+			desc = strings.TrimSpace(descTag.Text())
+		}
+		if desc == "" {
+			for _, alt := range Selectors.DescAlt {
+				if descTag := item.Find(alt).First(); descTag.Length() > 0 {
+					if t := strings.TrimSpace(descTag.Text()); t != "" {
+						desc = t
+						break
+					}
+				}
+			}
+		}
+		if desc == "" {
+			full := strings.TrimSpace(item.Text())
+			desc = strings.TrimSpace(strings.Replace(full, title, "", 1))
+		}
+
+		resultRank := rank
+		if isAd {
+			resultRank = adRank
+			adRank++
+		} else {
+			rank++
+		}
+
+		results = append(results, core.SearchResult{
+			Rank:         resultRank,
+			AbsoluteRank: absoluteRank,
+			URL:          href,
+			Title:        title,
+			Description:  desc,
+			Ad:           isAd,
+		})
+		absoluteRank++
+	})
+
+	// Re-rank sequentially after dedup so callers get a clean 1..N sequence
+	// (dedup may drop intermediate ranks when the same URL appears in
+	// multiple Baidu result-card variants on the same SERP).
+	deduped := core.DeduplicateResults(results)
+	organicIdx := 0
+	for i := range deduped {
+		if deduped[i].Ad {
+			continue
+		}
+		organicIdx++
+		deduped[i].Rank = organicIdx
+	}
+	return deduped
+}
+
+// canonicalBaiduURL returns the card's mu= destination when it is an absolute
+// http(s) URL. The attribute lives on the result-card container; when the title
+// link is nested, walk up to the nearest ancestor that carries it.
+func canonicalBaiduURL(item *goquery.Selection) string {
+	mu := strings.TrimSpace(firstAttrValue(item, "mu"))
+	if mu == "" {
+		if host := item.Closest("[mu]"); host.Length() > 0 {
+			mu = strings.TrimSpace(firstAttrValue(host, "mu"))
+		}
+	}
+	if strings.HasPrefix(mu, "http://") || strings.HasPrefix(mu, "https://") {
+		return mu
+	}
+	return ""
+}
+
+func firstAttrValue(item *goquery.Selection, name string) string {
+	if value, ok := item.Attr(name); ok {
+		return value
+	}
+	return ""
+}
+
+func baiduSelectionHasAdMarker(item *goquery.Selection) bool {
+	for _, selector := range Selectors.AdMarkers {
+		if item.Is(selector) || item.Find(selector).Length() > 0 {
+			return true
+		}
+	}
+
+	isAd := false
+	item.Find("span, i, em").EachWithBreak(func(_ int, marker *goquery.Selection) bool {
+		text := strings.TrimSpace(marker.Text())
+		if text == "广告" || text == "推广" || text == "商业推广" {
+			isAd = true
+			return false
+		}
+		return true
+	})
+	return isAd
+}

baidu/parse_html_test.go

@@ -0,0 +1,192 @@
+package baidu
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestParseBaiduHTML(t *testing.T) {
+	t.Parallel()
+
+	data, err := os.ReadFile("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("read fixture: %v", err)
+	}
+
+	results, err := ParseHTML(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+
+	rank := 0
+	for i, r := range results {
+		if r.Ad {
+			continue
+		}
+		rank++
+		if r.Rank != rank {
+			t.Fatalf("rank sequence broken at index %d: got %d, want %d", i, r.Rank, rank)
+		}
+		if r.URL == "" {
+			t.Fatalf("result %d: empty URL", i)
+		}
+		if r.Title == "" {
+			t.Fatalf("result %d: empty Title", i)
+		}
+	}
+	if rank == 0 {
+		t.Fatal("expected at least one organic result")
+	}
+}
+
+// TestParseBaiduHTMLParsesBaike locks in two fixes: baike/encyclopedia cards
+// (div.result-op.c-container, tpl=bk_polysemy) are parsed alongside organic
+// www_index cards instead of being dropped by first-selector-wins, and op
+// "People also search" cards (relative /s? links) are excluded as non-organic.
+func TestParseBaiduHTMLParsesBaike(t *testing.T) {
+	t.Parallel()
+
+	data, err := os.ReadFile("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("read fixture: %v", err)
+	}
+	results, err := ParseHTML(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+
+	foundBaike := false
+	for i, r := range results {
+		if strings.Contains(strings.ToLower(r.Title), "baike") ||
+			strings.Contains(strings.ToLower(r.Title), "encyclopedia") {
+			foundBaike = true
+			if strings.TrimSpace(r.Description) == "" {
+				t.Fatalf("baike result %d has empty description", i)
+			}
+		}
+		// Op cards link to relative on-site search; organic results must not.
+		if strings.HasPrefix(r.URL, "/") {
+			t.Fatalf("result %d has a relative (non-organic) URL: %s", i, r.URL)
+		}
+	}
+	if !foundBaike {
+		t.Fatal("expected a baidu baike/encyclopedia result to be parsed")
+	}
+}
+
+func TestParseBaiduHTMLEmpty(t *testing.T) {
+	t.Parallel()
+
+	results, err := ParseHTML(bytes.NewReader([]byte("")))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 0 {
+		t.Fatalf("expected zero results for empty HTML, got %d", len(results))
+	}
+}
+
+func TestParseBaiduHTMLFallbackSelectors(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<div id="content_left">
+  <div class="result-op c-container">
+    <h3><a href="https://example.com/result">Fallback Title</a></h3>
+    <div class="summary-gap_3Jb4I">Fallback description</div>
+  </div>
+</div>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 1 {
+		t.Fatalf("expected 1 result, got %d", len(results))
+	}
+	if results[0].URL != "https://example.com/result" {
+		t.Fatalf("unexpected URL: %s", results[0].URL)
+	}
+	if results[0].Title != "Fallback Title" {
+		t.Fatalf("unexpected title: %s", results[0].Title)
+	}
+	if results[0].Description != "Fallback description" {
+		t.Fatalf("unexpected description: %s", results[0].Description)
+	}
+}
+
+func TestParseBaiduHTMLFallsBackWhenEarlierSelectorHasNoResult(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<div id="content_left">
+  <div class="result c-container"></div>
+  <div class="result-op c-container">
+    <h3><a href="https://example.com/parseable">Parseable Title</a></h3>
+    <div class="summary-gap_3Jb4I">Parseable description</div>
+  </div>
+</div>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 1 {
+		t.Fatalf("expected 1 result, got %d", len(results))
+	}
+	if results[0].URL != "https://example.com/parseable" {
+		t.Fatalf("unexpected URL: %s", results[0].URL)
+	}
+}
+
+func TestParseBaiduHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<div id="content_left">
+  <div class="result c-container" data-tuiguang="1">
+    <h3><a href="https://ads.example.com">Sponsored Result</a></h3>
+    <div class="c-abstract">Paid snippet</div>
+  </div>
+  <div class="result c-container">
+    <h3><a href="https://organic.example.com/one">Organic One</a></h3>
+    <div class="c-abstract">Organic snippet one</div>
+  </div>
+  <div class="result c-container">
+    <h3><a href="https://organic.example.com/two">Organic Two</a></h3>
+    <div class="c-abstract">Organic snippet two</div>
+  </div>
+</div>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 3 {
+		t.Fatalf("expected 3 results, got %d", len(results))
+	}
+
+	organicRank := 0
+	adRank := 0
+	for _, r := range results {
+		if r.Ad {
+			adRank++
+			if r.Rank != adRank {
+				t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
+			}
+			continue
+		}
+		organicRank++
+		if r.Rank != organicRank {
+			t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
+		}
+	}
+	if organicRank != 2 {
+		t.Fatalf("organic count = %d, want 2", organicRank)
+	}
+	if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
+		t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
+	}
+}

baidu/search.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/go-rod/rod"
 	"github.com/karust/openserp/core"
-	"golang.org/x/time/rate"
 )
 
 type imageDataJson struct {
@@ -33,16 +32,6 @@ type imageDataJson struct {
 	}
 }
 
-var sel = struct {
-	Captcha string
-	Timeout string
-	Results string
-}{
-	Captcha: "div.passMod_dialog-body",
-	Timeout: "button.timeout-button",
-	Results: "div.c-container.new-pmd",
-}
-
 // Baidu implements core.SearchEngine for Baidu SERP pages.
 type Baidu struct {
 	core.Browser
@@ -64,41 +53,40 @@ func (baid *Baidu) Name() string {
 	return "baidu"
 }
 
-// GetRateLimiter returns a limiter configured from SearchEngineOptions.
-func (baid *Baidu) GetRateLimiter() *rate.Limiter {
-	ratelimit := rate.Every(baid.GetRatelimit())
-	return rate.NewLimiter(ratelimit, baid.RateBurst)
-}
-
 func (baid *Baidu) isCaptcha(page *rod.Page) bool {
-	has, _, _ := page.Has(sel.Captcha)
+	has, _, _ := page.Has(Selectors.Captcha)
 	return has
 }
 
 func (baid *Baidu) isTimeout(page *rod.Page) bool {
-	has, _, _ := page.Has(sel.Timeout)
+	has, _, _ := page.Has(Selectors.Timeout)
 	return has
 }
 
+func (baid *Baidu) classifyBlockPage(page *rod.Page, url string) error {
+	if baid.isCaptcha(page) {
+		baid.logger.Error("Captcha detected: %s", url)
+		return core.ErrCaptcha
+	}
+	if baid.isTimeout(page) {
+		baid.logger.Error("Timeout occurred: %s", url)
+		return core.ErrSearchTimeout
+	}
+	return nil
+}
+
 // Search executes a Baidu web search and returns normalized search results.
 // It may return core.ErrCaptcha or core.ErrSearchTimeout.
 func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
-	ctx = core.WithEngine(core.EnsureContext(ctx), baid.Name())
-	ctx = core.WithProfileRegion(ctx, query.LangCode)
-	ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
+	ctx = core.PrepareEngineContext(ctx, query, baid.Name(), true)
 	scoped := *baid
 	scoped.logger = baid.logger.WithRequest(ctx)
+	if scoped.Browser.WaitLoadTime == 0 || scoped.Browser.WaitLoadTime > 250*time.Millisecond {
+		scoped.Browser.WaitLoadTime = 250 * time.Millisecond
+	}
 	baid = &scoped
 
 	baid.logger.Debug("Starting search, query: %+v", query)
-	defer func() {
-		if recovered := recover(); recovered != nil {
-			err = core.RecoverEnginePanicWithContext(ctx, baid.Name(), recovered, baid.logger)
-			results = nil
-		}
-	}()
-
-	searchResults := []core.SearchResult{}
 
 	// Build URL from query struct to open in browser
 	url, err := BuildURL(query)
@@ -110,86 +98,80 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
 	if err != nil {
 		return nil, err
 	}
-	closePage := func() {
-		if baid.Browser.LeavePageOpen {
-			return
-		}
-		if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil {
-			baid.logger.Debug("Page close error: %v", closeErr)
-		}
-	}
+	defer core.DeferClosePage(ctx, page, &baid.Browser)()
 
-	searchRes, err := page.Timeout(baid.Timeout).Search(sel.Results)
+	searchResults, err := baid.waitForParsedSearchResults(ctx, page, url)
 	if err != nil {
-		closePage()
-		baid.logger.Error("Cannot parse search results: %s", err)
-		return nil, core.ErrParser
-	}
-
-	// Check why no results, maybe captcha?
-	if searchRes == nil {
-		closePage()
-
-		if baid.isCaptcha(page) {
-			baid.logger.Error("Captcha detected: %s", url)
-			return nil, core.ErrCaptcha
-		} else if baid.isTimeout(page) {
-			baid.logger.Error("Timeout occurred: %s", url)
-			return nil, core.ErrCaptcha
-		}
-		return nil, nil
-	}
-
-	resultElements, err := searchRes.All()
-	if err != nil {
-		closePage()
 		return nil, err
 	}
 
-	for i, r := range resultElements {
-		// Get URL
-		link, err := r.Element("a")
-		if err != nil {
-			if core.IsRodObjectNotFound(err) {
-				break
+	for i := range searchResults {
+		if searchResults[i].AbsoluteRank > 0 {
+			searchResults[i].AbsoluteRank += query.Start
+		}
+		if searchResults[i].Ad {
+			continue
+		}
+		searchResults[i].Rank = query.Start + searchResults[i].Rank
+	}
+	return searchResults, nil
+}
+
+func (baid *Baidu) waitForParsedSearchResults(ctx context.Context, page *rod.Page, url string) ([]core.SearchResult, error) {
+	timeout := baid.GetSelectorTimeout()
+	if timeout <= 0 {
+		timeout = 5 * time.Second
+	}
+	deadline := time.Now().Add(timeout)
+	var sawResultContainer bool
+	var lastErr error
+
+	for {
+		html, err := page.HTML()
+		if err == nil {
+			results, parseErr := ParseHTML(strings.NewReader(html))
+			if parseErr == nil && len(results) > 0 {
+				return results, nil
 			}
-			continue
-		}
-		linkText, err := link.Property("href")
-		if err != nil {
-			baid.logger.Debug("Missing href tag")
-			continue
+			lastErr = parseErr
+		} else {
+			lastErr = err
 		}
 
-		// Get title
-		title, err := link.Text()
-		if err != nil {
-			baid.logger.Debug("Failed to extract title")
-			title = "No title"
+		if core.HasAnySelector(page, baiduResultSelectors()) {
+			sawResultContainer = true
 		}
-
-		// Get description
-		desc, err := r.Text()
-		if err != nil {
-			desc = ""
+		if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
+			return nil, blockErr
+		}
+		if !time.Now().Before(deadline) {
+			break
+		}
+		if err := core.SleepContext(ctx, 120*time.Millisecond); err != nil {
+			return nil, err
 		}
-		desc = strings.ReplaceAll(desc, title, "")
-
-		gR := core.SearchResult{Rank: query.Start + i + 1, URL: linkText.String(), Title: title, Description: desc}
-		searchResults = append(searchResults, gR)
 	}
 
-	closePage()
-
-	return core.DeduplicateResults(searchResults), nil
+	if sawResultContainer {
+		if lastErr != nil {
+			baid.logger.Debug("Baidu result containers found but HTML parsing failed: %v", lastErr)
+		} else {
+			baid.logger.Debug("Baidu result containers found but no parseable organic results")
+		}
+		return nil, core.ErrParser
+	}
+	// The page never reached a recognizable state: no result containers, no
+	// captcha or timeout markers. Baidu hydrates result cards client-side and
+	// can exceed the selector deadline, so report a timeout rather than a
+	// successful empty SERP — callers must retry/skip, not trust 0 results.
+	baid.logger.Debug("No result containers or block markers within selector timeout")
+	return nil, core.ErrSearchTimeout
 }
 
 // SearchImage executes a Baidu image search and returns normalized image
 // results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
 func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
-	ctx = core.WithEngine(core.EnsureContext(ctx), baid.Name())
-	ctx = core.WithProfileRegion(ctx, query.LangCode)
-	ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
+	ctx = core.PrepareEngineContext(ctx, query, baid.Name(), true)
 	scoped := *baid
 	scoped.logger = baid.logger.WithRequest(ctx)
 	baid = &scoped
@@ -199,61 +181,46 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
 	searchResults := []core.SearchResult{}
 	searchPage := 0
 
-	for len(searchResults) < query.Limit {
+	// fetchPage loads one image-results page and appends parsed results.
+	// Returns (done, error): done=true ends the outer loop without error.
+	fetchPage := func() (bool, error) {
 		url, err := BuildImageURL(query, searchPage)
 		if err != nil {
-			return nil, err
+			return false, err
 		}
 
-		// Get anti-crawler cookies first, then reload page
+		// First load often seeds anti-crawler cookies; results tend to become
+		// available after one explicit reload.
 		page, err := baid.Navigate(ctx, url)
 		if err != nil {
-			return nil, err
+			return false, err
 		}
-		closePage := func() {
-			if baid.Browser.LeavePageOpen {
-				return
-			}
-			if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil {
-				baid.logger.Debug("Page close error: %v", closeErr)
-			}
-		}
-		if err := page.Reload(); err != nil {
-			closePage()
-			baid.logger.Error("Page reload failed: %s", err)
-			return nil, core.ErrSearchTimeout
-		}
-		if err := page.WaitLoad(); err != nil {
-			closePage()
-			baid.logger.Error("Page load wait failed: %s", err)
-			return nil, core.ErrSearchTimeout
+		defer core.DeferClosePage(ctx, page, &baid.Browser)()
+
+		jsonWaitTimeout := baid.GetSelectorTimeout()
+		if reloadErr := page.Reload(); reloadErr != nil {
+			return false, core.ErrSearchTimeout
 		}
 
-		result, err := page.Timeout(baid.Timeout).Search("body > pre")
+		preElements, _, err := core.WaitForElements(ctx, page, Selectors.ImageJSONRoot, jsonWaitTimeout)
 		if err != nil {
-			closePage()
+			if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
+				return false, blockErr
+			}
 			baid.logger.Error("Cannot parse search results: %s", err)
-			return nil, core.ErrParser
+			return false, core.ErrSearchTimeout
 		}
 
-		// Check why no results, maybe captcha?
-		if result == nil {
-			closePage()
-
-			if baid.isCaptcha(page) {
-				baid.logger.Error("Captcha detected: %s", url)
-				return nil, core.ErrCaptcha
-			} else if baid.isTimeout(page) {
-				baid.logger.Error("Timeout occurred: %s", url)
-				return nil, core.ErrCaptcha
+		if len(preElements) == 0 {
+			if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
+				return false, blockErr
 			}
-			return nil, nil
+			return true, nil
 		}
 
-		jsonText, err := result.First.Text()
+		jsonText, err := preElements[0].Text()
 		if err != nil {
-			closePage()
-			return nil, err
+			return false, err
 		}
 
 		var data imageDataJson
@@ -262,24 +229,23 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
 		jsonText = strings.ReplaceAll(jsonText, `\'`, "'")
 		matchNewlines, err := regexp.Compile(`[\r\n\t]`)
 		if err != nil {
-			closePage()
-			return nil, core.ErrParser
+			return false, core.ErrParser
 		}
 		escapeNewlines := func(s string) string {
 			return matchNewlines.ReplaceAllString(s, "\\n")
 		}
 		re, err := regexp.Compile(`"[^"\\]*(?:\\[\s\S][^"\\]*)*"`)
 		if err != nil {
-			closePage()
-			return nil, core.ErrParser
+			return false, core.ErrParser
 		}
 		fixedJson := re.ReplaceAllStringFunc(jsonText, escapeNewlines)
 
-		err = json.Unmarshal([]byte(fixedJson), &data)
-		if err != nil {
-			closePage()
+		if err := json.Unmarshal([]byte(fixedJson), &data); err != nil {
 			baid.logger.Error("Failed to unmarshal JSON: %v", err)
-			return nil, core.ErrParser
+			return false, core.ErrParser
+		}
+		if len(data.Data) == 0 {
+			return true, nil
 		}
 
 		for i, img := range data.Data {
@@ -300,21 +266,31 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
 					img.Type,
 					img.IsCopyright,
 				),
-				Ad: func() bool {
-					if img.AdType != "0" {
-						return true
-					} else {
-						return false
-					}
-				}(),
+				Ad: img.AdType != "0",
 			}
 			searchResults = append(searchResults, res)
+			if query.Limit > 0 && len(searchResults) >= query.Limit {
+				return true, nil
+			}
 		}
 
-		searchPage += 1
-
-		closePage()
+		return false, nil
 	}
 
-	return core.DeduplicateResults(searchResults), nil
+	for core.ShouldFetchResultPage(len(searchResults), query.Limit, searchPage) {
+		done, err := fetchPage()
+		if err != nil {
+			return nil, err
+		}
+		searchPage++
+		if done || !core.ShouldFetchResultPage(len(searchResults), query.Limit, searchPage) {
+			break
+		}
+	}
+
+	deduped := core.DeduplicateResults(searchResults)
+	if query.Limit > 0 && len(deduped) > query.Limit {
+		deduped = deduped[:query.Limit]
+	}
+	return deduped, nil
 }

baidu/search_integration_test.go

@@ -4,52 +4,14 @@
 package baidu
 
 import (
-	"context"
 	"testing"
 
 	"github.com/karust/openserp/core"
-	"github.com/karust/openserp/testutil"
 	"github.com/karust/openserp/testutil/ithelper"
 )
 
 func TestSearchBaidu(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	baid := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golang programming", Limit: 10}
-	results, err := baid.Search(context.Background(), query)
-	ithelper.HandleError(t, "baidu web search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first result title is empty")
-	}
-}
-
-func TestImageSearchBaidu(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	baid := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golden retriever puppy", Limit: 10}
-	results, err := baid.SearchImage(context.Background(), query)
-	ithelper.HandleError(t, "baidu image search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty image results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first image result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first image result title is empty")
-	}
+	ithelper.RunEngineTests(t, func(b *core.Browser) core.SearchEngine {
+		return New(*b, ithelper.EngineOptions())
+	})
 }

baidu/search_raw.go

@@ -1,117 +1,39 @@
 package baidu
 
 import (
+	"bytes"
 	"context"
+	"errors"
 	"fmt"
-	"net/http"
-	"strings"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/corpix/uarand"
 	"github.com/karust/openserp/core"
-	"github.com/sirupsen/logrus"
 )
 
-func baiduRequest(ctx context.Context, searchURL string, query core.Query) (*http.Response, error) {
-	baseClient, err := core.NewRawHTTPClient(query)
+func classifyBaiduRawHTML(body []byte) error {
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
 	if err != nil {
-		return nil, err
+		return err
 	}
-
-	req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
-	if err != nil {
-		return nil, err
+	if doc.Find(Selectors.Captcha).Length() > 0 || doc.Find(Selectors.Timeout).Length() > 0 {
+		return core.ErrCaptcha
 	}
-	req.Header.Set("User-Agent", uarand.GetRandom())
-	core.SetAcceptLanguageHeader(req, query.LangCode)
-
-	res, err := baseClient.Do(req)
-	if err != nil {
-		return nil, err
+	if doc.Find("div.content_none, div.nors").Length() > 0 {
+		return core.ErrEmptyResult
 	}
-	return res, nil
-}
-
-func baiduResultParser(response *http.Response) ([]core.SearchResult, error) {
-	doc, err := goquery.NewDocumentFromReader(response.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	results := []core.SearchResult{}
-	rank := 1
-
-	// Prefer organic result blocks from the main result column.
-	sel := doc.Find("#content_left .result.c-container")
-	if sel.Length() == 0 {
-		sel = doc.Find("div.c-container.new-pmd")
-	}
-
-	for i := range sel.Nodes {
-		item := sel.Eq(i)
-
-		// Find URL
-		titleTag := item.Find("h3").First()
-		if titleTag.Length() == 0 {
-			continue
-		}
-
-		linkTag := titleTag.Closest("a")
-		if linkTag.Length() == 0 {
-			linkTag = item.Find("a").First()
-		}
-		link, _ := linkTag.Attr("href")
-		link = strings.TrimSpace(link)
-
-		// Find title
-		title := strings.TrimSpace(titleTag.Text())
-
-		// Find description
-		descTag := item.Find(".c-abstract, .content-right_8Zs40, .summary-gap_3Jb4I").First()
-		desc := strings.TrimSpace(descTag.Text())
-		if desc == "" {
-			desc = strings.TrimSpace(item.Text())
-		}
-		desc = strings.ReplaceAll(desc, title, "")
-		desc = strings.TrimSpace(desc)
-
-		if link != "" && link != "#" && title != "" {
-			result := core.SearchResult{
-				Rank:        rank,
-				URL:         link,
-				Title:       title,
-				Description: desc,
-			}
-
-			results = append(results, result)
-			rank++
-		}
-	}
-
-	logrus.WithField("document_size", len(doc.Text())).Trace(
-		fmt.Sprintf("Baidu search document size: %d", len(doc.Text())),
-	)
-	return results, err
+	return nil
 }
 
 func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
-	ctx = core.EnsureContext(ctx)
-	ctx = core.WithEngine(ctx, "baidu")
-	ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
-	defer func() {
-		if recovered := recover(); recovered != nil {
-			err = core.RecoverEnginePanicWithContext(ctx, "baidu", recovered, nil)
-			results = nil
-		}
-	}()
+	ctx = core.PrepareEngineContext(ctx, query, "baidu", false)
 
-	googleURL, err := BuildURL(query)
+	searchURL, err := BuildURL(query)
 	if err != nil {
 		return nil, err
 	}
-	core.WithRequest(ctx).WithField("url", googleURL).Debug(fmt.Sprintf("Baidu URL built: %s", googleURL))
+	core.WithRequest(ctx).WithField("url", searchURL).Debug(fmt.Sprintf("Baidu URL built: %s", searchURL))
 
-	res, err := baiduRequest(ctx, googleURL, query)
+	res, err := core.RawSearchRequest(ctx, searchURL, query)
 	if err != nil {
 		return nil, err
 	}
@@ -120,13 +42,34 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
 		fmt.Sprintf("Baidu Raw response: code=%d", res.StatusCode),
 	)
 
-	parsedResults, err := baiduResultParser(res)
+	body, err := core.ReadRawSearchBody(res)
 	if err != nil {
 		return nil, err
 	}
+	htmlStatus := classifyBaiduRawHTML(body)
+	if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) {
+		return nil, htmlStatus
+	}
+
+	parsedResults, err := ParseHTML(bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	if len(parsedResults) == 0 {
+		if errors.Is(htmlStatus, core.ErrEmptyResult) {
+			return []core.SearchResult{}, nil
+		}
+		return nil, fmt.Errorf("%w: baidu raw search returned no parseable results", core.ErrParser)
+	}
 	if query.Start > 0 {
 		for i := range parsedResults {
-			parsedResults[i].Rank = query.Start + i + 1
+			if parsedResults[i].AbsoluteRank > 0 {
+				parsedResults[i].AbsoluteRank += query.Start
+			}
+			if parsedResults[i].Ad {
+				continue
+			}
+			parsedResults[i].Rank = query.Start + parsedResults[i].Rank
 		}
 	}
 	core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(

baidu/search_raw_test.go

@@ -1,76 +1,69 @@
 package baidu
 
 import (
+	"errors"
+	"io"
 	"testing"
 
+	"github.com/karust/openserp/core"
 	"github.com/karust/openserp/testutil"
 )
 
-func TestBaiduResultParserSnapshots(t *testing.T) {
+// TestBaiduParseHTMLFixtures covers the no-results and captcha fixtures.
+// The happy path is covered in TestParseBaiduHTML; this file ensures the
+// shared parser (used by both raw mode and the /baidu/parse endpoint) does
+// not over-extract on captcha or empty SERPs.
+func TestBaiduParseHTMLFixtures(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
-		name           string
-		fixture        string
-		minResultCount int
-		maxResultCount int
-		wantZero       bool
+		name     string
+		fixture  string
+		wantZero bool
 	}{
-		{
-			name:           "search results",
-			fixture:        "search_results.html",
-			minResultCount: 5,
-			maxResultCount: 30,
-		},
-		{
-			name:     "no results",
-			fixture:  "search_no_results.html",
-			wantZero: true,
-		},
-		{
-			name:     "captcha page",
-			fixture:  "search_captcha.html",
-			wantZero: true,
-		},
+		{name: "no results", fixture: "search_no_results.html", wantZero: true},
+		{name: "captcha page", fixture: "search_captcha.html", wantZero: true},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
-			results, err := baiduResultParser(testutil.ResponseFromFixture(t, tt.fixture))
+			results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body)
 			if err != nil {
-				t.Fatalf("baiduResultParser() error = %v", err)
+				t.Fatalf("ParseHTML() error = %v", err)
 			}
-
-			if tt.wantZero {
-				if len(results) != 0 {
-					t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results))
-				}
-				return
+			if tt.wantZero && len(results) != 0 {
+				t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results))
 			}
-
-			if len(results) < tt.minResultCount || len(results) > tt.maxResultCount {
-				t.Fatalf(
-					"unexpected result count for %s: got %d, want range [%d,%d]",
-					tt.fixture, len(results), tt.minResultCount, tt.maxResultCount,
-				)
-			}
-
-			testutil.AssertSequentialRanks(t, results)
-			testutil.AssertFirstResultFilled(t, results)
 		})
 	}
 }
 
-func TestBaiduResultParserEmptyHTML(t *testing.T) {
+func TestBaiduClassifyRawHTML(t *testing.T) {
 	t.Parallel()
 
-	results, err := baiduResultParser(testutil.ResponseFromString(""))
-	if err != nil {
-		t.Fatalf("baiduResultParser() error = %v", err)
+	tests := []struct {
+		name    string
+		fixture string
+		want    error
+	}{
+		{name: "no results", fixture: "search_no_results.html", want: core.ErrEmptyResult},
+		{name: "captcha page", fixture: "search_captcha.html", want: core.ErrCaptcha},
 	}
-	if len(results) != 0 {
-		t.Fatalf("expected zero results for empty HTML, got %d", len(results))
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			body, err := io.ReadAll(testutil.ResponseFromFixture(t, tt.fixture).Body)
+			if err != nil {
+				t.Fatalf("read fixture body: %v", err)
+			}
+			err = classifyBaiduRawHTML(body)
+			if !errors.Is(err, tt.want) {
+				t.Fatalf("expected %v for %s, got %v", tt.want, tt.fixture, err)
+			}
+		})
 	}
 }

baidu/selectors.go

@@ -0,0 +1,34 @@
+package baidu
+
+// Selectors is the single source of truth for Baidu SERP CSS selectors.
+var Selectors = struct {
+	Captcha       string
+	Timeout       string
+	Results       string
+	ResultsAlt    []string
+	AdMarkers     []string
+	ImageJSONRoot []string
+	Link          string
+	Desc          string
+	// DescAlt are additional description containers tried when Desc misses.
+	// Baidu varies abstract markup across feature blocks (info cards, news rows).
+	DescAlt []string
+}{
+	Captcha:       "div.passMod_dialog-wrapper",
+	Timeout:       "button.timeout-button",
+	Results:       "#content_left div.result.c-container",
+	ResultsAlt:    []string{"#content_left div.result-op.c-container", "div.c-container.new-pmd"},
+	AdMarkers:     []string{"[data-tuiguang]", "[data-click*='tuiguang']", ".ec-tuiguang", ".c-icon-bear-p"},
+	ImageJSONRoot: []string{"body > pre", "pre"},
+	Link:          "a",
+	Desc:          "div.c-abstract",
+	// DescAlt matches Baidu's hashed abstract containers by class *prefix*
+	// ([class*='summary-gap_']) rather than a frozen hash suffix
+	// (.summary-gap_3Jb4I): Baidu rotates the trailing hash per build (the same
+	// page already carries summary-gap_3Jb4I and summary-gap_68jXq), and the old
+	// content-right_8Zs40 suffix no longer appears at all. These two prefixes are
+	// specific enough to use as substrings. text_ is NOT: it is Baidu's generic
+	// text-styling class reused on dozens of nodes, so the baike abstract body is
+	// pinned to its exact .text_2NOr6 hash and tried last.
+	DescAlt: []string{"[class*='content-right_']", "[class*='summary-gap_']", "div.text_2NOr6"},
+}

baidu/serp_features_test.go

@@ -0,0 +1,94 @@
+package baidu
+
+import (
+	"bytes"
+	"os"
+	"testing"
+
+	"github.com/karust/openserp/core"
+)
+
+func TestParseHTMLFixtureExtractsRealFeatures(t *testing.T) {
+	t.Parallel()
+	f, err := os.Open("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("open fixture: %v", err)
+	}
+	defer f.Close()
+
+	results, err := ParseHTML(f)
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	assertFeatureType(t, results, core.ResultTypeRelatedSearches)
+}
+
+func TestParseHTMLExtractsSerpFeatures(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<div class="result-op c-container" tpl="ai_chat">
+  <div class="c-title">AI智能回答</div>
+  <div class="c-abstract">Baidu AI summary text.</div>
+  <a href="https://example.com/source">Source</a>
+</div>
+<div id="rs">
+  <table>
+    <tr>
+      <th><a href="https://example.com/related">baidu related search</a></th>
+    </tr>
+  </table>
+</div>
+<div id="content_left">
+  <div class="result c-container">
+    <h3><a href="https://example.com/result">Organic result</a></h3>
+    <div class="c-abstract">Snippet</div>
+  </div>
+</div>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	assertFeatureType(t, results, core.ResultTypeAISummary)
+	assertFeatureType(t, results, core.ResultTypeRelatedSearches)
+}
+
+func TestParseHTMLOrganicOnlyHasNoSerpFeatures(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<div id="content_left">
+  <div class="result c-container">
+    <h3><a href="https://example.com/result">Organic result</a></h3>
+    <div class="c-abstract">Snippet</div>
+  </div>
+</div>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	assertNoFeatures(t, results)
+}
+
+func assertFeatureType(t *testing.T, results []core.SearchResult, want core.ResultType) {
+	t.Helper()
+	for _, result := range results {
+		for _, feature := range result.Features {
+			if feature.Type == want {
+				return
+			}
+		}
+	}
+	t.Fatalf("expected feature type %q in %#v", want, results)
+}
+
+func assertNoFeatures(t *testing.T, results []core.SearchResult) {
+	t.Helper()
+	for _, result := range results {
+		if len(result.Features) > 0 {
+			t.Fatalf("expected no features, got %#v", result.Features)
+		}
+	}
+}

baidu/url.go

@@ -65,7 +65,7 @@ func BuildURL(q core.Query) (string, error) {
 		logrus.Warn("File search not supported")
 	}
 
-	if q.Limit != 0 {
+	if q.Limit > 10 {
 		params.Add("rn", strconv.Itoa(q.Limit))
 	}
 	if q.Start < 0 {
@@ -104,8 +104,10 @@ func BuildImageURL(q core.Query, pageNum int) (string, error) {
 		return "", errors.New("Empty query built")
 	}
 
-	if q.Limit != 0 {
-		params.Add("rn", "30")                     // Results per page
+	if q.Limit > 10 {
+		params.Add("rn", "30") // Results per page
+	}
+	if pageNum > 0 {
 		params.Add("pn", strconv.Itoa(pageNum*30)) // Offset
 	}
 

bing/bing_test.go

@@ -82,6 +82,9 @@ func TestBuildURL(t *testing.T) {
 				if got := params.Get("filters"); got != `ex1:"ez5_20454_20570"` {
 					t.Fatalf("unexpected filters value: %q", got)
 				}
+				if got := params.Get("count"); got != "" {
+					t.Fatalf("count should be omitted when Limit<=10, got %q", got)
+				}
 				// LangCode unset → no locale params; let Bing pick defaults
 				// from the request rather than biasing toward en-US.
 				for _, key := range []string{"mkt", "setlang", "cc"} {
@@ -117,6 +120,47 @@ func TestBuildURL(t *testing.T) {
 				}
 			},
 		},
+		{
+			name: "region overrides bing market country",
+			query: core.Query{
+				Text:     "weather",
+				LangCode: "en",
+				Region:   "DE",
+				Limit:    10,
+			},
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("mkt"); got != "en-DE" {
+					t.Fatalf("unexpected mkt value: %q", got)
+				}
+				if got := params.Get("setlang"); got != "en" {
+					t.Fatalf("unexpected setlang value: %q", got)
+				}
+				if got := params.Get("cc"); got != "DE" {
+					t.Fatalf("unexpected cc value: %q", got)
+				}
+			},
+		},
+		{
+			name: "region only sets bing country",
+			query: core.Query{
+				Text:   "weather",
+				Region: "RU",
+				Limit:  10,
+			},
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("cc"); got != "RU" {
+					t.Fatalf("unexpected cc value: %q", got)
+				}
+				if got := params.Get("mkt"); got != "" {
+					t.Fatalf("mkt should be omitted without language, got %q", got)
+				}
+				if got := params.Get("setlang"); got != "" {
+					t.Fatalf("setlang should be omitted without language, got %q", got)
+				}
+			},
+		},
 		{
 			name: "very large start",
 			query: core.Query{
@@ -231,6 +275,23 @@ func TestBuildImageURL(t *testing.T) {
 				}
 			},
 		},
+		{
+			name: "image region overrides market country",
+			query: core.Query{
+				Text:     "cats",
+				LangCode: "en",
+				Region:   "GB",
+			},
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("mkt"); got != "en-GB" {
+					t.Fatalf("unexpected mkt value: %q", got)
+				}
+				if got := params.Get("cc"); got != "GB" {
+					t.Fatalf("unexpected cc value: %q", got)
+				}
+			},
+		},
 		{
 			name:    "empty fields return error",
 			query:   core.Query{},

bing/features.go

@@ -0,0 +1,64 @@
+package bing
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"github.com/go-rod/rod"
+	"github.com/karust/openserp/core"
+)
+
+func extractBingFeatures(doc *goquery.Document) []core.SerpFeature {
+	features := core.ExtractSerpFeaturesBySelectors(doc, []core.SerpFeatureSelector{
+		{
+			Type: core.ResultTypeAnswerBox,
+			// Only treat a b_ans block as an answer box when it carries an
+			// actual answer/entity payload. A bare li.b_ans also wraps related
+			// modules ("Searches you might like", "Get a detailed look at ..."),
+			// so require a focus/fact/xl text node to be present.
+			Container:     []string{"li.b_ans:has(.b_focusTextLarge)", "li.b_ans:has(.b_focusLabel)", "li.b_ans:has(.b_xlText)", "li.b_ans:has(.b_factrow)"},
+			TitleSelector: []string{".b_focusLabel", "h2"},
+			TextSelector:  []string{".b_focusTextLarge", ".b_xlText", ".b_vPanel .b_factrow", ".b_caption p"},
+			LinkSelector:  []string{"a[href^='http']"},
+			Position:      1,
+			Confidence:    0.8,
+		},
+		{
+			Type:         core.ResultTypeRelatedQuestions,
+			Title:        "People also ask",
+			Container:    []string{".b_rrsr", ".rqnaacfacc", "li.b_ans:has(.df_alaskcr)"},
+			ItemSelector: []string{".df_qntext", ".rqnaacfacc a", "li a"},
+			LinkSelector: []string{"a[href^='http']"},
+			Confidence:   0.7,
+		},
+		{
+			Type:         core.ResultTypeRelatedSearches,
+			Title:        "Related searches",
+			Container:    []string{"#brsv3", "li.b_rs", "ol#b_rs"},
+			ItemSelector: []string{"li a", "a"},
+			LinkSelector: []string{"a[href^='http']", "a"},
+			Confidence:   0.75,
+		},
+		{
+			// Bing's "developer answer" / rich answer card is AI-generated
+			// ("This summary was generated using AI based on multiple online
+			// sources"). Title sits in h2.b_topTitle; cited sources are the
+			// numbered superscript anchors. Copilot chat (#b_sydConvCont) is kept
+			// as a fallback for SERPs that render the chat answer inline instead.
+			Type:          core.ResultTypeAISummary,
+			Title:         "AI answer",
+			Container:     []string{".developer_answercard_wrapper", "#b_sydConvCont", ".b_sydConvCont", "[data-testid='bing-chat-answer']"},
+			TitleSelector: []string{"h2.b_topTitle", ".b_sydAns"},
+			// The full generated answer lives in .devmag_card_content, split across
+			// many <p>/<li> inside span.devmag_cntnt_snip; take the wrapper's whole
+			// collapsed text so the body isn't truncated to the first paragraph.
+			TextSelector: []string{".devmag_card_content", ".rd_def_list", ".b_sydAns", "[data-testid='answer']", "p"},
+			LinkSelector: []string{".rd_cnt_srcs a[href^='http']", ".rd_gencon_attr a[href^='http']", "h2.b_topTitle a[href^='http']", "a[href^='http']"},
+			Position:     1,
+			Confidence:   0.6,
+		},
+	})
+	return core.DeduplicateSerpFeatures(features)
+}
+
+func extractBingFeaturesFromPage(page *rod.Page) []core.SerpFeature {
+	return core.FeaturesFromPage(page, extractBingFeatures)
+}

bing/parse_html.go

@@ -0,0 +1,125 @@
+package bing
+
+import (
+	"io"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/karust/openserp/core"
+)
+
+// ParseHTML parses a Bing SERP HTML document and returns search results.
+// Mirrors the rod-based parser in search.go but operates on a goquery doc.
+// No network I/O.
+func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(r)
+	if err != nil {
+		return nil, err
+	}
+	return parseBingDocument(doc), nil
+}
+
+func parseBingDocument(doc *goquery.Document) []core.SearchResult {
+	var results []core.SearchResult
+	rank := 1
+	adRank := 1
+	absoluteRank := 1
+
+	doc.Find(Selectors.ResultItems).Each(func(_ int, item *goquery.Selection) {
+		isAd := item.Is(Selectors.Ads)
+		isOrganic := item.Is(Selectors.Results)
+		if !isAd && !isOrganic {
+			return
+		}
+
+		titleSelector := Selectors.Title
+		if isAd {
+			titleSelector = Selectors.AdTitle
+		}
+
+		titleTag := item.Find(titleSelector).First()
+		if titleTag.Length() == 0 {
+			return
+		}
+
+		link, exists := titleTag.Attr("href")
+		if !exists || link == "" || link == "#" {
+			return
+		}
+
+		title := titleTag.Text()
+		if title == "" {
+			title = extractFirstText(item, Selectors.TitleFallbacks)
+		}
+		if title == "" {
+			return
+		}
+
+		desc := descriptionFromItem(item, title)
+
+		resultRank := rank
+		if isAd {
+			resultRank = adRank
+			adRank++
+		} else {
+			rank++
+		}
+
+		results = append(results, core.SearchResult{
+			Rank:         resultRank,
+			AbsoluteRank: absoluteRank,
+			URL:          link,
+			Title:        title,
+			Description:  desc,
+			Ad:           isAd,
+		})
+		absoluteRank++
+	})
+
+	return core.AttachFeaturesToFirstResult(core.DeduplicateResults(results), extractBingFeatures(doc))
+}
+
+func extractFirstText(item *goquery.Selection, selectors []string) string {
+	for _, selector := range selectors {
+		if tag := item.Find(selector).First(); tag.Length() > 0 {
+			if text := strings.TrimSpace(tag.Text()); text != "" {
+				return text
+			}
+			if label, exists := tag.Attr("aria-label"); exists {
+				if label = strings.TrimSpace(label); label != "" {
+					return label
+				}
+			}
+		}
+	}
+	return ""
+}
+
+// descriptionFromItem extracts a description using the same 4-step fallback
+// chain as the rod-based browser parser. Bing renders snippet text with heavy
+// source-indentation whitespace, so each candidate is whitespace-collapsed.
+func descriptionFromItem(item *goquery.Selection, title string) string {
+	if descTag := item.Find(Selectors.DescPrimary).First(); descTag.Length() > 0 {
+		if text := normalizeWhitespace(descTag.Text()); text != "" {
+			return text
+		}
+	}
+	if descTag := item.Find(Selectors.DescFallback).First(); descTag.Length() > 0 {
+		if text := normalizeWhitespace(descTag.Text()); text != "" {
+			return text
+		}
+	}
+	if descTag := item.Find(Selectors.DescAny).First(); descTag.Length() > 0 {
+		if text := normalizeWhitespace(descTag.Text()); text != "" {
+			return text
+		}
+	}
+	// Structural fallback: strip title from full text
+	return normalizeWhitespace(strings.Replace(item.Text(), title, "", 1))
+}
+
+// normalizeWhitespace collapses runs of whitespace (including the newlines and
+// indentation Bing leaves in snippet markup) into single spaces.
+func normalizeWhitespace(s string) string {
+	return strings.Join(strings.Fields(s), " ")
+}

bing/parse_html_test.go

@@ -0,0 +1,147 @@
+package bing
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestParseBingHTML(t *testing.T) {
+	t.Parallel()
+
+	data, err := os.ReadFile("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("read fixture: %v", err)
+	}
+
+	results, err := ParseHTML(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+
+	rank := 0
+	for i, r := range results {
+		if r.Ad {
+			continue
+		}
+		rank++
+		if r.Rank != rank {
+			t.Fatalf("rank sequence broken at index %d: got %d, want %d", i, r.Rank, rank)
+		}
+		if r.URL == "" {
+			t.Fatalf("result %d: empty URL", i)
+		}
+		if r.Title == "" {
+			t.Fatalf("result %d: empty Title", i)
+		}
+		if !strings.HasPrefix(r.URL, "http") {
+			t.Fatalf("result %d: URL not absolute: %s", i, r.URL)
+		}
+	}
+	if rank == 0 {
+		t.Fatal("expected at least one organic result")
+	}
+}
+
+func TestParseBingHTMLEmpty(t *testing.T) {
+	t.Parallel()
+
+	results, err := ParseHTML(bytes.NewReader([]byte("")))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 0 {
+		t.Fatalf("expected zero results for empty HTML, got %d", len(results))
+	}
+}
+
+func TestParseBingHTMLAds(t *testing.T) {
+	t.Parallel()
+
+	data, err := os.ReadFile("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("read fixture: %v", err)
+	}
+
+	results, err := ParseHTML(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+
+	var ads []struct{ url, title string }
+	for _, r := range results {
+		if r.Ad {
+			ads = append(ads, struct{ url, title string }{r.URL, r.Title})
+		}
+	}
+
+	for i, ad := range ads {
+		if ad.url == "" {
+			t.Fatalf("ad result %d: empty URL", i)
+		}
+		if ad.title == "" {
+			t.Fatalf("ad result %d: empty Title", i)
+		}
+	}
+}
+
+func TestParseBingHTMLMixedAdsKeepAbsoluteOrder(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<ol id="b_results">
+  <li class="b_algo">
+    <h2><a href="https://organic.example.com/one">Organic One</a></h2>
+    <div class="b_caption"><p>Organic snippet one</p></div>
+  </li>
+  <li class="b_ad">
+    <h2><a href="https://ads.example.com">Sponsored Result</a></h2>
+    <p>Paid snippet</p>
+  </li>
+  <li class="b_algo">
+    <h2><a href="https://organic.example.com/two">Organic Two</a></h2>
+    <div class="b_caption"><p>Organic snippet two</p></div>
+  </li>
+</ol>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 3 {
+		t.Fatalf("expected 3 results, got %d", len(results))
+	}
+	if results[0].Ad || !results[1].Ad || results[2].Ad {
+		t.Fatalf("unexpected ad ordering: %+v", results)
+	}
+	if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
+		t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
+	}
+	if results[2].Rank != 2 {
+		t.Fatalf("second organic rank = %d, want 2", results[2].Rank)
+	}
+}
+
+func TestParseBingHTMLTitleFallback(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<ol id="b_results">
+  <li class="b_algo">
+    <h2><a aria-label="Fallback Title" href="https://example.com/fallback"></a></h2>
+    <div class="b_caption"><p>Snippet</p></div>
+  </li>
+</ol>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 1 {
+		t.Fatalf("expected 1 result, got %d", len(results))
+	}
+	if results[0].Title != "Fallback Title" {
+		t.Fatalf("title = %q, want fallback", results[0].Title)
+	}
+}

bing/search.go

@@ -11,23 +11,8 @@ import (
 	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/proto"
 	"github.com/karust/openserp/core"
-	"golang.org/x/time/rate"
 )
 
-var sel = struct {
-	Captcha      []string
-	CookieBtn    string
-	Results      string
-	Ads          string
-	ImageResults string
-}{
-	Captcha:      []string{"div.captcha", "div.captcha_header"},
-	CookieBtn:    "button#bnp_btn_accept",
-	Results:      "li.b_algo",
-	Ads:          "li.b_ad",
-	ImageResults: "div.iuscp, div.isv",
-}
-
 // Bing implements core.SearchEngine for Bing SERP pages.
 type Bing struct {
 	core.Browser
@@ -49,14 +34,8 @@ func (bing *Bing) Name() string {
 	return "bing"
 }
 
-// GetRateLimiter returns a limiter configured from SearchEngineOptions.
-func (bing *Bing) GetRateLimiter() *rate.Limiter {
-	ratelimit := rate.Every(bing.GetRatelimit())
-	return rate.NewLimiter(ratelimit, bing.RateBurst)
-}
-
 func (bing *Bing) getTotalResults(page *rod.Page) (int, error) {
-	results, err := page.Timeout(bing.GetSelectorTimeout()).Elements(sel.Results)
+	results, err := page.Timeout(bing.GetSelectorTimeout()).Elements(Selectors.Results)
 	if err != nil {
 		return 0, errors.New("Cannot find result elements: " + err.Error())
 	}
@@ -75,7 +54,7 @@ func (bing *Bing) checkCaptcha(page *rod.Page) bool {
 		}
 	}
 
-	for _, selector := range sel.Captcha {
+	for _, selector := range Selectors.Captcha {
 		has, _, _ := page.Has(selector)
 		if has {
 			bing.logger.Debug("Captcha detected: %s", selector)
@@ -87,7 +66,7 @@ func (bing *Bing) checkCaptcha(page *rod.Page) bool {
 }
 
 func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error {
-	consentBtn, err := page.Timeout(bing.Timeout / 10).Element(sel.CookieBtn)
+	consentBtn, err := page.Timeout(bing.Timeout / 10).Element(Selectors.CookieBtn)
 	if err != nil {
 		return nil
 	}
@@ -98,23 +77,80 @@ func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error {
 	return core.SleepContext(ctx, 500*time.Millisecond)
 }
 
+func bingElementMatches(el *rod.Element, selector string) bool {
+	if el == nil {
+		return false
+	}
+	matches, err := el.Matches(selector)
+	return err == nil && matches
+}
+
+func (bing *Bing) parseResultElement(el *rod.Element, isAd bool, rank, absoluteRank int) (core.SearchResult, bool) {
+	titleSelector := Selectors.Title
+	if isAd {
+		titleSelector = Selectors.AdTitle
+	}
+
+	titleElem, err := el.Element(titleSelector)
+	if err != nil {
+		bing.logger.Debug("Missing title")
+		return core.SearchResult{}, false
+	}
+
+	href, err := titleElem.Property("href")
+	if err != nil {
+		bing.logger.Debug("Missing URL")
+		return core.SearchResult{}, false
+	}
+	url := strings.TrimSpace(href.String())
+	if url == "" || url == "#" || strings.HasPrefix(url, "javascript:") {
+		return core.SearchResult{}, false
+	}
+
+	title, _ := titleElem.Text()
+	title = strings.TrimSpace(title)
+	if title == "" {
+		title = core.FirstNonEmptyText(el, Selectors.TitleFallbacks...)
+	}
+	if title == "" {
+		title = core.FirstNonEmptyAttribute(el, "aria-label", Selectors.TitleFallbacks...)
+	}
+	if title == "" {
+		bing.logger.Debug("Missing title text")
+		return core.SearchResult{}, false
+	}
+
+	desc := ""
+	if descElem, err := el.Element(Selectors.DescPrimary); err == nil {
+		desc, _ = descElem.Text()
+	} else if descElem, err := el.Element(Selectors.DescFallback); err == nil {
+		desc, _ = descElem.Text()
+	} else if descElem, err := el.Element(Selectors.DescAny); err == nil {
+		desc, _ = descElem.Text()
+	} else {
+		fullText, _ := el.Text()
+		desc = strings.TrimSpace(strings.Replace(fullText, title, "", 1))
+	}
+
+	return core.SearchResult{
+		Rank:         rank,
+		AbsoluteRank: absoluteRank,
+		URL:          url,
+		Title:        title,
+		Description:  strings.TrimSpace(desc),
+		Ad:           isAd,
+	}, true
+}
+
 // Search executes a Bing web search and returns normalized search results.
 // It may return core.ErrCaptcha or core.ErrSearchTimeout.
 func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
-	ctx = core.WithEngine(core.EnsureContext(ctx), bing.Name())
-	ctx = core.WithProfileRegion(ctx, query.LangCode)
-	ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
+	ctx = core.PrepareEngineContext(ctx, query, bing.Name(), false)
 	scoped := *bing
 	scoped.logger = bing.logger.WithRequest(ctx)
 	bing = &scoped
 
 	bing.logger.Debug("Starting search, query: %+v", query)
-	defer func() {
-		if recovered := recover(); recovered != nil {
-			err = core.RecoverEnginePanicWithContext(ctx, bing.Name(), recovered, bing.logger)
-			results = nil
-		}
-	}()
 
 	searchResults := []core.SearchResult{}
 
@@ -127,19 +163,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
 	if err != nil {
 		return nil, err
 	}
-	defer func() {
-		if bing.Browser.LeavePageOpen {
-			return
-		}
-		if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil {
-			bing.logger.Debug("Page close error: %v", closeErr)
-		}
-	}()
-
-	if err := page.WaitLoad(); err != nil {
-		bing.logger.Error("Initial page load wait failed: %s", err)
-		return nil, core.ErrSearchTimeout
-	}
+	defer core.DeferClosePage(ctx, page, &bing.Browser)()
 
 	if bing.checkCaptcha(page) {
 		bing.logger.Error("Captcha detected: %s", url)
@@ -149,117 +173,59 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
 	if err := bing.acceptCookies(ctx, page); err != nil {
 		return nil, err
 	}
-	if err := page.WaitLoad(); err != nil {
-		bing.logger.Error("Post-consent page load wait failed: %s", err)
-		return nil, core.ErrSearchTimeout
-	}
 
-	organicElements, err := page.Timeout(bing.Timeout).Elements(sel.Results)
+	resultElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.ResultItems, Selectors.Results}, bing.GetSelectorTimeout())
 	if err != nil {
+		// Re-check captcha on timeout - Bing interstitials can render after WaitLoad.
+		if bing.checkCaptcha(page) {
+			bing.logger.Error("Captcha detected: %s", url)
+			return nil, core.ErrCaptcha
+		}
 		bing.logger.Error("Cannot parse organic results: %s", err)
-		return nil, core.ErrParser
-	}
-
-	adElements, err := page.Timeout(bing.Timeout).Elements(sel.Ads)
-	if err != nil {
-		bing.logger.Debug("No ads found")
+		return nil, core.ErrSearchTimeout
 	}
 
 	totalResults, err := bing.getTotalResults(page)
 	if err != nil {
 		bing.logger.Debug("Failed to get total results: %v", err)
 	}
-	bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements))
+	bing.logger.Info("Found %d organic result containers", totalResults)
 
 	rank := query.Start
-	for _, result := range organicElements {
-		srchRes := core.SearchResult{}
-
-		titleElem, err := result.Element("a")
-		if err != nil {
-			bing.logger.Debug("Missing title")
+	adRank := 1
+	absoluteRank := query.Start + 1
+	for _, result := range resultElements {
+		isAd := bingElementMatches(result, Selectors.Ads)
+		isOrganic := bingElementMatches(result, Selectors.Results)
+		if !isAd && !isOrganic {
 			continue
 		}
-		srchRes.Title, _ = titleElem.Text()
 
-		href, err := titleElem.Property("href")
-		if err != nil {
-			bing.logger.Debug("Missing URL")
+		resultRank := rank + 1
+		if isAd {
+			resultRank = adRank
+		}
+		srchRes, ok := bing.parseResultElement(result, isAd, resultRank, absoluteRank)
+		if !ok {
 			continue
 		}
-		srchRes.URL = href.String()
-
-		var desc string
-		if descElem, err := result.Element("div.b_caption p"); err == nil {
-			desc, _ = descElem.Text()
-		} else if descElem, err := result.Element("div.b_caption div"); err == nil {
-			desc, _ = descElem.Text()
-		} else if descElem, err := result.Element("p"); err == nil {
-			desc, _ = descElem.Text()
+		searchResults = append(searchResults, srchRes)
+		absoluteRank++
+		if isAd {
+			adRank++
 		} else {
-			fullText, _ := result.Text()
-			desc = strings.TrimSpace(strings.Replace(fullText, srchRes.Title, "", 1))
+			rank++
 		}
-		srchRes.Description = desc
-
-		rank++
-		srchRes.Rank = rank
-		srchRes.Ad = false
-
-		searchResults = append(searchResults, srchRes)
-	}
-
-	for _, adResult := range adElements {
-		srchRes := core.SearchResult{Ad: true}
-
-		titleElem, err := adResult.Element("h2 a")
-		if err != nil {
-			bing.logger.Debug("Ad missing title")
-			continue
-		}
-		srchRes.Title, _ = titleElem.Text()
-
-		href, err := titleElem.Property("href")
-		if err != nil {
-			bing.logger.Debug("Ad missing URL")
-			continue
-		}
-		srchRes.URL = href.String()
-
-		if descElem, err := adResult.Element("p"); err == nil {
-			srchRes.Description, _ = descElem.Text()
-		}
-
-		// Mark ads with negative rank
-		srchRes.Rank = -1
-		searchResults = append(searchResults, srchRes)
 	}
 
 	// Deduplicate results
 	deduped := core.DeduplicateResults(searchResults)
 
-	// Trim to exact limit if necessary (only organic results, not ads)
-	if query.Limit > 0 {
-		organicResults := []core.SearchResult{}
-		adResults := []core.SearchResult{}
+	deduped = core.LimitOrganicResults(deduped, query.Limit)
 
-		for _, result := range deduped {
-			if result.Ad {
-				adResults = append(adResults, result)
-			} else {
-				organicResults = append(organicResults, result)
-			}
-		}
-
-		// Trim organic results to limit
-		if len(organicResults) > query.Limit {
-			organicResults = organicResults[:query.Limit]
-		}
-
-		// Combine back: organic results + ads
-		deduped = append(organicResults, adResults...)
+	if query.Features {
+		deduped = core.AttachFeaturesToFirstResult(deduped, extractBingFeaturesFromPage(page))
 	}
-
 	return deduped, nil
 }
 
@@ -275,12 +241,20 @@ type BingImageData struct {
 	MURL   string `json:"murl"`   // Image URL
 }
 
+func resolveImageLinkElement(container *rod.Element) (*rod.Element, error) {
+	if container == nil {
+		return nil, errors.New("nil image container")
+	}
+	if core.HasAttribute(container, "m") {
+		return container, nil
+	}
+	return container.Element("a")
+}
+
 // SearchImage executes a Bing image search and returns normalized image
 // results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
 func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
-	ctx = core.WithEngine(core.EnsureContext(ctx), bing.Name())
-	ctx = core.WithProfileRegion(ctx, query.LangCode)
-	ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
+	ctx = core.PrepareEngineContext(ctx, query, bing.Name(), false)
 	scoped := *bing
 	scoped.logger = bing.logger.WithRequest(ctx)
 	bing = &scoped
@@ -299,19 +273,7 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
 	if err != nil {
 		return nil, err
 	}
-	defer func() {
-		if bing.Browser.LeavePageOpen {
-			return
-		}
-		if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil {
-			bing.logger.Debug("Page close error: %v", closeErr)
-		}
-	}()
-
-	if err := page.WaitLoad(); err != nil {
-		bing.logger.Error("Initial image page load wait failed: %s", err)
-		return nil, core.ErrSearchTimeout
-	}
+	defer core.DeferClosePage(ctx, page, &bing.Browser)()
 
 	// Check for captcha
 	if bing.checkCaptcha(page) {
@@ -324,18 +286,16 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
 		return nil, err
 	}
 
-	// Wait for image results to load
-	if err := page.WaitLoad(); err != nil {
-		bing.logger.Error("Image results load wait failed: %s", err)
-		return nil, core.ErrSearchTimeout
-	}
-	if err := core.SleepContext(ctx, 2*time.Second); err != nil {
-		return nil, err
-	}
-
-	// Find all image result containers using CSS selector
-	imageContainers, err := page.Timeout(bing.Timeout).Elements(sel.ImageResults)
+	imageContainers, _, err := core.WaitForElements(
+		ctx,
+		page,
+		[]string{Selectors.ImageResults},
+		bing.GetSelectorTimeout(),
+	)
 	if err != nil {
+		if bing.checkCaptcha(page) {
+			return nil, core.ErrCaptcha
+		}
 		bing.logger.Error("Cannot parse image results: %s", err)
 		return nil, core.ErrSearchTimeout
 	}
@@ -350,10 +310,9 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
 	for _, c := range imageContainers {
 		srchRes := core.SearchResult{}
 
-		// Get the <a> element inside the div
-		linkElem, err := c.Element("a")
+		linkElem, err := resolveImageLinkElement(c)
 		if err != nil {
-			bing.logger.Debug("Missing <a> element")
+			bing.logger.Debug("Missing image link element")
 			continue
 		}
 
@@ -400,6 +359,9 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
 		srchRes.Rank = rank
 
 		searchResults = append(searchResults, srchRes)
+		if query.Limit > 0 && len(searchResults) >= query.Limit {
+			break
+		}
 	}
 
 	return searchResults, nil

bing/search_integration_test.go

@@ -4,52 +4,14 @@
 package bing
 
 import (
-	"context"
 	"testing"
 
 	"github.com/karust/openserp/core"
-	"github.com/karust/openserp/testutil"
 	"github.com/karust/openserp/testutil/ithelper"
 )
 
 func TestSearchBing(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	bing := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golang programming", Limit: 10}
-	results, err := bing.Search(context.Background(), query)
-	ithelper.HandleError(t, "bing web search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first result title is empty")
-	}
-}
-
-func TestImageSearchBing(t *testing.T) {
-	testutil.RequireIntegration(t)
-
-	browser := ithelper.CreateBrowser(t)
-	bing := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golden retriever puppy", Limit: 10}
-	results, err := bing.SearchImage(context.Background(), query)
-	ithelper.HandleError(t, "bing image search", err)
-
-	if len(results) == 0 {
-		t.Fatal("returned empty image results")
-	}
-	if results[0].URL == "" {
-		t.Fatal("first image result URL is empty")
-	}
-	if results[0].Title == "" {
-		t.Fatal("first image result title is empty")
-	}
+	ithelper.RunEngineTests(t, func(b *core.Browser) core.SearchEngine {
+		return New(*b, ithelper.EngineOptions())
+	})
 }

bing/selectors.go

@@ -0,0 +1,35 @@
+package bing
+
+// Selectors is the single source of truth for Bing SERP CSS selectors.
+var Selectors = struct {
+	Captcha        []string
+	CookieBtn      string
+	ResultItems    string
+	Results        string
+	Ads            string
+	ImageResults   string
+	Title          string
+	TitleFallbacks []string
+	DescPrimary    string
+	DescFallback   string
+	DescAny        string
+	AdTitle        string
+}{
+	Captcha:      []string{"div.captcha", "div.captcha_header"},
+	CookieBtn:    "button#bnp_btn_accept",
+	// ResultItems matches the main-column children only, so carousels and
+	// "related searches" cards that reuse b_algo-style markup are excluded.
+	ResultItems:  "#b_results > li.b_algo, #b_results > li.b_ad",
+	Results:      "li.b_algo",
+	Ads:          "li.b_ad",
+	ImageResults: "a.iusc, div.iuscp, div.isv",
+	Title:        "h2 a",
+	// TitleFallbacks are tried when the primary Title selector matches but
+	// yields empty text (Bing occasionally renders an empty <h2><a/></h2>
+	// while the visible label sits in aria-label or h2).
+	TitleFallbacks: []string{"h2", "a[aria-label]"},
+	DescPrimary:    "div.b_caption p",
+	DescFallback:   "div.b_caption div",
+	DescAny:        "p",
+	AdTitle:        "h2 a",
+}

bing/serp_features_test.go

@@ -0,0 +1,116 @@
+package bing
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/karust/openserp/core"
+)
+
+// TestParseHTMLFixtureExtractsRealFeatures guards selectors against the
+// sanitized real-SERP fixture (related searches present; the noisy b_ans
+// "detailed look"/"searches you might like" modules must not be emitted as
+// answer boxes).
+func TestParseHTMLFixtureExtractsRealFeatures(t *testing.T) {
+	t.Parallel()
+	f, err := os.Open("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("open fixture: %v", err)
+	}
+	defer f.Close()
+
+	results, err := ParseHTML(f)
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	assertFeatureType(t, results, core.ResultTypeRelatedSearches)
+	// The fixture carries a developer/rich answer card flagged as AI-generated.
+	assertFeatureType(t, results, core.ResultTypeAISummary)
+	for _, r := range results {
+		for _, feature := range r.Features {
+			if feature.Type == core.ResultTypeAnswerBox && feature.Title == "Get a detailed look atpizza delivery" {
+				t.Fatalf("related-search module leaked into answer_box: %#v", feature)
+			}
+		}
+	}
+
+	// Descriptions must be whitespace-collapsed: Bing leaves raw newlines and
+	// source indentation in snippet markup, which previously surfaced verbatim.
+	for i, r := range results {
+		if strings.ContainsAny(r.Description, "\n\t") {
+			t.Fatalf("result %d description has raw whitespace: %q", i, r.Description)
+		}
+	}
+}
+
+func TestParseHTMLExtractsSerpFeatures(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<ol id="b_results">
+  <li class="b_ans">
+    <h2>Bing answer</h2>
+    <div class="b_focusTextLarge">Bing answer text.</div>
+    <div class="b_caption"><p>Source snippet</p></div>
+    <a href="https://example.com/source">Source</a>
+  </li>
+  <li class="b_rrsr">
+    <h2>People also ask</h2>
+    <ul>
+      <li><a href="https://example.com/question">What is OpenSERP?</a></li>
+    </ul>
+  </li>
+  <li class="b_algo">
+    <h2><a href="https://example.com/result">Organic result</a></h2>
+    <div class="b_caption"><p>Snippet</p></div>
+  </li>
+</ol>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	assertFeatureType(t, results, core.ResultTypeAnswerBox)
+	assertFeatureType(t, results, core.ResultTypeRelatedQuestions)
+}
+
+func TestParseHTMLOrganicOnlyHasNoSerpFeatures(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<ol id="b_results">
+  <li class="b_algo">
+    <h2><a href="https://example.com/result">Organic result</a></h2>
+    <div class="b_caption"><p>Snippet</p></div>
+  </li>
+</ol>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	assertNoFeatures(t, results)
+}
+
+func assertFeatureType(t *testing.T, results []core.SearchResult, want core.ResultType) {
+	t.Helper()
+	for _, result := range results {
+		for _, feature := range result.Features {
+			if feature.Type == want {
+				return
+			}
+		}
+	}
+	t.Fatalf("expected feature type %q in %#v", want, results)
+}
+
+func assertNoFeatures(t *testing.T, results []core.SearchResult) {
+	t.Helper()
+	for _, result := range results {
+		if len(result.Features) > 0 {
+			t.Fatalf("expected no features, got %#v", result.Features)
+		}
+	}
+}

bing/url.go

@@ -71,9 +71,13 @@ func BuildURL(q core.Query) (string, error) {
 		return "", errors.New("empty query built")
 	}
 
-	if locale, ok := bingLocale(q.LangCode); ok {
-		params.Add("mkt", locale.market)
-		params.Add("setlang", locale.language)
+	if locale, ok := bingLocale(q.LangCode, q.Region); ok {
+		if locale.market != "" {
+			params.Add("mkt", locale.market)
+		}
+		if locale.language != "" {
+			params.Add("setlang", locale.language)
+		}
 		params.Add("cc", locale.country)
 	}
 
@@ -85,7 +89,7 @@ func BuildURL(q core.Query) (string, error) {
 	if q.Start > 0 {
 		// Bing uses 1-based first-result index for pagination.
 		params.Add("first", strconv.Itoa(q.Start+1))
-	} else if q.Limit > 0 {
+	} else if q.Limit > 10 {
 		params.Add("count", strconv.Itoa(q.Limit))
 	}
 
@@ -117,15 +121,21 @@ type bingLocaleParams struct {
 // caller-supplied language code. It returns ok=false when the input is empty
 // so callers can omit Bing's locale parameters entirely instead of forcing a
 // default market that biases results toward en-US.
-func bingLocale(langCode string) (bingLocaleParams, bool) {
+func bingLocale(langCode, region string) (bingLocaleParams, bool) {
 	parsed := core.ParseLocale(langCode)
+	country := core.CountryFromRegion(region)
 	if parsed.Language == "" {
+		if country != "" {
+			return bingLocaleParams{country: country}, true
+		}
 		return bingLocaleParams{}, false
 	}
 
-	country := parsed.Country
 	if country == "" {
-		country = defaultBingCountry(parsed.Language)
+		country = parsed.Country
+		if country == "" {
+			country = defaultBingCountry(parsed.Language)
+		}
 	}
 	return bingLocaleParams{
 		language: parsed.Language,
@@ -225,9 +235,13 @@ func BuildImageURL(q core.Query) (string, error) {
 		return "", errors.New("empty query built")
 	}
 
-	if locale, ok := bingLocale(q.LangCode); ok {
-		params.Add("mkt", locale.market)
-		params.Add("setlang", locale.language)
+	if locale, ok := bingLocale(q.LangCode, q.Region); ok {
+		if locale.market != "" {
+			params.Add("mkt", locale.market)
+		}
+		if locale.language != "" {
+			params.Add("setlang", locale.language)
+		}
 		params.Add("cc", locale.country)
 	}
 

cmd/proxy_policy.go

@@ -13,6 +13,7 @@ func buildEngineProxyPolicyMap() map[string]string {
 		"baidu":      config.BaiduConfig.Proxy,
 		"bing":       config.BingConfig.Proxy,
 		"duckduckgo": config.DuckDuckGoConfig.Proxy,
+		"ecosia":     config.EcosiaConfig.Proxy,
 	}
 }
 

cmd/root.go

@@ -5,9 +5,11 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/karust/openserp/core"
 	browserprofile "github.com/karust/openserp/core/browser"
+	extractpkg "github.com/karust/openserp/extract"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -15,7 +17,7 @@ import (
 )
 
 const (
-	version               = "0.7.2"
+	version               = "0.8.3"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )
@@ -25,6 +27,7 @@ type Config struct {
 	App              AppConfig            `mapstructure:"app"`
 	Proxies          core.ProxiesConfig   `mapstructure:"proxies"`
 	Cache            CacheConfig          `mapstructure:"cache"`
+	Extract          extractpkg.Config    `mapstructure:"extract"`
 	Resilience       ResilienceConfig     `mapstructure:"resilience"`
 	CircuitBreaker   CircuitBreakerConfig `mapstructure:"circuit_breaker"`
 	CORS             CORSConfig           `mapstructure:"cors"`
@@ -35,6 +38,7 @@ type Config struct {
 	BaiduConfig      EngineConfig         `mapstructure:"baidu"`
 	BingConfig       EngineConfig         `mapstructure:"bing"`
 	DuckDuckGoConfig EngineConfig         `mapstructure:"duckduckgo"`
+	EcosiaConfig     EngineConfig         `mapstructure:"ecosia"`
 }
 
 type Config2Captcha struct {
@@ -52,16 +56,19 @@ type ServerConfig struct {
 }
 
 type AppConfig struct {
-	Timeout        int    `mapstructure:"timeout"`
-	BrowserPath    string `mapstructure:"browser_path"`
-	ProfilesJSON   string `mapstructure:"profiles"`
-	IsBrowserHead  bool   `mapstructure:"head"`
-	IsLeaveHead    bool   `mapstructure:"leave_head"`
-	IsLeakless     bool   `mapstructure:"leakless"`
-	BlockResources string `mapstructure:"block_resources"`
-	BlockTrackers  bool   `mapstructure:"block_trackers"`
-	DebugEndpoints bool   `mapstructure:"debug_endpoints"`
-	LogFormat      string `mapstructure:"log_format"`
+	Timeout        int           `mapstructure:"timeout"`
+	BrowserPath    string        `mapstructure:"browser_path"`
+	ProfilesJSON   string        `mapstructure:"profiles"`
+	IsBrowserHead  bool          `mapstructure:"head"`
+	IsLeaveHead    bool          `mapstructure:"leave_head"`
+	IsLeakless     bool          `mapstructure:"leakless"`
+	BlockResources string        `mapstructure:"block_resources"`
+	BlockTrackers  bool          `mapstructure:"block_trackers"`
+	DebugEndpoints bool          `mapstructure:"debug_endpoints"`
+	LogFormat      string        `mapstructure:"log_format"`
+	MaxProcesses   int           `mapstructure:"max_processes"`
+	IdleTTL        time.Duration `mapstructure:"idle_ttl"`
+	MegaTimeout    time.Duration `mapstructure:"mega_timeout"`
 }
 
 type EngineConfig struct {
@@ -148,11 +155,61 @@ var RootCmd = &cobra.Command{
 		config.App.LogFormat = logFormat
 
 		core.InitLogger(config.Server.IsVerbose, config.Server.IsDebug, config.App.LogFormat)
-		logrus.WithField("config", fmt.Sprintf("%+v", config)).Debug("Final config")
+		logrus.WithField("config", sanitizedConfigForLog(config)).Debug("Final config")
 		return nil
 	},
 }
 
+func sanitizedConfigForLog(cfg Config) map[string]interface{} {
+	return map[string]interface{}{
+		"server": cfg.Server,
+		"app": map[string]interface{}{
+			"timeout":         cfg.App.Timeout,
+			"browser_path":    cfg.App.BrowserPath != "",
+			"profiles":        cfg.App.ProfilesJSON != "",
+			"head":            cfg.App.IsBrowserHead,
+			"leave_head":      cfg.App.IsLeaveHead,
+			"leakless":        cfg.App.IsLeakless,
+			"block_resources": cfg.App.BlockResources,
+			"block_trackers":  cfg.App.BlockTrackers,
+			"debug_endpoints": cfg.App.DebugEndpoints,
+			"log_format":      cfg.App.LogFormat,
+			"max_processes":   cfg.App.MaxProcesses,
+			"idle_ttl":        cfg.App.IdleTTL.String(),
+			"mega_timeout":    cfg.App.MegaTimeout.String(),
+		},
+		"proxies": map[string]interface{}{
+			"global":                  maskedProxyForLog(cfg.Proxies.Global),
+			"entries":                 len(cfg.Proxies.Entries),
+			"allow_request_proxy_url": cfg.Proxies.AllowRequestProxyURL,
+			"health":                  cfg.Proxies.Health,
+			"lanes":                   cfg.Proxies.Lanes,
+		},
+		"cache":           cfg.Cache,
+		"extract":         cfg.Extract,
+		"resilience":      cfg.Resilience,
+		"circuit_breaker": cfg.CircuitBreaker,
+		"cors":            cfg.CORS,
+		"captcha":         cfg.Captcha,
+		"2captcha": map[string]interface{}{
+			"apikey_configured": strings.TrimSpace(cfg.Config2Capcha.ApiKey) != "",
+		},
+		"google":     cfg.GoogleConfig,
+		"yandex":     cfg.YandexConfig,
+		"baidu":      cfg.BaiduConfig,
+		"bing":       cfg.BingConfig,
+		"duckduckgo": cfg.DuckDuckGoConfig,
+		"ecosia":     cfg.EcosiaConfig,
+	}
+}
+
+func maskedProxyForLog(proxyURL string) string {
+	if strings.TrimSpace(proxyURL) == "" {
+		return ""
+	}
+	return core.MaskProxyURL(proxyURL)
+}
+
 // Bind each cobra flag to its associated viper configuration (config file and environment variable)
 func bindFlags(cmd *cobra.Command, vpr *viper.Viper) {
 	cmd.Flags().VisitAll(func(flg *pflag.Flag) {
@@ -259,15 +316,11 @@ func initializeConfig(cmd *cobra.Command) error {
 		return fmt.Errorf("invalid proxies config: %w", err)
 	}
 
-	if config.Server.IsDebug {
-		logrus.Debug("Viper config:")
-		v.Debug()
-	}
 	return nil
 }
 
 func validateEngineProxyTags(v *viper.Viper) error {
-	for _, engineName := range []string{"google", "yandex", "baidu", "bing", "duckduckgo"} {
+	for _, engineName := range []string{"google", "yandex", "baidu", "bing", "duckduckgo", "ecosia"} {
 		key := engineName + ".proxy"
 		if !v.IsSet(key) {
 			continue
@@ -340,13 +393,26 @@ func setConfigDefaults(v *viper.Viper) {
 	v.SetDefault("app.block_resources", "")
 	v.SetDefault("app.block_trackers", false)
 	v.SetDefault("app.debug_endpoints", false)
+	v.SetDefault("app.max_processes", 4)
+	v.SetDefault("app.idle_ttl", "10m")
+	v.SetDefault("app.mega_timeout", "90s")
 
 	v.SetDefault("proxies.entries", []interface{}{})
 	v.SetDefault("proxies.global", "")
+	v.SetDefault("proxies.allow_request_proxy_url", false)
 	v.SetDefault("proxies.health.failure_threshold", core.DefaultProxyFailureThreshold)
+	v.SetDefault("proxies.lanes.enabled", true)
+	v.SetDefault("proxies.lanes.max_lanes", core.DefaultProxyLaneMaxLanes)
+	v.SetDefault("proxies.lanes.drop_cookies_on_challenge", true)
 
 	v.SetDefault("cache.ttl_seconds", 300)
 	v.SetDefault("cache.max_size", 1000)
+	v.SetDefault("extract.enabled", true)
+	v.SetDefault("extract.default_mode", "auto")
+	v.SetDefault("extract.timeout", "20s")
+	v.SetDefault("extract.max_bytes", 2*1024*1024)
+	v.SetDefault("extract.max_concurrent", 2)
+	v.SetDefault("extract.allow_private_networks", false)
 	// Keep stage2 defaults stable even when config file is absent.
 	v.SetDefault("resilience.max_retries", 3)
 	v.SetDefault("resilience.allow_endpoint_fallback", false)
@@ -356,7 +422,7 @@ func setConfigDefaults(v *viper.Viper) {
 	v.SetDefault("cors.enabled", true)
 	v.SetDefault("cors.allow_origins", "*")
 	v.SetDefault("cors.allow_methods", "GET, POST, OPTIONS")
-	v.SetDefault("cors.allow_headers", "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Request-ID, X-Tenant")
+	v.SetDefault("cors.allow_headers", "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Proxy-URL, X-Proxy-Country, X-Proxy-Class, X-Proxy-Provider, X-Proxy-Session-ID, X-Request-ID, X-Tenant")
 	v.SetDefault("cors.max_age", 86400)
 	v.SetDefault("captcha.solver_enabled", false)
 }

cmd/search.go

@@ -12,6 +12,7 @@ import (
 	"github.com/karust/openserp/bing"
 	"github.com/karust/openserp/core"
 	"github.com/karust/openserp/duckduckgo"
+	"github.com/karust/openserp/ecosia"
 	"github.com/karust/openserp/google"
 	"github.com/karust/openserp/yandex"
 	"github.com/sirupsen/logrus"
@@ -21,7 +22,7 @@ import (
 var searchCMD = &cobra.Command{
 	Use:     "search",
 	Aliases: []string{"find"},
-	Short:   "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo)",
+	Short:   "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo, ecosia)",
 	Args:    cobra.MatchAll(cobra.OnlyValidArgs, cobra.ExactArgs(2)),
 	Run:     search,
 }
@@ -99,7 +100,10 @@ func search(cmd *cobra.Command, args []string) {
 
 func searchBrowser(engineType string, query core.Query, browserProxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) ([]core.SearchResult, error) {
 	var engine core.SearchEngine
-	blockedResourceTypes := core.MustParseBlockedResourceTypes(config.App.BlockResources)
+	blockedResourceTypes, err := core.ParseBlockedResourceTypes(config.App.BlockResources)
+	if err != nil {
+		return nil, fmt.Errorf("invalid block_resources config: %w", err)
+	}
 	if core.IsAuthenticatedSocksProxyURL(browserProxyURL) {
 		return nil, fmt.Errorf(
 			"%w: browser runtime does not support authenticated SOCKS proxy %s",
@@ -142,6 +146,8 @@ func searchBrowser(engineType string, query core.Query, browserProxyURL string,
 		engine = bing.New(*browser, config.BingConfig.SearchEngineOptions)
 	case "duckduckgo":
 		engine = duckduckgo.New(*browser, config.DuckDuckGoConfig.SearchEngineOptions)
+	case "ecosia":
+		engine = ecosia.New(*browser, config.EcosiaConfig.SearchEngineOptions)
 	default:
 		return nil, fmt.Errorf("no %q search engine found", engineType)
 	}
@@ -160,6 +166,8 @@ func searchRaw(engineType string, query core.Query) ([]core.SearchResult, error)
 		return google.Search(ctx, query)
 	case "baidu":
 		return baidu.Search(ctx, query)
+	case "ecosia":
+		return ecosia.Search(ctx, query)
 	case "bing":
 		logrus.Warn("Bing does not support raw HTTP requests mode. Please use browser mode instead.")
 		return nil, fmt.Errorf("bing does not support raw requests mode")

cmd/serve.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
+	"net/url"
 	"os"
 	"os/signal"
 	"strings"
@@ -15,6 +17,7 @@ import (
 	"github.com/karust/openserp/bing"
 	"github.com/karust/openserp/core"
 	"github.com/karust/openserp/duckduckgo"
+	"github.com/karust/openserp/ecosia"
 	"github.com/karust/openserp/google"
 	"github.com/karust/openserp/yandex"
 	"github.com/sirupsen/logrus"
@@ -24,7 +27,9 @@ import (
 
 // rawEngine implements SearchEngine interface for raw HTTP requests
 type rawEngine struct {
-	name string
+	name      string
+	limiterMu sync.Mutex
+	limiter   *rate.Limiter
 }
 
 func (r *rawEngine) Search(ctx context.Context, q core.Query) ([]core.SearchResult, error) {
@@ -37,6 +42,8 @@ func (r *rawEngine) Search(ctx context.Context, q core.Query) ([]core.SearchResu
 		return yandex.Search(ctx, q)
 	case "baidu":
 		return baidu.Search(ctx, q)
+	case "ecosia":
+		return ecosia.Search(ctx, q)
 	default:
 		return nil, fmt.Errorf("unsupported engine: %s", r.name)
 	}
@@ -55,8 +62,13 @@ func (r *rawEngine) IsInitialized() bool {
 }
 
 func (r *rawEngine) GetRateLimiter() *rate.Limiter {
-	// Use default rate limiter for raw requests
-	return rate.NewLimiter(rate.Every(time.Second), 5)
+	r.limiterMu.Lock()
+	defer r.limiterMu.Unlock()
+	if r.limiter == nil {
+		// Use default rate limiter for raw requests.
+		r.limiter = rate.NewLimiter(rate.Every(time.Second), 5)
+	}
+	return r.limiter
 }
 
 var serveCMD = &cobra.Command{
@@ -100,6 +112,7 @@ func serve(cmd *cobra.Command, args []string) {
 			&rawEngine{name: "google"},
 			&rawEngine{name: "yandex"},
 			&rawEngine{name: "baidu"},
+			&rawEngine{name: "ecosia"},
 		)
 		if err := listenWithGracefulShutdown(serv, nil); err != nil {
 			logrus.Error(err)
@@ -112,13 +125,14 @@ func serve(cmd *cobra.Command, args []string) {
 	baseOpts.CaptchaSolverEnabled = captchaSolverEnabled
 	baseOpts.CaptchaSolverApiKey = captchaSolverAPIKey
 
-	engines, closeBrowsers, err := buildBrowserEngines(baseOpts, proxyCfg)
+	engines, closeBrowsers, browserResolver, err := buildBrowserEngines(baseOpts, proxyCfg)
 	if err != nil {
 		logrus.Error(err)
 		return
 	}
 
 	serverOpts := buildServerOptions(corsCfg, proxyCfg, fingerprintBrowserOpts)
+	serverOpts.BrowserResolver = browserResolver
 	serv := core.NewServerWithOptions(config.Server.Host, config.Server.Port, serverOpts, engines...)
 	if err := listenWithGracefulShutdown(serv, closeBrowsers); err != nil {
 		logrus.Error(err)
@@ -144,6 +158,14 @@ func buildFingerprintBrowserOptions() core.BrowserOpts {
 }
 
 func buildServerOptions(corsCfg core.CORSConfig, proxyCfg core.ProxyConfig, fingerprintBrowserOpts core.BrowserOpts) core.ServerOptions {
+	retryCfg := core.RetryConfig{
+		MaxRetries:     config.Resilience.MaxRetries,
+		InitialBackoff: 1 * time.Second,
+		MaxBackoff:     30 * time.Second,
+		BackoffFactor:  2.0,
+	}
+	engineTimeout := time.Duration(config.App.Timeout) * time.Second
+
 	return core.ServerOptions{
 		CacheTTL:               time.Duration(config.Cache.TTLSeconds) * time.Second,
 		CacheMaxSize:           config.Cache.MaxSize,
@@ -153,13 +175,11 @@ func buildServerOptions(corsCfg core.CORSConfig, proxyCfg core.ProxyConfig, fing
 		EnableDebugEndpoints:   config.App.DebugEndpoints,
 		FingerprintArtifactDir: core.DefaultFingerprintArtifactDir,
 		FingerprintBrowserOpts: fingerprintBrowserOpts,
+		MegaTimeout:            config.App.MegaTimeout,
+		RequestTimeout:         core.RequestTimeoutForRetries(engineTimeout, retryCfg),
+		Extract:                config.Extract,
 		Resilience: core.ResilientConfig{
-			Retry: core.RetryConfig{
-				MaxRetries:     config.Resilience.MaxRetries,
-				InitialBackoff: 1 * time.Second,
-				MaxBackoff:     30 * time.Second,
-				BackoffFactor:  2.0,
-			},
+			Retry: retryCfg,
 			CircuitBreaker: core.CircuitBreakerConfig{
 				FailureThreshold: config.CircuitBreaker.Failures,
 				RecoveryTimeout:  time.Duration(config.CircuitBreaker.RecoverySeconds) * time.Second,
@@ -235,60 +255,284 @@ func isServerNotRunningError(err error) bool {
 	return strings.Contains(strings.ToLower(err.Error()), "server is not running")
 }
 
+// pooledBrowser is one Chrome process in the pool, dedicated to a single proxy
+// auth identity (or to the shared no-auth/unauth path when launchProxyURL=="").
+type pooledBrowser struct {
+	browser        *core.Browser
+	launchProxyURL string
+	lastUsedAt     time.Time
+}
+
+// browserPool keeps a bounded set of Chrome processes keyed by proxy auth
+// identity (scheme+host+port+username). Each entry was launched with its own
+// `l.Proxy(...)` so Chrome handles 407 natively for the main document AND all
+// subresources. Direct and unauthenticated proxies share one entry whose Chrome
+// was launched without a process-level proxy; per-BrowserContext ProxyServer is
+// applied at request time for unauthenticated request-URL proxies.
 type browserPool struct {
-	mu      sync.Mutex
-	base    core.BrowserOpts
-	browser map[string]*core.Browser
+	mu        sync.Mutex
+	base      core.BrowserOpts
+	laneStore *core.LaneStore
+
+	maxProcesses int
+	idleTTL      time.Duration
+
+	browsers map[string]*pooledBrowser
+
+	evictedLRU  int
+	evictedIdle int
+
+	stopSweeper chan struct{}
+	sweeperDone chan struct{}
 }
 
-func newBrowserPool(base core.BrowserOpts) *browserPool {
-	return &browserPool{
-		base:    base,
-		browser: map[string]*core.Browser{},
+const directBrowserKey = "direct"
+
+func newBrowserPool(base core.BrowserOpts, defaultLaunchProxyURL string, laneStore *core.LaneStore, maxProcesses int, idleTTL time.Duration) *browserPool {
+	base.ProxyLaneStore = laneStore
+	if maxProcesses <= 0 {
+		maxProcesses = 4
 	}
+	pool := &browserPool{
+		base:         base,
+		laneStore:    laneStore,
+		maxProcesses: maxProcesses,
+		idleTTL:      idleTTL,
+		browsers:     map[string]*pooledBrowser{},
+		stopSweeper:  make(chan struct{}),
+		sweeperDone:  make(chan struct{}),
+	}
+	// A configured global proxy (legacy) becomes a pre-bound entry on the
+	// shared "direct" key so requests without a per-request proxy still use it.
+	if launchURL := strings.TrimSpace(defaultLaunchProxyURL); launchURL != "" {
+		pool.browsers[directBrowserKey] = &pooledBrowser{
+			launchProxyURL: launchURL,
+			lastUsedAt:     time.Now(),
+		}
+	}
+	if idleTTL > 0 {
+		go pool.sweepIdle()
+	} else {
+		close(pool.sweeperDone)
+	}
+	return pool
 }
 
-func (p *browserPool) get(proxyURL string) (*core.Browser, error) {
-	key := strings.TrimSpace(proxyURL)
-	if key == "" {
-		key = "direct"
+// browserPoolKey derives the pool key from a request's proxy URL. Authenticated
+// HTTP/HTTPS proxies get their own Chrome keyed by scheme+host+port+username.
+// Empty/unauthenticated/SOCKS request URLs fall through to the shared
+// "direct" Chrome.
+func browserPoolKey(requestProxyURL string) string {
+	requestProxyURL = strings.TrimSpace(requestProxyURL)
+	if requestProxyURL == "" {
+		return directBrowserKey
+	}
+	normalized, err := core.NormalizeProxyURL(requestProxyURL)
+	if err != nil || normalized == "" {
+		return directBrowserKey
+	}
+	parsed, err := url.Parse(normalized)
+	if err != nil {
+		return directBrowserKey
+	}
+	if parsed.Scheme != "http" && parsed.Scheme != "https" {
+		// Authenticated SOCKS is rejected upstream; unauthenticated SOCKS goes
+		// through the per-context proxy path on the shared Chrome.
+		return directBrowserKey
+	}
+	if parsed.User == nil {
+		return directBrowserKey
+	}
+	username := parsed.User.Username()
+	return fmt.Sprintf("%s|%s|%s", parsed.Scheme, parsed.Host, username)
+}
+
+// browserLaunchURL returns the URL to pass to launcher.Proxy for a given
+// request URL, or "" when the launch should be unproxied (direct + unauth).
+func browserLaunchURL(requestProxyURL string) string {
+	requestProxyURL = strings.TrimSpace(requestProxyURL)
+	if requestProxyURL == "" {
+		return ""
+	}
+	normalized, err := core.NormalizeProxyURL(requestProxyURL)
+	if err != nil || normalized == "" {
+		return ""
+	}
+	parsed, err := url.Parse(normalized)
+	if err != nil {
+		return ""
+	}
+	if parsed.Scheme != "http" && parsed.Scheme != "https" {
+		return ""
+	}
+	if parsed.User == nil {
+		return ""
+	}
+	return normalized
+}
+
+// get returns a Chrome that can route the supplied request URL. For
+// authenticated HTTP(S) proxies it returns the dedicated Chrome (launching one
+// if needed). For everything else it returns the shared "direct" Chrome.
+func (p *browserPool) get(requestProxyURL string) (*core.Browser, error) {
+	key := browserPoolKey(requestProxyURL)
+	launchURL := ""
+	if key != directBrowserKey {
+		launchURL = browserLaunchURL(requestProxyURL)
 	}
 
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
-	if b, ok := p.browser[key]; ok {
-		return b, nil
+	if entry, ok := p.browsers[key]; ok && entry.browser != nil {
+		entry.lastUsedAt = time.Now()
+		return entry.browser, nil
+	}
+
+	// Use any pre-bound launchProxyURL on the existing entry (e.g. legacy
+	// global proxy) when the caller didn't supply one.
+	if entry, ok := p.browsers[key]; ok && entry.browser == nil {
+		if launchURL == "" {
+			launchURL = entry.launchProxyURL
+		}
 	}
 
 	opts := p.base
-	opts.ProxyURL = proxyURL
-	b, err := core.NewBrowser(opts)
+	opts.ProxyURL = launchURL
+	browser, err := core.NewBrowser(opts)
 	if err != nil {
 		return nil, err
 	}
 
-	// Reuse one launched browser per unique effective proxy so startup stays lazy
-	// and engines with identical proxy policy don't spawn duplicate browser processes.
-	p.browser[key] = b
-	return b, nil
+	p.browsers[key] = &pooledBrowser{
+		browser:        browser,
+		launchProxyURL: launchURL,
+		lastUsedAt:     time.Now(),
+	}
+	p.evictLRULocked()
+	return browser, nil
+}
+
+func (p *browserPool) evictLRULocked() {
+	for len(p.browsers) > p.maxProcesses {
+		var (
+			oldestKey string
+			oldest    time.Time
+			found     bool
+		)
+		for key, entry := range p.browsers {
+			if !found || entry.lastUsedAt.Before(oldest) {
+				oldestKey = key
+				oldest = entry.lastUsedAt
+				found = true
+			}
+		}
+		if !found {
+			return
+		}
+		entry := p.browsers[oldestKey]
+		delete(p.browsers, oldestKey)
+		p.evictedLRU++
+		go closePooledBrowser(entry, "lru")
+	}
+}
+
+func (p *browserPool) sweepIdle() {
+	defer close(p.sweeperDone)
+	interval := p.idleTTL / 4
+	if interval < time.Second {
+		interval = time.Second
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-p.stopSweeper:
+			return
+		case now := <-ticker.C:
+			p.mu.Lock()
+			for key, entry := range p.browsers {
+				if entry.browser == nil {
+					continue
+				}
+				if now.Sub(entry.lastUsedAt) < p.idleTTL {
+					continue
+				}
+				delete(p.browsers, key)
+				p.evictedIdle++
+				go closePooledBrowser(entry, "idle")
+			}
+			p.mu.Unlock()
+		}
+	}
+}
+
+func closePooledBrowser(entry *pooledBrowser, reason string) {
+	if entry == nil || entry.browser == nil {
+		return
+	}
+	if err := entry.browser.Close(); err != nil {
+		logrus.WithError(err).WithField("evict_reason", reason).Debug("Browser pool: close evicted browser failed")
+	}
+}
+
+func (p *browserPool) dropLaneCookies(ctx context.Context, engineName string, q core.Query) {
+	if p == nil || p.laneStore == nil {
+		return
+	}
+	laneKey := core.ProxyLaneKeyForTenant(engineName, core.TenantFromContext(ctx), q, q.ProxyURL)
+	p.laneStore.DropCookies(laneKey)
+}
+
+func (p *browserPool) laneStats() core.LaneStats {
+	if p == nil || p.laneStore == nil {
+		return core.LaneStats{}
+	}
+	return p.laneStore.Stats()
+}
+
+func (p *browserPool) browserStats() core.BrowserPoolStats {
+	if p == nil {
+		return core.BrowserPoolStats{}
+	}
+	p.mu.Lock()
+	active := 0
+	for _, entry := range p.browsers {
+		if entry.browser != nil {
+			active++
+		}
+	}
+	stats := core.BrowserPoolStats{
+		Active:      active,
+		Max:         p.maxProcesses,
+		EvictedLRU:  p.evictedLRU,
+		EvictedIdle: p.evictedIdle,
+	}
+	p.mu.Unlock()
+	return stats
 }
 
 func (p *browserPool) close() error {
+	if p == nil {
+		return nil
+	}
+	close(p.stopSweeper)
+	<-p.sweeperDone
+
 	p.mu.Lock()
-	browsers := make([]*core.Browser, 0, len(p.browser))
-	for key, b := range p.browser {
-		browsers = append(browsers, b)
-		delete(p.browser, key)
+	entries := make([]*pooledBrowser, 0, len(p.browsers))
+	for key, entry := range p.browsers {
+		entries = append(entries, entry)
+		delete(p.browsers, key)
 	}
 	p.mu.Unlock()
 
 	var closeErr error
-	for _, browser := range browsers {
-		if browser == nil {
+	for _, entry := range entries {
+		if entry == nil || entry.browser == nil {
 			continue
 		}
-		if err := browser.Close(); err != nil {
+		if err := entry.browser.Close(); err != nil {
 			closeErr = errors.Join(closeErr, err)
 		}
 	}
@@ -297,17 +541,26 @@ func (p *browserPool) close() error {
 
 type pooledBrowserEngine struct {
 	name    string
-	limiter *rate.Limiter
 	opts    core.SearchEngineOptions
 	factory func(core.Browser, core.SearchEngineOptions) core.SearchEngine
 	pool    *browserPool
 
-	mu      sync.Mutex
-	engines map[string]core.SearchEngine
+	reportLaneStats bool
+}
+
+// parsableEngine wraps pooledBrowserEngine and additionally satisfies
+// core.HTMLParser for engines that have a stateless HTML parse function.
+type parsableEngine struct {
+	*pooledBrowserEngine
+	parseHTMLFn func(io.Reader) ([]core.SearchResult, error)
+}
+
+func (e *parsableEngine) ParseHTML(r io.Reader) ([]core.SearchResult, error) {
+	return e.parseHTMLFn(r)
 }
 
 func (e *pooledBrowserEngine) Search(ctx context.Context, q core.Query) ([]core.SearchResult, error) {
-	engine, err := e.getOrCreate(q.ProxyURL)
+	engine, err := e.resolveEngine(q)
 	if err != nil {
 		return nil, err
 	}
@@ -315,7 +568,7 @@ func (e *pooledBrowserEngine) Search(ctx context.Context, q core.Query) ([]core.
 }
 
 func (e *pooledBrowserEngine) SearchImage(ctx context.Context, q core.Query) ([]core.SearchResult, error) {
-	engine, err := e.getOrCreate(q.ProxyURL)
+	engine, err := e.resolveEngine(q)
 	if err != nil {
 		return nil, err
 	}
@@ -331,36 +584,45 @@ func (e *pooledBrowserEngine) Name() string {
 }
 
 func (e *pooledBrowserEngine) GetRateLimiter() *rate.Limiter {
-	return e.limiter
+	return e.opts.GetRateLimiter()
 }
 
-func (e *pooledBrowserEngine) getOrCreate(proxyURL string) (core.SearchEngine, error) {
-	key := strings.TrimSpace(proxyURL)
-	if key == "" {
-		key = "direct"
+func (e *pooledBrowserEngine) DropProxyLaneCookies(ctx context.Context, q core.Query) {
+	e.pool.dropLaneCookies(ctx, e.name, q)
+}
+
+func (e *pooledBrowserEngine) ProxyLaneStats() core.LaneStats {
+	if !e.reportLaneStats {
+		return core.LaneStats{}
 	}
+	return e.pool.laneStats()
+}
 
-	e.mu.Lock()
-	defer e.mu.Unlock()
-
-	if engine, ok := e.engines[key]; ok {
-		return engine, nil
+func (e *pooledBrowserEngine) BrowserPoolStats() core.BrowserPoolStats {
+	if !e.reportLaneStats {
+		return core.BrowserPoolStats{}
 	}
+	return e.pool.browserStats()
+}
 
-	browser, err := e.pool.get(proxyURL)
+// resolveEngine builds a fresh engine wrapper around the pool-resolved Browser.
+// The wrapper is intentionally not cached: pool eviction can replace the Chrome
+// behind a key, and a cached engine would carry a stale Browser value (closed
+// connection, dead browserAddr). Engines are thin wrappers, so per-call
+// construction is cheap.
+func (e *pooledBrowserEngine) resolveEngine(q core.Query) (core.SearchEngine, error) {
+	browser, err := e.pool.get(q.ProxyURL)
 	if err != nil {
 		return nil, err
 	}
-
-	engine := e.factory(*browser, e.opts)
-	e.engines[key] = engine
-	return engine, nil
+	return e.factory(*browser, e.opts), nil
 }
 
 type browserEngineSpec struct {
-	name    string
-	opts    core.SearchEngineOptions
-	factory func(core.Browser, core.SearchEngineOptions) core.SearchEngine
+	name        string
+	opts        core.SearchEngineOptions
+	factory     func(core.Browser, core.SearchEngineOptions) core.SearchEngine
+	parseHTMLFn func(io.Reader) ([]core.SearchResult, error)
 }
 
 func browserEngineSpecs() []browserEngineSpec {
@@ -371,6 +633,7 @@ func browserEngineSpecs() []browserEngineSpec {
 			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
 				return google.New(browser, opts)
 			},
+			parseHTMLFn: google.ParseHTML,
 		},
 		{
 			name: "yandex",
@@ -378,6 +641,7 @@ func browserEngineSpecs() []browserEngineSpec {
 			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
 				return yandex.New(browser, opts)
 			},
+			parseHTMLFn: yandex.ParseHTML,
 		},
 		{
 			name: "baidu",
@@ -385,6 +649,7 @@ func browserEngineSpecs() []browserEngineSpec {
 			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
 				return baidu.New(browser, opts)
 			},
+			parseHTMLFn: baidu.ParseHTML,
 		},
 		{
 			name: "bing",
@@ -392,6 +657,7 @@ func browserEngineSpecs() []browserEngineSpec {
 			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
 				return bing.New(browser, opts)
 			},
+			parseHTMLFn: bing.ParseHTML,
 		},
 		{
 			name: "duckduckgo",
@@ -399,34 +665,63 @@ func browserEngineSpecs() []browserEngineSpec {
 			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
 				return duckduckgo.New(browser, opts)
 			},
+			parseHTMLFn: duckduckgo.ParseHTML,
+		},
+		{
+			name: "ecosia",
+			opts: config.EcosiaConfig.SearchEngineOptions,
+			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
+				return ecosia.New(browser, opts)
+			},
+			parseHTMLFn: ecosia.ParseHTML,
 		},
 	}
 }
 
-func buildBrowserEngines(baseOpts core.BrowserOpts, proxyCfg core.ProxyConfig) ([]core.SearchEngine, func() error, error) {
-	pool := newBrowserPool(baseOpts)
+func buildBrowserEngines(baseOpts core.BrowserOpts, proxyCfg core.ProxyConfig) ([]core.SearchEngine, func() error, core.BrowserResolver, error) {
+	launchProxyURL := ""
+	if strings.TrimSpace(proxyCfg.Proxies.Global) != "" && !proxyCfg.Proxies.AllowRequestProxyURL {
+		launchProxyURL = proxyCfg.Proxies.Global
+	}
+	var laneStore *core.LaneStore
+	if proxyCfg.Proxies.Lanes.Enabled {
+		laneStore = core.NewLaneStore(proxyCfg.Proxies.Lanes.MaxLanes)
+	}
+	maxProcesses := config.App.MaxProcesses
+	if maxProcesses <= 0 {
+		maxProcesses = 4
+	}
+	idleTTL := config.App.IdleTTL
+	if idleTTL < 0 {
+		idleTTL = 0
+	}
+	pool := newBrowserPool(baseOpts, launchProxyURL, laneStore, maxProcesses, idleTTL)
 	specs := browserEngineSpecs()
 
 	engines := make([]core.SearchEngine, 0, len(specs))
-	for _, spec := range specs {
+	for idx, spec := range specs {
 		policy := resolveEngineProxyPolicy(proxyCfg, spec.name)
 		if err := validateBrowserProxyPolicy(proxyCfg, policy); err != nil {
-			return nil, nil, fmt.Errorf("browser proxy validation failed for engine %s: %w", spec.name, err)
+			return nil, nil, nil, fmt.Errorf("browser proxy validation failed for engine %s: %w", spec.name, err)
 		}
 
 		opts := spec.opts
 		opts.Init()
-		engines = append(engines, &pooledBrowserEngine{
-			name:    spec.name,
-			limiter: rate.NewLimiter(rate.Every(opts.GetRatelimit()), opts.RateBurst),
-			opts:    opts,
-			factory: spec.factory,
-			pool:    pool,
-			engines: map[string]core.SearchEngine{},
-		})
+		base := &pooledBrowserEngine{
+			name:            spec.name,
+			opts:            opts,
+			factory:         spec.factory,
+			pool:            pool,
+			reportLaneStats: idx == 0,
+		}
+		if spec.parseHTMLFn != nil {
+			engines = append(engines, &parsableEngine{pooledBrowserEngine: base, parseHTMLFn: spec.parseHTMLFn})
+		} else {
+			engines = append(engines, base)
+		}
 	}
 
-	return engines, pool.close, nil
+	return engines, pool.close, pool.get, nil
 }
 
 func validateBrowserProxyPolicy(proxyCfg core.ProxyConfig, policy core.ProxyPolicy) error {

cmd/serve_test.go

@@ -3,10 +3,130 @@ package cmd
 import (
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/karust/openserp/core"
 )
 
+func TestRawEngineCachesRateLimiter(t *testing.T) {
+	engine := &rawEngine{name: "google"}
+	first := engine.GetRateLimiter()
+	if first == nil {
+		t.Fatal("expected limiter")
+	}
+	if second := engine.GetRateLimiter(); second != first {
+		t.Fatal("expected rawEngine to return the cached limiter")
+	}
+}
+
+func TestBrowserPoolKey(t *testing.T) {
+	cases := []struct {
+		name string
+		raw  string
+		want string
+	}{
+		{"empty -> direct", "", directBrowserKey},
+		{"unauth http -> direct", "http://proxy.example:8080", directBrowserKey},
+		{"unauth socks -> direct", "socks5://proxy.example:1080", directBrowserKey},
+		{"auth socks -> direct (rejected upstream)", "socks5://user:pass@proxy.example:1080", directBrowserKey},
+		{"auth http", "http://user:pass@proxy.example:8080", "http|proxy.example:8080|user"},
+		{"auth https different scheme", "https://user:pass@proxy.example:8443", "https|proxy.example:8443|user"},
+		{"different password same key", "http://user:other-pass@proxy.example:8080", "http|proxy.example:8080|user"},
+		{"different user different key", "http://user2:pass@proxy.example:8080", "http|proxy.example:8080|user2"},
+		{"different host different key", "http://user:pass@proxy2.example:8080", "http|proxy2.example:8080|user"},
+		{"different port different key", "http://user:pass@proxy.example:9090", "http|proxy.example:9090|user"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := browserPoolKey(tc.raw); got != tc.want {
+				t.Fatalf("browserPoolKey(%q) = %q, want %q", tc.raw, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestBrowserLaunchURL(t *testing.T) {
+	cases := []struct {
+		name string
+		raw  string
+		want string
+	}{
+		{"empty -> empty", "", ""},
+		{"unauth http -> empty (per-context path)", "http://proxy.example:8080", ""},
+		{"unauth socks -> empty", "socks5://proxy.example:1080", ""},
+		{"auth http -> normalized", "http://user:pass@proxy.example:8080", "http://user:pass@proxy.example:8080"},
+		{"auth https -> normalized", "https://u:p@proxy.example:8443", "https://u:p@proxy.example:8443"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := browserLaunchURL(tc.raw)
+			if got != tc.want {
+				t.Fatalf("browserLaunchURL(%q) = %q, want %q", tc.raw, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestBrowserPoolEvictLRU(t *testing.T) {
+	// Pre-populate with bare entries (browser=nil) so we exercise eviction
+	// without launching real Chrome. closePooledBrowser handles nil safely.
+	pool := &browserPool{
+		maxProcesses: 2,
+		browsers:     map[string]*pooledBrowser{},
+		stopSweeper:  make(chan struct{}),
+		sweeperDone:  make(chan struct{}),
+	}
+	close(pool.sweeperDone)
+
+	now := time.Now()
+	pool.browsers["a"] = &pooledBrowser{lastUsedAt: now.Add(-3 * time.Second)}
+	pool.browsers["b"] = &pooledBrowser{lastUsedAt: now.Add(-2 * time.Second)}
+	pool.browsers["c"] = &pooledBrowser{lastUsedAt: now.Add(-1 * time.Second)}
+
+	pool.mu.Lock()
+	pool.evictLRULocked()
+	pool.mu.Unlock()
+
+	if _, ok := pool.browsers["a"]; ok {
+		t.Fatal("expected oldest entry 'a' to be evicted")
+	}
+	if _, ok := pool.browsers["b"]; !ok {
+		t.Fatal("expected entry 'b' to remain")
+	}
+	if _, ok := pool.browsers["c"]; !ok {
+		t.Fatal("expected entry 'c' to remain")
+	}
+	if pool.evictedLRU != 1 {
+		t.Fatalf("expected 1 LRU eviction, got %d", pool.evictedLRU)
+	}
+}
+
+func TestBrowserPoolBrowserStats(t *testing.T) {
+	pool := &browserPool{
+		maxProcesses: 4,
+		browsers:     map[string]*pooledBrowser{},
+		stopSweeper:  make(chan struct{}),
+		sweeperDone:  make(chan struct{}),
+	}
+	close(pool.sweeperDone)
+
+	// Pre-bound entry without a launched browser should not count as active.
+	pool.browsers["pre-bound"] = &pooledBrowser{launchProxyURL: "http://u:p@proxy.example:8080", lastUsedAt: time.Now()}
+	pool.evictedLRU = 2
+	pool.evictedIdle = 5
+
+	stats := pool.browserStats()
+	if stats.Max != 4 {
+		t.Fatalf("expected max=4, got %d", stats.Max)
+	}
+	if stats.Active != 0 {
+		t.Fatalf("expected active=0 (entry has no live browser), got %d", stats.Active)
+	}
+	if stats.EvictedLRU != 2 || stats.EvictedIdle != 5 {
+		t.Fatalf("unexpected stats: %#v", stats)
+	}
+}
+
 func TestValidateBrowserProxyPolicyRejectsAuthenticatedSocks(t *testing.T) {
 	tests := []struct {
 		name     string

config.yaml

@@ -7,16 +7,29 @@ server:
   insecure: true # Allow insecure TLS connections
 
 app:
-  # json|text. Empty means auto: text in debug mode, json otherwise.
-  log_format: "text"
-  timeout: 15 # Browser/search timeout in seconds
+  log_format: "text" # json|text
+  timeout: 15 # Browser/search timeout in seconds, per attempt; the request deadline is derived from this x retries
   browser_path: "" # Custom browser binary path (chrome/chromium/edge..)
-  profiles: "" # Optional JSON file path overriding built-in browser profiles
-  head: false # Show browser UI (headful mode)
+  profiles: "" # Overriding built-in browser profiles
+  head: false # Headful mode
   leakless: false # Force browser process cleanup after request
-  leave_head: false # Keep tabs open after request for debugging
+  leave_head: false # Keep tabs open after request
+  max_processes: 6 # Concurrent Chrome processes
+  idle_ttl: 5m # close a Chrome that has not served traffic for this long
+
+  mega_timeout: 90s # max total wait for /mega/* requests; slow engines return partial results
+  block_trackers: true
+  block_resources: "image,font,css,media"
+
+extract:
+  enabled: true
+  default_mode: auto # auto|fast|rendered
+  timeout: 20s
+  max_bytes: 2000000
+  max_concurrent: 2
 
 proxies:
+  allow_request_proxy_url: false
   # Force a single proxy for all engines.
   # Same behavior as passing --proxy on the CLI.
   #global: http://127.0.0.1:8080
@@ -28,14 +41,18 @@ proxies:
   #  - url: socks5h://127.0.0.1:1080
   #    tags: [eu]
   health:
-    failure_threshold: 3 # Disable proxy after this many consecutive failures
+    failure_threshold: 2 # Disable proxy after this many consecutive failures
+  lanes:
+    enabled: true # Reuse browser profile/cookies per engine + proxy session ID
+    max_lanes: 100 # LRU cap for sticky lanes kept in worker memory
+    drop_cookies_on_challenge: true # Clear lane cookies on captcha/challenge only
 
 cache:
-  ttl_seconds: 60 # Dedicated endpoint cache TTL in seconds (0 disables cache)
+  ttl_seconds: 120 # Dedicated endpoint cache TTL in seconds (0 disables cache)
   max_size: 1000 # Maximum cached dedicated responses before oldest-entry eviction
 
 resilience:
-  max_retries: 2 # Retry attempts per engine request (0 disables retries)
+  max_retries: 1 # Retry attempts per engine request (0 disables retries)
   allow_endpoint_fallback: false # Keep dedicated endpoints engine-pure by default
 
 # circuit_breaker:
@@ -47,7 +64,7 @@ cors:
   enabled: true
   allow_origins: "*"
   allow_methods: "GET, POST, OPTIONS"
-  allow_headers: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy"
+  allow_headers: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Proxy-URL, X-Proxy-Country, X-Proxy-Class, X-Proxy-Provider, X-Proxy-Session-ID, X-Request-ID, X-Tenant"
   max_age: 86400
 
 # 2captcha:
@@ -57,24 +74,29 @@ captcha:
   solver_enabled: false # Global captcha solver gate (requires 2captcha.apikey)
 
 google:
-  rate_requests: 4 # Allowed average requests per minute
-  rate_burst: 2 # Burst requests before limiter applies
-  captcha: true # Engine-level solver flag (also requires captcha.solver_enabled=true)
+  rate_requests: 60 # Allowed average requests per minute
+  rate_burst: 3 # Burst requests before limiter applies
+  #captcha: true # Engine-level solver flag (also requires captcha.solver_enabled=true)
 
 yandex:
-  rate_requests: 4
-  rate_burst: 2
+  rate_requests: 60
+  rate_burst: 3
 
 baidu:
-  rate_requests: 4
-  rate_burst: 2
+  rate_requests: 60
+  rate_burst: 3
   # No proxy tag means direct traffic
 
 bing:
-  rate_requests: 4
-  rate_burst: 2
+  rate_requests: 60
+  rate_burst: 3
   # No proxy tag means direct traffic
 
 duckduckgo:
-  rate_requests: 4
-  rate_burst: 2
+  rate_requests: 60
+  rate_burst: 3
+
+ecosia:
+  rate_requests: 60
+  rate_burst: 3
+  # No proxy tag means direct traffic

core/browser/patch.js

@@ -1,73 +1,273 @@
 // Stealth patches injected via EvalOnNewDocument.
 // Arguments are injected as a leading const block by the Go caller:
-//   const __langs = [...];   // navigator_langs from profile
-//   const __w = 1920;        // viewport width
-//   const __h = 1080;        // viewport height
+//   const __langs = [...];          // navigator_langs from profile
+//   const __w = 1920;               // viewport/content width
+//   const __h = 955;                // viewport/content height
+//   const __screenW = 1920;         // screen.width
+//   const __screenH = 1080;         // screen.height
+//   const __availW = 1920;          // screen.availWidth
+//   const __availH = 1040;          // screen.availHeight
+//   const __availTop = 0;           // screen.availTop
+//   const __outerW = 1920;          // window.outerWidth
+//   const __outerH = 1040;          // window.outerHeight
+//   const __webglVendor = "...";    // UNMASKED_VENDOR_WEBGL spoof
+//   const __webglRenderer = "...";  // UNMASKED_RENDERER_WEBGL spoof
+//
+// Scope: only patches that fix detectors with high ROI and low introspection
+// surface. Notably absent: Function.prototype.toString proxy, custom Worker
+// constructor, mass plugin/mimeType arrays, iframe contentWindow patching.
+// Those triggered server-side detection on Google in commit 612e0dc.
 (() => {
   'use strict';
 
-  // --- navigator.language / navigator.languages ---
-  //
-  // CDP Network.setUserAgentOverride(acceptLanguage) sets the HTTP header but
-  // does NOT update navigator.language / navigator.languages in JS. Those are
-  // read from the browser profile at context creation. On Linux headless they
-  // reflect the system ICU locale (usually "en-US" regardless of profile).
-  //
-  // We patch them here. The key to being undetectable: define with a getter
-  // first (requires configurable:true), then immediately lock the descriptor
-  // back to configurable:false so it looks exactly like a native property.
-  const primary = __langs.length ? __langs[0] : 'en-US';
-
   const sealGetter = (target, prop, fn) => {
     try {
       Object.defineProperty(target, prop, {
         get: fn,
         set: undefined,
         enumerable: true,
-        configurable: true,   // must be true to set a getter
-      });
-      Object.defineProperty(target, prop, {
-        configurable: false,  // seal: now indistinguishable from native
+        configurable: true,
       });
+      Object.defineProperty(target, prop, { configurable: false });
     } catch (_) {}
   };
 
+  // --- navigator.webdriver ---
+  // headless Chrome sets this to true. Delete it so getter returns undefined.
+  // (We rely on `disable-blink-features=AutomationControlled` already turning
+  // this off via the launcher; this is belt-and-braces in case it leaks.)
+  try {
+    if (typeof navigator.webdriver !== 'undefined') {
+      delete Object.getPrototypeOf(navigator).webdriver;
+    }
+  } catch (_) {}
+
+  // --- navigator.language / navigator.languages ---
+  // CDP setUserAgentOverride sets the HTTP header but not these JS props.
+  const primary = __langs.length ? __langs[0] : 'en-US';
   const patchLangs = (target) => {
     if (!target) return;
     sealGetter(target, 'language', () => primary);
     sealGetter(target, 'languages', () => Object.freeze(__langs.slice()));
   };
-
-  // Patch the navigator instance (Linux headless stores own-props here)
-  // and Navigator.prototype (other platforms / future Chrome versions).
-  patchLangs(navigator);
   patchLangs(Object.getPrototypeOf(navigator));
   if (typeof WorkerNavigator !== 'undefined') {
     patchLangs(WorkerNavigator.prototype);
   }
 
   // --- screen dimensions ---
-  //
   // EmulationSetDeviceMetricsOverride sets the CSS viewport but leaves
-  // window.screen.* at headless defaults. Checkers compare screen size
-  // against viewport and flag the mismatch as automation.
-  //
-  // Screen properties live on Screen.prototype as non-configurable getters.
-  // Patching the prototype makes them look native.
-  const patchScreen = () => {
-    const proto = typeof Screen !== 'undefined' ? Screen.prototype : null;
-    if (!proto) return;
-    sealGetter(proto, 'width',       () => __w);
-    sealGetter(proto, 'height',      () => __h);
-    sealGetter(proto, 'availWidth',  () => __w);
-    sealGetter(proto, 'availHeight', () => __h);
-    sealGetter(proto, 'availLeft',   () => 0);
-    sealGetter(proto, 'availTop',    () => 0);
-  };
-  patchScreen();
+  // window.screen.* at headless defaults.
+  const screenProto = typeof Screen !== 'undefined' ? Screen.prototype : null;
+  if (screenProto) {
+    sealGetter(screenProto, 'width', () => __screenW);
+    sealGetter(screenProto, 'height', () => __screenH);
+    sealGetter(screenProto, 'availWidth', () => __availW);
+    sealGetter(screenProto, 'availHeight', () => __availH);
+    sealGetter(screenProto, 'availLeft', () => 0);
+    sealGetter(screenProto, 'availTop', () => __availTop);
+    // Headless reports colorDepth/pixelDepth=24 already in most builds, but
+    // some checks see 0 in WSL/Docker. Lock to 24 which matches real Chrome.
+    sealGetter(screenProto, 'colorDepth', () => 24);
+    sealGetter(screenProto, 'pixelDepth', () => 24);
+  }
+  if (typeof window !== 'undefined') {
+    sealGetter(window, 'outerWidth', () => __outerW);
+    sealGetter(window, 'outerHeight', () => __outerH);
+  }
 
-  // window.outerWidth/Height are own configurable properties; Chrome normally
-  // sets them to match the OS window size. In headless they are 0.
-  sealGetter(window, 'outerWidth',  () => __w);
-  sealGetter(window, 'outerHeight', () => __h);
+  // --- WebGL vendor/renderer spoof ---
+  // Patches getParameter(37445=UNMASKED_VENDOR_WEBGL, 37446=UNMASKED_RENDERER_WEBGL)
+  // on WebGLRenderingContext.prototype and WebGL2RenderingContext.prototype.
+  // Only delegates to the original for all other params, so behavioral
+  // signals (extension list, real pixel rendering) still pass through.
+  const patchWebGLProto = (proto) => {
+    if (!proto) return;
+    const original = proto.getParameter;
+    if (typeof original !== 'function') return;
+    const replacement = function getParameter(parameter) {
+      if (parameter === 37445) return __webglVendor;
+      if (parameter === 37446) return __webglRenderer;
+      return original.apply(this, arguments);
+    };
+    try {
+      Object.defineProperty(proto, 'getParameter', {
+        value: replacement,
+        writable: true,
+        enumerable: false,
+        configurable: true,
+      });
+    } catch (_) {}
+  };
+  if (typeof WebGLRenderingContext !== 'undefined') {
+    patchWebGLProto(WebGLRenderingContext.prototype);
+  }
+  if (typeof WebGL2RenderingContext !== 'undefined') {
+    patchWebGLProto(WebGL2RenderingContext.prototype);
+  }
+
+  // --- navigator.plugins / mimeTypes ---
+  // Match real Chrome 136 exactly: 5 plugins (all "internal-pdf-viewer"), each
+  // exposing application/pdf + text/pdf. navigator.mimeTypes dedupes to 2.
+  // Each MimeType.enabledPlugin must back-reference the FIRST plugin that owns
+  // that type (Chrome's invariant). Plugin order is fixed.
+  try {
+    const PluginArrayProto = typeof PluginArray !== 'undefined' ? PluginArray.prototype : null;
+    const PluginProto = typeof Plugin !== 'undefined' ? Plugin.prototype : null;
+    const MimeTypeArrayProto = typeof MimeTypeArray !== 'undefined' ? MimeTypeArray.prototype : null;
+    const MimeTypeProto = typeof MimeType !== 'undefined' ? MimeType.prototype : null;
+
+    if (PluginArrayProto && PluginProto && MimeTypeArrayProto && MimeTypeProto) {
+      const pluginNames = [
+        'PDF Viewer',
+        'Chrome PDF Viewer',
+        'Chromium PDF Viewer',
+        'Microsoft Edge PDF Viewer',
+        'WebKit built-in PDF',
+      ];
+      const mimeSpecs = [
+        { type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format' },
+        { type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format' },
+      ];
+
+      // Two MimeType instances, each enabledPlugin points to the first plugin
+      // (PDF Viewer) per Chrome's invariant: navigator.mimeTypes[i].enabledPlugin
+      // === navigator.plugins[0] for every PDF mime.
+      const sharedMimes = mimeSpecs.map((spec) => {
+        const m = Object.create(MimeTypeProto);
+        Object.defineProperty(m, 'type', { value: spec.type, enumerable: true });
+        Object.defineProperty(m, 'suffixes', { value: spec.suffixes, enumerable: true });
+        Object.defineProperty(m, 'description', { value: spec.description, enumerable: true });
+        return m;
+      });
+
+      const plugins = pluginNames.map((name) => {
+        const p = Object.create(PluginProto);
+        Object.defineProperty(p, 'name', { value: name, enumerable: true });
+        Object.defineProperty(p, 'filename', { value: 'internal-pdf-viewer', enumerable: true });
+        Object.defineProperty(p, 'description', { value: 'Portable Document Format', enumerable: true });
+        Object.defineProperty(p, 'length', { value: sharedMimes.length, enumerable: true });
+        sharedMimes.forEach((m, i) => {
+          Object.defineProperty(p, String(i), { value: m, enumerable: true });
+          Object.defineProperty(p, m.type, { value: m });
+        });
+        return p;
+      });
+
+      // Set enabledPlugin AFTER plugins are constructed, pointing to plugins[0].
+      sharedMimes.forEach((m) => {
+        Object.defineProperty(m, 'enabledPlugin', { value: plugins[0], enumerable: true });
+      });
+
+      const pluginArr = Object.create(PluginArrayProto);
+      Object.defineProperty(pluginArr, 'length', { value: plugins.length, enumerable: true });
+      plugins.forEach((p, i) => {
+        Object.defineProperty(pluginArr, String(i), { value: p, enumerable: true });
+        Object.defineProperty(pluginArr, p.name, { value: p });
+      });
+
+      const mimeArr = Object.create(MimeTypeArrayProto);
+      Object.defineProperty(mimeArr, 'length', { value: sharedMimes.length, enumerable: true });
+      sharedMimes.forEach((m, i) => {
+        Object.defineProperty(mimeArr, String(i), { value: m, enumerable: true });
+        Object.defineProperty(mimeArr, m.type, { value: m });
+      });
+
+      sealGetter(Object.getPrototypeOf(navigator), 'plugins', () => pluginArr);
+      sealGetter(Object.getPrototypeOf(navigator), 'mimeTypes', () => mimeArr);
+    }
+  } catch (_) {}
+
+  // --- navigator.permissions.query notifications fix ---
+  // headless returns 'denied' for notifications when Notification.permission is
+  // 'default'. Real Chrome returns 'prompt' in that case. Sannysoft checks this
+  // mismatch (permissions_new / headchr_permissions).
+  try {
+    if (navigator.permissions && typeof navigator.permissions.query === 'function') {
+      const proto = Object.getPrototypeOf(navigator.permissions);
+      const desc = Object.getOwnPropertyDescriptor(proto, 'query');
+      if (desc && typeof desc.value === 'function') {
+        const original = desc.value;
+        const replacement = function query(parameters) {
+          if (parameters && parameters.name === 'notifications' &&
+              typeof Notification !== 'undefined' && Notification.permission === 'default') {
+            return Promise.resolve({ state: 'prompt', onchange: null });
+          }
+          return original.apply(this, arguments);
+        };
+        Object.defineProperty(proto, 'query', {
+          value: replacement,
+          writable: desc.writable,
+          enumerable: desc.enumerable,
+          configurable: desc.configurable,
+        });
+      }
+    }
+  } catch (_) {}
+
+  // --- getBoundingClientRect / getClientRects subpixel jitter ---
+  // Headless Chrome returns integer-valued rects; real Chrome returns subpixel
+  // floats due to CSS layout fractions. Fingerprinters hash rect tuples; even
+  // a sub-pixel offset breaks the canonical "headless rect" hash.
+  // Jitter is deterministic per-element (based on element identity) so the
+  // same element returns the same value across calls within the page lifetime.
+  try {
+    const rectProto = typeof DOMRect !== 'undefined' ? DOMRect.prototype : null;
+    const elProto = typeof Element !== 'undefined' ? Element.prototype : null;
+    if (rectProto && elProto) {
+      const wmJitter = new WeakMap();
+      const jitterFor = (el) => {
+        let j = wmJitter.get(el);
+        if (!j) {
+          // Tiny noise in [-0.05, +0.05) — well below visual threshold but
+          // changes hash output. Generated once per element.
+          j = {
+            x: (Math.random() - 0.5) * 0.1,
+            y: (Math.random() - 0.5) * 0.1,
+          };
+          wmJitter.set(el, j);
+        }
+        return j;
+      };
+      const origGBCR = elProto.getBoundingClientRect;
+      Object.defineProperty(elProto, 'getBoundingClientRect', {
+        value: function getBoundingClientRect() {
+          const r = origGBCR.apply(this, arguments);
+          const j = jitterFor(this);
+          // DOMRect is mutable; nudge x/y. width/height left intact so layout
+          // calculations don't drift.
+          try { r.x = r.x + j.x; r.y = r.y + j.y; } catch (_) {}
+          return r;
+        },
+        writable: true,
+        enumerable: false,
+        configurable: true,
+      });
+    }
+  } catch (_) {}
+
+  // --- window.chrome.runtime stub ---
+  // Real Chrome exposes window.chrome with a .runtime sub-object.
+  // headless leaves window.chrome empty, which sannysoft (chrome_new,
+  // headchr_chrome_obj) flags. A minimal runtime stub satisfies the check
+  // without touching method behavior.
+  try {
+    if (typeof window !== 'undefined') {
+      if (!window.chrome) {
+        Object.defineProperty(window, 'chrome', { value: {}, writable: true, configurable: true });
+      }
+      if (window.chrome && !window.chrome.runtime) {
+        Object.defineProperty(window.chrome, 'runtime', {
+          value: {
+            OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', UPDATE: 'update' },
+            OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' },
+            PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', WIN: 'win' },
+          },
+          writable: true,
+          enumerable: true,
+          configurable: true,
+        });
+      }
+    }
+  } catch (_) {}
 })();

core/browser/profile.go

@@ -4,8 +4,10 @@ import (
 	_ "embed"
 	"encoding/json"
 	"fmt"
+	"hash/fnv"
 	"os"
 	"runtime"
+	"slices"
 	"strings"
 	"sync"
 )
@@ -35,6 +37,10 @@ type Profile struct {
 	Locale          string         `json:"locale"`
 	Timezone        string         `json:"timezone"`
 	Viewport        Viewport       `json:"viewport"`
+	WebGLVendor     string         `json:"webgl_vendor"`
+	WebGLRenderer   string         `json:"webgl_renderer"`
+	Tags            []string       `json:"tags"`
+	Weight          int            `json:"weight"`
 }
 
 type catalogConfig struct {
@@ -44,10 +50,10 @@ type catalogConfig struct {
 }
 
 const (
-	ProfileChromeWinUS   = "chrome-win-us"
+	ProfileChromeWinUS   = "chrome-win-uhd620"
 	ProfileChromeWinRU   = "chrome-win-ru"
-	ProfileChromeMacUS   = "chrome-macos-us"
-	ProfileChromeLinuxUS = "chrome-linux-us"
+	ProfileChromeMacUS   = "chrome-macos-intel-iris"
+	ProfileChromeLinuxUS = "chrome-linux-mesa-uhd620"
 	ProfileChromeLinuxRU = "chrome-linux-ru"
 )
 
@@ -182,7 +188,19 @@ func Catalog() []Profile {
 	return out
 }
 
+// SelectProfile returns a deterministic profile for the given engine and region.
+// It respects lane_profile_ids overrides and falls back to the OS-preferred default.
+// Used by tests and single-instance callers; internally delegates to SelectProfileForSession with empty salt.
 func SelectProfile(engine string, region string) Profile {
+	return SelectProfileForSession(engine, region, "")
+}
+
+// SelectProfileForSession picks a profile for (engine, region, salt).
+// If a lane_profile_ids override exists it is always honoured.
+// Empty salt picks the first eligible profile (same as SelectProfile).
+// Non-empty salt uses weighted selection seeded by FNV-1a hash of salt,
+// giving each session a stable but varied profile.
+func SelectProfileForSession(engine, region, salt string) Profile {
 	engine = NormalizeEngine(engine)
 	region = NormalizeRegion(region)
 	if region == "" {
@@ -196,7 +214,101 @@ func SelectProfile(engine string, region string) Profile {
 	if ok {
 		return profileByID(profileID)
 	}
-	return profileByID(defaultProfileID(region))
+
+	pool := eligibleProfiles(engine, region)
+	return pickWeighted(pool, salt)
+}
+
+type weightedProfile struct {
+	profile Profile
+	weight  int
+}
+
+// eligibleProfiles builds the weighted pool for (engine, region).
+// Linux profiles are preferred 4x on linux runtime; Windows 4x on windows; macOS 4x on darwin.
+// Profiles tagged "ru" are included only when region == "ru"; "ru"-tagged profiles are excluded otherwise.
+func eligibleProfiles(engine, region string) []weightedProfile {
+	profileCatalogMu.RLock()
+	snap := make([]Profile, 0, len(catalog))
+	for _, p := range catalog {
+		snap = append(snap, p)
+	}
+	profileCatalogMu.RUnlock()
+
+	// Stable ordering so empty-salt picks are deterministic across map iterations.
+	slices.SortFunc(snap, func(a, b Profile) int {
+		return strings.Compare(a.ID, b.ID)
+	})
+
+	goos := runtime.GOOS
+
+	var pool []weightedProfile
+	for _, p := range snap {
+		isRu := slices.Contains(p.Tags, "ru")
+		if region == "ru" && !isRu {
+			continue
+		}
+		if region != "ru" && isRu {
+			continue
+		}
+
+		w := p.Weight
+		if w <= 0 {
+			w = 1
+		}
+
+		platformLower := strings.ToLower(p.Platform)
+		switch goos {
+		case "linux":
+			if platformLower == "linux" {
+				w *= 4
+			}
+		case "windows":
+			if platformLower == "windows" {
+				w *= 4
+			}
+		case "darwin":
+			if platformLower == "macos" {
+				w *= 4
+			}
+		}
+
+		pool = append(pool, weightedProfile{profile: p, weight: w})
+	}
+
+	return pool
+}
+
+// pickWeighted selects a profile from pool using FNV-1a hash of salt modulo total weight.
+// Empty salt returns the first profile in the pool (deterministic for tests).
+func pickWeighted(pool []weightedProfile, salt string) Profile {
+	if len(pool) == 0 {
+		return profileByID(defaultProfileID("us"))
+	}
+	if salt == "" {
+		return pool[0].profile
+	}
+
+	total := 0
+	for _, wp := range pool {
+		total += wp.weight
+	}
+	if total <= 0 {
+		return pool[0].profile
+	}
+
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(salt))
+	idx := int(h.Sum32()) % total
+
+	cumulative := 0
+	for _, wp := range pool {
+		cumulative += wp.weight
+		if idx < cumulative {
+			return wp.profile
+		}
+	}
+	return pool[len(pool)-1].profile
 }
 
 func LaneKey(engine string, region string) string {
@@ -255,6 +367,16 @@ func NormalizeRegion(region string) string {
 	}
 }
 
+// ProfileByID looks up a profile by exact ID. Returns (profile, true) when found,
+// (zero, false) when the ID is not in the catalog. Unlike the internal profileByID,
+// it does not fall back to a default; the caller decides what to do on miss.
+func ProfileByID(profileID string) (Profile, bool) {
+	profileCatalogMu.RLock()
+	defer profileCatalogMu.RUnlock()
+	profile, ok := catalog[strings.TrimSpace(profileID)]
+	return profile, ok
+}
+
 func profileByID(profileID string) Profile {
 	profileCatalogMu.RLock()
 	defer profileCatalogMu.RUnlock()

core/browser/profile_coherence_test.go

@@ -5,9 +5,11 @@ package browser_test
 
 import (
 	"context"
+	_ "embed"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 	"time"
 
@@ -17,6 +19,9 @@ import (
 	"github.com/karust/openserp/testutil"
 )
 
+//go:embed profile_surface_test.js
+var profileSurfaceScript string
+
 func TestProfileCoherence(t *testing.T) {
 	testutil.RequireIntegration(t)
 
@@ -59,9 +64,9 @@ func TestProfileCoherence(t *testing.T) {
 
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
-			expected := browserprofile.SelectProfile(tc.engine, tc.region)
 			ctx := core.WithEngine(context.Background(), tc.engine)
 			ctx = core.WithProfileRegion(ctx, tc.region)
+			ctx = core.WithBrowserProfileUsage(ctx)
 
 			page, err := browser.Navigate(ctx, fixture.URL)
 			if err != nil {
@@ -78,6 +83,8 @@ func TestProfileCoherence(t *testing.T) {
 				t.Fatalf("collect profile surfaces: %v", err)
 			}
 
+			expected := selectedProfileFromContext(t, ctx)
+			expected.UserAgent = expectedUserAgentForRuntime(expected.UserAgent, got.UserAgent)
 			if got.UserAgent != expected.UserAgent {
 				t.Fatalf("navigator.userAgent mismatch:\nexpected: %s\nactual:   %s", expected.UserAgent, got.UserAgent)
 			}
@@ -120,63 +127,48 @@ func TestProfileCoherence(t *testing.T) {
 			if got.WorkerTimezone != got.Timezone {
 				t.Fatalf("worker timezone mismatch: main %q worker %q", got.Timezone, got.WorkerTimezone)
 			}
+			if got.WorkerWebGLVendor != expected.WebGLVendor {
+				t.Fatalf("worker WebGL vendor mismatch: expected %q got %q", expected.WebGLVendor, got.WorkerWebGLVendor)
+			}
+			if got.WorkerWebGLRenderer != expected.WebGLRenderer {
+				t.Fatalf("worker WebGL renderer mismatch: expected %q got %q", expected.WebGLRenderer, got.WorkerWebGLRenderer)
+			}
+			if got.InnerHeight >= got.OuterHeight {
+				t.Fatalf("innerHeight should be smaller than outerHeight, got inner=%d outer=%d", got.InnerHeight, got.OuterHeight)
+			}
+			if got.OuterHeight > got.ScreenAvailHeight {
+				t.Fatalf("outerHeight should fit in screen.availHeight, got outer=%d avail=%d", got.OuterHeight, got.ScreenAvailHeight)
+			}
+			if got.ScreenAvailHeight >= got.ScreenHeight {
+				t.Fatalf("screen.availHeight should be smaller than screen.height, got avail=%d screen=%d", got.ScreenAvailHeight, got.ScreenHeight)
+			}
 		})
 	}
 }
 
 type profileSurface struct {
-	UserAgent                string   `json:"userAgent"`
-	Platform                 string   `json:"platform"`
-	NavigatorPlatform        string   `json:"navigatorPlatform"`
-	NavigatorLanguages       []string `json:"navigatorLanguages"`
-	Timezone                 string   `json:"timezone"`
-	Locale                   string   `json:"locale"`
-	WebdriverType            string   `json:"webdriverType"`
-	WebdriverOwnPropPresent  bool     `json:"webdriverOwnPropPresent"`
-	WorkerUserAgent          string   `json:"workerUserAgent"`
-	WorkerPlatform           string   `json:"workerPlatform"`
-	WorkerNavigatorLangs     []string `json:"workerNavigatorLangs"`
-	WorkerTimezone           string   `json:"workerTimezone"`
+	UserAgent               string   `json:"userAgent"`
+	Platform                string   `json:"platform"`
+	NavigatorPlatform       string   `json:"navigatorPlatform"`
+	NavigatorLanguages      []string `json:"navigatorLanguages"`
+	Timezone                string   `json:"timezone"`
+	Locale                  string   `json:"locale"`
+	WebdriverType           string   `json:"webdriverType"`
+	WebdriverOwnPropPresent bool     `json:"webdriverOwnPropPresent"`
+	WorkerUserAgent         string   `json:"workerUserAgent"`
+	WorkerPlatform          string   `json:"workerPlatform"`
+	WorkerNavigatorLangs    []string `json:"workerNavigatorLangs"`
+	WorkerTimezone          string   `json:"workerTimezone"`
+	WorkerWebGLVendor       string   `json:"workerWebGLVendor"`
+	WorkerWebGLRenderer     string   `json:"workerWebGLRenderer"`
+	InnerHeight             int      `json:"innerHeight"`
+	OuterHeight             int      `json:"outerHeight"`
+	ScreenHeight            int      `json:"screenHeight"`
+	ScreenAvailHeight       int      `json:"screenAvailHeight"`
 }
 
 func browserProfileSurface(page *rod.Page) (profileSurface, error) {
-	result, err := page.Eval(`async () => {
-		const workerData = await new Promise((resolve) => {
-			try {
-				const source = "self.onmessage=()=>{self.postMessage({userAgent:self.navigator.userAgent||'',platform:self.navigator.platform||'',navigatorLanguages:Array.from(self.navigator.languages||[]),timezone:Intl.DateTimeFormat().resolvedOptions().timeZone||''});};";
-				const blob = new Blob([source], { type: 'application/javascript' });
-				const url = URL.createObjectURL(blob);
-				const worker = new Worker(url);
-				worker.onmessage = (event) => {
-					resolve(event.data || {});
-					worker.terminate();
-					URL.revokeObjectURL(url);
-				};
-				worker.onerror = () => {
-					resolve({});
-					worker.terminate();
-					URL.revokeObjectURL(url);
-				};
-				worker.postMessage('run');
-			} catch (_) {
-				resolve({});
-			}
-		});
-		return {
-			userAgent: navigator.userAgent || "",
-			platform: navigator.userAgentData ? (navigator.userAgentData.platform || "") : "",
-			navigatorPlatform: navigator.platform || "",
-			navigatorLanguages: Array.from(navigator.languages || []),
-			timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || "",
-			locale: Intl.DateTimeFormat().resolvedOptions().locale || "",
-			webdriverType: typeof navigator.webdriver,
-			webdriverOwnPropPresent: Object.getOwnPropertyNames(navigator).includes('webdriver'),
-			workerUserAgent: workerData.userAgent || "",
-			workerPlatform: workerData.platform || "",
-			workerNavigatorLangs: Array.from(workerData.navigatorLanguages || []),
-			workerTimezone: workerData.timezone || "",
-		};
-	}`)
+	result, err := page.Eval(profileSurfaceScript)
 	if err != nil {
 		return profileSurface{}, err
 	}
@@ -198,3 +190,42 @@ func expectedNavigatorPlatform(platform string) string {
 		return "Linux x86_64"
 	}
 }
+
+func selectedProfileFromContext(t *testing.T, ctx context.Context) browserprofile.Profile {
+	t.Helper()
+
+	ids := core.BrowserProfileIDsFromContext(ctx)
+	if len(ids) == 0 {
+		t.Fatal("expected selected browser profile id")
+	}
+	profile, ok := browserprofile.ProfileByID(ids[0])
+	if !ok {
+		t.Fatalf("selected browser profile %q not found", ids[0])
+	}
+	return profile
+}
+
+func expectedUserAgentForRuntime(profileUserAgent, runtimeUserAgent string) string {
+	runtimeChrome := chromeToken(runtimeUserAgent)
+	if runtimeChrome == "" {
+		return profileUserAgent
+	}
+	profileChrome := chromeToken(profileUserAgent)
+	if profileChrome == "" {
+		return profileUserAgent
+	}
+	return strings.Replace(profileUserAgent, profileChrome, runtimeChrome, 1)
+}
+
+func chromeToken(userAgent string) string {
+	const prefix = "Chrome/"
+	start := strings.Index(userAgent, prefix)
+	if start < 0 {
+		return ""
+	}
+	end := strings.IndexByte(userAgent[start:], ' ')
+	if end < 0 {
+		return userAgent[start:]
+	}
+	return userAgent[start : start+end]
+}

core/browser/profile_surface_test.js

@@ -0,0 +1,66 @@
+async () => {
+  const workerData = await new Promise((resolve) => {
+    try {
+      const source = [
+        "self.onmessage = () => {",
+        "let webGLVendor = '';",
+        "let webGLRenderer = '';",
+        "try {",
+        "const canvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : null;",
+        "const gl = canvas ? (canvas.getContext('webgl') || canvas.getContext('experimental-webgl') || canvas.getContext('webgl2')) : null;",
+        "const debugInfo = gl && gl.getExtension('WEBGL_debug_renderer_info');",
+        "if (gl && debugInfo) {",
+        "webGLVendor = gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL) || '';",
+        "webGLRenderer = gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL) || '';",
+        "}",
+        "} catch (_) {}",
+        "self.postMessage({",
+        "userAgent: self.navigator.userAgent || '',",
+        "platform: self.navigator.platform || '',",
+        "navigatorLanguages: Array.from(self.navigator.languages || []),",
+        "timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || '',",
+        "webGLVendor,",
+        "webGLRenderer,",
+        "});",
+        "};",
+      ].join("\n");
+      const blob = new Blob([source], { type: "application/javascript" });
+      const url = URL.createObjectURL(blob);
+      const worker = new Worker(url);
+      worker.onmessage = (event) => {
+        resolve(event.data || {});
+        worker.terminate();
+        URL.revokeObjectURL(url);
+      };
+      worker.onerror = () => {
+        resolve({});
+        worker.terminate();
+        URL.revokeObjectURL(url);
+      };
+      worker.postMessage("run");
+    } catch (_) {
+      resolve({});
+    }
+  });
+
+  return {
+    userAgent: navigator.userAgent || "",
+    platform: navigator.userAgentData ? (navigator.userAgentData.platform || "") : "",
+    navigatorPlatform: navigator.platform || "",
+    navigatorLanguages: Array.from(navigator.languages || []),
+    timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || "",
+    locale: Intl.DateTimeFormat().resolvedOptions().locale || "",
+    webdriverType: typeof navigator.webdriver,
+    webdriverOwnPropPresent: Object.getOwnPropertyNames(navigator).includes("webdriver"),
+    workerUserAgent: workerData.userAgent || "",
+    workerPlatform: workerData.platform || "",
+    workerNavigatorLangs: Array.from(workerData.navigatorLanguages || []),
+    workerTimezone: workerData.timezone || "",
+    workerWebGLVendor: workerData.webGLVendor || "",
+    workerWebGLRenderer: workerData.webGLRenderer || "",
+    innerHeight: window.innerHeight || 0,
+    outerHeight: window.outerHeight || 0,
+    screenHeight: window.screen ? (window.screen.height || 0) : 0,
+    screenAvailHeight: window.screen ? (window.screen.availHeight || 0) : 0,
+  };
+}

core/browser/profile_test.go

@@ -3,37 +3,30 @@ package browser
 import (
 	"os"
 	"path/filepath"
+	"slices"
 	"testing"
 )
 
 func TestSelectProfile(t *testing.T) {
 	tests := []struct {
-		name         string
-		engine       string
-		region       string
-		wantLocale   string
-		wantTimezone string
+		name   string
+		engine string
+		region string
 	}{
 		{
-			name:         "google ru lane",
-			engine:       "google",
-			region:       "ru",
-			wantLocale:   "ru-RU",
-			wantTimezone: "Europe/Moscow",
+			name:   "google ru lane",
+			engine: "google",
+			region: "ru",
 		},
 		{
-			name:         "yandex defaults to ru",
-			engine:       "yandex",
-			region:       "",
-			wantLocale:   "ru-RU",
-			wantTimezone: "Europe/Moscow",
+			name:   "yandex defaults to ru",
+			engine: "yandex",
+			region: "",
 		},
 		{
-			name:         "google default lane uses us profile",
-			engine:       "google",
-			region:       "en",
-			wantLocale:   "en-US",
-			wantTimezone: "America/New_York",
+			name:   "google default lane uses us profile",
+			engine: "google",
+			region: "en",
 		},
 	}
 
@@ -43,19 +36,47 @@ func TestSelectProfile(t *testing.T) {
 			if profile.ID == "" {
 				t.Fatal("expected non-empty profile ID")
 			}
-			if profile.Locale != tt.wantLocale {
-				t.Fatalf("expected locale %q, got %q", tt.wantLocale, profile.Locale)
-			}
-			if profile.Timezone != tt.wantTimezone {
-				t.Fatalf("expected timezone %q, got %q", tt.wantTimezone, profile.Timezone)
-			}
 			if profile.Platform == "" {
 				t.Fatal("expected non-empty platform")
 			}
+			if profile.Viewport.Width <= 0 || profile.Viewport.Height <= 0 {
+				t.Fatalf("expected positive viewport, got %+v", profile.Viewport)
+			}
 		})
 	}
 }
 
+func TestSelectProfileForSession(t *testing.T) {
+	t.Run("same salt returns same profile", func(t *testing.T) {
+		first := SelectProfileForSession("google", "us", "session-abc")
+		for i := 0; i < 10; i++ {
+			got := SelectProfileForSession("google", "us", "session-abc")
+			if got.ID != first.ID {
+				t.Fatalf("iteration %d: expected %q, got %q", i, first.ID, got.ID)
+			}
+		}
+	})
+
+	t.Run("rotation produces multiple distinct profiles", func(t *testing.T) {
+		seen := map[string]bool{}
+		for i := 0; i < 100; i++ {
+			salt := "session-" + string(rune('a'+i%26)) + string(rune('0'+i/26%10))
+			p := SelectProfileForSession("google", "us", salt)
+			seen[p.ID] = true
+		}
+		if len(seen) < 3 {
+			t.Fatalf("expected at least 3 distinct profiles, got %d: %v", len(seen), seen)
+		}
+	})
+
+	t.Run("ru region returns ru-tagged profile", func(t *testing.T) {
+		p := SelectProfileForSession("yandex", "ru", "some-session")
+		if !slices.Contains(p.Tags, "ru") {
+			t.Fatalf("expected ru-tagged profile, got ID=%q tags=%v", p.ID, p.Tags)
+		}
+	})
+}
+
 func TestNormalizeRegion(t *testing.T) {
 	tests := []struct {
 		input string
@@ -106,7 +127,9 @@ func TestLoadProfilesFromJSON(t *testing.T) {
 				"navigator_langs": ["ru-RU"],
 				"locale": "ru-RU",
 				"timezone": "Europe/Moscow",
-				"viewport": {"width": 1920, "height": 1080}
+				"viewport": {"width": 1920, "height": 1080},
+				"tags": ["linux", "ru"],
+				"weight": 2
 			}
 		],
 		"lane_profile_ids": {
@@ -135,6 +158,12 @@ func TestLoadProfilesFromJSON(t *testing.T) {
 	if profile.Timezone != "Europe/Moscow" {
 		t.Fatalf("expected timezone Europe/Moscow, got %q", profile.Timezone)
 	}
+	if !slices.Contains(profile.Tags, "ru") {
+		t.Fatalf("expected tags to contain 'ru', got %v", profile.Tags)
+	}
+	if profile.Weight != 2 {
+		t.Fatalf("expected weight 2, got %d", profile.Weight)
+	}
 }
 
 func snapshotProfileState() (map[string]Profile, map[string]string, map[string]string) {

core/browser/profiles.json

@@ -1,35 +1,185 @@
 {
   "profiles": [
     {
-      "id": "chrome-win-us",
-      "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+      "id": "chrome-linux-mesa-uhd620",
+      "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
       "uach_brands": [
-        {
-          "brand": "Not_A Brand",
-          "version": "24"
-        },
-        {
-          "brand": "Chromium",
-          "version": "136"
-        },
-        {
-          "brand": "Google Chrome",
-          "version": "136"
-        }
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
       ],
       "uach_full_version_list": [
-        {
-          "brand": "Not_A Brand",
-          "version": "24.0.0.0"
-        },
-        {
-          "brand": "Chromium",
-          "version": "136.0.0.0"
-        },
-        {
-          "brand": "Google Chrome",
-          "version": "136.0.0.0"
-        }
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
+      ],
+      "platform": "Linux",
+      "platform_version": "6.1.0",
+      "architecture": "x86",
+      "bitness": "64",
+      "mobile": false,
+      "accept_language": "en-US,en;q=0.9",
+      "navigator_langs": ["en-US"],
+      "locale": "en-US",
+      "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080},
+      "webgl_vendor": "Intel Inc.",
+      "webgl_renderer": "Mesa Intel(R) UHD Graphics 620 (KBL GT2)",
+      "tags": ["linux", "mesa", "integrated"],
+      "weight": 3
+    },
+    {
+      "id": "chrome-linux-mesa-iris-xe",
+      "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+      "uach_brands": [
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
+      ],
+      "uach_full_version_list": [
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
+      ],
+      "platform": "Linux",
+      "platform_version": "6.1.0",
+      "architecture": "x86",
+      "bitness": "64",
+      "mobile": false,
+      "accept_language": "en-US,en;q=0.9",
+      "navigator_langs": ["en-US"],
+      "locale": "en-US",
+      "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080},
+      "webgl_vendor": "Intel Inc.",
+      "webgl_renderer": "Mesa Intel(R) Graphics (RPL-S)",
+      "tags": ["linux", "mesa", "integrated"],
+      "weight": 2
+    },
+    {
+      "id": "chrome-linux-nvidia",
+      "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+      "uach_brands": [
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
+      ],
+      "uach_full_version_list": [
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
+      ],
+      "platform": "Linux",
+      "platform_version": "6.1.0",
+      "architecture": "x86",
+      "bitness": "64",
+      "mobile": false,
+      "accept_language": "en-US,en;q=0.9",
+      "navigator_langs": ["en-US"],
+      "locale": "en-US",
+      "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080},
+      "webgl_vendor": "Google Inc. (NVIDIA Corporation)",
+      "webgl_renderer": "ANGLE (NVIDIA Corporation, NVIDIA GeForce GTX 1660 SUPER/PCIe/SSE2, OpenGL 4.5.0 NVIDIA 535.86.05)",
+      "tags": ["linux", "nvidia"],
+      "weight": 2
+    },
+    {
+      "id": "chrome-linux-amd",
+      "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+      "uach_brands": [
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
+      ],
+      "uach_full_version_list": [
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
+      ],
+      "platform": "Linux",
+      "platform_version": "6.1.0",
+      "architecture": "x86",
+      "bitness": "64",
+      "mobile": false,
+      "accept_language": "en-US,en;q=0.9",
+      "navigator_langs": ["en-US"],
+      "locale": "en-US",
+      "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080},
+      "webgl_vendor": "Google Inc. (AMD)",
+      "webgl_renderer": "ANGLE (AMD, AMD Radeon RX 6600 (radeonsi, navi23, ACO, DRM 3.42.0, 5.15.0-91-generic), OpenGL 4.6 (Core Profile))",
+      "tags": ["linux", "amd"],
+      "weight": 1
+    },
+    {
+      "id": "chrome-linux-swiftshader",
+      "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+      "uach_brands": [
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
+      ],
+      "uach_full_version_list": [
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
+      ],
+      "platform": "Linux",
+      "platform_version": "6.1.0",
+      "architecture": "x86",
+      "bitness": "64",
+      "mobile": false,
+      "accept_language": "en-US,en;q=0.9",
+      "navigator_langs": ["en-US"],
+      "locale": "en-US",
+      "timezone": "America/New_York",
+      "viewport": {"width": 1366, "height": 768},
+      "webgl_vendor": "Google Inc. (Google)",
+      "webgl_renderer": "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device (Subzero) (0x0000C0DE)), SwiftShader driver)",
+      "tags": ["linux", "swiftshader", "headless"],
+      "weight": 1
+    },
+    {
+      "id": "chrome-linux-ru",
+      "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+      "uach_brands": [
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
+      ],
+      "uach_full_version_list": [
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
+      ],
+      "platform": "Linux",
+      "platform_version": "6.0.0",
+      "architecture": "x86",
+      "bitness": "64",
+      "mobile": false,
+      "accept_language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
+      "navigator_langs": ["ru-RU"],
+      "locale": "ru-RU",
+      "timezone": "Europe/Moscow",
+      "viewport": {"width": 1920, "height": 1080},
+      "webgl_vendor": "Intel Inc.",
+      "webgl_renderer": "Mesa Intel(R) UHD Graphics 620 (KBL GT2)",
+      "tags": ["linux", "ru"],
+      "weight": 1
+    },
+    {
+      "id": "chrome-win-uhd620",
+      "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+      "uach_brands": [
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
+      ],
+      "uach_full_version_list": [
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
       ],
       "platform": "Windows",
       "platform_version": "15.0.0",
@@ -37,46 +187,111 @@
       "bitness": "64",
       "mobile": false,
       "accept_language": "en-US,en;q=0.9",
-      "navigator_langs": [
-        "en-US"
-      ],
+      "navigator_langs": ["en-US"],
       "locale": "en-US",
       "timezone": "America/New_York",
-      "viewport": {
-        "width": 1920,
-        "height": 1080
-      }
+      "viewport": {"width": 1920, "height": 1080},
+      "webgl_vendor": "Intel Inc.",
+      "webgl_renderer": "ANGLE (Intel, Intel(R) UHD Graphics 620 Direct3D11 vs_5_0 ps_5_0, D3D11)",
+      "tags": ["windows"],
+      "weight": 2
+    },
+    {
+      "id": "chrome-win-nvidia",
+      "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+      "uach_brands": [
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
+      ],
+      "uach_full_version_list": [
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
+      ],
+      "platform": "Windows",
+      "platform_version": "15.0.0",
+      "architecture": "x86",
+      "bitness": "64",
+      "mobile": false,
+      "accept_language": "en-US,en;q=0.9",
+      "navigator_langs": ["en-US"],
+      "locale": "en-US",
+      "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080},
+      "webgl_vendor": "Google Inc. (NVIDIA)",
+      "webgl_renderer": "ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)",
+      "tags": ["windows", "nvidia"],
+      "weight": 2
+    },
+    {
+      "id": "chrome-win-iris-xe",
+      "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+      "uach_brands": [
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
+      ],
+      "uach_full_version_list": [
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
+      ],
+      "platform": "Windows",
+      "platform_version": "15.0.0",
+      "architecture": "x86",
+      "bitness": "64",
+      "mobile": false,
+      "accept_language": "en-US,en;q=0.9",
+      "navigator_langs": ["en-US"],
+      "locale": "en-US",
+      "timezone": "America/New_York",
+      "viewport": {"width": 1536, "height": 864},
+      "webgl_vendor": "Google Inc. (Intel)",
+      "webgl_renderer": "ANGLE (Intel, Intel(R) Iris(R) Xe Graphics Direct3D11 vs_5_0 ps_5_0, D3D11)",
+      "tags": ["windows", "integrated"],
+      "weight": 1
+    },
+    {
+      "id": "chrome-win-amd",
+      "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+      "uach_brands": [
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
+      ],
+      "uach_full_version_list": [
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
+      ],
+      "platform": "Windows",
+      "platform_version": "15.0.0",
+      "architecture": "x86",
+      "bitness": "64",
+      "mobile": false,
+      "accept_language": "en-US,en;q=0.9",
+      "navigator_langs": ["en-US"],
+      "locale": "en-US",
+      "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080},
+      "webgl_vendor": "Google Inc. (AMD)",
+      "webgl_renderer": "ANGLE (AMD, AMD Radeon RX 6600 Direct3D11 vs_5_0 ps_5_0, D3D11)",
+      "tags": ["windows", "amd"],
+      "weight": 1
     },
     {
       "id": "chrome-win-ru",
       "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
       "uach_brands": [
-        {
-          "brand": "Not_A Brand",
-          "version": "24"
-        },
-        {
-          "brand": "Chromium",
-          "version": "136"
-        },
-        {
-          "brand": "Google Chrome",
-          "version": "136"
-        }
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
       ],
       "uach_full_version_list": [
-        {
-          "brand": "Not_A Brand",
-          "version": "24.0.0.0"
-        },
-        {
-          "brand": "Chromium",
-          "version": "136.0.0.0"
-        },
-        {
-          "brand": "Google Chrome",
-          "version": "136.0.0.0"
-        }
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
       ],
       "platform": "Windows",
       "platform_version": "15.0.0",
@@ -84,46 +299,83 @@
       "bitness": "64",
       "mobile": false,
       "accept_language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
-      "navigator_langs": [
-        "ru-RU"
-      ],
+      "navigator_langs": ["ru-RU"],
       "locale": "ru-RU",
       "timezone": "Europe/Moscow",
-      "viewport": {
-        "width": 1920,
-        "height": 1080
-      }
+      "viewport": {"width": 1920, "height": 1080},
+      "webgl_vendor": "Intel Inc.",
+      "webgl_renderer": "ANGLE (Intel, Intel(R) UHD Graphics 620 Direct3D11 vs_5_0 ps_5_0, D3D11)",
+      "tags": ["windows", "ru"],
+      "weight": 1
     },
     {
-      "id": "chrome-macos-us",
+      "id": "chrome-macos-m1",
       "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
       "uach_brands": [
-        {
-          "brand": "Not_A Brand",
-          "version": "24"
-        },
-        {
-          "brand": "Chromium",
-          "version": "136"
-        },
-        {
-          "brand": "Google Chrome",
-          "version": "136"
-        }
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
       ],
       "uach_full_version_list": [
-        {
-          "brand": "Not_A Brand",
-          "version": "24.0.0.0"
-        },
-        {
-          "brand": "Chromium",
-          "version": "136.0.0.0"
-        },
-        {
-          "brand": "Google Chrome",
-          "version": "136.0.0.0"
-        }
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
+      ],
+      "platform": "macOS",
+      "platform_version": "13.0.0",
+      "architecture": "arm",
+      "bitness": "64",
+      "mobile": false,
+      "accept_language": "en-US,en;q=0.9",
+      "navigator_langs": ["en-US"],
+      "locale": "en-US",
+      "timezone": "America/New_York",
+      "viewport": {"width": 1440, "height": 900},
+      "webgl_vendor": "Google Inc. (Apple)",
+      "webgl_renderer": "ANGLE (Apple, ANGLE Metal Renderer: Apple M1, Unspecified Version)",
+      "tags": ["macos", "apple-silicon"],
+      "weight": 2
+    },
+    {
+      "id": "chrome-macos-m2",
+      "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+      "uach_brands": [
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
+      ],
+      "uach_full_version_list": [
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
+      ],
+      "platform": "macOS",
+      "platform_version": "14.0.0",
+      "architecture": "arm",
+      "bitness": "64",
+      "mobile": false,
+      "accept_language": "en-US,en;q=0.9",
+      "navigator_langs": ["en-US"],
+      "locale": "en-US",
+      "timezone": "America/New_York",
+      "viewport": {"width": 1512, "height": 982},
+      "webgl_vendor": "Google Inc. (Apple)",
+      "webgl_renderer": "ANGLE (Apple, ANGLE Metal Renderer: Apple M2, Unspecified Version)",
+      "tags": ["macos", "apple-silicon"],
+      "weight": 2
+    },
+    {
+      "id": "chrome-macos-intel-iris",
+      "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+      "uach_brands": [
+        {"brand": "Not_A Brand", "version": "24"},
+        {"brand": "Chromium", "version": "136"},
+        {"brand": "Google Chrome", "version": "136"}
+      ],
+      "uach_full_version_list": [
+        {"brand": "Not_A Brand", "version": "24.0.0.0"},
+        {"brand": "Chromium", "version": "136.0.0.0"},
+        {"brand": "Google Chrome", "version": "136.0.0.0"}
       ],
       "platform": "macOS",
       "platform_version": "14.0.0",
@@ -131,109 +383,14 @@
       "bitness": "64",
       "mobile": false,
       "accept_language": "en-US,en;q=0.9",
-      "navigator_langs": [
-        "en-US"
-      ],
-      "locale": "en-US",
-      "timezone": "America/Los_Angeles",
-      "viewport": {
-        "width": 1680,
-        "height": 1050
-      }
-    },
-    {
-      "id": "chrome-linux-us",
-      "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
-      "uach_brands": [
-        {
-          "brand": "Not_A Brand",
-          "version": "24"
-        },
-        {
-          "brand": "Chromium",
-          "version": "136"
-        },
-        {
-          "brand": "Google Chrome",
-          "version": "136"
-        }
-      ],
-      "uach_full_version_list": [
-        {
-          "brand": "Not_A Brand",
-          "version": "24.0.0.0"
-        },
-        {
-          "brand": "Chromium",
-          "version": "136.0.0.0"
-        },
-        {
-          "brand": "Google Chrome",
-          "version": "136.0.0.0"
-        }
-      ],
-      "platform": "Linux",
-      "platform_version": "6.0.0",
-      "architecture": "x86",
-      "bitness": "64",
-      "mobile": false,
-      "accept_language": "en-US,en;q=0.9",
-      "navigator_langs": [
-        "en-US"
-      ],
+      "navigator_langs": ["en-US"],
       "locale": "en-US",
       "timezone": "America/New_York",
-      "viewport": {
-        "width": 1920,
-        "height": 1080
-      }
-    },
-    {
-      "id": "chrome-linux-ru",
-      "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
-      "uach_brands": [
-        {
-          "brand": "Not_A Brand",
-          "version": "24"
-        },
-        {
-          "brand": "Chromium",
-          "version": "136"
-        },
-        {
-          "brand": "Google Chrome",
-          "version": "136"
-        }
-      ],
-      "uach_full_version_list": [
-        {
-          "brand": "Not_A Brand",
-          "version": "24.0.0.0"
-        },
-        {
-          "brand": "Chromium",
-          "version": "136.0.0.0"
-        },
-        {
-          "brand": "Google Chrome",
-          "version": "136.0.0.0"
-        }
-      ],
-      "platform": "Linux",
-      "platform_version": "6.0.0",
-      "architecture": "x86",
-      "bitness": "64",
-      "mobile": false,
-      "accept_language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
-      "navigator_langs": [
-        "ru-RU"
-      ],
-      "locale": "ru-RU",
-      "timezone": "Europe/Moscow",
-      "viewport": {
-        "width": 1920,
-        "height": 1080
-      }
+      "viewport": {"width": 1680, "height": 1050},
+      "webgl_vendor": "Intel Inc.",
+      "webgl_renderer": "Intel Iris OpenGL Engine",
+      "tags": ["macos", "integrated"],
+      "weight": 1
     }
   ],
   "lane_profile_ids": {},

core/browser_profile_usage.go

@@ -0,0 +1,62 @@
+package core
+
+import (
+	"context"
+	"slices"
+	"strings"
+	"sync"
+)
+
+type browserProfileUsageContextKey struct{}
+
+type browserProfileUsageTracker struct {
+	mu  sync.Mutex
+	ids []string
+}
+
+func WithBrowserProfileUsage(ctx context.Context) context.Context {
+	ctx = EnsureContext(ctx)
+	if browserProfileUsageFromContext(ctx) != nil {
+		return ctx
+	}
+	return context.WithValue(ctx, browserProfileUsageContextKey{}, &browserProfileUsageTracker{})
+}
+
+func SetBrowserProfileID(ctx context.Context, profileID string) {
+	profileID = strings.TrimSpace(profileID)
+	if profileID == "" {
+		return
+	}
+	tracker := browserProfileUsageFromContext(ctx)
+	if tracker == nil {
+		return
+	}
+
+	tracker.mu.Lock()
+	defer tracker.mu.Unlock()
+	if slices.Contains(tracker.ids, profileID) {
+		return
+	}
+	tracker.ids = append(tracker.ids, profileID)
+}
+
+func BrowserProfileIDsFromContext(ctx context.Context) []string {
+	tracker := browserProfileUsageFromContext(ctx)
+	if tracker == nil {
+		return nil
+	}
+
+	tracker.mu.Lock()
+	defer tracker.mu.Unlock()
+	out := make([]string, len(tracker.ids))
+	copy(out, tracker.ids)
+	return out
+}
+
+func browserProfileUsageFromContext(ctx context.Context) *browserProfileUsageTracker {
+	if ctx == nil {
+		return nil
+	}
+	tracker, _ := ctx.Value(browserProfileUsageContextKey{}).(*browserProfileUsageTracker)
+	return tracker
+}

core/browser_resource_blocking_test.go

@@ -51,6 +51,25 @@ func TestShouldBlockResourceType(t *testing.T) {
 	}
 }
 
+func TestProxyAuthFetchPatternsOnlyInterceptDocuments(t *testing.T) {
+	patterns := proxyAuthFetchPatterns()
+	if len(patterns) != 2 {
+		t.Fatalf("expected 2 proxy auth fetch patterns, got %d", len(patterns))
+	}
+
+	for _, pattern := range patterns {
+		if pattern.URLPattern != "http://*/*" && pattern.URLPattern != "https://*/*" {
+			t.Fatalf("unexpected proxy auth URL pattern: %q", pattern.URLPattern)
+		}
+		if pattern.ResourceType != proto.NetworkResourceTypeDocument {
+			t.Fatalf("expected document-only proxy auth interception, got %s", pattern.ResourceType)
+		}
+		if pattern.RequestStage != proto.FetchRequestStageRequest {
+			t.Fatalf("expected request-stage proxy auth interception, got %s", pattern.RequestStage)
+		}
+	}
+}
+
 func TestParseBlockedResourceTypes(t *testing.T) {
 	got, err := ParseBlockedResourceTypes("image,font,css,js,media")
 	if err != nil {

core/browser_test.go

@@ -6,6 +6,7 @@ package core
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -103,6 +104,162 @@ func TestNavigateUsesIsolatedBrowserContext(t *testing.T) {
 	}
 }
 
+func TestNavigateReusesCookiesForSameProxyLane(t *testing.T) {
+	testutil.RequireIntegration(t)
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/cookies/set":
+			http.SetCookie(w, &http.Cookie{Name: "openserp_lane", Value: "same-lane", Path: "/"})
+			_, _ = w.Write([]byte("cookie-set"))
+		case "/cookies":
+			_, _ = w.Write([]byte(r.Header.Get("Cookie")))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	browser, err := NewBrowser(BrowserOpts{
+		IsHeadless:     true,
+		IsLeakless:     false,
+		Timeout:        15 * time.Second,
+		ProxyLaneStore: NewLaneStore(10),
+	})
+	if err != nil {
+		t.Fatalf("failed initializing browser: %s", err)
+	}
+	defer closeTestBrowser(t, browser)
+
+	laneCtx := WithProxyLaneKey(WithEngine(context.Background(), "google"), ProxyLaneKey{Engine: "google", SessionID: "sid-a"})
+	pageA, err := browser.Navigate(laneCtx, srv.URL+"/cookies/set")
+	if err != nil {
+		t.Fatalf("navigate cookie setter: %v", err)
+	}
+	if err := ClosePageWithTimeout(context.Background(), pageA, time.Second); err != nil {
+		t.Fatalf("close setter page: %v", err)
+	}
+
+	pageB, err := browser.Navigate(laneCtx, srv.URL+"/cookies")
+	if err != nil {
+		t.Fatalf("navigate cookie reader: %v", err)
+	}
+	defer func() {
+		if err := ClosePageWithTimeout(context.Background(), pageB, time.Second); err != nil {
+			t.Logf("close reader page: %v", err)
+		}
+	}()
+
+	body, err := pageB.Timeout(5 * time.Second).Element("body")
+	if err != nil {
+		t.Fatalf("read response body: %v", err)
+	}
+	cookieHeader, err := body.Text()
+	if err != nil {
+		t.Fatalf("extract response text: %v", err)
+	}
+	if !strings.Contains(cookieHeader, "openserp_lane=same-lane") {
+		t.Fatalf("expected same proxy lane to restore cookie, got %q", cookieHeader)
+	}
+}
+
+func TestNavigateDoesNotShareCookiesAcrossProxyLanes(t *testing.T) {
+	testutil.RequireIntegration(t)
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/cookies/set":
+			http.SetCookie(w, &http.Cookie{Name: "openserp_lane", Value: "lane-a", Path: "/"})
+			_, _ = w.Write([]byte("cookie-set"))
+		case "/cookies":
+			_, _ = w.Write([]byte(r.Header.Get("Cookie")))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	browser, err := NewBrowser(BrowserOpts{
+		IsHeadless:     true,
+		IsLeakless:     false,
+		Timeout:        15 * time.Second,
+		ProxyLaneStore: NewLaneStore(10),
+	})
+	if err != nil {
+		t.Fatalf("failed initializing browser: %s", err)
+	}
+	defer closeTestBrowser(t, browser)
+
+	laneA := WithProxyLaneKey(WithEngine(context.Background(), "google"), ProxyLaneKey{Engine: "google", SessionID: "sid-a"})
+	laneB := WithProxyLaneKey(WithEngine(context.Background(), "google"), ProxyLaneKey{Engine: "google", SessionID: "sid-b"})
+	pageA, err := browser.Navigate(laneA, srv.URL+"/cookies/set")
+	if err != nil {
+		t.Fatalf("navigate cookie setter: %v", err)
+	}
+	if err := ClosePageWithTimeout(context.Background(), pageA, time.Second); err != nil {
+		t.Fatalf("close setter page: %v", err)
+	}
+
+	pageB, err := browser.Navigate(laneB, srv.URL+"/cookies")
+	if err != nil {
+		t.Fatalf("navigate cookie reader: %v", err)
+	}
+	defer func() {
+		if err := ClosePageWithTimeout(context.Background(), pageB, time.Second); err != nil {
+			t.Logf("close reader page: %v", err)
+		}
+	}()
+
+	body, err := pageB.Timeout(5 * time.Second).Element("body")
+	if err != nil {
+		t.Fatalf("read response body: %v", err)
+	}
+	cookieHeader, err := body.Text()
+	if err != nil {
+		t.Fatalf("extract response text: %v", err)
+	}
+	if strings.Contains(cookieHeader, "openserp_lane=lane-a") {
+		t.Fatalf("cookie leaked across proxy lanes; got header %q", cookieHeader)
+	}
+}
+
+func TestNavigateClassifiesMainDocumentStatus(t *testing.T) {
+	testutil.RequireIntegration(t)
+
+	tests := []struct {
+		name   string
+		status int
+		want   error
+	}{
+		{name: "blocked", status: http.StatusForbidden, want: ErrBlocked},
+		{name: "rate limited", status: http.StatusTooManyRequests, want: ErrRateLimited},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				w.WriteHeader(tt.status)
+				_, _ = w.Write([]byte("classified"))
+			}))
+			defer srv.Close()
+
+			browser, err := NewBrowser(BrowserOpts{IsHeadless: true, IsLeakless: false, Timeout: 15 * time.Second})
+			if err != nil {
+				t.Fatalf("failed initializing browser: %s", err)
+			}
+			defer closeTestBrowser(t, browser)
+
+			page, err := browser.Navigate(context.Background(), srv.URL)
+			if page != nil {
+				_ = ClosePageWithTimeout(context.Background(), page, time.Second)
+			}
+			if !errors.Is(err, tt.want) {
+				t.Fatalf("expected %v, got %v", tt.want, err)
+			}
+		})
+	}
+}
+
 func TestFingerprintDetectors(t *testing.T) {
 	testutil.RequireIntegration(t)
 	if strings.TrimSpace(os.Getenv(botFingerprintTestsEnv)) != "1" {

core/browser_unit_test.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	browserprofile "github.com/karust/openserp/core/browser"
 )
@@ -60,6 +61,40 @@ func TestResolveBrowserBinaryPathRejectsInvalidExplicit(t *testing.T) {
 	}
 }
 
+func TestMinPositiveDuration(t *testing.T) {
+	tests := []struct {
+		name string
+		a    time.Duration
+		b    time.Duration
+		want time.Duration
+	}{
+		{name: "both positive", a: 30 * time.Second, b: 2 * time.Second, want: 2 * time.Second},
+		{name: "first unset", a: 0, b: 2 * time.Second, want: 2 * time.Second},
+		{name: "second unset", a: 30 * time.Second, b: 0, want: 30 * time.Second},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := minPositiveDuration(tt.a, tt.b); got != tt.want {
+				t.Fatalf("minPositiveDuration() = %s, want %s", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestApplyProfileLanguageHintRewritesTimezone(t *testing.T) {
+	profile := browserprofile.Profile{
+		AcceptLanguage: "ru-RU,ru;q=0.9",
+		NavigatorLangs: []string{"ru-RU"},
+		Locale:         "ru-RU",
+		Timezone:       "Europe/Moscow",
+	}
+	got := applyProfileLanguageHint(profile, "de-DE")
+	if got.Timezone != "Europe/Berlin" {
+		t.Fatalf("expected timezone Europe/Berlin, got %q", got.Timezone)
+	}
+}
+
 func TestApplyProfileLanguageHint(t *testing.T) {
 	base := browserprofile.Profile{
 		AcceptLanguage: "en-US,en;q=0.9",

core/cache.go

@@ -4,6 +4,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 )
@@ -35,25 +36,58 @@ func NewResponseCache(ttl time.Duration, maxSize int) *ResponseCache {
 }
 
 func BuildCacheKey(engine string, action string, q Query) string {
+	country, class, provider := cacheProxyMarket(q)
 	raw := fmt.Sprintf(
-		"%s|%s|%s|%s|%s|%s|%s|%d|%d|%t|%t|%s",
-		engine,
-		action,
-		q.Text,
-		q.LangCode,
-		q.DateInterval,
-		q.Filetype,
-		q.Site,
+		"%s|%s|%s|%s|%s|%s|%s|%s|%d|%d|%t|%t|%s|%s|%s",
+		cacheToken(engine),
+		cacheToken(action),
+		strings.TrimSpace(q.Text),
+		cacheToken(q.LangCode),
+		cacheToken(q.Region),
+		strings.TrimSpace(q.DateInterval),
+		cacheToken(q.Filetype),
+		cacheToken(q.Site),
 		q.Limit,
 		q.Start,
 		q.Filter,
-		q.Answers,
-		q.ProxyOverride,
+		q.Features,
+		country,
+		class,
+		provider,
 	)
 	hash := sha256.Sum256([]byte(raw))
 	return hex.EncodeToString(hash[:])
 }
 
+func cacheToken(value string) string {
+	return strings.ToLower(strings.TrimSpace(value))
+}
+
+func cacheProxyMarket(q Query) (country string, class string, provider string) {
+	country = cacheToken(q.ProxyCountry)
+	if country == "" {
+		// Region is a stronger market signal than LangCode; LangCode is the last
+		// fallback. TODO: Use explicit balancer market metadata everywhere.
+		if region := CountryFromRegion(q.Region); region != "" {
+			country = strings.ToLower(region)
+		} else {
+			country = cacheToken(q.LangCode)
+		}
+	}
+	return country,
+		cacheToken(q.ProxyClass),
+		cacheToken(q.ProxyProvider)
+}
+
+func ShouldBypassCacheForProxyMarket(q Query) bool {
+	if strings.TrimSpace(q.ProxyURL) == "" && strings.TrimSpace(q.ProxyOverride) == "" {
+		return false
+	}
+	return strings.TrimSpace(q.ProxyCountry) == "" &&
+		strings.TrimSpace(q.ProxyClass) == "" &&
+		strings.TrimSpace(q.ProxyProvider) == ""
+}
+
 func (c *ResponseCache) Get(key string) ([]byte, bool) {
 	c.mu.Lock()
 	defer c.mu.Unlock()

core/cache_test.go

@@ -5,6 +5,28 @@ import (
 	"time"
 )
 
+func TestCacheProxyMarketFallback(t *testing.T) {
+	tests := []struct {
+		name    string
+		q       Query
+		country string
+	}{
+		{name: "proxy country wins", q: Query{ProxyCountry: "DE", Region: "RU", LangCode: "EN"}, country: "de"},
+		{name: "region country fallback", q: Query{Region: "RU", LangCode: "EN"}, country: "ru"},
+		{name: "region locale fallback", q: Query{Region: "en-GB", LangCode: "EN"}, country: "gb"},
+		{name: "lang code last resort", q: Query{LangCode: "EN"}, country: "en"},
+		{name: "yandex numeric region ignored for market", q: Query{Region: "213", LangCode: "RU"}, country: "ru"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			country, _, _ := cacheProxyMarket(tt.q)
+			if country != tt.country {
+				t.Fatalf("cacheProxyMarket country = %q, want %q", country, tt.country)
+			}
+		})
+	}
+}
+
 func TestResponseCacheSetAndGet(t *testing.T) {
 	cache := NewResponseCache(5*time.Second, 10)
 	key := BuildCacheKey("google", "search", Query{Text: "golang", Limit: 10})
@@ -104,7 +126,7 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) {
 		Limit:    10,
 		Start:    0,
 		Filter:   true,
-		Answers:  false,
+		Features: false,
 	}
 
 	baseKey := BuildCacheKey("google", "search", base)
@@ -112,13 +134,24 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) {
 	if same := BuildCacheKey("google", "search", base); same != baseKey {
 		t.Fatal("expected deterministic key for same query")
 	}
+	if changed := BuildCacheKey("google", "search", Query{
+		Text:     "golang",
+		LangCode: "EN",
+		Region:   "US",
+		Limit:    10,
+		Start:    0,
+		Filter:   true,
+		Features: false,
+	}); changed == baseKey {
+		t.Fatal("expected region to affect cache key")
+	}
 	if changed := BuildCacheKey("google", "search", Query{
 		Text:     "golang",
 		LangCode: "EN",
 		Limit:    20,
 		Start:    0,
 		Filter:   true,
-		Answers:  false,
+		Features: false,
 	}); changed == baseKey {
 		t.Fatal("expected limit to affect cache key")
 	}
@@ -128,7 +161,7 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) {
 		Limit:    10,
 		Start:    10,
 		Filter:   true,
-		Answers:  false,
+		Features: false,
 	}); changed == baseKey {
 		t.Fatal("expected start to affect cache key")
 	}
@@ -138,7 +171,7 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) {
 		Limit:    10,
 		Start:    0,
 		Filter:   false,
-		Answers:  false,
+		Features: false,
 	}); changed == baseKey {
 		t.Fatal("expected filter to affect cache key")
 	}
@@ -148,8 +181,107 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) {
 		Limit:    10,
 		Start:    0,
 		Filter:   true,
-		Answers:  true,
+		Features: true,
 	}); changed == baseKey {
-		t.Fatal("expected answers to affect cache key")
+		t.Fatal("expected features to affect cache key")
+	}
+}
+
+func TestBuildCacheKeyNormalizesStableFields(t *testing.T) {
+	base := BuildCacheKey("google", "search", Query{
+		Text:         " golang ",
+		LangCode:     "EN",
+		Region:       " us ",
+		Filetype:     "PDF",
+		Site:         "EXAMPLE.COM",
+		Limit:        10,
+		ProxyCountry: " US ",
+		ProxyClass:   " Residential ",
+	})
+	same := BuildCacheKey("Google", "Search", Query{
+		Text:         "golang",
+		LangCode:     "en",
+		Region:       "US",
+		Filetype:     "pdf",
+		Site:         "example.com",
+		Limit:        10,
+		ProxyCountry: "us",
+		ProxyClass:   "residential",
+	})
+	if same != base {
+		t.Fatal("expected cache key to normalize engine/action, locale, filters, and proxy market fields")
+	}
+}
+
+func TestBuildCacheKeyUsesProxyMarketNotSessionOrURL(t *testing.T) {
+	base := Query{
+		Text:           "golang",
+		LangCode:       "EN",
+		Limit:          10,
+		ProxyURL:       "http://user:password-a@proxy-a:8080",
+		ProxyCountry:   " US ",
+		ProxyClass:     " Residential ",
+		ProxyProvider:  " WebShare ",
+		ProxySessionID: "sid-a",
+	}
+	baseKey := BuildCacheKey("google", "search", base)
+
+	sameMarket := base
+	sameMarket.ProxyURL = "http://user:password-b@proxy-b:8080"
+	sameMarket.ProxySessionID = "sid-b"
+	if got := BuildCacheKey("google", "search", sameMarket); got != baseKey {
+		t.Fatal("expected proxy URL and session id not to affect cache key")
+	}
+
+	differentCountry := base
+	differentCountry.ProxyCountry = "de"
+	if got := BuildCacheKey("google", "search", differentCountry); got == baseKey {
+		t.Fatal("expected proxy country to affect cache key")
+	}
+
+	differentClass := base
+	differentClass.ProxyClass = "datacenter"
+	if got := BuildCacheKey("google", "search", differentClass); got == baseKey {
+		t.Fatal("expected proxy class to affect cache key")
+	}
+
+	differentProvider := base
+	differentProvider.ProxyProvider = "brightdata"
+	if got := BuildCacheKey("google", "search", differentProvider); got == baseKey {
+		t.Fatal("expected proxy provider to affect cache key")
+	}
+}
+
+func TestBuildCacheKeyFallsBackToLanguageWhenCountryAbsent(t *testing.T) {
+	base := Query{Text: "golang", LangCode: "en", Limit: 10}
+	baseKey := BuildCacheKey("google", "search", base)
+
+	changed := base
+	changed.LangCode = "de"
+	if got := BuildCacheKey("google", "search", changed); got == baseKey {
+		t.Fatal("expected language fallback to affect cache key when proxy country is absent")
+	}
+
+	withCountry := base
+	withCountry.ProxyCountry = "us"
+	changedWithCountry := withCountry
+	changedWithCountry.LangCode = "de"
+	if got := BuildCacheKey("google", "search", changedWithCountry); got == BuildCacheKey("google", "search", withCountry) {
+		t.Fatal("expected language itself to remain part of the cache key")
+	}
+}
+
+func TestShouldBypassCacheForProxyMarket(t *testing.T) {
+	if !ShouldBypassCacheForProxyMarket(Query{ProxyURL: "http://proxy.example:8080"}) {
+		t.Fatal("expected request proxy without market metadata to bypass cache")
+	}
+	if !ShouldBypassCacheForProxyMarket(Query{ProxyOverride: "us"}) {
+		t.Fatal("expected tag override without market metadata to bypass cache")
+	}
+	if ShouldBypassCacheForProxyMarket(Query{ProxyURL: "http://proxy.example:8080", ProxyCountry: "us"}) {
+		t.Fatal("expected explicit country market metadata to allow cache")
+	}
+	if ShouldBypassCacheForProxyMarket(Query{Text: "golang"}) {
+		t.Fatal("expected direct query without proxy override to allow cache")
 	}
 }

core/circuit_breaker.go

@@ -50,6 +50,8 @@ type CircuitBreaker struct {
 	config          CircuitBreakerConfig
 	failureCount    int
 	successCount    int
+	successLatency  time.Duration
+	successSamples  int64
 	lastFailureTime time.Time
 	lastStateChange time.Time
 }
@@ -85,9 +87,18 @@ func (cb *CircuitBreaker) AllowRequest(ctx context.Context) bool {
 }
 
 func (cb *CircuitBreaker) RecordSuccess(ctx context.Context) {
+	cb.RecordSuccessDuration(ctx, 0)
+}
+
+func (cb *CircuitBreaker) RecordSuccessDuration(ctx context.Context, elapsed time.Duration) {
 	cb.mu.Lock()
 	defer cb.mu.Unlock()
 
+	if elapsed > 0 {
+		cb.successLatency += elapsed
+		cb.successSamples++
+	}
+
 	switch cb.state {
 	case CircuitHalfOpen:
 		cb.successCount++
@@ -155,10 +166,22 @@ func (cb *CircuitBreaker) Stats() map[string]interface{} {
 		}
 		stats["retry_in"] = retryInSeconds
 	}
+	if cb.successSamples > 0 {
+		stats["avg_response_ms"] = int64((cb.successLatency / time.Duration(cb.successSamples)) / time.Millisecond)
+	}
 
 	return stats
 }
 
+func (cb *CircuitBreaker) AvgSuccessLatency() (time.Duration, bool) {
+	cb.mu.RLock()
+	defer cb.mu.RUnlock()
+	if cb.successSamples == 0 {
+		return 0, false
+	}
+	return cb.successLatency / time.Duration(cb.successSamples), true
+}
+
 func (cb *CircuitBreaker) setState(state CircuitState) {
 	cb.state = state
 	cb.lastStateChange = time.Now()

core/circuit_breaker_test.go

@@ -139,6 +139,13 @@ func TestCircuitBreaker_Stats(t *testing.T) {
 	if _, ok := stats["retry_in"]; ok {
 		t.Fatalf("did not expect retry_in in closed state, got: %v", stats["retry_in"])
 	}
+	latencyCB := NewCircuitBreaker("latency-engine", DefaultCircuitBreakerConfig())
+	latencyCB.RecordSuccessDuration(context.Background(), 25*time.Millisecond)
+	latencyStats := latencyCB.Stats()
+	avg, ok := latencyStats["avg_response_ms"].(int64)
+	if !ok || avg <= 0 {
+		t.Fatalf("expected avg_response_ms int64 > 0, got: %v (%T)", latencyStats["avg_response_ms"], latencyStats["avg_response_ms"])
+	}
 
 	openCfg := CircuitBreakerConfig{FailureThreshold: 1, RecoveryTimeout: time.Second, SuccessThreshold: 1}
 	openCB := NewCircuitBreaker("open-engine", openCfg)

core/common.go

@@ -8,9 +8,11 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"golang.org/x/time/rate"
 )
 
 // ErrCaptcha is returned when the engine detects a captcha challenge page.
@@ -45,6 +47,12 @@ var ErrTimeout = errors.New("timeout")
 // It is not a failure; the proxy stays healthy and no credit is charged.
 var ErrEmptyResult = errors.New("empty_result")
 
+// ErrBlocked is returned when the search engine blocks the browser request.
+var ErrBlocked = errors.New("blocked")
+
+// ErrRateLimited is returned when the search engine returns an HTTP rate limit.
+var ErrRateLimited = errors.New("rate_limited")
+
 // IsProxyNetworkError reports whether err is a network-level error that
 // indicates a faulty proxy (connect failure, auth rejection, or timeout).
 // Parser drift, captcha pages, and engine errors must NOT degrade proxy health.
@@ -88,9 +96,14 @@ func classifyProxyNetworkError(err error) error {
 
 // SearchResult represents one normalized result item returned by any engine.
 type SearchResult struct {
-	// Rank is a 1-based position in engine output. Some engines use negative
-	// ranks for non-organic blocks such as ads or instant answers.
+	// Rank is the 1-based position within this result type. For SEO callers,
+	// organic rank must not be shifted by ads.
 	Rank int `json:"rank"`
+	// AbsoluteRank is the 1-based position in the mixed SERP stream.
+	AbsoluteRank int `json:"absolute_rank,omitempty"`
+	// Type is the SERP block type when an engine can classify a non-standard
+	// SERP module without changing the public SearchEngine interface.
+	Type ResultType `json:"type,omitempty"`
 	// URL is the canonical result URL.
 	URL string `json:"url"`
 	// Title is the result headline shown on the SERP.
@@ -99,6 +112,8 @@ type SearchResult struct {
 	Description string `json:"description"`
 	// Ad reports whether the result is sponsored.
 	Ad bool `json:"ad"`
+	// Features carries extracted SERP modules alongside the legacy result stream.
+	Features []SerpFeature `json:"-"`
 }
 
 // DeduplicateResults removes items with duplicate URLs and returns a result set
@@ -111,14 +126,15 @@ func DeduplicateResults(results []SearchResult) []SearchResult {
 		if result.URL == "" {
 			continue
 		}
-		if !unique[result.URL] {
-			unique[result.URL] = true
+		key := resultDedupKey(result)
+		if !unique[key] {
+			unique[key] = true
 			deduped = append(deduped, result)
 		}
 	}
 
 	sort.Slice(deduped, func(i, j int) bool {
-		return deduped[i].Rank < deduped[j].Rank
+		return resultLess(deduped[i], deduped[j])
 	})
 	return deduped
 }
@@ -133,11 +149,99 @@ func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]Search
 	}
 
 	sort.Slice(searchResults, func(i, j int) bool {
-		return searchResults[i].Rank < searchResults[j].Rank
+		return resultLess(searchResults[i], searchResults[j])
 	})
 	return &searchResults
 }
 
+// CountOrganicResults returns the number of non-ad results in a mixed SERP.
+func CountOrganicResults(results []SearchResult) int {
+	count := 0
+	for _, result := range results {
+		if !result.Ad {
+			count++
+		}
+	}
+	return count
+}
+
+// OrganicLimitReached reports whether enough organic results have been
+// collected to satisfy limit. A non-positive limit means "no limit", so it
+// is never reached and pagination continues until the engine runs out.
+func OrganicLimitReached(results []SearchResult, limit int) bool {
+	return limit > 0 && CountOrganicResults(results) >= limit
+}
+
+// ShouldFetchResultPage reports whether a paginated engine should fetch another
+// SERP page. Small/default limits should use the first SERP page as-is instead
+// of chasing a target count across multiple page loads.
+func ShouldFetchResultPage(collected, limit, pagesFetched int) bool {
+	if pagesFetched <= 0 {
+		return true
+	}
+	if limit > 0 && collected >= limit {
+		return false
+	}
+	return limit > defaultQueryLimit
+}
+
+// LimitOrganicResults keeps all ads and at most limit non-ad results.
+func LimitOrganicResults(results []SearchResult, limit int) []SearchResult {
+	if limit <= 0 {
+		return results
+	}
+	out := make([]SearchResult, 0, len(results))
+	organicCount := 0
+	for _, result := range results {
+		if result.Ad {
+			out = append(out, result)
+			continue
+		}
+		if organicCount >= limit {
+			continue
+		}
+		organicCount++
+		out = append(out, result)
+	}
+	return out
+}
+
+func resultDedupKey(result SearchResult) string {
+	resultType := "organic"
+	if result.Ad {
+		resultType = "ad"
+	}
+	return resultType + "\x00" + result.URL
+}
+
+func resultLess(left, right SearchResult) bool {
+	leftPos := resultSortPosition(left)
+	rightPos := resultSortPosition(right)
+	if leftPos != rightPos {
+		return leftPos < rightPos
+	}
+	if left.Ad != right.Ad {
+		return left.Ad
+	}
+	if left.Rank != right.Rank {
+		return left.Rank < right.Rank
+	}
+	return left.URL < right.URL
+}
+
+func resultSortPosition(result SearchResult) int {
+	if result.AbsoluteRank > 0 {
+		return result.AbsoluteRank
+	}
+	if result.Rank < 0 {
+		return -result.Rank
+	}
+	if result.Rank > 0 {
+		return result.Rank
+	}
+	return int(^uint(0) >> 1)
+}
+
 // Query holds request parameters used by HTTP handlers and search engines.
 // Example minimal query: Query{Text: "golang", Limit: 10}.
 type Query struct {
@@ -145,6 +249,9 @@ type Query struct {
 	Text string
 	// LangCode is an engine language hint such as "EN", "DE", or "RU".
 	LangCode string
+	// Region is an engine market/location hint. Yandex accepts numeric lr IDs;
+	// global engines accept country-style hints such as "RU" or "en-RU".
+	Region string
 	// DateInterval filters by date range in YYYYMMDD..YYYYMMDD format.
 	// Example: "20250101..20250331".
 	DateInterval string
@@ -160,16 +267,54 @@ type Query struct {
 	// Filter controls duplicate filtering when supported by the engine.
 	// For Google, false includes similar results and true hides them.
 	Filter bool
-	// Answers enables parsing answer modules when supported by the engine.
-	// Such entries may be returned with negative rank values.
-	Answers bool
+	// Features enables parsing SERP feature modules (AI summaries, answer boxes,
+	// people-also-ask, related searches) on the browser Search path when
+	// supported by the engine. Such entries may be returned with non-positive
+	// internal rank values.
+	Features bool
+	// Extract fetches and embeds cleaned target-page content for top results.
+	Extract bool
+	// ExtractTop limits how many top results are enriched when Extract is true.
+	ExtractTop int
+	// ExtractMode selects auto, fast, or rendered extraction.
+	ExtractMode string
+	// ExtractMinRunes overrides the auto-mode escalation floor (0 = default).
+	ExtractMinRunes int
 	// ProxyURL is a direct proxy URL used by raw HTTP search paths.
 	ProxyURL string
+	// ProxyCountry identifies the proxy market country for cache/error metadata.
+	ProxyCountry string
+	// ProxyClass identifies the proxy class such as datacenter or residential.
+	ProxyClass string
+	// ProxyProvider identifies the upstream proxy provider.
+	ProxyProvider string
+	// ProxySessionID identifies a sticky balancer session/lane.
+	ProxySessionID string
 	// ProxyOverride is a request-scoped proxy policy override (tag or "direct"),
 	// typically parsed from the X-Use-Proxy header.
 	ProxyOverride string
 	// Insecure enables insecure TLS for request/browser execution.
 	Insecure bool
+	// GuardPrivateNetworks rejects raw HTTP targets that resolve to private,
+	// loopback, link-local, multicast, or otherwise non-public addresses.
+	GuardPrivateNetworks bool
+}
+
+// String renders Query for logs with the proxy URL credentials masked. The
+// default %+v formatter calls this method, so logging Query through %v/%+v
+// never leaks proxy passwords.
+func (q Query) String() string {
+	maskedProxyURL := ""
+	if q.ProxyURL != "" {
+		maskedProxyURL = MaskProxyURL(q.ProxyURL)
+	}
+	return fmt.Sprintf(
+		"{Text:%s LangCode:%s Region:%s DateInterval:%s Filetype:%s Site:%s Limit:%d Start:%d Filter:%t Features:%t Extract:%t ExtractTop:%d ExtractMode:%s ProxyURL:%s ProxyCountry:%s ProxyClass:%s ProxyProvider:%s ProxySessionID:%s ProxyOverride:%s Insecure:%t}",
+		q.Text, q.LangCode, q.Region, q.DateInterval, q.Filetype, q.Site,
+		q.Limit, q.Start, q.Filter, q.Features, q.Extract, q.ExtractTop, q.ExtractMode,
+		maskedProxyURL, q.ProxyCountry, q.ProxyClass, q.ProxyProvider,
+		q.ProxySessionID, q.ProxyOverride, q.Insecure,
+	)
 }
 
 // ComputePagination translates an absolute start offset into page index and
@@ -195,17 +340,23 @@ func (q Query) IsEmpty() bool {
 // MaxQueryLimit is the maximum allowed value for the limit parameter.
 const MaxQueryLimit = 100
 
+// defaultQueryLimit is the assumed limit when a request omits it (InitFromContext)
+// and the fallback used by pagination math for internally-built queries that
+// leave Limit unset.
+const defaultQueryLimit = 10
+
 // InitFromContext populates Query from HTTP query parameters and request
 // headers. It validates numeric/boolean inputs and returns an *APIError for
 // invalid client input (400) or a plain error for internal failures.
 func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
-	searchQuery.Text = reqCtx.Query("text")
-	searchQuery.LangCode = reqCtx.Query("lang")
-	searchQuery.DateInterval = reqCtx.Query("date")
-	searchQuery.Filetype = reqCtx.Query("file")
-	searchQuery.Site = reqCtx.Query("site")
+	searchQuery.Text = strings.TrimSpace(reqCtx.Query("text"))
+	searchQuery.LangCode = strings.TrimSpace(reqCtx.Query("lang"))
+	searchQuery.Region = strings.TrimSpace(reqCtx.Query("region"))
+	searchQuery.DateInterval = strings.TrimSpace(reqCtx.Query("date"))
+	searchQuery.Filetype = strings.TrimSpace(reqCtx.Query("file"))
+	searchQuery.Site = strings.TrimSpace(reqCtx.Query("site"))
 
-	limitRaw := reqCtx.Query("limit", "25")
+	limitRaw := reqCtx.Query("limit", strconv.Itoa(defaultQueryLimit))
 	limit, err := strconv.Atoi(limitRaw)
 	if err != nil {
 		return errInvalidLimit("limit must be an integer")
@@ -230,15 +381,55 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
 		return errInvalidParam(fmt.Sprintf("filter: %v", err))
 	}
 
-	searchQuery.Answers, err = strconv.ParseBool(reqCtx.Query("answers", "0"))
+	searchQuery.Features, err = strconv.ParseBool(reqCtx.Query("features", "1"))
 	if err != nil {
-		return errInvalidParam(fmt.Sprintf("answers: %v", err))
+		return errInvalidParam(fmt.Sprintf("features: %v", err))
+	}
+	searchQuery.Extract, err = strconv.ParseBool(reqCtx.Query("extract", "0"))
+	if err != nil {
+		return errInvalidParam(fmt.Sprintf("extract: %v", err))
+	}
+	searchQuery.ExtractTop = 3
+	if raw := strings.TrimSpace(reqCtx.Query("extract_top")); raw != "" {
+		extractTop, err := strconv.Atoi(raw)
+		if err != nil {
+			return errInvalidParam("extract_top must be an integer")
+		}
+		if extractTop < 1 {
+			extractTop = 1
+		}
+		if extractTop > 5 {
+			extractTop = 5
+		}
+		searchQuery.ExtractTop = extractTop
+	}
+	searchQuery.ExtractMode = strings.ToLower(strings.TrimSpace(reqCtx.Query("extract_mode", "auto")))
+	switch searchQuery.ExtractMode {
+	case "auto", "fast", "rendered":
+	default:
+		return errInvalidParam("extract_mode must be one of auto, fast, rendered")
+	}
+	searchQuery.ExtractMinRunes, err = parseNonNegativeIntQuery(reqCtx.Query("min_runes"), 0)
+	if err != nil {
+		return errInvalidParam("min_runes must be a non-negative integer")
 	}
 
 	searchQuery.ProxyOverride, err = NormalizeProxyRequestOverride(reqCtx.Get("X-Use-Proxy"))
 	if err != nil {
 		return errInvalidParam(fmt.Sprintf("X-Use-Proxy: %v", err))
 	}
+	rawProxyURL := strings.TrimSpace(reqCtx.Get("X-Proxy-URL"))
+	if rawProxyURL != "" {
+		normalized, err := NormalizeProxyURL(rawProxyURL)
+		if err != nil {
+			return errInvalidParam(fmt.Sprintf("X-Proxy-URL: %v", err))
+		}
+		searchQuery.ProxyURL = normalized
+	}
+	searchQuery.ProxyCountry = strings.ToLower(strings.TrimSpace(reqCtx.Get("X-Proxy-Country")))
+	searchQuery.ProxyClass = strings.ToLower(strings.TrimSpace(reqCtx.Get("X-Proxy-Class")))
+	searchQuery.ProxyProvider = strings.ToLower(strings.TrimSpace(reqCtx.Get("X-Proxy-Provider")))
+	searchQuery.ProxySessionID = strings.TrimSpace(reqCtx.Get("X-Proxy-Session-ID"))
 
 	if searchQuery.IsEmpty() {
 		return errEmptyQuery()
@@ -260,8 +451,18 @@ type SearchEngineOptions struct {
 	// IsSolveCaptcha enables automatic captcha solving when engine support and
 	// solver credentials are configured.
 	IsSolveCaptcha bool `mapstructure:"captcha"`
+
+	limiterState *rateLimiterState
 }
 
+type rateLimiterState struct {
+	limiter *rate.Limiter
+	every   time.Duration
+	burst   int
+}
+
+var searchEngineOptionsLimiterMu sync.Mutex
+
 // Init sets default option values when fields are zero.
 func (o *SearchEngineOptions) Init() {
 	if o.RateRequests == 0 {
@@ -279,10 +480,30 @@ func (o *SearchEngineOptions) Init() {
 }
 
 // GetRatelimit returns the interval between two allowed requests.
+// Call Init() first so RateRequests / RateTime are non-zero.
 func (o *SearchEngineOptions) GetRatelimit() time.Duration {
 	return (time.Duration(o.RateTime) * time.Second) / time.Duration(o.RateRequests)
 }
 
+// GetRateLimiter returns a cached limiter configured from SearchEngineOptions.
+// Call Init() first so RateBurst is non-zero. Do not copy SearchEngineOptions
+// after first use; the limiter state is intentionally shared by each engine.
+func (o *SearchEngineOptions) GetRateLimiter() *rate.Limiter {
+	every := o.GetRatelimit()
+	burst := o.RateBurst
+	searchEngineOptionsLimiterMu.Lock()
+	defer searchEngineOptionsLimiterMu.Unlock()
+	if o.limiterState == nil {
+		o.limiterState = &rateLimiterState{}
+	}
+	if o.limiterState.limiter == nil || o.limiterState.every != every || o.limiterState.burst != burst {
+		o.limiterState.limiter = rate.NewLimiter(rate.Every(every), burst)
+		o.limiterState.every = every
+		o.limiterState.burst = burst
+	}
+	return o.limiterState.limiter
+}
+
 // GetSelectorTimeout returns the selector wait timeout as time.Duration.
 func (o *SearchEngineOptions) GetSelectorTimeout() time.Duration {
 	return time.Duration(o.SelectorTimeout) * time.Second

core/common_test.go

@@ -0,0 +1,58 @@
+package core
+
+import "testing"
+
+func TestShouldFetchResultPage(t *testing.T) {
+	tests := []struct {
+		name         string
+		collected    int
+		limit        int
+		pagesFetched int
+		want         bool
+	}{
+		{
+			name:         "always fetch first page",
+			limit:        10,
+			pagesFetched: 0,
+			want:         true,
+		},
+		{
+			name:         "unset limit stops after first page",
+			collected:    8,
+			limit:        0,
+			pagesFetched: 1,
+			want:         false,
+		},
+		{
+			name:         "default limit stops after first page even if short",
+			collected:    8,
+			limit:        10,
+			pagesFetched: 1,
+			want:         false,
+		},
+		{
+			name:         "larger limit can fetch another short page",
+			collected:    8,
+			limit:        11,
+			pagesFetched: 1,
+			want:         true,
+		},
+		{
+			name:         "larger limit stops when satisfied",
+			collected:    11,
+			limit:        11,
+			pagesFetched: 1,
+			want:         false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ShouldFetchResultPage(tt.collected, tt.limit, tt.pagesFetched)
+			if got != tt.want {
+				t.Fatalf("ShouldFetchResultPage(%d, %d, %d) = %t, want %t",
+					tt.collected, tt.limit, tt.pagesFetched, got, tt.want)
+			}
+		})
+	}
+}

core/context.go

@@ -36,3 +36,29 @@ func SleepContext(ctx context.Context, d time.Duration) error {
 func IsContextDone(err error) bool {
 	return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
 }
+
+// PrepareEngineContext applies request-scoped metadata expected by all engine
+// search implementations.
+func PrepareEngineContext(ctx context.Context, query Query, engineName string, minimalBrowserProfile bool) context.Context {
+	ctx = WithEngine(EnsureContext(ctx), engineName)
+	ctx = WithProfileRegion(ctx, profileRegionHint(query))
+	if minimalBrowserProfile {
+		ctx = WithMinimalBrowserProfile(ctx)
+	}
+	return WithQueryHash(ctx, QueryHashFromQuery(query))
+}
+
+// profileRegionHint picks the strongest market signal for browser fingerprint
+// matching. An explicit country-code Region wins (e.g. region=DE → de or en-DE),
+// since it's what the user asked the engine to localize to. Engine-native
+// numeric region IDs (Yandex lr) are ignored here and we fall back to LangCode.
+func profileRegionHint(q Query) string {
+	country := CountryFromRegion(q.Region)
+	if country == "" {
+		return q.LangCode
+	}
+	if lang := ParseLocale(q.LangCode).Language; lang != "" {
+		return lang + "-" + country
+	}
+	return country
+}

core/context_test.go

@@ -0,0 +1,26 @@
+package core
+
+import "testing"
+
+func TestProfileRegionHint(t *testing.T) {
+	tests := []struct {
+		name string
+		q    Query
+		want string
+	}{
+		{name: "empty", q: Query{}, want: ""},
+		{name: "lang only", q: Query{LangCode: "en"}, want: "en"},
+		{name: "region country combines with lang", q: Query{LangCode: "en", Region: "DE"}, want: "en-DE"},
+		{name: "region locale combines with lang", q: Query{LangCode: "en", Region: "en-GB"}, want: "en-GB"},
+		{name: "region country without lang", q: Query{Region: "DE"}, want: "DE"},
+		{name: "yandex numeric region falls back to lang", q: Query{LangCode: "ru", Region: "213"}, want: "ru"},
+		{name: "yandex numeric region without lang stays empty", q: Query{Region: "213"}, want: ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := profileRegionHint(tt.q); got != tt.want {
+				t.Fatalf("profileRegionHint = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}

core/enrichment_domain.go

@@ -45,15 +45,9 @@ func EnrichDomainInfo(domain string) *DomainInfo {
 	cfg := loadEnrichmentDomains()
 
 	info := &DomainInfo{
-		TLD:           tld,
-		SLD:           sld,
-		IsGov:         isGovTLD(domain, tld),
-		IsEdu:         isEduTLD(domain, tld),
-		IsMil:         isMilTLD(tld),
-		IsNews:        cfg.NewsDomains[domain],
-		IsForum:       cfg.ForumDomains[domain],
-		IsMarketplace: cfg.MarketplaceDomains[domain],
-		IsSocial:      cfg.SocialDomains[domain],
+		TLD:      tld,
+		SLD:      sld,
+		Category: domainCategory(domain, tld, cfg),
 	}
 	return info
 }
@@ -67,6 +61,9 @@ func ClassifyURL(rawURL, domain string) *Classification {
 
 	contentType := classifyContentType(rawURL)
 	sourceHint := classifySourceHint(domain)
+	if contentType == "webpage" && sourceHint == "" {
+		return nil
+	}
 
 	return &Classification{
 		ContentType: contentType,
@@ -74,6 +71,27 @@ func ClassifyURL(rawURL, domain string) *Classification {
 	}
 }
 
+func domainCategory(domain, tld string, cfg enrichmentDomainsConfig) string {
+	switch {
+	case isGovTLD(domain, tld):
+		return "gov"
+	case isEduTLD(domain, tld):
+		return "edu"
+	case isMilTLD(tld):
+		return "mil"
+	case cfg.NewsDomains[domain]:
+		return "news"
+	case cfg.ForumDomains[domain]:
+		return "forum"
+	case cfg.MarketplaceDomains[domain]:
+		return "marketplace"
+	case cfg.SocialDomains[domain]:
+		return "social"
+	default:
+		return ""
+	}
+}
+
 // splitDomain returns (public suffix, registrable domain label).
 func splitDomain(domain string) (tld, sld string) {
 	domain = normalizeDomain(domain)
@@ -140,9 +158,20 @@ func classifyContentType(rawURL string) string {
 
 func classifySourceHint(domain string) string {
 	cfg := loadEnrichmentDomains()
-	if hint, ok := cfg.DomainSourceHints[normalizeDomain(domain)]; ok {
+	domain = normalizeDomain(domain)
+	if hint, ok := cfg.DomainSourceHints[domain]; ok {
 		return hint
 	}
+	// Fall back to the registrable domain so subdomain hosts (e.g.
+	// megadeth.fandom.com, zh.m.wikipedia.org) match a hint keyed on the
+	// registrable domain (fandom.com, wikipedia.org).
+	if _, sld := splitDomain(domain); sld != "" {
+		if registrable, err := publicsuffix.EffectiveTLDPlusOne(domain); err == nil {
+			if hint, ok := cfg.DomainSourceHints[registrable]; ok {
+				return hint
+			}
+		}
+	}
 	return ""
 }
 

core/enrichment_domains.yaml

@@ -1,6 +1,11 @@
 domain_source_hints:
   wikipedia.org: encyclopedia
   en.wikipedia.org: encyclopedia
+  zh.wikipedia.org: encyclopedia
+  baike.baidu.com: encyclopedia
+  wiki.mbalib.com: encyclopedia
+  britannica.com: encyclopedia
+  fandom.com: encyclopedia
   github.com: code_repository
   gitlab.com: code_repository
   stackoverflow.com: qa_forum

core/errors.go

@@ -5,22 +5,29 @@ import "fmt"
 // APIError represents a client-facing error with a stable machine-readable reason code.
 type APIError struct {
 	HTTPStatus int
+	ErrorCode  string
 	Reason     string
 	Message    string
+	Meta       map[string]interface{}
 }
 
 func (e *APIError) Error() string {
+	if e.Reason == "" {
+		return e.Message
+	}
 	return fmt.Sprintf("%s: %s", e.Reason, e.Message)
 }
 
 // Common validation reason codes.
 const (
-	ReasonInvalidLimit  = "INVALID_LIMIT"
-	ReasonInvalidStart  = "INVALID_START"
-	ReasonInvalidParam  = "INVALID_PARAM"
-	ReasonEmptyQuery    = "EMPTY_QUERY"
-	ReasonNoEngines     = "NO_ENGINES"
-	ReasonUnknownFormat = "UNKNOWN_FORMAT"
+	ReasonInvalidLimit            = "INVALID_LIMIT"
+	ReasonInvalidStart            = "INVALID_START"
+	ReasonInvalidParam            = "INVALID_PARAM"
+	ReasonEmptyQuery              = "EMPTY_QUERY"
+	ReasonNoEngines               = "NO_ENGINES"
+	ReasonUnknownFormat           = "UNKNOWN_FORMAT"
+	ReasonRequestProxyURLDisabled = "REQUEST_PROXY_URL_DISABLED"
+	ReasonUnsupportedProxyScheme  = "UNSUPPORTED_PROXY_SCHEME"
 )
 
 func errInvalidLimit(msg string) *APIError {

core/feature_selectors.go

@@ -0,0 +1,274 @@
+package core
+
+import (
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// blockLevelTags are HTML elements whose boundaries should become line breaks
+// when flattening a feature container to text, so block content (headings,
+// paragraphs, list items, code blocks) does not fuse into the neighbouring text.
+// div/section are deliberately excluded: some engines (e.g. Google's streaming
+// AI Overview) wrap every word in its own <div>, which would otherwise put each
+// word on its own line. Structure there comes from p/h*/li/br instead.
+var blockLevelTags = map[atom.Atom]bool{
+	atom.P: true, atom.Br: true, atom.Li: true,
+	atom.Tr: true, atom.Pre: true,
+	atom.H1: true, atom.H2: true, atom.H3: true, atom.H4: true, atom.H5: true, atom.H6: true,
+	atom.Blockquote: true,
+}
+
+// blockAwareText flattens a selection to text while inserting line breaks at
+// block-element boundaries, then collapses horizontal whitespace per line and
+// drops blank lines. The result keeps logical structure (one line per heading/
+// paragraph/list item) instead of fusing words across element edges, which is
+// what goquery's raw .Text() does.
+func blockAwareText(sel *goquery.Selection) string {
+	var sb strings.Builder
+	for _, node := range sel.Nodes {
+		writeNodeText(&sb, node)
+	}
+	lines := strings.Split(sb.String(), "\n")
+	cleaned := make([]string, 0, len(lines))
+	for _, line := range lines {
+		if line = cleanFeatureText(line); line != "" {
+			cleaned = append(cleaned, line)
+		}
+	}
+	return strings.Join(cleaned, "\n")
+}
+
+func isASCIISpace(b byte) bool {
+	return b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\f' || b == '\v'
+}
+
+func writeNodeText(sb *strings.Builder, node *html.Node) {
+	switch node.Type {
+	case html.TextNode:
+		// Collapse whitespace inside the text node (including source-formatting
+		// newlines) to single spaces, so only the block-boundary breaks inserted
+		// below survive. Preserve a single leading/trailing space so adjacent
+		// inline fragments ("global " + "fetch()") keep their word gap.
+		text := node.Data
+		collapsed := strings.Join(strings.Fields(text), " ")
+		if collapsed == "" {
+			return
+		}
+		if len(text) > 0 && isASCIISpace(text[0]) {
+			sb.WriteByte(' ')
+		}
+		sb.WriteString(collapsed)
+		if len(text) > 0 && isASCIISpace(text[len(text)-1]) {
+			sb.WriteByte(' ')
+		}
+		return
+	case html.ElementNode:
+		if node.DataAtom == atom.Script || node.DataAtom == atom.Style {
+			return
+		}
+		block := blockLevelTags[node.DataAtom]
+		if block {
+			sb.WriteByte('\n')
+		}
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			writeNodeText(sb, child)
+		}
+		if block {
+			sb.WriteByte('\n')
+		}
+	default:
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			writeNodeText(sb, child)
+		}
+	}
+}
+
+// SerpFeatureSelector describes one engine-native SERP module shape.
+type SerpFeatureSelector struct {
+	Type          ResultType
+	Title         string
+	Container     []string
+	TitleSelector []string
+	TextSelector  []string
+	ItemSelector  []string
+	LinkSelector  []string
+	Position      int
+	Confidence    float64
+	// SingleMatch emits at most one feature for this spec: the first container
+	// node (across Container selectors, in order) that yields content. Use it
+	// for modules whose container selector also matches nested sub-panels, which
+	// would otherwise fragment one logical module into many features.
+	SingleMatch bool
+}
+
+// ExtractSerpFeaturesBySelectors converts engine-native SERP module markup into
+// normalized features. It is intentionally conservative: a matched container is
+// emitted only when it yields text, grouped items, or source links.
+func ExtractSerpFeaturesBySelectors(doc *goquery.Document, specs []SerpFeatureSelector) []SerpFeature {
+	var features []SerpFeature
+	for _, spec := range specs {
+		matched := false
+		for _, selector := range spec.Container {
+			if spec.SingleMatch && matched {
+				break
+			}
+			doc.Find(selector).EachWithBreak(func(_ int, container *goquery.Selection) bool {
+				feature := SerpFeature{
+					Type:       spec.Type,
+					Title:      firstNonEmpty(spec.Title, firstSelectedText(container, spec.TitleSelector)),
+					Text:       firstSelectedText(container, spec.TextSelector),
+					Items:      selectedFeatureItems(container, spec.ItemSelector),
+					Links:      selectedFeatureLinks(container, spec.LinkSelector),
+					Confidence: spec.Confidence,
+				}
+				if spec.Position > 0 {
+					feature.Position = &Position{Absolute: spec.Position}
+				}
+				if feature.Text == "" && len(feature.Items) == 0 && len(feature.Links) == 0 {
+					return true
+				}
+				features = append(features, feature)
+				matched = true
+				// Stop after the first content-bearing container when SingleMatch.
+				return !spec.SingleMatch
+			})
+		}
+	}
+	return DeduplicateSerpFeatures(features)
+}
+
+// AttachFeaturesToFirstResult keeps ParseHTML signatures unchanged while
+// letting server response building split features onto the new top-level field.
+func AttachFeaturesToFirstResult(results []SearchResult, features []SerpFeature) []SearchResult {
+	if len(features) == 0 {
+		return results
+	}
+	if len(results) == 0 {
+		return []SearchResult{{Features: features}}
+	}
+	results[0].Features = append(results[0].Features, features...)
+	return results
+}
+
+// DeduplicateSerpFeatures removes duplicate modules emitted by overlapping
+// selectors while preserving original order.
+func DeduplicateSerpFeatures(features []SerpFeature) []SerpFeature {
+	seen := map[string]struct{}{}
+	unique := make([]SerpFeature, 0, len(features))
+	for _, feature := range features {
+		key := serpFeatureKey(feature)
+		if _, ok := seen[key]; ok {
+			continue
+		}
+		seen[key] = struct{}{}
+		unique = append(unique, feature)
+	}
+	return unique
+}
+
+func firstSelectedText(container *goquery.Selection, selectors []string) string {
+	for _, selector := range selectors {
+		var text string
+		container.Find(selector).EachWithBreak(func(_ int, item *goquery.Selection) bool {
+			text = blockAwareText(item)
+			return text == ""
+		})
+		if text != "" {
+			return text
+		}
+	}
+	return ""
+}
+
+func selectedFeatureItems(container *goquery.Selection, selectors []string) []FeatureItem {
+	var items []FeatureItem
+	for _, selector := range selectors {
+		container.Find(selector).Each(func(_ int, item *goquery.Selection) {
+			text := cleanFeatureText(item.Text())
+			title := firstAttr(item, "data-q", "data-title", "aria-label", "title")
+			// Some modules (e.g. Google PAA) carry the question in an attribute
+			// and render the answer lazily, so the element text can be empty.
+			if text == "" {
+				text = cleanFeatureText(title)
+			}
+			if text == "" {
+				return
+			}
+			link := firstAttr(item, "href", "data-url", "data-link")
+			items = append(items, FeatureItem{
+				Title: strings.TrimSpace(title),
+				Text:  text,
+				Link:  strings.TrimSpace(link),
+			})
+		})
+		if len(items) > 0 {
+			break
+		}
+	}
+	return items
+}
+
+func selectedFeatureLinks(container *goquery.Selection, selectors []string) []FeatureLink {
+	var links []FeatureLink
+	for _, selector := range selectors {
+		container.Find(selector).Each(func(_ int, item *goquery.Selection) {
+			href := strings.TrimSpace(firstAttr(item, "href", "data-url", "data-link"))
+			if href == "" {
+				return
+			}
+			title := cleanFeatureText(firstAttr(item, "data-title", "aria-label", "title"))
+			if title == "" {
+				title = cleanFeatureText(item.Text())
+			}
+			links = append(links, FeatureLink{Title: title, URL: href})
+		})
+		if len(links) > 0 {
+			break
+		}
+	}
+	return links
+}
+
+func firstAttr(item *goquery.Selection, names ...string) string {
+	for _, name := range names {
+		value, ok := item.Attr(name)
+		if ok && strings.TrimSpace(value) != "" {
+			return value
+		}
+	}
+	return ""
+}
+
+func firstNonEmpty(values ...string) string {
+	for _, value := range values {
+		if strings.TrimSpace(value) != "" {
+			return strings.TrimSpace(value)
+		}
+	}
+	return ""
+}
+
+func cleanFeatureText(value string) string {
+	return strings.Join(strings.Fields(value), " ")
+}
+
+func serpFeatureKey(feature SerpFeature) string {
+	firstLink := ""
+	if len(feature.Links) > 0 {
+		firstLink = feature.Links[0].URL
+	}
+	firstItem := ""
+	if len(feature.Items) > 0 {
+		firstItem = feature.Items[0].Text + "|" + feature.Items[0].Link
+	}
+	return strings.Join([]string{
+		string(feature.Type),
+		strings.ToLower(cleanFeatureText(feature.Title)),
+		strings.ToLower(cleanFeatureText(feature.Text)),
+		strings.ToLower(firstItem),
+		strings.ToLower(firstLink),
+	}, "|")
+}

core/format_markdown.go

@@ -12,48 +12,130 @@ func RenderMarkdown(env *Envelope) []byte {
 
 	enginesStr := strings.Join(env.Query.EnginesRequested, ", ")
 	fmt.Fprintf(&b, "# Search results for %q\n\n", env.Query.Text)
-	fmt.Fprintf(&b, "**Query:** %s · **Engines:** %s · **Took:** %dms\n\n",
+	fmt.Fprintf(&b, "**Query:** %s - **Engines:** %s - **Took:** %dms\n\n",
 		env.Query.Text, enginesStr, env.Meta.TookMs)
 
 	if len(env.Meta.EnginesFailed) > 0 {
-		fmt.Fprintf(&b, "> ⚠️ Engines that failed: %s\n\n", strings.Join(env.Meta.EnginesFailed, ", "))
+		fmt.Fprintf(&b, "> Engines that failed: %s\n\n", strings.Join(env.Meta.EnginesFailed, ", "))
 	}
 
+	renderMarkdownFeatures(&b, env.SerpFeatures, featureRenderOrderBeforeResults())
+
+	if len(env.Results) > 0 {
+		b.WriteString("## Results\n\n")
+	}
 	for i, r := range env.Results {
-		fmt.Fprintf(&b, "## %d. %s\n\n", i+1, escapeMarkdown(r.Title))
+		fmt.Fprintf(&b, "### %d. %s\n\n", i+1, escapeMarkdown(r.Title))
 		typeLabel := string(r.Type)
-		if r.IsAd {
-			typeLabel = "ad"
-		}
-		fmt.Fprintf(&b, "**%s** · %s\n\n", r.DisplayURL, typeLabel)
+		fmt.Fprintf(&b, "**%s** - %s\n\n", r.DisplayURL, typeLabel)
 		if r.Snippet != "" {
 			fmt.Fprintf(&b, "%s\n\n", r.Snippet)
 		}
-		fmt.Fprintf(&b, "→ %s\n\n", r.URL)
+		fmt.Fprintf(&b, "-> %s\n\n", r.URL)
+		if r.Extracted != nil && r.Extracted.Content != "" {
+			b.WriteString("#### Extracted content\n\n")
+			b.WriteString(shiftMarkdownHeadings(r.Extracted.Content, 4))
+			b.WriteString("\n\n")
+		}
 	}
 
+	renderMarkdownFeatures(&b, env.SerpFeatures, featureRenderOrderAfterResults(env.SerpFeatures))
+
 	return []byte(b.String())
 }
 
+func shiftMarkdownHeadings(markdown string, minLevel int) string {
+	lines := strings.Split(markdown, "\n")
+	for i, line := range lines {
+		trimmed := strings.TrimLeft(line, " ")
+		indent := line[:len(line)-len(trimmed)]
+		if !strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+		count := 0
+		for count < len(trimmed) && trimmed[count] == '#' {
+			count++
+		}
+		if count == 0 || count >= len(trimmed) || trimmed[count] != ' ' {
+			continue
+		}
+		target := count + minLevel
+		if target > 6 {
+			target = 6
+		}
+		lines[i] = indent + strings.Repeat("#", target) + trimmed[count:]
+	}
+	return strings.TrimSpace(strings.Join(lines, "\n"))
+}
+
 // RenderMarkdownImage formats an ImageEnvelope as Markdown.
 func RenderMarkdownImage(env *ImageEnvelope) []byte {
 	var b strings.Builder
 
 	enginesStr := strings.Join(env.Query.EnginesRequested, ", ")
 	fmt.Fprintf(&b, "# Image results for %q\n\n", env.Query.Text)
-	fmt.Fprintf(&b, "**Query:** %s · **Engines:** %s · **Took:** %dms\n\n",
+	fmt.Fprintf(&b, "**Query:** %s - **Engines:** %s - **Took:** %dms\n\n",
 		env.Query.Text, enginesStr, env.Meta.TookMs)
 
 	for i, r := range env.Results {
 		fmt.Fprintf(&b, "## %d. %s\n\n", i+1, escapeMarkdown(r.Title))
 		fmt.Fprintf(&b, "**Source:** %s\n\n", r.Source.Domain)
-		fmt.Fprintf(&b, "→ Image: %s\n", r.Image.URL)
-		fmt.Fprintf(&b, "→ Page: %s\n\n", r.Source.PageURL)
+		fmt.Fprintf(&b, "-> Image: %s\n", r.Image.URL)
+		fmt.Fprintf(&b, "-> Page: %s\n\n", r.Source.PageURL)
 	}
 
 	return []byte(b.String())
 }
 
+func renderMarkdownFeatures(b *strings.Builder, features []SerpFeature, order []ResultType) {
+	forEachFeatureInOrder(features, order, func(feature SerpFeature) {
+		renderMarkdownFeature(b, feature)
+	})
+}
+
+func renderMarkdownFeature(b *strings.Builder, feature SerpFeature) {
+	heading := featureHeading(feature)
+	if feature.Type == ResultTypeKnowledgePanel && feature.Title != "" {
+		heading += " - " + feature.Title
+	}
+	fmt.Fprintf(b, "## %s\n\n", heading)
+	if feature.Type == ResultTypeFeaturedSnippet && feature.Text != "" {
+		fmt.Fprintf(b, "> %s\n", feature.Text)
+		if len(feature.Links) > 0 {
+			fmt.Fprintf(b, "> - [%s](%s)\n", escapeMarkdown(feature.Links[0].Title), feature.Links[0].URL)
+		}
+		b.WriteString("\n")
+		return
+	}
+	if feature.Text != "" {
+		fmt.Fprintf(b, "%s\n\n", feature.Text)
+	}
+	if len(feature.Items) > 0 {
+		for _, item := range feature.Items {
+			switch {
+			case item.Title != "" && item.Text != "":
+				fmt.Fprintf(b, "- **%s** - %s\n", escapeMarkdown(item.Title), item.Text)
+			case item.Text != "":
+				fmt.Fprintf(b, "- %s\n", item.Text)
+			case item.Title != "":
+				fmt.Fprintf(b, "- %s\n", escapeMarkdown(item.Title))
+			}
+		}
+		b.WriteString("\n")
+	}
+	if len(feature.Links) > 0 {
+		b.WriteString("Sources:\n")
+		for _, link := range feature.Links {
+			title := link.Title
+			if title == "" {
+				title = link.URL
+			}
+			fmt.Fprintf(b, "- [%s](%s)\n", escapeMarkdown(title), link.URL)
+		}
+		b.WriteString("\n")
+	}
+}
+
 func escapeMarkdown(s string) string {
 	replacer := strings.NewReplacer(
 		"*", `\*`,

core/format_text.go

@@ -21,14 +21,26 @@ func RenderText(env *Envelope) []byte {
 	}
 	b.WriteString("\n")
 
+	renderTextFeatures(&b, env.SerpFeatures, featureRenderOrderBeforeResults())
+
+	if len(env.Results) > 0 {
+		b.WriteString("Results\n\n")
+	}
 	for i, r := range env.Results {
 		fmt.Fprintf(&b, "[%d] %s (%s)\n", i+1, r.Title, r.Domain)
 		if r.Snippet != "" {
 			fmt.Fprintf(&b, "%s\n", r.Snippet)
 		}
 		fmt.Fprintf(&b, "URL: %s\n\n", r.URL)
+		if r.Extracted != nil && r.Extracted.Content != "" {
+			b.WriteString("Extracted content:\n")
+			b.WriteString(r.Extracted.Content)
+			b.WriteString("\n\n")
+		}
 	}
 
+	renderTextFeatures(&b, env.SerpFeatures, featureRenderOrderAfterResults(env.SerpFeatures))
+
 	return []byte(b.String())
 }
 
@@ -47,17 +59,14 @@ func RenderTextImage(env *ImageEnvelope) []byte {
 	return []byte(b.String())
 }
 
-// RenderNDJSON formats an Envelope as newline-delimited JSON (one Result per line).
-// The envelope meta is omitted from the body; clients should read response headers.
+// RenderNDJSON formats an Envelope as newline-delimited JSON.
 func RenderNDJSON(env *Envelope) []byte {
 	var b strings.Builder
 	for _, r := range env.Results {
-		data, err := json.Marshal(r)
-		if err != nil {
-			continue
-		}
-		b.Write(data)
-		b.WriteByte('\n')
+		writeNDJSONLine(&b, "result", r)
+	}
+	for _, feature := range env.SerpFeatures {
+		writeNDJSONLine(&b, "feature", feature)
 	}
 	return []byte(b.String())
 }
@@ -66,12 +75,164 @@ func RenderNDJSON(env *Envelope) []byte {
 func RenderNDJSONImage(env *ImageEnvelope) []byte {
 	var b strings.Builder
 	for _, r := range env.Results {
-		data, err := json.Marshal(r)
-		if err != nil {
-			continue
-		}
-		b.Write(data)
-		b.WriteByte('\n')
+		writeNDJSONLine(&b, "result", r)
 	}
 	return []byte(b.String())
 }
+
+func renderTextFeatures(b *strings.Builder, features []SerpFeature, order []ResultType) {
+	forEachFeatureInOrder(features, order, func(feature SerpFeature) {
+		renderTextFeature(b, feature)
+	})
+}
+
+// forEachFeatureInOrder invokes render for every feature whose Type appears in
+// order, type by type. It is the single iteration shared by the text and
+// markdown renderers.
+func forEachFeatureInOrder(features []SerpFeature, order []ResultType, render func(SerpFeature)) {
+	for _, featureType := range order {
+		for _, feature := range features {
+			if feature.Type == featureType {
+				render(feature)
+			}
+		}
+	}
+}
+
+func renderTextFeature(b *strings.Builder, feature SerpFeature) {
+	heading := featureHeading(feature)
+	if feature.Type == ResultTypeKnowledgePanel && feature.Title != "" {
+		heading += " - " + feature.Title
+	}
+	fmt.Fprintf(b, "%s\n", heading)
+	if feature.Text != "" {
+		fmt.Fprintf(b, "%s", feature.Text)
+		if len(feature.Links) == 1 {
+			fmt.Fprintf(b, " (source: %s)", feature.Links[0].URL)
+		}
+		b.WriteString("\n")
+	}
+	for _, item := range feature.Items {
+		switch {
+		case item.Title != "" && item.Text != "":
+			fmt.Fprintf(b, "- %s - %s\n", item.Title, item.Text)
+		case item.Text != "":
+			fmt.Fprintf(b, "- %s\n", item.Text)
+		case item.Title != "":
+			fmt.Fprintf(b, "- %s\n", item.Title)
+		}
+	}
+	if len(feature.Links) > 1 {
+		b.WriteString("Sources:\n")
+		for _, link := range feature.Links {
+			fmt.Fprintf(b, "- %s\n", link.URL)
+		}
+	}
+	b.WriteString("\n")
+}
+
+func writeNDJSONLine(b *strings.Builder, kind string, value any) {
+	data, err := json.Marshal(value)
+	if err != nil {
+		return
+	}
+	var object map[string]any
+	if err := json.Unmarshal(data, &object); err != nil {
+		return
+	}
+	object["kind"] = kind
+	data, err = json.Marshal(object)
+	if err != nil {
+		return
+	}
+	b.Write(data)
+	b.WriteByte('\n')
+}
+
+// featureRenderOrderBeforeResults lists the feature sections rendered above the
+// results list, in fixed order (spec: AI summary -> answer box -> featured
+// snippet -> PAA -> related questions -> knowledge panel -> results -> ...).
+func featureRenderOrderBeforeResults() []ResultType {
+	return []ResultType{
+		ResultTypeAISummary,
+		ResultTypeAnswerBox,
+		ResultTypeFeaturedSnippet,
+		ResultTypePeopleAlsoAsk,
+		ResultTypeRelatedQuestions,
+		ResultTypeKnowledgePanel,
+	}
+}
+
+// featureRenderOrderAfterResults lists the feature sections rendered below the
+// results list (related searches and the module gallery), in fixed order. Any
+// feature type present in features but absent from both fixed orders is appended
+// at the end, so a newly added feature enum is never silently dropped from text
+// or markdown output.
+func featureRenderOrderAfterResults(features []SerpFeature) []ResultType {
+	order := []ResultType{
+		ResultTypeRelatedSearches,
+		ResultTypeNews,
+		ResultTypeVideo,
+		ResultTypeVideos,
+		ResultTypeShopping,
+		ResultTypeImagesInline,
+		ResultTypeLocal,
+		ResultTypeSitelinks,
+		ResultTypeCalculator,
+		ResultTypeWeather,
+		ResultTypeDictionary,
+	}
+	placed := make(map[ResultType]bool, len(order)+len(featureRenderOrderBeforeResults()))
+	for _, t := range featureRenderOrderBeforeResults() {
+		placed[t] = true
+	}
+	for _, t := range order {
+		placed[t] = true
+	}
+	for _, feature := range features {
+		if !placed[feature.Type] {
+			order = append(order, feature.Type)
+			placed[feature.Type] = true
+		}
+	}
+	return order
+}
+
+func featureHeading(feature SerpFeature) string {
+	switch feature.Type {
+	case ResultTypeAISummary:
+		return "AI summary"
+	case ResultTypeAnswerBox:
+		return "Answer box"
+	case ResultTypeFeaturedSnippet:
+		return "Featured snippet"
+	case ResultTypePeopleAlsoAsk:
+		return "People also ask"
+	case ResultTypeRelatedQuestions:
+		return "Related questions"
+	case ResultTypeKnowledgePanel:
+		return "Knowledge panel"
+	case ResultTypeRelatedSearches:
+		return "Related searches"
+	case ResultTypeNews:
+		return "News"
+	case ResultTypeVideo, ResultTypeVideos:
+		return "Videos"
+	case ResultTypeShopping:
+		return "Shopping"
+	case ResultTypeImagesInline:
+		return "Images"
+	case ResultTypeLocal:
+		return "Local pack"
+	case ResultTypeSitelinks:
+		return "Sitelinks"
+	case ResultTypeCalculator:
+		return "Calculator"
+	case ResultTypeWeather:
+		return "Weather"
+	case ResultTypeDictionary:
+		return "Dictionary"
+	default:
+		return strings.ReplaceAll(string(feature.Type), "_", " ")
+	}
+}

core/fpcheck/detectors/custom.go

@@ -2,7 +2,6 @@ package detectors
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/url"
 	"strings"
@@ -13,17 +12,37 @@ import (
 )
 
 const customDetectorName = "custom"
+const defaultCustomSelector = "body"
 
 type Custom struct {
 	targetURL string
+	selector  string
+}
+
+type customPayload struct {
+	Found        bool   `json:"found"`
+	Error        string `json:"error"`
+	Title        string `json:"title"`
+	URL          string `json:"url"`
+	ReadyState   string `json:"readyState"`
+	Selector     string `json:"selector"`
+	SelectedText string `json:"selectedText"`
+	SelectedHTML string `json:"selectedHTML"`
 }
 
 func NewCustom(rawURL string) (fpcheck.Detector, error) {
+	return NewCustomWithSelector(rawURL, "")
+}
+
+func NewCustomWithSelector(rawURL string, selector string) (fpcheck.Detector, error) {
 	normalized, err := normalizeCustomURL(rawURL)
 	if err != nil {
 		return nil, err
 	}
-	return Custom{targetURL: normalized}, nil
+	return Custom{
+		targetURL: normalized,
+		selector:  normalizeCustomSelector(selector),
+	}, nil
 }
 
 func (c Custom) Name() string {
@@ -34,57 +53,76 @@ func (c Custom) URL() string {
 	return c.targetURL
 }
 
+func (c Custom) Selector() string {
+	return normalizeCustomSelector(c.selector)
+}
+
 func (c Custom) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
+	selector := c.Selector()
+	var payload customPayload
 	err := waitFor(ctx, 15*time.Second, 200*time.Millisecond, func() (bool, error) {
-		hasBody, _, err := page.Has("pre")
+		current, err := extractCustomPayload(page, selector)
 		if err != nil {
 			return false, err
 		}
-		return hasBody, nil
+		if !current.Found {
+			return false, nil
+		}
+		payload = current
+		return true, nil
 	})
 	if err != nil {
-		return nil, "", fmt.Errorf("custom page readiness: %w", err)
+		return nil, "", fmt.Errorf("custom page readiness for selector %q: %w", selector, err)
 	}
 
-	res, err := page.Eval(`() => {
-		const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
-		return {
-			title: document.title || "",
-			url: location.href || "",
-			readyState: document.readyState || "",
-			bodyText: normalize(document.body ? document.body.innerText || document.body.textContent || "" : ""),
-			html: document.documentElement ? document.documentElement.outerHTML || "" : "",
-		};
-	}`)
-	if err != nil {
-		return nil, "", err
-	}
-
-	var payload struct {
-		Title      string `json:"title"`
-		URL        string `json:"url"`
-		ReadyState string `json:"readyState"`
-		BodyText   string `json:"bodyText"`
-		HTML       string `json:"html"`
-	}
-	if err := res.Value.Unmarshal(&payload); err != nil {
-		return nil, "", fmt.Errorf("decode custom detector payload: %w", err)
-	}
-
-	payload.BodyText = strings.TrimSpace(payload.BodyText)
-	payload.HTML = strings.TrimSpace(payload.HTML)
-
-	rawOut, err := json.MarshalIndent(payload, "", "  ")
-	if err != nil {
-		return nil, "", fmt.Errorf("encode custom detector payload: %w", err)
+	rawOut := strings.TrimSpace(payload.SelectedText)
+	if rawOut == "" {
+		rawOut = strings.TrimSpace(payload.SelectedHTML)
 	}
 
 	return map[string]fpcheck.Detection{
-		"raw_page_output": {
+		"selected_page_output": {
 			Detected:    false,
-			Description: "raw page payload captured",
+			Description: fmt.Sprintf("captured selector %q", payload.Selector),
 		},
-	}, string(rawOut), nil
+	}, rawOut, nil
+}
+
+func extractCustomPayload(page *rod.Page, selector string) (customPayload, error) {
+	res, err := page.Timeout(2*time.Second).Eval(`(selector) => {
+		const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
+		let selected = null;
+		try {
+			selected = document.querySelector(selector);
+		 } catch (err) {
+			return {
+				found: false,
+				error: err && err.message ? err.message : String(err),
+				selector,
+			};
+		}
+		return {
+			found: !!selected,
+			title: document.title || "",
+			url: location.href || "",
+			readyState: document.readyState || "",
+			selector,
+			selectedText: normalize(selected ? selected.innerText || selected.textContent || "" : ""),
+			selectedHTML: selected ? selected.innerHTML || "" : "",
+		};
+	}`, selector)
+	if err != nil {
+		return customPayload{}, err
+	}
+
+	var payload customPayload
+	if err := res.Value.Unmarshal(&payload); err != nil {
+		return customPayload{}, fmt.Errorf("decode custom detector payload: %w", err)
+	}
+	if strings.TrimSpace(payload.Error) != "" {
+		return customPayload{}, fmt.Errorf("query selector %q: %s", selector, payload.Error)
+	}
+	return payload, nil
 }
 
 func normalizeCustomURL(rawURL string) (string, error) {
@@ -106,3 +144,11 @@ func normalizeCustomURL(rawURL string) (string, error) {
 
 	return parsed.String(), nil
 }
+
+func normalizeCustomSelector(selector string) string {
+	trimmed := strings.TrimSpace(selector)
+	if trimmed == "" {
+		return defaultCustomSelector
+	}
+	return trimmed
+}

core/fpcheck/detectors/registry.go

@@ -28,13 +28,17 @@ func All() []fpcheck.Detector {
 }
 
 func Select(name string, customURL string) ([]fpcheck.Detector, error) {
+	return SelectWithCustomSelector(name, customURL, "")
+}
+
+func SelectWithCustomSelector(name string, customURL string, customSelector string) ([]fpcheck.Detector, error) {
 	trimmed := strings.ToLower(strings.TrimSpace(name))
 	if trimmed == "" || trimmed == "all" {
 		return All(), nil
 	}
 
 	if IsCustom(trimmed) {
-		customDetector, err := NewCustom(customURL)
+		customDetector, err := NewCustomWithSelector(customURL, customSelector)
 		if err != nil {
 			return nil, err
 		}

core/fpcheck/detectors/registry_test.go

@@ -23,6 +23,28 @@ func TestSelectCustomAcceptsHTTPSURL(t *testing.T) {
 	if detectorList[0].URL() != "https://localhost:9000" {
 		t.Fatalf("expected normalized custom URL to be preserved, got %q", detectorList[0].URL())
 	}
+	custom, ok := detectorList[0].(Custom)
+	if !ok {
+		t.Fatalf("expected custom detector type, got %T", detectorList[0])
+	}
+	if custom.Selector() != "body" {
+		t.Fatalf("expected default custom selector body, got %q", custom.Selector())
+	}
+}
+
+func TestSelectCustomAcceptsSelector(t *testing.T) {
+	detectorList, err := SelectWithCustomSelector("custom", "https://localhost:9000", "pre")
+	if err != nil {
+		t.Fatalf("expected custom detector URL to be accepted, got %v", err)
+	}
+
+	custom, ok := detectorList[0].(Custom)
+	if !ok {
+		t.Fatalf("expected custom detector type, got %T", detectorList[0])
+	}
+	if custom.Selector() != "pre" {
+		t.Fatalf("expected custom selector pre, got %q", custom.Selector())
+	}
 }
 
 func TestNamesIncludesCustom(t *testing.T) {

core/fpcheck/runner.go

@@ -15,8 +15,9 @@ import (
 
 // RunOptions controls detector run behavior.
 type RunOptions struct {
-	ArtifactDir     string
-	WaitBeforeClose time.Duration
+	ArtifactDir       string
+	WaitBeforeExtract time.Duration
+	WaitBeforeClose   time.Duration
 }
 
 // Run navigates the given browser to detector URL, extracts verdicts,
@@ -53,9 +54,13 @@ func RunWithOptions(ctx context.Context, browser BrowserNavigator, detector Dete
 		if options.WaitBeforeClose > 0 {
 			_ = sleepWithContext(ctx, options.WaitBeforeClose)
 		}
-		closePageWithTimeout(context.Background(), page, time.Second)
+		closePageWithTimeout(context.Background(), browser, page, time.Second)
 	}()
 
+	if options.WaitBeforeExtract > 0 {
+		_ = sleepWithContext(ctx, options.WaitBeforeExtract)
+	}
+
 	detections, rawNotes, err := detector.Extract(ctx, page)
 	if err != nil {
 		_ = saveScreenshot(page, screenshotPath)
@@ -124,10 +129,18 @@ func saveScreenshot(page *rod.Page, path string) error {
 	return nil
 }
 
-func closePageWithTimeout(ctx context.Context, page *rod.Page, timeout time.Duration) {
+type pageCloser interface {
+	ClosePage(context.Context, *rod.Page, time.Duration) error
+}
+
+func closePageWithTimeout(ctx context.Context, browser BrowserNavigator, page *rod.Page, timeout time.Duration) {
 	if page == nil {
 		return
 	}
+	if closer, ok := browser.(pageCloser); ok {
+		_ = closer.ClosePage(ctx, page, timeout)
+		return
+	}
 	if timeout <= 0 {
 		timeout = time.Second
 	}

core/html_parser.go

@@ -0,0 +1,10 @@
+package core
+
+import "io"
+
+// HTMLParser is implemented by engines that can parse a SERP HTML document
+// without a live browser. Used to expose POST /parse/{engine} endpoints.
+type HTMLParser interface {
+	Name() string
+	ParseHTML(io.Reader) ([]SearchResult, error)
+}

core/http_client.go

@@ -3,6 +3,7 @@ package core
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -10,10 +11,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/corpix/uarand"
 	utls "github.com/refraction-networking/utls"
 )
 
-const rawHTTPTimeout = 10 * time.Second
+const rawHTTPTimeout = 30 * time.Second
 
 // SetAcceptLanguageHeader sets the Accept-Language header from a lang code.
 // No-op when the code has no language subtag.
@@ -37,6 +39,58 @@ func DrainAndCloseResponse(resp *http.Response) {
 	_ = resp.Body.Close()
 }
 
+// RawSearchRequest builds and executes a raw-mode SERP HTTP GET. It uses the
+// shared raw HTTP client (TLS fingerprinting, network usage tracking, proxy
+// support), randomizes the User-Agent, and applies the Accept-Language header
+// derived from the query locale. The caller owns the returned response and
+// must drain/close it (see DrainAndCloseResponse).
+func RawSearchRequest(ctx context.Context, searchURL string, query Query) (*http.Response, error) {
+	if query.GuardPrivateNetworks {
+		if err := ValidatePublicHTTPURL(ctx, searchURL); err != nil {
+			return nil, err
+		}
+	}
+	client, err := NewRawHTTPClient(query)
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", uarand.GetRandom())
+	SetAcceptLanguageHeader(req, query.LangCode)
+	return client.Do(req)
+}
+
+func ReadRawSearchBody(resp *http.Response) ([]byte, error) {
+	if resp == nil {
+		return nil, fmt.Errorf("%w: nil raw search response", ErrEngineInternal)
+	}
+	if err := ClassifySearchHTTPStatus(resp.StatusCode); err != nil {
+		return nil, err
+	}
+	return io.ReadAll(resp.Body)
+}
+
+func ClassifySearchHTTPStatus(status int) error {
+	switch status {
+	case 0:
+		return nil
+	case http.StatusForbidden, http.StatusUnauthorized:
+		return ErrBlocked
+	case http.StatusTooManyRequests:
+		return ErrRateLimited
+	}
+	if status >= 500 {
+		return fmt.Errorf("%w: search engine returned HTTP %d", ErrBlocked, status)
+	}
+	if status < 200 || status >= 300 {
+		return fmt.Errorf("%w: search engine returned HTTP %d", ErrParser, status)
+	}
+	return nil
+}
+
 func NewRawHTTPClient(query Query) (*http.Client, error) {
 	transport, err := newRawTransport(query)
 	if err != nil {
@@ -44,17 +98,30 @@ func NewRawHTTPClient(query Query) (*http.Client, error) {
 	}
 	roundTripper := http.RoundTripper(transport)
 	if transport.Proxy != nil {
-		roundTripper = proxyErrorTransport{base: transport}
+		roundTripper = proxyErrorTransport{base: roundTripper}
 	}
 
-	return &http.Client{
+	client := &http.Client{
 		Transport: roundTripper,
 		Timeout:   rawHTTPTimeout,
-	}, nil
+	}
+	if query.GuardPrivateNetworks {
+		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+			return ValidatePublicHTTPURL(req.Context(), req.URL.String())
+		}
+	}
+	return client, nil
 }
 
 func newRawTransport(query Query) (*http.Transport, error) {
-	transport := &http.Transport{}
+	dialContext := dialNetworkUsageConn
+	if query.GuardPrivateNetworks {
+		dialContext = guardedDialNetworkUsageConn
+	}
+
+	transport := &http.Transport{
+		DialContext: dialContext,
+	}
 	if query.Insecure {
 		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 	}
@@ -71,13 +138,15 @@ func newRawTransport(query Query) (*http.Transport, error) {
 
 		// Keep proxied requests on the standard transport path so SOCKS5/SOCKS5H
 		// resolution and routing are handled by the configured proxy correctly.
+		// The extract SSRF guard validates the target URL before the request and
+		// on redirects; the proxy address itself may legitimately be local.
+		transport.DialContext = dialNetworkUsageConn
 		transport.Proxy = http.ProxyURL(parsed)
 		return transport, nil
 	}
 
 	transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
-		dialer := &net.Dialer{}
-		rawConn, err := dialer.DialContext(ctx, network, addr)
+		rawConn, err := dialContext(ctx, network, addr)
 		if err != nil {
 			return nil, err
 		}
@@ -86,9 +155,15 @@ func newRawTransport(query Query) (*http.Transport, error) {
 		config := &utls.Config{
 			ServerName:         hostname,
 			InsecureSkipVerify: query.Insecure,
+			NextProtos:         []string{"http/1.1"},
 		}
 
 		uconn := utls.UClient(rawConn, config, utls.HelloChrome_Auto)
+		if err := uconn.BuildHandshakeState(); err != nil {
+			rawConn.Close()
+			return nil, err
+		}
+		forceHTTP1ALPN(uconn)
 		if err := uconn.Handshake(); err != nil {
 			rawConn.Close()
 			return nil, err
@@ -100,6 +175,44 @@ func newRawTransport(query Query) (*http.Transport, error) {
 	return transport, nil
 }
 
+func forceHTTP1ALPN(conn *utls.UConn) {
+	for _, ext := range conn.Extensions {
+		if alpn, ok := ext.(*utls.ALPNExtension); ok {
+			alpn.AlpnProtocols = []string{"http/1.1"}
+			return
+		}
+	}
+	conn.Extensions = append(conn.Extensions, &utls.ALPNExtension{AlpnProtocols: []string{"http/1.1"}})
+}
+
+func dialNetworkUsageConn(ctx context.Context, network, addr string) (net.Conn, error) {
+	dialer := &net.Dialer{}
+	conn, err := dialer.DialContext(ctx, network, addr)
+	if err != nil {
+		return nil, err
+	}
+	return networkUsageConn{Conn: conn, ctx: ctx}, nil
+}
+
+func guardedDialNetworkUsageConn(ctx context.Context, network, addr string) (net.Conn, error) {
+	conn, err := GuardedDialContext(ctx, network, addr)
+	if err != nil {
+		return nil, err
+	}
+	return networkUsageConn{Conn: conn, ctx: ctx}, nil
+}
+
+type networkUsageConn struct {
+	net.Conn
+	ctx context.Context
+}
+
+func (c networkUsageConn) Read(p []byte) (int, error) {
+	n, err := c.Conn.Read(p)
+	AddNetworkBytes(c.ctx, int64(n))
+	return n, err
+}
+
 type proxyErrorTransport struct {
 	base http.RoundTripper
 }

core/http_client_test.go

@@ -1,8 +1,11 @@
 package core
 
 import (
+	"context"
+	"errors"
 	"io"
 	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
 )
@@ -33,7 +36,100 @@ func TestDrainAndCloseResponseDrainsAndCloses(t *testing.T) {
 	}
 }
 
+func TestClassifySearchHTTPStatus(t *testing.T) {
+	tests := []struct {
+		name   string
+		status int
+		want   error
+	}{
+		{name: "unknown browser status", status: 0, want: nil},
+		{name: "ok", status: http.StatusOK, want: nil},
+		{name: "blocked", status: http.StatusForbidden, want: ErrBlocked},
+		{name: "rate limited", status: http.StatusTooManyRequests, want: ErrRateLimited},
+		{name: "server error", status: http.StatusBadGateway, want: ErrBlocked},
+		{name: "unexpected status", status: http.StatusNotFound, want: ErrParser},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ClassifySearchHTTPStatus(tt.status)
+			if tt.want == nil {
+				if err != nil {
+					t.Fatalf("expected nil error, got %v", err)
+				}
+				return
+			}
+			if !errors.Is(err, tt.want) {
+				t.Fatalf("expected %v, got %v", tt.want, err)
+			}
+		})
+	}
+}
+
 func TestDrainAndCloseResponseNilSafe(t *testing.T) {
 	DrainAndCloseResponse(nil)
 	DrainAndCloseResponse(&http.Response{})
 }
+
+func TestRawHTTPClientTracksNetworkBytes(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		_, _ = w.Write([]byte("payload"))
+	}))
+	defer server.Close()
+
+	client, err := NewRawHTTPClient(Query{})
+	if err != nil {
+		t.Fatalf("new raw client: %v", err)
+	}
+
+	ctx := WithNetworkUsage(context.Background())
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("do request: %v", err)
+	}
+	defer DrainAndCloseResponse(resp)
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("read body: %v", err)
+	}
+	if string(body) != "payload" {
+		t.Fatalf("unexpected body: %q", string(body))
+	}
+	if got := NetworkBytesFromContext(ctx); got < int64(len(body)) {
+		t.Fatalf("expected tracked bytes >= body length, got %d", got)
+	}
+}
+
+func TestRawHTTPClientTracksProxyErrorBytes(t *testing.T) {
+	proxyBody := "proxy auth required"
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusProxyAuthRequired)
+		_, _ = w.Write([]byte(proxyBody))
+	}))
+	defer proxy.Close()
+
+	client, err := NewRawHTTPClient(Query{ProxyURL: proxy.URL})
+	if err != nil {
+		t.Fatalf("new raw client: %v", err)
+	}
+
+	ctx := WithNetworkUsage(context.Background())
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://example.com/", nil)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+	resp, err := client.Do(req)
+	DrainAndCloseResponse(resp)
+	if !errors.Is(err, ErrProxyAuth) {
+		t.Fatalf("expected proxy auth error, got %v", err)
+	}
+	if got := NetworkBytesFromContext(ctx); got < int64(len(proxyBody)) {
+		t.Fatalf("expected tracked bytes >= proxy body length, got %d", got)
+	}
+}

core/locale.go

@@ -1,15 +1,11 @@
 package core
 
-import "strings"
+import "github.com/karust/openserp/core/region"
 
 // Locale is a parsed language/region pair derived from a BCP47-style code.
-// Language is the lowercase 2-letter language subtag (e.g. "en", "de").
-// Country is the uppercase 2-letter region subtag (e.g. "US", "DE"); it may be
-// empty when the input had no region and the caller did not request a default.
-type Locale struct {
-	Language string
-	Country  string
-}
+// The canonical definition lives in the dependency-free core/region subpackage;
+// this alias preserves the historical core.Locale name for existing callers.
+type Locale = region.Locale
 
 var defaultLocaleCountryByLanguage = map[string]string{
 	"en": "US",
@@ -29,26 +25,15 @@ var defaultLocaleCountryByLanguage = map[string]string{
 }
 
 // ParseLocale parses a language code such as "en", "EN-us", or "de_AT" into a
-// Locale. Returns the zero value when the input is empty or has no language
-// subtag. Country is uppercased; Language is lowercased.
+// Locale. See region.ParseLocale for details.
 func ParseLocale(code string) Locale {
-	code = strings.TrimSpace(code)
-	if code == "" {
-		return Locale{}
-	}
-	code = strings.ReplaceAll(code, "_", "-")
+	return region.ParseLocale(code)
+}
 
-	parts := strings.Split(code, "-")
-	language := strings.ToLower(strings.TrimSpace(parts[0]))
-	if language == "" {
-		return Locale{}
-	}
-
-	country := ""
-	if len(parts) > 1 {
-		country = strings.ToUpper(strings.TrimSpace(parts[1]))
-	}
-	return Locale{Language: language, Country: country}
+// CountryFromRegion extracts a two-letter country/market code from a region
+// hint. See region.CountryFromRegion for details.
+func CountryFromRegion(hint string) string {
+	return region.CountryFromRegion(hint)
 }
 
 // PrimaryLanguageTag returns the BCP47 primary tag for a lang code, filling in
@@ -69,6 +54,37 @@ func PrimaryLanguageTag(langCode string) string {
 	return locale.Language + "-" + country
 }
 
+var defaultTimezoneByCountry = map[string]string{
+	"US": "America/New_York", "GB": "Europe/London", "DE": "Europe/Berlin",
+	"FR": "Europe/Paris", "ES": "Europe/Madrid", "IT": "Europe/Rome",
+	"RU": "Europe/Moscow", "BR": "America/Sao_Paulo", "JP": "Asia/Tokyo",
+	"CN": "Asia/Shanghai", "KR": "Asia/Seoul", "IN": "Asia/Kolkata",
+	"AU": "Australia/Sydney", "CA": "America/Toronto",
+	"MX": "America/Mexico_City", "PL": "Europe/Warsaw",
+	"NL": "Europe/Amsterdam", "TR": "Europe/Istanbul",
+	"AR": "America/Argentina/Buenos_Aires", "SA": "Asia/Riyadh",
+	"BE": "Europe/Brussels", "KZ": "Asia/Almaty", "UA": "Europe/Kyiv",
+}
+
+// TimezoneForLocale returns an IANA timezone for a locale's country, or for
+// the default country of the language when country is empty. Returns "" for
+// unknown locales — caller should retain the existing profile timezone.
+func TimezoneForLocale(loc Locale) string {
+	if loc.Country != "" {
+		if tz, ok := defaultTimezoneByCountry[loc.Country]; ok {
+			return tz
+		}
+	}
+	if loc.Language != "" {
+		if country, ok := defaultLocaleCountryByLanguage[loc.Language]; ok {
+			if tz, ok2 := defaultTimezoneByCountry[country]; ok2 {
+				return tz
+			}
+		}
+	}
+	return ""
+}
+
 // BuildAcceptLanguageHeader formats an Accept-Language value from a lang code.
 // Example: "de" -> "de-DE,de;q=0.9", "en-GB" -> "en-GB,en;q=0.9", "sw" -> "sw".
 func BuildAcceptLanguageHeader(langCode string) string {

core/locale_test.go

@@ -1,6 +1,8 @@
 package core
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestParseLocale(t *testing.T) {
 	tests := []struct {
@@ -32,6 +34,30 @@ func TestParseLocale(t *testing.T) {
 	}
 }
 
+func TestCountryFromRegion(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{name: "empty", in: "", want: ""},
+		{name: "country lowercase", in: "ru", want: "RU"},
+		{name: "country uppercase", in: "US", want: "US"},
+		{name: "locale dash", in: "en-GB", want: "GB"},
+		{name: "locale underscore", in: "de_AT", want: "AT"},
+		{name: "numeric engine region is not country", in: "213", want: ""},
+		{name: "unknown shape", in: "moscow", want: ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := CountryFromRegion(tt.in); got != tt.want {
+				t.Fatalf("CountryFromRegion(%q) = %q, want %q", tt.in, got, tt.want)
+			}
+		})
+	}
+}
+
 func TestBuildAcceptLanguageHeader(t *testing.T) {
 	tests := []struct {
 		name string
@@ -55,3 +81,25 @@ func TestBuildAcceptLanguageHeader(t *testing.T) {
 		})
 	}
 }
+
+func TestTimezoneForLocale(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{name: "de-DE explicit country", in: "de-DE", want: "Europe/Berlin"},
+		{name: "pt-BR explicit country", in: "pt-BR", want: "America/Sao_Paulo"},
+		{name: "bare de via default country", in: "de", want: "Europe/Berlin"},
+		{name: "unknown locale", in: "xx-YY", want: ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := TimezoneForLocale(ParseLocale(tt.in))
+			if got != tt.want {
+				t.Fatalf("TimezoneForLocale(%q) = %q, want %q", tt.in, got, tt.want)
+			}
+		})
+	}
+}

core/logger.go

@@ -77,6 +77,11 @@ func RequestIDFromContext(ctx context.Context) string {
 	return strings.TrimSpace(value)
 }
 
+func TenantFromContext(ctx context.Context) string {
+	value, _ := EnsureContext(ctx).Value(tenantContextKey).(string)
+	return strings.TrimSpace(value)
+}
+
 func WithRequest(ctx context.Context) *logrus.Entry {
 	ctx = EnsureContext(ctx)
 	fields := logrus.Fields{}
@@ -93,6 +98,12 @@ func WithRequest(ctx context.Context) *logrus.Entry {
 	if queryHash, ok := ctx.Value(queryHashContextKey).(string); ok && strings.TrimSpace(queryHash) != "" {
 		fields["query_hash"] = strings.TrimSpace(queryHash)
 	}
+	profileIDs := BrowserProfileIDsFromContext(ctx)
+	if len(profileIDs) == 1 {
+		fields["browser_profile_id"] = profileIDs[0]
+	} else if len(profileIDs) > 1 {
+		fields["browser_profile_ids"] = strings.Join(profileIDs, ",")
+	}
 
 	return logrus.WithFields(fields)
 }
@@ -116,6 +127,7 @@ func QueryHashFromQuery(q Query) string {
 		q.Site,
 		q.Filetype,
 		q.LangCode,
+		q.Region,
 		q.DateInterval,
 	}, "|")
 	return QueryHash(raw)

core/middleware.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"time"
@@ -11,10 +12,12 @@ import (
 )
 
 type JSONErrorResponse struct {
-	Error   string `json:"error"`
-	Code    int    `json:"code"`
-	Message string `json:"message,omitempty"`
-	Reason  string `json:"reason,omitempty"`
+	Error     string                 `json:"error"`
+	Code      int                    `json:"code"`
+	RequestID string                 `json:"request_id,omitempty"`
+	Message   string                 `json:"message,omitempty"`
+	Reason    string                 `json:"reason,omitempty"`
+	Meta      map[string]interface{} `json:"meta,omitempty"`
 }
 
 type CORSConfig struct {
@@ -24,11 +27,16 @@ type CORSConfig struct {
 	MaxAge       int
 }
 
+const browserProfileIDHeader = "X-Browser-Profile-Id"
+const useProfileHeader = "X-Use-Profile"
+
+const exposedResponseHeaders = "X-Request-ID, X-Cache, X-Fallback-Engine, X-Proxy-Mode, X-Proxy-Tag, X-Proxy-Used, X-Network-Bytes, " + browserProfileIDHeader
+
 func DefaultCORSConfig() CORSConfig {
 	return CORSConfig{
 		AllowOrigins: "*",
 		AllowMethods: "GET, POST, OPTIONS",
-		AllowHeaders: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Request-ID, X-Tenant",
+		AllowHeaders: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Proxy-URL, X-Proxy-Country, X-Proxy-Class, X-Proxy-Provider, X-Proxy-Session-ID, X-Request-ID, X-Tenant, X-Use-Profile",
 		MaxAge:       86400,
 	}
 }
@@ -55,6 +63,21 @@ func RequestContextMiddleware() fiber.Handler {
 	}
 }
 
+// RequestTimeoutMiddleware bounds wall-clock time per request by attaching a
+// deadline to the user context, which fasthttp never cancels on client
+// disconnect. /mega/* (MegaTimeout) and /extract (batch budget) are exempt.
+func RequestTimeoutMiddleware(timeout time.Duration) fiber.Handler {
+	return func(c *fiber.Ctx) error {
+		if strings.HasPrefix(c.Path(), "/mega/") || c.Path() == "/extract" {
+			return c.Next()
+		}
+		ctx, cancel := context.WithTimeout(c.UserContext(), timeout)
+		defer cancel()
+		c.SetUserContext(ctx)
+		return c.Next()
+	}
+}
+
 func CORSMiddleware(cfg CORSConfig) fiber.Handler {
 	cfg = normalizeCORSConfig(cfg)
 
@@ -63,6 +86,7 @@ func CORSMiddleware(cfg CORSConfig) fiber.Handler {
 		c.Set("Access-Control-Allow-Methods", cfg.AllowMethods)
 		c.Set("Access-Control-Allow-Headers", cfg.AllowHeaders)
 		c.Set("Access-Control-Max-Age", fmt.Sprintf("%d", cfg.MaxAge))
+		c.Set("Access-Control-Expose-Headers", exposedResponseHeaders)
 
 		if c.Method() == "OPTIONS" {
 			return c.SendStatus(fiber.StatusNoContent)
@@ -118,6 +142,7 @@ func RequestLoggerMiddleware() fiber.Handler {
 		if query := c.Query("text"); query != "" {
 			logFields["query_hash"] = QueryHash(query)
 		}
+		addProxyLogFields(c, logFields)
 
 		entry := WithRequest(c.UserContext()).WithFields(logFields)
 		if status >= 500 {
@@ -132,24 +157,61 @@ func RequestLoggerMiddleware() fiber.Handler {
 	}
 }
 
+func addProxyLogFields(c *fiber.Ctx, fields logrus.Fields) {
+	if country := strings.ToLower(strings.TrimSpace(c.Get("X-Proxy-Country"))); country != "" {
+		fields["proxy_country"] = country
+	}
+	if class := strings.ToLower(strings.TrimSpace(c.Get("X-Proxy-Class"))); class != "" {
+		fields["proxy_class"] = class
+	}
+	if provider := strings.ToLower(strings.TrimSpace(c.Get("X-Proxy-Provider"))); provider != "" {
+		fields["proxy_provider"] = provider
+	}
+
+	if sessionID := strings.TrimSpace(c.Get("X-Proxy-Session-ID")); sessionID != "" {
+		fields["proxy_session_id"] = sessionID
+	}
+
+	if proxyURL := strings.TrimSpace(c.Get("X-Proxy-URL")); proxyURL != "" {
+		fields["proxy_used"] = MaskProxyURL(proxyURL)
+	}
+
+	if laneKey := proxyLaneKeyFromContext(c.UserContext()); !laneKey.Empty() {
+		fields["lane_id"] = laneKey.ID()
+	}
+}
+
 func JSONErrorMiddleware() fiber.ErrorHandler {
 	return func(c *fiber.Ctx, err error) error {
 		code := fiber.StatusInternalServerError
+		errorCode := ""
 		reason := ""
+		var meta map[string]interface{}
 
 		if e, ok := err.(*fiber.Error); ok {
 			code = e.Code
 		}
 		if apiErr, ok := err.(*APIError); ok {
 			code = apiErr.HTTPStatus
+			errorCode = apiErr.ErrorCode
 			reason = apiErr.Reason
+			meta = apiErr.Meta
+		}
+		if errorCode == "" {
+			errorCode = statusText(code)
+		}
+		requestID := RequestIDFromContext(c.UserContext())
+		if requestID == "" {
+			requestID = strings.TrimSpace(c.Get("X-Request-ID"))
 		}
 
 		resp := JSONErrorResponse{
-			Error:   statusText(code),
-			Code:    code,
-			Message: err.Error(),
-			Reason:  reason,
+			Error:     errorCode,
+			Code:      code,
+			RequestID: requestID,
+			Message:   err.Error(),
+			Reason:    reason,
+			Meta:      meta,
 		}
 
 		c.Set("Content-Type", "application/json")

core/middleware_test.go

@@ -63,6 +63,9 @@ func TestCORSMiddleware_UsesConfiguredHeaders(t *testing.T) {
 	if got := resp.Header.Get("Access-Control-Max-Age"); got != "1200" {
 		t.Fatalf("unexpected max-age: %q", got)
 	}
+	if got := resp.Header.Get("Access-Control-Expose-Headers"); !strings.Contains(got, "X-Network-Bytes") || !strings.Contains(got, "X-Browser-Profile-Id") {
+		t.Fatalf("expected exposed headers to include network and browser profile headers, got %q", got)
+	}
 }
 
 func TestCORSMiddleware_OPTIONSReturnsNoContent(t *testing.T) {

core/network_guard.go

@@ -0,0 +1,159 @@
+package core
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"net/url"
+	"strings"
+)
+
+// ErrTargetNotAllowed marks URL-guard policy rejections (bad scheme, missing
+// host, or a target resolving to a non-public IP). Handlers match it with
+// errors.Is to report a client error instead of an upstream failure.
+var ErrTargetNotAllowed = errors.New("target not allowed")
+
+var carrierGradeNATPrefix = netip.MustParsePrefix("100.64.0.0/10")
+
+// GuardedDialContext dials only public IP targets. Hostnames are resolved first
+// and the returned connection is made to the vetted IP, so DNS rebinding cannot
+// swap in a private address between validation and dial.
+func GuardedDialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+	ips, err := resolvePublicDialTargets(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+
+	dialer := &net.Dialer{}
+	var lastErr error
+	for _, ip := range ips {
+		if !ipMatchesNetwork(ip, network) {
+			continue
+		}
+		conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
+		if err == nil {
+			return conn, nil
+		}
+		lastErr = err
+	}
+	if lastErr != nil {
+		return nil, lastErr
+	}
+	return nil, fmt.Errorf("no public IPs available for %s", host)
+}
+
+func ValidatePublicHTTPURL(ctx context.Context, rawURL string) error {
+	parsed, err := validateHTTPURL(rawURL)
+	if err != nil {
+		return err
+	}
+	return validatePublicHost(ctx, parsed.Hostname())
+}
+
+func validateHTTPURL(rawURL string) (*url.URL, error) {
+	parsed, err := url.Parse(strings.TrimSpace(rawURL))
+	if err != nil {
+		return nil, fmt.Errorf("%w: invalid URL: %v", ErrTargetNotAllowed, err)
+	}
+	if parsed.Scheme != "http" && parsed.Scheme != "https" {
+		return nil, fmt.Errorf("%w: unsupported URL scheme %q: only http and https are allowed", ErrTargetNotAllowed, parsed.Scheme)
+	}
+	if parsed.Hostname() == "" {
+		return nil, fmt.Errorf("%w: URL host is required", ErrTargetNotAllowed)
+	}
+	return parsed, nil
+}
+
+// resolveHostIPs resolves host (or parses a literal IP) and partitions the
+// addresses by the public-IP policy.
+func resolveHostIPs(ctx context.Context, host string) (public, blocked []netip.Addr, err error) {
+	if ip, parseErr := netip.ParseAddr(host); parseErr == nil {
+		ip = ip.Unmap()
+		if isPublicIP(ip) {
+			return []netip.Addr{ip}, nil, nil
+		}
+		return nil, []netip.Addr{ip}, nil
+	}
+	ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host)
+	if err != nil {
+		return nil, nil, err
+	}
+	for _, ip := range ips {
+		ip = ip.Unmap()
+		if isPublicIP(ip) {
+			public = append(public, ip)
+		} else {
+			blocked = append(blocked, ip)
+		}
+	}
+	return public, blocked, nil
+}
+
+func resolvePublicDialTargets(ctx context.Context, host string) ([]netip.Addr, error) {
+	public, blocked, err := resolveHostIPs(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+	if len(public) == 0 {
+		if len(blocked) == 0 {
+			return nil, fmt.Errorf("%w: target host %q resolved to no IP addresses", ErrTargetNotAllowed, host)
+		}
+		return nil, fmt.Errorf("%w: target host %q resolves only to non-public IPs: %s", ErrTargetNotAllowed, host, joinAddrs(blocked))
+	}
+	return public, nil
+}
+
+// validatePublicHost rejects a host when any of its addresses is non-public —
+// stricter than the dial guard on purpose. The rendered path hands the URL to
+// Chrome, which resolves DNS on its own, so a mixed public/private record set
+// must fail closed here rather than rely on dial-time pinning.
+func validatePublicHost(ctx context.Context, host string) error {
+	public, blocked, err := resolveHostIPs(ctx, host)
+	if err != nil {
+		return err
+	}
+	if len(blocked) > 0 {
+		return fmt.Errorf("%w: target host %q resolves to non-public IPs: %s", ErrTargetNotAllowed, host, joinAddrs(blocked))
+	}
+	if len(public) == 0 {
+		return fmt.Errorf("%w: target host %q resolved to no IP addresses", ErrTargetNotAllowed, host)
+	}
+	return nil
+}
+
+func joinAddrs(addrs []netip.Addr) string {
+	parts := make([]string, len(addrs))
+	for i, addr := range addrs {
+		parts[i] = addr.String()
+	}
+	return strings.Join(parts, ", ")
+}
+
+func isPublicIP(ip netip.Addr) bool {
+	ip = ip.Unmap()
+	return ip.IsValid() &&
+		!ip.IsUnspecified() &&
+		!ip.IsLoopback() &&
+		!ip.IsPrivate() &&
+		!ip.IsLinkLocalUnicast() &&
+		!ip.IsLinkLocalMulticast() &&
+		!ip.IsMulticast() &&
+		!carrierGradeNATPrefix.Contains(ip)
+}
+
+func ipMatchesNetwork(ip netip.Addr, network string) bool {
+	switch network {
+	case "tcp4":
+		return ip.Is4()
+	case "tcp6":
+		return ip.Is6()
+	default:
+		return true
+	}
+}

core/network_guard_test.go

@@ -0,0 +1,42 @@
+package core
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+)
+
+func TestValidatePublicHTTPURLRejectsPrivateTargets(t *testing.T) {
+	tests := []string{
+		"http://127.0.0.1/",
+		"http://[::1]/",
+		"http://10.0.0.1/",
+		"http://172.16.0.1/",
+		"http://192.168.1.1/",
+		"http://169.254.169.254/",
+		"http://100.64.0.1/",
+	}
+
+	for _, rawURL := range tests {
+		t.Run(rawURL, func(t *testing.T) {
+			err := ValidatePublicHTTPURL(context.Background(), rawURL)
+			if !errors.Is(err, ErrTargetNotAllowed) {
+				t.Fatalf("expected ErrTargetNotAllowed, got %v", err)
+			}
+			if !strings.Contains(strings.ToLower(err.Error()), "non-public") {
+				t.Fatalf("expected non-public error, got %v", err)
+			}
+		})
+	}
+}
+
+func TestValidatePublicHTTPURLRejectsUnsupportedScheme(t *testing.T) {
+	err := ValidatePublicHTTPURL(context.Background(), "file:///etc/passwd")
+	if !errors.Is(err, ErrTargetNotAllowed) {
+		t.Fatalf("expected ErrTargetNotAllowed, got %v", err)
+	}
+	if !strings.Contains(err.Error(), "only http and https") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}

core/network_usage.go

@@ -0,0 +1,44 @@
+package core
+
+import (
+	"context"
+	"sync/atomic"
+)
+
+type networkUsageContextKey struct{}
+
+type networkUsageTracker struct {
+	bytes atomic.Int64
+}
+
+func WithNetworkUsage(ctx context.Context) context.Context {
+	ctx = EnsureContext(ctx)
+	if networkUsageFromContext(ctx) != nil {
+		return ctx
+	}
+	return context.WithValue(ctx, networkUsageContextKey{}, &networkUsageTracker{})
+}
+
+func AddNetworkBytes(ctx context.Context, n int64) {
+	if n <= 0 {
+		return
+	}
+	if tracker := networkUsageFromContext(ctx); tracker != nil {
+		tracker.bytes.Add(n)
+	}
+}
+
+func NetworkBytesFromContext(ctx context.Context) int64 {
+	if tracker := networkUsageFromContext(ctx); tracker != nil {
+		return tracker.bytes.Load()
+	}
+	return 0
+}
+
+func networkUsageFromContext(ctx context.Context) *networkUsageTracker {
+	if ctx == nil {
+		return nil
+	}
+	tracker, _ := ctx.Value(networkUsageContextKey{}).(*networkUsageTracker)
+	return tracker
+}

core/page_helpers.go

@@ -0,0 +1,185 @@
+package core
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/go-rod/rod"
+)
+
+// pollInterval is how often WaitForElements re-probes selectors while the
+// page hydrates. Short enough to feel snappy, long enough not to hammer CDP.
+const pollInterval = 120 * time.Millisecond
+
+// WaitForElements probes the supplied CSS selectors until one returns at least
+// one matching element or timeout elapses. It exists because rod's
+// page.Search/Elements and the surrounding WaitLoad/WaitStable do not wait for
+// a *specific* selector to hydrate — modern SPA SERPs (DDG, Bing, Google)
+// regularly fire `load` and even reach DOM-stable before result rows render,
+// causing parsers to see an empty page on the first probe and forcing the
+// caller's retry layer to reload.
+//
+// The probe loop returns as soon as a selector matches, returning the matched
+// elements and the selector that hit. On timeout it returns ErrSearchTimeout
+// so callers can disambiguate between "no results" / "captcha" by inspecting
+// the page directly.
+func WaitForElements(ctx context.Context, page *rod.Page, selectors []string, timeout time.Duration) (rod.Elements, string, error) {
+	if page == nil {
+		return nil, "", ErrSearchTimeout
+	}
+	ctx = EnsureContext(ctx)
+	if timeout <= 0 {
+		timeout = 2 * time.Second
+	}
+
+	probe := func() (rod.Elements, string) {
+		for _, selector := range selectors {
+			elements, err := page.Elements(selector)
+			if err != nil || len(elements) == 0 {
+				continue
+			}
+			return elements, selector
+		}
+		return nil, ""
+	}
+
+	if elements, selector := probe(); len(elements) > 0 {
+		return elements, selector, nil
+	}
+
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if err := ctx.Err(); err != nil {
+			return nil, "", err
+		}
+		if elements, selector := probe(); len(elements) > 0 {
+			return elements, selector, nil
+		}
+		if err := SleepContext(ctx, pollInterval); err != nil {
+			return nil, "", err
+		}
+	}
+	return nil, "", ErrSearchTimeout
+}
+
+// HasAnySelector returns true if at least one of the supplied selectors
+// currently matches in the page DOM. It does not wait — pair with
+// WaitForElements when hydration may be in flight.
+func HasAnySelector(page *rod.Page, selectors []string) bool {
+	if page == nil {
+		return false
+	}
+	for _, selector := range selectors {
+		has, _, err := page.Has(selector)
+		if err == nil && has {
+			return true
+		}
+	}
+	return false
+}
+
+// DeferClosePage returns a cleanup function that closes page unless the browser
+// is configured to leave pages open for debugging.
+func DeferClosePage(ctx context.Context, page *rod.Page, browser *Browser) func() {
+	return func() {
+		if browser != nil && browser.LeavePageOpen {
+			return
+		}
+		if err := ClosePageWithTimeout(ctx, page, time.Second); err != nil {
+			WithRequest(ctx).WithError(err).Debug("Page close error")
+		}
+	}
+}
+
+// HasAttribute reports whether el carries attr (regardless of value).
+func HasAttribute(el *rod.Element, attr string) bool {
+	if el == nil {
+		return false
+	}
+	v, err := el.Attribute(attr)
+	return err == nil && v != nil
+}
+
+// FirstNonEmptyText returns the trimmed text of the first selector under root
+// that yields non-empty content. Empty string if none match.
+func FirstNonEmptyText(root *rod.Element, selectors ...string) string {
+	if root == nil {
+		return ""
+	}
+	for _, selector := range selectors {
+		el, err := root.Element(selector)
+		if err != nil {
+			continue
+		}
+		text, err := el.Text()
+		if err != nil {
+			continue
+		}
+		if trimmed := strings.TrimSpace(text); trimmed != "" {
+			return trimmed
+		}
+	}
+	return ""
+}
+
+// ClosestMatching walks up the ancestor chain (including el itself) and returns
+// the first element matching selector, or nil if none is found within maxHops.
+// rod has no native Closest helper, so this is a bounded walk used by parsers
+// that need to recover a wrapping <a> from a nested title node.
+func ClosestMatching(el *rod.Element, selector string, maxHops int) *rod.Element {
+	if el == nil || selector == "" {
+		return nil
+	}
+	current := el
+	for hop := 0; hop <= maxHops; hop++ {
+		if matches, err := current.Matches(selector); err == nil && matches {
+			return current
+		}
+		parent, err := current.Parent()
+		if err != nil || parent == nil {
+			return nil
+		}
+		current = parent
+	}
+	return nil
+}
+
+// FirstNonEmptyAttribute returns the trimmed value of attr from the first
+// selector under root whose attribute is non-empty.
+func FirstNonEmptyAttribute(root *rod.Element, attr string, selectors ...string) string {
+	if root == nil {
+		return ""
+	}
+	for _, selector := range selectors {
+		el, err := root.Element(selector)
+		if err != nil {
+			continue
+		}
+		value, err := el.Attribute(attr)
+		if err != nil || value == nil {
+			continue
+		}
+		if trimmed := strings.TrimSpace(*value); trimmed != "" {
+			return trimmed
+		}
+	}
+	return ""
+}
+
+// FeaturesFromPage renders a live rod page to HTML and runs a document-level
+// feature extractor over it. Every engine's browser path shares this boilerplate
+// (page.HTML -> goquery doc -> extract), so it lives here rather than being
+// copied per engine. Returns nil on any rendering/parse error.
+func FeaturesFromPage(page *rod.Page, extract func(*goquery.Document) []SerpFeature) []SerpFeature {
+	html, err := page.HTML()
+	if err != nil {
+		return nil
+	}
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		return nil
+	}
+	return extract(doc)
+}

core/profile_context.go

@@ -8,6 +8,8 @@ import (
 type profileContextKey string
 
 const profileRegionContextKey profileContextKey = "profile_region"
+const forcedProfileIDContextKey profileContextKey = "forced_profile_id"
+const minimalProfileContextKey profileContextKey = "minimal_profile"
 
 func WithProfileRegion(ctx context.Context, region string) context.Context {
 	region = strings.TrimSpace(region)
@@ -26,3 +28,25 @@ func engineFromContext(ctx context.Context) string {
 	value, _ := EnsureContext(ctx).Value(engineContextKey).(string)
 	return strings.TrimSpace(value)
 }
+
+func WithForcedProfileID(ctx context.Context, profileID string) context.Context {
+	profileID = strings.TrimSpace(profileID)
+	if profileID == "" {
+		return EnsureContext(ctx)
+	}
+	return context.WithValue(EnsureContext(ctx), forcedProfileIDContextKey, profileID)
+}
+
+func forcedProfileIDFromContext(ctx context.Context) string {
+	value, _ := EnsureContext(ctx).Value(forcedProfileIDContextKey).(string)
+	return strings.TrimSpace(value)
+}
+
+func WithMinimalBrowserProfile(ctx context.Context) context.Context {
+	return context.WithValue(EnsureContext(ctx), minimalProfileContextKey, true)
+}
+
+func minimalBrowserProfileFromContext(ctx context.Context) bool {
+	value, _ := EnsureContext(ctx).Value(minimalProfileContextKey).(bool)
+	return value
+}

core/proxy.go

@@ -18,6 +18,7 @@ const (
 	ProxyRuntimeRaw              = "raw"
 	ProxyModeOff                 = "off"
 	ProxyModeTagPool             = "tag_pool"
+	ProxyModeRequestURL          = "request_url"
 	DefaultProxyFailureThreshold = 3
 	ProxyOverrideDirect          = "direct"
 	// ProxyPoolQuarantineDuration is how long an exhausted tag pool stays quarantined
@@ -49,9 +50,11 @@ type ProxiesHealthConfig struct {
 }
 
 type ProxiesConfig struct {
-	Global  string              `json:"global,omitempty" mapstructure:"global"`
-	Entries []ProxyEntryConfig  `json:"entries" mapstructure:"entries"`
-	Health  ProxiesHealthConfig `json:"health" mapstructure:"health"`
+	Global               string              `json:"global,omitempty" mapstructure:"global"`
+	Entries              []ProxyEntryConfig  `json:"entries" mapstructure:"entries"`
+	Health               ProxiesHealthConfig `json:"health" mapstructure:"health"`
+	AllowRequestProxyURL bool                `json:"allow_request_proxy_url" mapstructure:"allow_request_proxy_url"`
+	Lanes                ProxyLanesConfig    `json:"lanes" mapstructure:"lanes"`
 }
 
 type ProxyConfig struct {
@@ -80,12 +83,15 @@ type ProxyEngineStats struct {
 }
 
 type ProxyStats struct {
-	ConfiguredCount int                         `json:"configured_count"`
-	HealthyCount    int                         `json:"healthy_count"`
-	UnhealthyCount  int                         `json:"unhealthy_count"`
-	Tags            map[string]ProxyTagSummary  `json:"tags"`
-	Entries         []ProxyStatsEntry           `json:"entries"`
-	Engines         map[string]ProxyEngineStats `json:"engines,omitempty"`
+	ConfiguredCount        int                         `json:"configured_count"`
+	HealthyCount           int                         `json:"healthy_count"`
+	UnhealthyCount         int                         `json:"unhealthy_count"`
+	RequestProxyURLEnabled bool                        `json:"request_proxy_url_enabled"`
+	Lanes                  LaneStats                   `json:"lanes"`
+	BrowserProcesses       BrowserPoolStats            `json:"browser_processes"`
+	Tags                   map[string]ProxyTagSummary  `json:"tags"`
+	Entries                []ProxyStatsEntry           `json:"entries"`
+	Engines                map[string]ProxyEngineStats `json:"engines,omitempty"`
 }
 
 type proxyState struct {
@@ -110,6 +116,7 @@ func DefaultProxiesConfig() ProxiesConfig {
 		Global:  "",
 		Entries: []ProxyEntryConfig{},
 		Health:  ProxiesHealthConfig{FailureThreshold: DefaultProxyFailureThreshold},
+		Lanes:   DefaultProxyLanesConfig(),
 	}
 }
 
@@ -203,6 +210,7 @@ func NormalizeProxiesConfig(cfg ProxiesConfig) (ProxiesConfig, error) {
 
 	cfg.Entries = normalizedEntries
 	cfg.Health = ProxiesHealthConfig{FailureThreshold: failureThreshold}
+	cfg.Lanes = NormalizeProxyLanesConfig(cfg.Lanes)
 	return cfg, nil
 }
 

core/proxy_context.go

@@ -0,0 +1,37 @@
+package core
+
+import (
+	"context"
+	"strings"
+)
+
+type proxyContextKey string
+
+const requestProxyURLContextKey proxyContextKey = "request_proxy_url"
+const proxyLaneKeyContextKey proxyContextKey = "proxy_lane_key"
+
+func WithRequestProxyURL(ctx context.Context, proxyURL string) context.Context {
+	proxyURL = strings.TrimSpace(proxyURL)
+	if proxyURL == "" {
+		return EnsureContext(ctx)
+	}
+	return context.WithValue(EnsureContext(ctx), requestProxyURLContextKey, proxyURL)
+}
+
+func requestProxyURLFromContext(ctx context.Context) string {
+	value, _ := EnsureContext(ctx).Value(requestProxyURLContextKey).(string)
+	return strings.TrimSpace(value)
+}
+
+func WithProxyLaneKey(ctx context.Context, key ProxyLaneKey) context.Context {
+	key = NormalizeProxyLaneKey(key)
+	if key.Empty() {
+		return EnsureContext(ctx)
+	}
+	return context.WithValue(EnsureContext(ctx), proxyLaneKeyContextKey, key)
+}
+
+func proxyLaneKeyFromContext(ctx context.Context) ProxyLaneKey {
+	value, _ := EnsureContext(ctx).Value(proxyLaneKeyContextKey).(ProxyLaneKey)
+	return NormalizeProxyLaneKey(value)
+}

core/proxy_lane.go

@@ -0,0 +1,289 @@
+package core
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/go-rod/rod/lib/proto"
+	browserprofile "github.com/karust/openserp/core/browser"
+)
+
+const DefaultProxyLaneMaxLanes = 100
+
+type ProxyLanesConfig struct {
+	Enabled                bool `json:"enabled" mapstructure:"enabled"`
+	MaxLanes               int  `json:"max_lanes" mapstructure:"max_lanes"`
+	DropCookiesOnChallenge bool `json:"drop_cookies_on_challenge" mapstructure:"drop_cookies_on_challenge"`
+}
+
+type ProxyLaneKey struct {
+	Tenant    string
+	Engine    string
+	SessionID string
+}
+
+type LaneStats struct {
+	Active         int `json:"active"`
+	EvictedLRU     int `json:"evicted_lru"`
+	CookiesDropped int `json:"cookies_dropped"`
+}
+
+// BrowserPoolStats describes the live state of the per-process browser pool that
+// keeps one Chrome per authenticated upstream proxy identity. Reported via
+// /stats/proxy as `browser_processes`.
+type BrowserPoolStats struct {
+	Active      int `json:"active"`
+	Max         int `json:"max"`
+	EvictedLRU  int `json:"evicted_lru"`
+	EvictedIdle int `json:"evicted_idle"`
+}
+
+type laneState struct {
+	Key        ProxyLaneKey
+	Profile    browserprofile.Profile
+	Cookies    []*proto.NetworkCookie
+	LastUsedAt time.Time
+}
+
+type LaneStore struct {
+	mu             sync.Mutex
+	maxLanes       int
+	lanes          map[ProxyLaneKey]*laneState
+	evictedLRU     int
+	cookiesDropped int
+}
+
+func DefaultProxyLanesConfig() ProxyLanesConfig {
+	return ProxyLanesConfig{
+		Enabled:                true,
+		MaxLanes:               DefaultProxyLaneMaxLanes,
+		DropCookiesOnChallenge: true,
+	}
+}
+
+func NormalizeProxyLanesConfig(cfg ProxyLanesConfig) ProxyLanesConfig {
+	if cfg.MaxLanes <= 0 {
+		cfg.MaxLanes = DefaultProxyLaneMaxLanes
+	}
+	return cfg
+}
+
+func NewLaneStore(maxLanes int) *LaneStore {
+	if maxLanes <= 0 {
+		maxLanes = DefaultProxyLaneMaxLanes
+	}
+	return &LaneStore{
+		maxLanes: maxLanes,
+		lanes:    map[ProxyLaneKey]*laneState{},
+	}
+}
+
+func (s *LaneStore) Profile(key ProxyLaneKey, create func() browserprofile.Profile) browserprofile.Profile {
+	if s == nil || key.Empty() {
+		if create == nil {
+			return browserprofile.Profile{}
+		}
+		return create()
+	}
+
+	now := time.Now()
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if state, ok := s.lanes[key]; ok {
+		state.LastUsedAt = now
+		return state.Profile
+	}
+
+	profile := browserprofile.Profile{}
+	if create != nil {
+		profile = create()
+	}
+	s.lanes[key] = &laneState{Key: key, Profile: profile, LastUsedAt: now}
+	s.evictLRULocked()
+	return profile
+}
+
+func (s *LaneStore) Cookies(key ProxyLaneKey) []*proto.NetworkCookie {
+	if s == nil || key.Empty() {
+		return nil
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	state, ok := s.lanes[key]
+	if !ok {
+		return nil
+	}
+	state.LastUsedAt = time.Now()
+	return cloneCookies(state.Cookies)
+}
+
+func (s *LaneStore) SaveCookies(key ProxyLaneKey, cookies []*proto.NetworkCookie) {
+	if s == nil || key.Empty() {
+		return
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	state, ok := s.lanes[key]
+	if !ok {
+		state = &laneState{Key: key}
+		s.lanes[key] = state
+	}
+	state.Cookies = cloneCookies(cookies)
+	state.LastUsedAt = time.Now()
+	s.evictLRULocked()
+}
+
+func (s *LaneStore) DropCookies(key ProxyLaneKey) {
+	if s == nil || key.Empty() {
+		return
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	state, ok := s.lanes[key]
+	if !ok {
+		return
+	}
+	if len(state.Cookies) > 0 {
+		s.cookiesDropped++
+	}
+	state.Cookies = nil
+	state.LastUsedAt = time.Now()
+}
+
+func (s *LaneStore) Stats() LaneStats {
+	if s == nil {
+		return LaneStats{}
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return LaneStats{
+		Active:         len(s.lanes),
+		EvictedLRU:     s.evictedLRU,
+		CookiesDropped: s.cookiesDropped,
+	}
+}
+
+func (s *LaneStore) evictLRULocked() {
+	for len(s.lanes) > s.maxLanes {
+		var (
+			oldestKey ProxyLaneKey
+			oldest    time.Time
+			hasOldest bool
+		)
+		for key, state := range s.lanes {
+			if !hasOldest || state.LastUsedAt.Before(oldest) {
+				oldestKey = key
+				oldest = state.LastUsedAt
+				hasOldest = true
+			}
+		}
+		if !hasOldest {
+			return
+		}
+		delete(s.lanes, oldestKey)
+		s.evictedLRU++
+	}
+}
+
+func (k ProxyLaneKey) Empty() bool {
+	return strings.TrimSpace(k.Engine) == "" || strings.TrimSpace(k.SessionID) == ""
+}
+
+func (k ProxyLaneKey) ID() string {
+	k = NormalizeProxyLaneKey(k)
+	if k.Empty() {
+		return ""
+	}
+	if k.Tenant != "" {
+		return k.Tenant + ":" + k.Engine + ":" + k.SessionID
+	}
+	return k.Engine + ":" + k.SessionID
+}
+
+func NormalizeProxyLaneKey(key ProxyLaneKey) ProxyLaneKey {
+	return ProxyLaneKey{
+		Tenant:    strings.TrimSpace(key.Tenant),
+		Engine:    normalizeEngineName(key.Engine),
+		SessionID: strings.TrimSpace(key.SessionID),
+	}
+}
+
+func ProxyLaneKeyForTenant(engine string, tenant string, q Query, proxyURL string) ProxyLaneKey {
+	sessionID := strings.TrimSpace(q.ProxySessionID)
+	if sessionID == "" {
+		sessionID = proxyLaneIDFromProxyURL(proxyURL)
+	}
+	return NormalizeProxyLaneKey(ProxyLaneKey{Tenant: tenant, Engine: engine, SessionID: sessionID})
+}
+
+func proxyLaneIDFromProxyURL(raw string) string {
+	normalized, err := NormalizeProxyURL(raw)
+	if err != nil || normalized == "" {
+		return ""
+	}
+	parsed, err := url.Parse(normalized)
+	if err != nil {
+		return ""
+	}
+	username := ""
+	if parsed.User != nil {
+		username = parsed.User.Username()
+	}
+	sum := sha256.Sum256([]byte(parsed.Host + "|" + username))
+	return hex.EncodeToString(sum[:])[:16]
+}
+
+func cloneCookies(cookies []*proto.NetworkCookie) []*proto.NetworkCookie {
+	if len(cookies) == 0 {
+		return nil
+	}
+	out := make([]*proto.NetworkCookie, 0, len(cookies))
+	for _, cookie := range cookies {
+		if cookie == nil {
+			continue
+		}
+		cloned := *cookie
+		out = append(out, &cloned)
+	}
+	return out
+}
+
+func cookieParams(cookies []*proto.NetworkCookie) []*proto.NetworkCookieParam {
+	if len(cookies) == 0 {
+		return nil
+	}
+	params := make([]*proto.NetworkCookieParam, 0, len(cookies))
+	for _, cookie := range cookies {
+		if cookie == nil {
+			continue
+		}
+		sourcePort := cookie.SourcePort
+		params = append(params, &proto.NetworkCookieParam{
+			Name:         cookie.Name,
+			Value:        cookie.Value,
+			Domain:       cookie.Domain,
+			Path:         cookie.Path,
+			Secure:       cookie.Secure,
+			HTTPOnly:     cookie.HTTPOnly,
+			SameSite:     cookie.SameSite,
+			Expires:      cookie.Expires,
+			Priority:     cookie.Priority,
+			SameParty:    cookie.SameParty,
+			SourceScheme: cookie.SourceScheme,
+			SourcePort:   &sourcePort,
+			PartitionKey: cookie.PartitionKey,
+		})
+	}
+	return params
+}

core/proxy_lane_test.go

@@ -0,0 +1,101 @@
+package core
+
+import (
+	"testing"
+	"time"
+
+	"github.com/go-rod/rod/lib/proto"
+	browserprofile "github.com/karust/openserp/core/browser"
+)
+
+func TestLaneStoreReusesCookiesBySession(t *testing.T) {
+	store := NewLaneStore(10)
+	key := ProxyLaneKey{Engine: "google", SessionID: "sid-a"}
+	cookies := []*proto.NetworkCookie{{Name: "sid", Value: "a", Domain: "example.com", Path: "/"}}
+
+	store.SaveCookies(key, cookies)
+	got := store.Cookies(key)
+	if len(got) != 1 || got[0].Name != "sid" || got[0].Value != "a" {
+		t.Fatalf("expected saved cookie, got %#v", got)
+	}
+
+	other := store.Cookies(ProxyLaneKey{Engine: "google", SessionID: "sid-b"})
+	if len(other) != 0 {
+		t.Fatalf("expected different SID to be cookie-clean, got %#v", other)
+	}
+}
+
+func TestLaneStoreDropCookiesPreservesProfile(t *testing.T) {
+	store := NewLaneStore(10)
+	key := ProxyLaneKey{Engine: "google", SessionID: "sid-a"}
+	profile := store.Profile(key, func() browserprofile.Profile {
+		return browserprofile.Profile{ID: "profile-a"}
+	})
+	if profile.ID != "profile-a" {
+		t.Fatalf("expected initial profile, got %#v", profile)
+	}
+
+	store.SaveCookies(key, []*proto.NetworkCookie{{Name: "sid", Value: "a", Domain: "example.com", Path: "/"}})
+	store.DropCookies(key)
+
+	if got := store.Cookies(key); len(got) != 0 {
+		t.Fatalf("expected cookies to be dropped, got %#v", got)
+	}
+	profile = store.Profile(key, func() browserprofile.Profile {
+		return browserprofile.Profile{ID: "profile-b"}
+	})
+	if profile.ID != "profile-a" {
+		t.Fatalf("expected profile to be preserved after cookie drop, got %#v", profile)
+	}
+	if stats := store.Stats(); stats.CookiesDropped != 1 {
+		t.Fatalf("expected cookies_dropped=1, got %#v", stats)
+	}
+}
+
+func TestLaneStoreEvictsLRU(t *testing.T) {
+	store := NewLaneStore(2)
+	keyA := ProxyLaneKey{Engine: "google", SessionID: "a"}
+	keyB := ProxyLaneKey{Engine: "google", SessionID: "b"}
+	keyC := ProxyLaneKey{Engine: "google", SessionID: "c"}
+
+	store.SaveCookies(keyA, []*proto.NetworkCookie{{Name: "sid", Value: "a"}})
+	time.Sleep(time.Millisecond)
+	store.SaveCookies(keyB, []*proto.NetworkCookie{{Name: "sid", Value: "b"}})
+	time.Sleep(time.Millisecond)
+	_ = store.Cookies(keyB)
+	time.Sleep(time.Millisecond)
+	store.SaveCookies(keyC, []*proto.NetworkCookie{{Name: "sid", Value: "c"}})
+
+	if got := store.Cookies(keyA); len(got) != 0 {
+		t.Fatalf("expected oldest lane A to be evicted, got %#v", got)
+	}
+	if got := store.Cookies(keyB); len(got) != 1 {
+		t.Fatalf("expected lane B to remain, got %#v", got)
+	}
+	if stats := store.Stats(); stats.Active != 2 || stats.EvictedLRU != 1 {
+		t.Fatalf("unexpected lane stats: %#v", stats)
+	}
+}
+
+func TestProxyLaneKeyForOmitsPassword(t *testing.T) {
+	a := ProxyLaneKeyForTenant("Google", "", Query{}, "http://user:pass-a@proxy.example:8080")
+	b := ProxyLaneKeyForTenant("google", "", Query{}, "http://user:pass-b@proxy.example:8080")
+	if a.Empty() || b.Empty() {
+		t.Fatalf("expected derived lane keys, got %#v %#v", a, b)
+	}
+	if a != b {
+		t.Fatalf("expected password changes not to affect lane key: %#v %#v", a, b)
+	}
+}
+
+func TestProxyLaneKeyIncludesTenant(t *testing.T) {
+	q := Query{ProxySessionID: "sid-a"}
+	a := ProxyLaneKeyForTenant("google", "tenant-a", q, "http://proxy.example:8080")
+	b := ProxyLaneKeyForTenant("google", "tenant-b", q, "http://proxy.example:8080")
+	if a == b {
+		t.Fatalf("expected different tenants to produce different lane keys: %#v", a)
+	}
+	if got := a.ID(); got != "tenant-a:google:sid-a" {
+		t.Fatalf("unexpected tenant lane id: %q", got)
+	}
+}

core/proxy_per_context_test.go

@@ -0,0 +1,82 @@
+//go:build integration
+// +build integration
+
+package core
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/karust/openserp/testutil"
+)
+
+// TestPerContextProxyIsolation verifies the supported per-request proxy path:
+// a shared Chrome (launched without a process-level proxy) routes each request
+// through the unauthenticated proxy from its context, with no cross-context
+// leakage between concurrent navigations.
+//
+// Authenticated per-request proxies are intentionally NOT supported on a
+// shared browser: Chrome's Fetch-based proxy auth is browser-global, so
+// concurrent contexts with different credentials race and fail with
+// ERR_INVALID_AUTH_CREDENTIALS. The server routes authenticated proxies to a
+// dedicated Chrome process per auth identity instead (see browserPool in
+// cmd/serve.go).
+func TestPerContextProxyIsolation(t *testing.T) {
+	testutil.RequireIntegration(t)
+
+	newProxy := func(name string) *httptest.Server {
+		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			_, _ = fmt.Fprintf(w, "<html><body>%s</body></html>", name)
+		}))
+	}
+
+	proxyA := newProxy("proxy-a")
+	defer proxyA.Close()
+	proxyB := newProxy("proxy-b")
+	defer proxyB.Close()
+
+	browser, err := NewBrowser(BrowserOpts{IsHeadless: true, Timeout: 20 * time.Second})
+	if err != nil {
+		t.Fatalf("create browser: %v", err)
+	}
+	defer closeTestBrowser(t, browser)
+
+	run := func(proxyURL, want string) error {
+		ctx := WithRequestProxyURL(context.Background(), proxyURL)
+		page, err := browser.Navigate(ctx, "http://per-context-proxy.invalid/")
+		if err != nil {
+			return err
+		}
+		defer func() {
+			_ = ClosePageWithTimeout(context.Background(), page, time.Second)
+		}()
+		body, err := page.Timeout(5 * time.Second).Element("body")
+		if err != nil {
+			return err
+		}
+		text, err := body.Text()
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(text, want) {
+			return fmt.Errorf("expected response from %s, got %q", want, text)
+		}
+		return nil
+	}
+
+	errCh := make(chan error, 2)
+	go func() { errCh <- run(proxyA.URL, "proxy-a") }()
+	go func() { errCh <- run(proxyB.URL, "proxy-b") }()
+
+	for i := 0; i < 2; i++ {
+		if err := <-errCh; err != nil {
+			t.Fatalf("proxied navigation failed: %v", err)
+		}
+	}
+}

core/proxy_test.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"strings"
 	"testing"
 	"time"
 
@@ -189,6 +190,24 @@ func TestMaskProxyURLRedactsCredentials(t *testing.T) {
 	}
 }
 
+func TestBrowserOptsLogFieldsRedactSecrets(t *testing.T) {
+	fields := browserOptsLogFields(BrowserOpts{
+		ProxyURL:             "http://user:sentinel-password@proxy.example:8080",
+		CaptchaSolverApiKey:  "captcha-secret",
+		CaptchaSolverEnabled: true,
+	})
+	rendered := fmt.Sprintf("%v", fields)
+	if strings.Contains(rendered, "sentinel-password") || strings.Contains(rendered, "captcha-secret") {
+		t.Fatalf("browser option log fields leaked secret: %s", rendered)
+	}
+	if fields["proxy"] != "http://proxy.example:8080" {
+		t.Fatalf("expected masked proxy field, got %#v", fields["proxy"])
+	}
+	if fields["captcha_solver_has_key"] != true {
+		t.Fatalf("expected captcha key presence boolean")
+	}
+}
+
 func TestProxyURLForBrowserLaunchStripsCredentials(t *testing.T) {
 	u, err := url.Parse("http://user:pass@127.0.0.1:18888")
 	if err != nil {
@@ -325,6 +344,34 @@ func TestNewRawHTTPClientSocks5hUsesProxyDNS(t *testing.T) {
 	}
 }
 
+func TestNewRawHTTPClientDirectTLSUsesHTTP1(t *testing.T) {
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(r.Proto))
+	}))
+	server.EnableHTTP2 = true
+	server.StartTLS()
+	defer server.Close()
+
+	client, err := NewRawHTTPClient(Query{Insecure: true})
+	if err != nil {
+		t.Fatalf("new raw http client: %v", err)
+	}
+
+	resp, err := client.Get(server.URL)
+	if err != nil {
+		t.Fatalf("expected direct TLS request to succeed, got %v", err)
+	}
+	defer DrainAndCloseResponse(resp)
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("read response body: %v", err)
+	}
+	if string(body) != "HTTP/1.1" {
+		t.Fatalf("expected raw client to use HTTP/1.1, got %q", string(body))
+	}
+}
+
 func TestClassifyProxyNetworkError(t *testing.T) {
 	tests := []struct {
 		name string
@@ -356,6 +403,18 @@ func TestClassifyProxyNetworkError(t *testing.T) {
 	}
 }
 
+func TestClassifyMainDocumentStatus(t *testing.T) {
+	if !errors.Is(classifyMainDocumentStatus(http.StatusForbidden), ErrBlocked) {
+		t.Fatal("expected 403 to classify as ErrBlocked")
+	}
+	if !errors.Is(classifyMainDocumentStatus(http.StatusTooManyRequests), ErrRateLimited) {
+		t.Fatal("expected 429 to classify as ErrRateLimited")
+	}
+	if classifyMainDocumentStatus(http.StatusOK) != nil {
+		t.Fatal("expected 200 to remain unclassified")
+	}
+}
+
 func TestNewRawHTTPClientClassifiesProxyAuthFailure(t *testing.T) {
 	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusProxyAuthRequired)

core/rate_limiter_test.go

@@ -0,0 +1,38 @@
+package core
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestSearchEngineOptionsGetRateLimiterCachesLimiterAndPaces(t *testing.T) {
+	opts := SearchEngineOptions{
+		RateRequests:    20,
+		RateTime:        1,
+		RateBurst:       1,
+		SelectorTimeout: 1,
+	}
+
+	limiter := opts.GetRateLimiter()
+	if limiter == nil {
+		t.Fatal("expected limiter")
+	}
+	if got := opts.GetRateLimiter(); got != limiter {
+		t.Fatal("expected GetRateLimiter to return the cached limiter")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	if err := limiter.Wait(ctx); err != nil {
+		t.Fatalf("first wait failed: %v", err)
+	}
+
+	start := time.Now()
+	if err := limiter.Wait(ctx); err != nil {
+		t.Fatalf("second wait failed: %v", err)
+	}
+	if elapsed := time.Since(start); elapsed < 40*time.Millisecond {
+		t.Fatalf("second wait elapsed %s, want pacing near 50ms", elapsed)
+	}
+}

core/region.go

@@ -0,0 +1,18 @@
+package core
+
+import "github.com/karust/openserp/core/region"
+
+// Region resolution lives in the dependency-free github.com/karust/openserp/core/region
+// subpackage so lightweight consumers that only need geotargeting can import it
+// without pulling in core's browser/scraping dependencies. The aliases and
+// wrappers below preserve the historical core.* API for existing callers.
+
+// RegionTarget is the resolved, per-engine targeting for a user-supplied region
+// hint. See region.RegionTarget for details.
+type RegionTarget = region.RegionTarget
+
+// ResolveRegion turns a free-text region hint into per-engine targeting.
+// See region.ResolveRegion for accepted inputs and semantics.
+func ResolveRegion(hint string) RegionTarget {
+	return region.ResolveRegion(hint)
+}

core/region/region.go

@@ -0,0 +1,287 @@
+// Package region resolves a free-text region hint into per-engine search
+// targeting (Google UULE canonical names, Yandex lr IDs, ISO country codes).
+//
+// It is deliberately dependency-free (standard library only) so lightweight
+// consumers that only need geotargeting can import it without pulling in the
+// headless-browser/scraping stack the rest of github.com/karust/openserp/core
+// depends on. The parent core package re-exports these symbols for backward
+// compatibility.
+package region
+
+import (
+	"encoding/base64"
+	"strings"
+)
+
+// Locale is a parsed language/region pair derived from a BCP47-style code.
+// Language is the lowercase 2-letter language subtag (e.g. "en", "de").
+// Country is the uppercase 2-letter region subtag (e.g. "US", "DE"); it may be
+// empty when the input had no region and the caller did not request a default.
+type Locale struct {
+	Language string
+	Country  string
+}
+
+// ParseLocale parses a language code such as "en", "EN-us", or "de_AT" into a
+// Locale. Returns the zero value when the input is empty or has no language
+// subtag. Country is uppercased; Language is lowercased.
+func ParseLocale(code string) Locale {
+	code = strings.TrimSpace(code)
+	if code == "" {
+		return Locale{}
+	}
+	code = strings.ReplaceAll(code, "_", "-")
+
+	parts := strings.Split(code, "-")
+	language := strings.ToLower(strings.TrimSpace(parts[0]))
+	if language == "" {
+		return Locale{}
+	}
+
+	country := ""
+	if len(parts) > 1 {
+		country = strings.ToUpper(strings.TrimSpace(parts[1]))
+	}
+	return Locale{Language: language, Country: country}
+}
+
+// CountryFromRegion extracts a two-letter country/market code from a region
+// hint. It accepts "ru", "RU", "en-RU", and "en_RU"; engine-native numeric
+// region IDs intentionally return empty here.
+func CountryFromRegion(region string) string {
+	region = strings.TrimSpace(region)
+	if region == "" {
+		return ""
+	}
+	region = strings.ReplaceAll(region, "_", "-")
+	if len(region) == 2 && isAlphaASCII(region[0]) && isAlphaASCII(region[1]) {
+		return strings.ToUpper(region)
+	}
+
+	locale := ParseLocale(region)
+	if len(locale.Country) == 2 && isAlphaASCII(locale.Country[0]) && isAlphaASCII(locale.Country[1]) {
+		return locale.Country
+	}
+	return ""
+}
+
+func isAlphaASCII(ch byte) bool {
+	return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')
+}
+
+// RegionTarget is the resolved, per-engine targeting for a user-supplied region
+// hint. Engines read the field relevant to them: Google uses GoogleCanonical to
+// build a UULE, Yandex uses YandexLR. Country is the ISO 3166-1 alpha-2 code
+// when one could be derived, useful as a coarse market signal.
+//
+// A field left empty means "no better signal than the raw input" — callers
+// should fall back to their previous behavior (e.g. gl= from locale, or
+// dropping the parameter entirely). Resolution never fails: an unrecognized
+// region simply yields empty engine fields rather than an error.
+type RegionTarget struct {
+	// Raw is the trimmed original input, preserved for engines that pass it
+	// through (e.g. Yandex numeric lr IDs).
+	Raw string
+	// Country is the ISO 3166-1 alpha-2 code (uppercase) when derivable, else "".
+	Country string
+	// GoogleCanonical is the exact Google geotargets canonical location name
+	// (e.g. "Berlin,Berlin,Germany") suitable for UULE v1 encoding, else "".
+	GoogleCanonical string
+	// YandexLR is the Yandex lr region ID (e.g. "213"), else "".
+	YandexLR string
+}
+
+// yandexLRByCountry maps an ISO country code to a Yandex lr region ID. Yandex
+// only exposes a limited set of country-level regions; cities require numeric
+// lr IDs passed through verbatim.
+var yandexLRByCountry = map[string]string{
+	"AT": "113", "AU": "211", "BE": "114", "BR": "94", "CA": "95",
+	"CH": "126", "DE": "96", "DK": "203", "ES": "204", "FI": "123",
+	"FR": "124", "GB": "102", "IE": "10063", "IN": "994", "IT": "205",
+	"JP": "137", "KR": "135", "MX": "20271", "NL": "118", "NO": "119",
+	"PL": "120", "RU": "225", "SE": "127", "SG": "10105", "TR": "983",
+	"UA": "187", "UK": "102", "US": "84", "ZA": "10021",
+}
+
+// cityCanonical maps a normalized bare city name to its exact Google geotargets
+// canonical name (used to build a UULE). Only UULE-bearing city targeting needs
+// a name table — country/state targeting rides on gl= and never needs one.
+//
+// The list is deliberately small and hand-curated: bare city names are
+// ambiguous (e.g. "London" exists in CA/GB/US), so we only auto-resolve a
+// prominence list to its "obvious" match. Canonical names below are copied
+// verbatim from Google's geotargets data; any other city can still be targeted
+// by passing its full canonical name ("City,Region,Country") directly.
+var cityCanonical = map[string]string{
+	"amsterdam":      "Amsterdam,North Holland,Netherlands",
+	"athens":         "Athens,Athens,Attica,Greece",
+	"austin":         "Austin,Texas,United States",
+	"bangalore":      "Bengaluru,Karnataka,India",
+	"barcelona":      "Barcelona,Barcelona,Catalonia,Spain",
+	"beijing":        "Beijing,Beijing,China",
+	"berlin":         "Berlin,Berlin,Germany",
+	"birmingham":     "Birmingham,West Midlands,England,United Kingdom",
+	"boston":         "Boston,Massachusetts,United States",
+	"brussels":       "Brussels,Brussels,Belgium",
+	"buenos aires":   "Buenos Aires,Buenos Aires,Argentina",
+	"cairo":          "Cairo,Cairo Governorate,Egypt",
+	"chicago":        "Chicago,Illinois,United States",
+	"copenhagen":     "Copenhagen,Capital Region of Denmark,Denmark",
+	"dallas":         "Dallas,Texas,United States",
+	"delhi":          "Delhi,Delhi,India",
+	"dubai":          "Dubai,Dubai,United Arab Emirates",
+	"dublin":         "Dublin,County Dublin,Ireland",
+	"frankfurt":      "Frankfurt am Main,Hessen,Germany",
+	"hamburg":        "Hamburg,Hamburg,Germany",
+	"helsinki":       "Helsinki,Helsinki,Uusimaa,Finland",
+	"hong kong":      "Hong Kong,Hong Kong",
+	"istanbul":       "Istanbul,Istanbul,Turkiye",
+	"johannesburg":   "Johannesburg,Gauteng,South Africa",
+	"kyiv":           "Kyiv,Kyiv city,Ukraine",
+	"lisbon":         "Lisbon,Lisbon,Lisbon,Portugal",
+	"london":         "London,England,United Kingdom",
+	"los angeles":    "Los Angeles,California,United States",
+	"lyon":           "Lyon,Auvergne-Rhone-Alpes,France",
+	"madrid":         "Madrid,Community of Madrid,Spain",
+	"manchester":     "Manchester,England,United Kingdom",
+	"marseille":      "Marseille,Provence-Alpes-Cote d'Azur,France",
+	"melbourne":      "Melbourne,Victoria,Australia",
+	"mexico city":    "Mexico City,Mexico City,Mexico",
+	"miami":          "Miami,Florida,United States",
+	"milan":          "Milan,Milan,Lombardy,Italy",
+	"montreal":       "Montreal,Montreal,Quebec,Canada",
+	"moscow":         "Moscow,Moscow,Russia",
+	"mumbai":         "Mumbai,Maharashtra,India",
+	"munich":         "Munich,Bavaria,Germany",
+	"new york":       "New York,New York,United States",
+	"osaka":          "Osaka,Osaka,Japan",
+	"oslo":           "Oslo,Oslo,Norway",
+	"paris":          "Paris,Paris,Ile-de-France,France",
+	"prague":         "Prague,Prague,Czechia",
+	"rio de janeiro": "Rio de Janeiro,State of Rio de Janeiro,Brazil",
+	"rome":           "Rome,Rome,Lazio,Italy",
+	"san francisco":  "San Francisco,California,United States",
+	"sao paulo":      "Sao Paulo,State of Sao Paulo,Brazil",
+	"seattle":        "Seattle,Washington,United States",
+	"seoul":          "Seoul,Seoul,South Korea",
+	"shanghai":       "Shanghai,Shanghai,China",
+	"singapore":      "Singapore,Singapore",
+	"stockholm":      "Stockholm,Stockholm County,Sweden",
+	"sydney":         "Sydney,New South Wales,Australia",
+	"tokyo":          "Tokyo,Tokyo,Japan",
+	"toronto":        "Toronto,Toronto,Ontario,Canada",
+	"vancouver":      "Vancouver,British Columbia,Canada",
+	"vienna":         "Vienna,Vienna,Vienna,Austria",
+	"warsaw":         "Warsaw,Warsaw,Masovian Voivodeship,Poland",
+	"washington":     "Washington,District of Columbia,United States",
+	"zurich":         "Zurich,Zurich,Switzerland",
+}
+
+// ResolveRegion turns a free-text region hint into per-engine targeting. It
+// never errors: unrecognized input yields a RegionTarget with empty engine
+// fields, leaving callers to fall back to their defaults.
+//
+// Accepted inputs, in priority order:
+//   - Numeric (e.g. "213"): a Yandex-native lr ID. Passed through as YandexLR.
+//   - A 2-letter country or BCP47-style locale (e.g. "DE", "en-GB"): resolved to
+//     a country and its Yandex lr. No Google canonical — country targeting rides
+//     on gl=, not UULE.
+//   - A bare curated city name (e.g. "Berlin"): resolved to its canonical name.
+//   - A full "City,Region,Country" canonical name typed verbatim (>=2 commas):
+//     passed through to Google as-is.
+func ResolveRegion(region string) RegionTarget {
+	region = strings.TrimSpace(region)
+	t := RegionTarget{Raw: region}
+	if region == "" {
+		return t
+	}
+
+	// Yandex-native numeric lr IDs: pass through, nothing else to derive.
+	if isDigitsOnly(region) {
+		t.YandexLR = region
+		return t
+	}
+
+	// Country / locale code (e.g. "DE", "en-GB"). Country-level targeting is
+	// conveyed via Country (Google uses gl=, Yandex the country lr); we
+	// deliberately do NOT emit a Google canonical/UULE for a whole country.
+	if cc := CountryFromRegion(region); cc != "" {
+		t.Country = cc
+		t.YandexLR = yandexLRByCountry[cc]
+		return t
+	}
+
+	// Bare curated city name (e.g. "Berlin", "New York").
+	if c := cityCanonical[strings.ToLower(region)]; c != "" {
+		t.GoogleCanonical = c
+		return t
+	}
+
+	// Looks like a full "City,Region,Country" canonical name the caller typed
+	// verbatim (>=2 commas). Pass it through to Google as-is; Google ignores it
+	// if it isn't a real canonical name, which is the caller's responsibility.
+	if strings.Count(region, ",") >= 2 {
+		t.GoogleCanonical = region
+		return t
+	}
+
+	return t
+}
+
+func isDigitsOnly(s string) bool {
+	if s == "" {
+		return false
+	}
+	for _, r := range s {
+		if r < '0' || r > '9' {
+			return false
+		}
+	}
+	return true
+}
+
+const googleUULEPrefix = "w+CAIQICI"
+
+var googleUULELengthAlphabet = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_")
+
+// GoogleUULE builds a Google UULE v1 value for a region hint, returning "" when
+// no UULE applies.
+//
+// UULE only takes effect when the encoded string is an exact Google geotargets
+// canonical name; a bare free-text city is silently ignored by Google. We
+// therefore resolve the region first and encode the resolved canonical name.
+// Country-level hints (e.g. "DE") and Yandex numeric lr IDs intentionally
+// produce no UULE — those are conveyed via gl= or are not applicable to Google.
+//
+// As a best-effort fallback, a free-text region that resolves to neither a
+// country, a numeric lr, nor a known canonical name is encoded verbatim; Google
+// ignores it if it is not a real canonical name, matching prior behavior.
+func GoogleUULE(region string) string {
+	target := ResolveRegion(region)
+	canonical := target.GoogleCanonical
+	if canonical == "" {
+		if target.Country != "" || target.YandexLR != "" {
+			return ""
+		}
+		canonical = strings.TrimSpace(region)
+	}
+	return EncodeGoogleUULE(canonical)
+}
+
+// EncodeGoogleUULE encodes a canonical location name into a UULE v1 value:
+// a fixed prefix, a single length-tag character, then base64(name). Returns ""
+// for an empty name or one too long for the length alphabet.
+func EncodeGoogleUULE(canonical string) string {
+	length := len([]rune(canonical))
+	if length <= 0 || length >= len(googleUULELengthAlphabet) {
+		return ""
+	}
+	return googleUULEPrefix + string(googleUULELengthAlphabet[length]) + base64.StdEncoding.EncodeToString([]byte(canonical))
+}
+
+// YandexLR converts a public region hint to a Yandex lr ID, returning "" when
+// none applies. Numeric input is passed through; a 2-letter country or locale
+// is mapped via the country table.
+func YandexLR(region string) string {
+	return ResolveRegion(region).YandexLR
+}

core/region/region_test.go

@@ -0,0 +1,112 @@
+package region
+
+import "testing"
+
+func TestResolveRegion(t *testing.T) {
+	tests := []struct {
+		name      string
+		region    string
+		country   string
+		canonical string
+		lr        string
+	}{
+		{
+			name:      "bare curated city resolves to canonical name",
+			region:    "Berlin",
+			canonical: "Berlin,Berlin,Germany",
+		},
+		{
+			name:      "curated city is case-insensitive",
+			region:    "  berlin ",
+			canonical: "Berlin,Berlin,Germany",
+		},
+		{
+			name:      "ambiguous curated city picks obvious match",
+			region:    "London",
+			canonical: "London,England,United Kingdom",
+		},
+		{
+			name:      "full canonical name passes through verbatim",
+			region:    "Smalltown,Some Region,Faraway",
+			canonical: "Smalltown,Some Region,Faraway",
+		},
+		{
+			name:    "bare country code sets country and yandex lr but no uule",
+			region:  "DE",
+			country: "DE",
+			lr:      "96",
+		},
+		{
+			name:    "locale code resolves country",
+			region:  "en-GB",
+			country: "GB",
+			lr:      "102",
+		},
+		{
+			name:   "numeric region is a yandex lr passthrough",
+			region: "213",
+			lr:     "213",
+		},
+		{
+			name:   "empty region resolves to nothing",
+			region: "",
+		},
+		{
+			name:   "unknown bare name resolves to nothing",
+			region: "Nowhereville",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ResolveRegion(tt.region)
+			if got.Country != tt.country {
+				t.Errorf("Country = %q, want %q", got.Country, tt.country)
+			}
+			if got.GoogleCanonical != tt.canonical {
+				t.Errorf("GoogleCanonical = %q, want %q", got.GoogleCanonical, tt.canonical)
+			}
+			if got.YandexLR != tt.lr {
+				t.Errorf("YandexLR = %q, want %q", got.YandexLR, tt.lr)
+			}
+		})
+	}
+}
+
+func TestGoogleUULE(t *testing.T) {
+	// A curated bare city must encode its canonical name, not the raw input —
+	// Google ignores a UULE whose decoded value is not a real geotargets name.
+	if got := GoogleUULE("Berlin"); got != EncodeGoogleUULE("Berlin,Berlin,Germany") {
+		t.Errorf("GoogleUULE(Berlin) = %q, want canonical encoding", got)
+	}
+	// Country codes and numeric lr IDs ride on gl=/lr=, never UULE.
+	if got := GoogleUULE("DE"); got != "" {
+		t.Errorf("GoogleUULE(DE) = %q, want empty", got)
+	}
+	if got := GoogleUULE("213"); got != "" {
+		t.Errorf("GoogleUULE(213) = %q, want empty", got)
+	}
+	if got := GoogleUULE(""); got != "" {
+		t.Errorf("GoogleUULE(empty) = %q, want empty", got)
+	}
+}
+
+func TestYandexLR(t *testing.T) {
+	cases := map[string]string{
+		"213":   "213",
+		" 213 ": "213",
+		"RU":    "225",
+		"de":    "96",
+		"en-RU": "225",
+		"en_GB": "102",
+		"UK":    "102",
+		"ZZ":    "",
+		"21a":   "",
+		"":      "",
+	}
+	for in, want := range cases {
+		if got := YandexLR(in); got != want {
+			t.Errorf("YandexLR(%q) = %q, want %q", in, got, want)
+		}
+	}
+}

core/region_test.go

@@ -0,0 +1,18 @@
+package core
+
+import "testing"
+
+// The full ResolveRegion behavior is exercised in the core/region subpackage.
+// This test only verifies the core.* re-export wiring stays intact.
+func TestResolveRegionReExport(t *testing.T) {
+	got := ResolveRegion("Berlin")
+	if got.GoogleCanonical != "Berlin,Berlin,Germany" {
+		t.Errorf("GoogleCanonical = %q, want %q", got.GoogleCanonical, "Berlin,Berlin,Germany")
+	}
+	if cc := CountryFromRegion("en-GB"); cc != "GB" {
+		t.Errorf("CountryFromRegion(en-GB) = %q, want GB", cc)
+	}
+	if loc := ParseLocale("de_AT"); loc.Language != "de" || loc.Country != "AT" {
+		t.Errorf("ParseLocale(de_AT) = %+v, want {de AT}", loc)
+	}
+}

core/resilient.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/sirupsen/logrus"
 )
@@ -35,6 +36,18 @@ type ResilientConfig struct {
 	Proxy          ProxyConfig
 }
 
+type proxyLaneCookieDropper interface {
+	DropProxyLaneCookies(context.Context, Query)
+}
+
+type proxyLaneStatser interface {
+	ProxyLaneStats() LaneStats
+}
+
+type browserPoolStatser interface {
+	BrowserPoolStats() BrowserPoolStats
+}
+
 func DefaultResilientConfig() ResilientConfig {
 	return ResilientConfig{
 		Retry:          DefaultRetryConfig(),
@@ -157,11 +170,12 @@ func (rs *ResilientSearcher) searchWithProtection(ctx context.Context, engine Se
 	policy := rs.effectivePolicyForQuery(engine.Name(), q)
 	attemptMeta := rs.baseProxyMeta(policy)
 
+	startedAt := time.Now()
 	result := RetryableSearch(ctx, rs.retryCfg, engine.Name(), func(callCtx context.Context) ([]SearchResult, error) {
 		limiter := engine.GetRateLimiter()
 		if limiter != nil {
 			if err := limiter.Wait(callCtx); err != nil {
-				return nil, err
+				return nil, normalizeLimiterWaitErr(callCtx, err)
 			}
 		}
 
@@ -174,6 +188,10 @@ func (rs *ResilientSearcher) searchWithProtection(ctx context.Context, engine Se
 		case ProxyModeOff:
 			attemptQuery.ProxyURL = ""
 			attemptMeta.Used = "direct"
+		case ProxyModeRequestURL:
+			proxyURL = q.ProxyURL
+			attemptQuery.ProxyURL = proxyURL
+			attemptMeta.Used = MaskProxyURL(proxyURL)
 		case ProxyModeTagPool:
 			proxyURL = rs.selectProxyForQuery(policy, q, engineCtx)
 			if proxyURL == "" {
@@ -184,45 +202,187 @@ func (rs *ResilientSearcher) searchWithProtection(ctx context.Context, engine Se
 			attemptMeta.Used = MaskProxyURL(proxyURL)
 		}
 
-		var (
-			results []SearchResult
-			err     error
-		)
-		if isImage {
-			results, err = engine.SearchImage(callCtx, attemptQuery)
-		} else {
-			results, err = engine.Search(callCtx, attemptQuery)
-		}
+		requestCtx := proxyRequestContext(callCtx, engine.Name(), attemptQuery)
+		results, err := invokeEngine(requestCtx, engine, attemptQuery, isImage)
 
 		if reportToRegistry {
 			rs.reportProxyAttempt(engineCtx, proxyURL, err)
 		}
+		if err != nil && errors.Is(err, ErrCaptcha) && rs.proxyCfg.Proxies.Lanes.DropCookiesOnChallenge {
+			// Recompute lane key only to gate the call: empty key means we have no
+			// session to drop cookies for. The dropper recomputes the key itself
+			// when it actually needs to mutate lane state.
+			if !ProxyLaneKeyForTenant(engine.Name(), TenantFromContext(callCtx), attemptQuery, attemptQuery.ProxyURL).Empty() {
+				if dropper, ok := engine.(proxyLaneCookieDropper); ok {
+					dropper.DropProxyLaneCookies(callCtx, attemptQuery)
+				}
+			}
+		}
 
 		return results, err
 	})
 
 	if result.Err != nil {
-		if !errors.Is(result.Err, ErrProxyUnavailable) {
+		if shouldRecordCircuitFailure(result.Err) {
 			cb.RecordFailure(engineCtx)
 		}
 		return nil, attemptMeta, result.Err
 	}
 
-	cb.RecordSuccess(engineCtx)
+	cb.RecordSuccessDuration(engineCtx, time.Since(startedAt))
 	return result.Results, attemptMeta, nil
 }
 
+func shouldRecordCircuitFailure(err error) bool {
+	return err != nil &&
+		!IsContextDone(err) &&
+		!errors.Is(err, ErrProxyUnavailable) &&
+		!errors.Is(err, ErrCircuitOpen)
+}
+
+// normalizeLimiterWaitErr maps rate.Limiter.Wait failures back to the caller's
+// context error. Wait reports a bare "would exceed context deadline" error that
+// errors.Is cannot trace to the context, so without this an impatient client
+// would be counted as an engine failure by the circuit breaker.
+func normalizeLimiterWaitErr(ctx context.Context, err error) error {
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return ctxErr
+	}
+	if _, hasDeadline := ctx.Deadline(); hasDeadline {
+		return fmt.Errorf("%w: %v", context.DeadlineExceeded, err)
+	}
+	return err
+}
+
+// invokeEngine is the single panic-recovery point for every engine call made
+// through the resilient pipeline (browser, raw, and any future engine method).
+// A rod/CDP panic surfaces as ErrEngineInternal instead of killing the process.
+func invokeEngine(ctx context.Context, engine SearchEngine, q Query, isImage bool) (results []SearchResult, err error) {
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			results = nil
+			err = RecoverEnginePanicWithContext(ctx, engine.Name(), recovered, nil)
+		}
+	}()
+
+	if isImage {
+		return engine.SearchImage(ctx, q)
+	}
+	return engine.Search(ctx, q)
+}
+
 // SearchAllParallel applies retry/circuit protections per engine for mega search.
 // Returns results, list of engines that responded, and list of engines that failed.
 func (rs *ResilientSearcher) SearchAllParallel(ctx context.Context, q Query, engines []SearchEngine) ([]MegaSearchResult, []string, []string) {
-	return rs.runParallel(ctx, q, engines, false)
+	results, responded, failed, _ := rs.runParallelDetailed(ctx, q, engines, false)
+	return results, responded, failed
 }
 
 func (rs *ResilientSearcher) SearchAllImageParallel(ctx context.Context, q Query, engines []SearchEngine) ([]MegaSearchResult, []string, []string) {
-	return rs.runParallel(ctx, q, engines, true)
+	results, responded, failed, _ := rs.runParallelDetailed(ctx, q, engines, true)
+	return results, responded, failed
 }
 
-func (rs *ResilientSearcher) runParallel(ctx context.Context, q Query, engines []SearchEngine, isImage bool) ([]MegaSearchResult, []string, []string) {
+func (rs *ResilientSearcher) searchAllParallelDetailed(ctx context.Context, q Query, engines []SearchEngine) ([]MegaSearchResult, []string, []EngineErrorDetail) {
+	results, responded, _, errors := rs.runParallelDetailed(ctx, q, engines, false)
+	return results, responded, errors
+}
+
+func (rs *ResilientSearcher) searchAllImageParallelDetailed(ctx context.Context, q Query, engines []SearchEngine) ([]MegaSearchResult, []string, []EngineErrorDetail) {
+	results, responded, _, errors := rs.runParallelDetailed(ctx, q, engines, true)
+	return results, responded, errors
+}
+
+func (rs *ResilientSearcher) searchAnyDetailed(ctx context.Context, q Query, engines []SearchEngine, isImage bool) ([]MegaSearchResult, []string, []EngineErrorDetail) {
+	ctx = EnsureContext(ctx)
+
+	engineErrors := make([]EngineErrorDetail, 0, len(engines))
+	for _, engine := range engines {
+		if err := ctx.Err(); err != nil {
+			engineErrors = append(engineErrors, engineErrorDetail(engine.Name(), err, q))
+			break
+		}
+		if !engine.IsInitialized() {
+			engineErrors = append(engineErrors, engineErrorDetail(engine.Name(), fmt.Errorf("not initialized"), q))
+			continue
+		}
+
+		results, _, err := rs.searchWithProtection(ctx, engine, q, isImage)
+		if err != nil {
+			engineErrors = append(engineErrors, engineErrorDetail(engine.Name(), err, q))
+			continue
+		}
+
+		mega := make([]MegaSearchResult, len(results))
+		for i, r := range results {
+			mega[i] = MegaSearchResult{SearchResult: r, Engine: engine.Name()}
+		}
+		return mega, []string{engine.Name()}, engineErrors
+	}
+
+	if engineErrors == nil {
+		engineErrors = []EngineErrorDetail{}
+	}
+	return []MegaSearchResult{}, []string{}, engineErrors
+}
+
+func (rs *ResilientSearcher) searchFastestDetailed(ctx context.Context, q Query, engines []SearchEngine, isImage bool) ([]MegaSearchResult, []string, []EngineErrorDetail) {
+	ctx = EnsureContext(ctx)
+
+	var (
+		fastest      SearchEngine
+		bestLatency  = time.Duration(1<<63 - 1)
+		foundLatency bool
+		unavailable  []EngineErrorDetail
+		candidates   []SearchEngine
+	)
+
+	for _, engine := range engines {
+		cb := rs.cbManager.Get(engine.Name())
+		if !engine.IsInitialized() {
+			unavailable = append(unavailable, engineErrorDetail(engine.Name(), fmt.Errorf("not initialized"), q))
+			continue
+		}
+
+		engineCtx := WithEngine(ctx, engine.Name())
+		if !cb.AllowRequest(engineCtx) {
+			unavailable = append(unavailable, engineErrorDetail(engine.Name(), ErrCircuitOpen, q))
+			continue
+		}
+
+		candidates = append(candidates, engine)
+		if avg, ok := cb.AvgSuccessLatency(); ok {
+			if !foundLatency || avg < bestLatency {
+				bestLatency = avg
+				fastest = engine
+				foundLatency = true
+			}
+		}
+	}
+
+	if len(candidates) == 0 {
+		if unavailable == nil {
+			unavailable = []EngineErrorDetail{}
+		}
+		return []MegaSearchResult{}, []string{}, unavailable
+	}
+	if fastest == nil {
+		fastest = candidates[0]
+	}
+
+	results, _, err := rs.searchWithProtection(ctx, fastest, q, isImage)
+	if err != nil {
+		return []MegaSearchResult{}, []string{}, []EngineErrorDetail{engineErrorDetail(fastest.Name(), err, q)}
+	}
+
+	mega := make([]MegaSearchResult, len(results))
+	for i, r := range results {
+		mega[i] = MegaSearchResult{SearchResult: r, Engine: fastest.Name()}
+	}
+	return mega, []string{fastest.Name()}, []EngineErrorDetail{}
+}
+
+func (rs *ResilientSearcher) runParallelDetailed(ctx context.Context, q Query, engines []SearchEngine, isImage bool) ([]MegaSearchResult, []string, []string, []EngineErrorDetail) {
 	ctx = EnsureContext(ctx)
 
 	type engineResult struct {
@@ -268,14 +428,37 @@ func (rs *ResilientSearcher) runParallel(ctx context.Context, q Query, engines [
 
 	var allResults []MegaSearchResult
 	var responded, failed []string
+	var engineErrors []EngineErrorDetail
 
-	for i := 0; i < started; i++ {
-		res := <-resultCh
-		if res.err != nil {
-			failed = append(failed, res.name)
-		} else {
-			responded = append(responded, res.name)
-			allResults = append(allResults, res.results...)
+	// Collect results, but bail out early if the parent context is cancelled
+	// (e.g. mega-search hits its aggregate deadline). Engines whose work has
+	// not yet returned a result are reported as failed with the context error
+	// so the caller can return partial results to the client instead of
+	// blocking on a slow/stuck engine.
+	pending := map[string]struct{}{}
+	for _, eng := range engines {
+		pending[eng.Name()] = struct{}{}
+	}
+	collected := 0
+collectLoop:
+	for collected < started {
+		select {
+		case res := <-resultCh:
+			collected++
+			delete(pending, res.name)
+			if res.err != nil {
+				failed = append(failed, res.name)
+				engineErrors = append(engineErrors, engineErrorDetail(res.name, res.err, q))
+			} else {
+				responded = append(responded, res.name)
+				allResults = append(allResults, res.results...)
+			}
+		case <-ctx.Done():
+			for name := range pending {
+				failed = append(failed, name)
+				engineErrors = append(engineErrors, engineErrorDetail(name, ctx.Err(), q))
+			}
+			break collectLoop
 		}
 	}
 
@@ -285,7 +468,10 @@ func (rs *ResilientSearcher) runParallel(ctx context.Context, q Query, engines [
 	if failed == nil {
 		failed = []string{}
 	}
-	return allResults, responded, failed
+	if engineErrors == nil {
+		engineErrors = []EngineErrorDetail{}
+	}
+	return allResults, responded, failed, engineErrors
 }
 
 func (rs *ResilientSearcher) GetCircuitBreakerStats() []map[string]interface{} {
@@ -294,16 +480,21 @@ func (rs *ResilientSearcher) GetCircuitBreakerStats() []map[string]interface{} {
 
 func (rs *ResilientSearcher) GetProxyStats() ProxyStats {
 	stats := ProxyStats{
-		ConfiguredCount: 0,
-		HealthyCount:    0,
-		UnhealthyCount:  0,
-		Tags:            map[string]ProxyTagSummary{},
-		Entries:         []ProxyStatsEntry{},
+		ConfiguredCount:        0,
+		HealthyCount:           0,
+		UnhealthyCount:         0,
+		RequestProxyURLEnabled: rs.proxyCfg.Proxies.AllowRequestProxyURL,
+		Lanes:                  rs.proxyLaneStats(),
+		Tags:                   map[string]ProxyTagSummary{},
+		Entries:                []ProxyStatsEntry{},
 	}
 
 	if rs.proxyRegistry != nil {
 		stats = rs.proxyRegistry.BuildStats()
 	}
+	stats.RequestProxyURLEnabled = rs.proxyCfg.Proxies.AllowRequestProxyURL
+	stats.Lanes = rs.proxyLaneStats()
+	stats.BrowserProcesses = rs.browserPoolStats()
 
 	engines := map[string]ProxyEngineStats{}
 	for _, engine := range rs.engines {
@@ -332,6 +523,38 @@ func (rs *ResilientSearcher) GetProxyStats() ProxyStats {
 	return stats
 }
 
+func (rs *ResilientSearcher) proxyLaneStats() LaneStats {
+	var out LaneStats
+	for _, engine := range rs.engines {
+		statser, ok := engine.(proxyLaneStatser)
+		if !ok {
+			continue
+		}
+		stats := statser.ProxyLaneStats()
+		out.Active += stats.Active
+		out.EvictedLRU += stats.EvictedLRU
+		out.CookiesDropped += stats.CookiesDropped
+	}
+	return out
+}
+
+// browserPoolStats reports the first non-zero browser pool stats found across
+// engines. The pool is shared across engines, so reading from any engine that
+// exposes it is sufficient; other engines' implementations return zero values.
+func (rs *ResilientSearcher) browserPoolStats() BrowserPoolStats {
+	for _, engine := range rs.engines {
+		statser, ok := engine.(browserPoolStatser)
+		if !ok {
+			continue
+		}
+		stats := statser.BrowserPoolStats()
+		if stats.Max > 0 || stats.Active > 0 || stats.EvictedLRU > 0 || stats.EvictedIdle > 0 {
+			return stats
+		}
+	}
+	return BrowserPoolStats{}
+}
+
 func (rs *ResilientSearcher) ResolveMegaProxyMeta(q Query, engines []SearchEngine) ProxyExecutionMeta {
 	if len(engines) == 0 {
 		return ProxyExecutionMeta{Mode: ProxyModeOff, Used: "direct"}
@@ -347,6 +570,9 @@ func (rs *ResilientSearcher) ResolveMegaProxyMeta(q Query, engines []SearchEngin
 			hasOff = true
 			continue
 		}
+		if policy.Mode == ProxyModeRequestURL {
+			return ProxyExecutionMeta{Mode: ProxyModeRequestURL, Used: MaskProxyURL(q.ProxyURL)}
+		}
 
 		allOff = false
 		if policy.Tag != "" {
@@ -406,10 +632,17 @@ func (rs *ResilientSearcher) effectivePolicyForEngine(engineName string) ProxyPo
 
 func (rs *ResilientSearcher) effectivePolicyForQuery(engineName string, q Query) ProxyPolicy {
 	switch q.ProxyOverride {
-	case "":
-		return rs.effectivePolicyForEngine(engineName)
 	case ProxyOverrideDirect:
 		return ProxyPolicy{Mode: ProxyModeOff}
+	}
+
+	if strings.TrimSpace(q.ProxyURL) != "" && rs.proxyCfg.Proxies.AllowRequestProxyURL {
+		return ProxyPolicy{Mode: ProxyModeRequestURL}
+	}
+
+	switch q.ProxyOverride {
+	case "":
+		return rs.effectivePolicyForEngine(engineName)
 	default:
 		return ProxyPolicy{Mode: ProxyModeTagPool, Tag: q.ProxyOverride}
 	}
@@ -451,4 +684,18 @@ func (rs *ResilientSearcher) selectProxyForQuery(policy ProxyPolicy, q Query, ct
 	return rs.selectProxyForTag(ctx, policy.Tag)
 }
 
+func proxyRequestContext(ctx context.Context, engineName string, q Query) context.Context {
+	ctx = WithRequestProxyURL(ctx, q.ProxyURL)
+	if q.ProxyURL == "" {
+		return ctx
+	}
+	if laneKey := ProxyLaneKeyForTenant(engineName, TenantFromContext(ctx), q, q.ProxyURL); !laneKey.Empty() {
+		ctx = WithProxyLaneKey(ctx, laneKey)
+		if q.ProxyCountry != "" {
+			ctx = WithProfileRegion(ctx, q.ProxyCountry)
+		}
+	}
+	return ctx
+}
+
 var ErrAllEnginesFailed = fmt.Errorf("all search engines failed")

core/resilient_context_test.go

@@ -73,3 +73,88 @@ func TestResilientSearchPrimary_CancelledContextStopsWithin100ms(t *testing.T) {
 		t.Fatal("search did not stop after context cancellation")
 	}
 }
+
+type staticErrorEngine struct {
+	name string
+	err  error
+}
+
+func (e *staticErrorEngine) Name() string { return e.name }
+
+func (e *staticErrorEngine) IsInitialized() bool { return true }
+
+func (e *staticErrorEngine) GetRateLimiter() *rate.Limiter { return nil }
+
+func (e *staticErrorEngine) Search(context.Context, Query) ([]SearchResult, error) {
+	return nil, e.err
+}
+
+func (e *staticErrorEngine) SearchImage(context.Context, Query) ([]SearchResult, error) {
+	return nil, e.err
+}
+
+func TestResilientSearchPrimary_ContextErrorsDoNotOpenCircuit(t *testing.T) {
+	engine := &staticErrorEngine{name: "cancelled", err: context.Canceled}
+	cfg := DefaultResilientConfig()
+	cfg.Retry.MaxRetries = 0
+	cfg.CircuitBreaker.FailureThreshold = 1
+	rs := NewResilientSearcher([]SearchEngine{engine}, cfg)
+
+	_, _, _, err := rs.SearchPrimary(context.Background(), engine, Query{Text: "cancel-me"})
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+	if state := rs.cbManager.Get(engine.Name()).State(); state != CircuitClosed {
+		t.Fatalf("circuit state = %s, want closed", state)
+	}
+}
+
+type slowLimiterEngine struct {
+	staticErrorEngine
+	limiter *rate.Limiter
+}
+
+func (e *slowLimiterEngine) GetRateLimiter() *rate.Limiter { return e.limiter }
+
+func TestResilientSearchPrimary_LimiterDeadlineDoesNotOpenCircuit(t *testing.T) {
+	// Drained limiter with a 1h refill: any deadline-bounded Wait fails with
+	// rate's bare "would exceed context deadline" error, not a context error.
+	limiter := rate.NewLimiter(rate.Every(time.Hour), 1)
+	if !limiter.Allow() {
+		t.Fatal("expected initial burst token")
+	}
+	engine := &slowLimiterEngine{
+		staticErrorEngine: staticErrorEngine{name: "ratelimited"},
+		limiter:           limiter,
+	}
+	cfg := DefaultResilientConfig()
+	cfg.Retry.MaxRetries = 0
+	cfg.CircuitBreaker.FailureThreshold = 1
+	rs := NewResilientSearcher([]SearchEngine{engine}, cfg)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+	_, _, _, err := rs.SearchPrimary(ctx, engine, Query{Text: "limited"})
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("expected context.DeadlineExceeded, got %v", err)
+	}
+	if state := rs.cbManager.Get(engine.Name()).State(); state != CircuitClosed {
+		t.Fatalf("circuit state = %s, want closed", state)
+	}
+}
+
+func TestResilientSearchPrimary_EngineFailureStillOpensCircuit(t *testing.T) {
+	engine := &staticErrorEngine{name: "parser", err: ErrParser}
+	cfg := DefaultResilientConfig()
+	cfg.Retry.MaxRetries = 0
+	cfg.CircuitBreaker.FailureThreshold = 1
+	rs := NewResilientSearcher([]SearchEngine{engine}, cfg)
+
+	_, _, _, err := rs.SearchPrimary(context.Background(), engine, Query{Text: "break-me"})
+	if !errors.Is(err, ErrParser) {
+		t.Fatalf("expected ErrParser, got %v", err)
+	}
+	if state := rs.cbManager.Get(engine.Name()).State(); state != CircuitOpen {
+		t.Fatalf("circuit state = %s, want open", state)
+	}
+}

core/response.go

@@ -6,16 +6,26 @@ import "time"
 type QueryEcho struct {
 	Text             string   `json:"text"`
 	Lang             string   `json:"lang,omitempty"`
+	Region           string   `json:"region,omitempty"`
 	EnginesRequested []string `json:"engines_requested"`
 }
 
 // ResponseMeta carries request-level metadata for observability and debugging.
 type ResponseMeta struct {
-	RequestID     string   `json:"request_id"`
-	RequestedAt   string   `json:"requested_at"`
-	TookMs        int64    `json:"took_ms"`
-	EnginesFailed []string `json:"engines_failed"`
-	Version       string   `json:"version"`
+	RequestID        string              `json:"request_id"`
+	RequestedAt      string              `json:"requested_at"`
+	TookMs           int64               `json:"took_ms"`
+	EnginesResponded []string            `json:"engines_responded,omitempty"`
+	EnginesFailed    []string            `json:"engines_failed"`
+	EngineErrors     []EngineErrorDetail `json:"engine_errors,omitempty"`
+	Version          string              `json:"version"`
+}
+
+// EngineErrorDetail is a client-facing, sanitized per-engine failure summary.
+type EngineErrorDetail struct {
+	Engine  string `json:"engine"`
+	Error   string `json:"error"`
+	Message string `json:"message,omitempty"`
 }
 
 // Pagination carries cursor information for client-side loop termination.
@@ -25,12 +35,13 @@ type Pagination struct {
 	NextStart int  `json:"next_start"`
 }
 
-// Envelope is the top-level v1 response wrapper for all search endpoints.
+// Envelope is the top-level v2 response wrapper for all search endpoints.
 type Envelope struct {
-	Query      QueryEcho    `json:"query"`
-	Meta       ResponseMeta `json:"meta"`
-	Results    []Result     `json:"results"`
-	Pagination Pagination   `json:"pagination"`
+	Query        QueryEcho     `json:"query"`
+	Meta         ResponseMeta  `json:"meta"`
+	Results      []Result      `json:"results"`
+	SerpFeatures []SerpFeature `json:"serp_features"`
+	Pagination   Pagination    `json:"pagination"`
 	// Clusters is only populated by /mega/search (see clusters.go).
 	Clusters *[]Cluster `json:"clusters,omitempty"`
 }
@@ -55,7 +66,7 @@ type ClusterOccurrence struct {
 	ResultID string `json:"result_id"`
 }
 
-// ImageEnvelope is the top-level v1 response wrapper for image search endpoints.
+// ImageEnvelope is the top-level v2 response wrapper for image search endpoints.
 type ImageEnvelope struct {
 	Query      QueryEcho     `json:"query"`
 	Meta       ResponseMeta  `json:"meta"`
@@ -63,7 +74,7 @@ type ImageEnvelope struct {
 	Pagination Pagination    `json:"pagination"`
 }
 
-const apiVersion = "1.0"
+const apiVersion = "2.1"
 
 // NewEnvelope builds a fresh Envelope pre-filled with query echo and an open
 // meta block. Call Finalize before serializing.
@@ -72,6 +83,7 @@ func NewEnvelope(q Query, requestID string, startedAt time.Time, engines []strin
 		Query: QueryEcho{
 			Text:             q.Text,
 			Lang:             q.LangCode,
+			Region:           q.Region,
 			EnginesRequested: engines,
 		},
 		Meta: ResponseMeta{
@@ -80,8 +92,9 @@ func NewEnvelope(q Query, requestID string, startedAt time.Time, engines []strin
 			EnginesFailed: []string{},
 			Version:       apiVersion,
 		},
-		Results:    []Result{},
-		Pagination: Pagination{},
+		Results:      []Result{},
+		SerpFeatures: []SerpFeature{},
+		Pagination:   Pagination{},
 	}
 }
 
@@ -91,6 +104,7 @@ func NewImageEnvelope(q Query, requestID string, startedAt time.Time, engines []
 		Query: QueryEcho{
 			Text:             q.Text,
 			Lang:             q.LangCode,
+			Region:           q.Region,
 			EnginesRequested: engines,
 		},
 		Meta: ResponseMeta{
@@ -110,23 +124,33 @@ func (e *Envelope) Finalize(startedAt time.Time, q Query) {
 
 	limit := q.Limit
 	if limit <= 0 {
-		limit = 25
+		limit = defaultQueryLimit
 	}
 	page := q.Start/limit + 1
 	e.Pagination = Pagination{
 		Page:      page,
-		HasMore:   len(e.Results) >= limit,
+		HasMore:   countNonAdResults(e.Results) >= limit,
 		NextStart: q.Start + limit,
 	}
 }
 
+func countNonAdResults(results []Result) int {
+	count := 0
+	for _, result := range results {
+		if result.Type != ResultTypeAd {
+			count++
+		}
+	}
+	return count
+}
+
 // Finalize stamps the elapsed time and computes pagination fields.
 func (e *ImageEnvelope) Finalize(startedAt time.Time, q Query) {
 	e.Meta.TookMs = time.Since(startedAt).Milliseconds()
 
 	limit := q.Limit
 	if limit <= 0 {
-		limit = 25
+		limit = defaultQueryLimit
 	}
 	page := q.Start/limit + 1
 	e.Pagination = Pagination{

core/response_builder.go

@@ -9,6 +9,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 )
 
 const responseIDBytes = 8
@@ -24,7 +25,7 @@ type EnrichContext struct {
 	Query  Query
 }
 
-// EnrichResult converts a raw engine result into the v1 Result shape.
+// EnrichResult converts a raw engine result into the v2 Result shape.
 func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
 	normalizedURL := normalizeURL(raw.URL)
 	domain := extractDomain(normalizedURL)
@@ -35,24 +36,36 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
 	}
 
 	resultType := ResultTypeOrganic
+	if raw.Type != "" {
+		if validType, warning := ValidateResultType(raw.Type); warning == "" {
+			resultType = validType
+		}
+	}
 	if raw.Ad {
 		resultType = ResultTypeAd
 	}
-	// Google answer boxes use negative rank; promote to answer_box type.
-	if raw.Rank <= 0 && !raw.Ad {
+	// Backward compatibility for older parsers: Google answer boxes used
+	// negative rank before SearchResult carried an explicit type hint.
+	if raw.Rank <= 0 && !raw.Ad && raw.Type == "" {
 		resultType = ResultTypeAnswerBox
 	}
-
-	limit := ctx.Query.Limit
-	if limit <= 0 {
-		limit = 25
+	rank := raw.Rank
+	if rank < 0 {
+		if raw.Ad {
+			rank = -rank
+		} else {
+			rank = 0
+		}
+	}
+
+	absolute := computeResultPosition(raw, ctx.Query.Start)
+	if absolute <= 0 {
+		absolute = rank
 	}
-	page := ctx.Query.Start/limit + 1
-	absolute, onPage := computeResultPosition(raw.Rank, ctx.Query.Start)
 
 	result := Result{
 		ID:         buildResultID(ctx.Engine, normalizedURL),
-		Rank:       raw.Rank,
+		Rank:       rank,
 		Type:       resultType,
 		Title:      raw.Title,
 		URL:        normalizedURL,
@@ -60,13 +73,10 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
 		Snippet:    raw.Description,
 		Domain:     domain,
 		Favicon:    favicon,
-		IsAd:       raw.Ad,
-		Position: Position{
-			Absolute: absolute,
-			Page:     page,
-			OnPage:   onPage,
-		},
-		Engine: ctx.Engine,
+		Engine:     ctx.Engine,
+	}
+	if absolute > 0 {
+		result.Position = &Position{Absolute: absolute}
 	}
 
 	result.DomainInfo = EnrichDomainInfo(domain)
@@ -75,7 +85,97 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
 	return result
 }
 
-// EnrichImageResult converts a raw engine result into the v1 ImageResult shape.
+// AppendEnrichedSearchResult preserves the legacy results[] surface while
+// copying any extracted SERP features onto the top-level feature surface.
+func AppendEnrichedSearchResult(env *Envelope, raw SearchResult, ctx EnrichContext, extractedAt time.Time) {
+	var sourceResultID string
+	if raw.URL != "" || raw.Title != "" || raw.Description != "" || raw.Rank != 0 {
+		result := EnrichResult(raw, ctx)
+		env.Results = append(env.Results, result)
+		sourceResultID = result.ID
+	}
+
+	for _, rawFeature := range raw.Features {
+		env.SerpFeatures = append(env.SerpFeatures, EnrichSerpFeature(rawFeature, ctx.Engine, sourceResultID, extractedAt))
+	}
+	if len(raw.Features) == 0 && sourceResultID != "" && shouldMirrorResultAsFeature(raw.Type) {
+		result := env.Results[len(env.Results)-1]
+		env.SerpFeatures = append(env.SerpFeatures, EnrichSerpFeature(SerpFeature{
+			Type:     result.Type,
+			Title:    result.Title,
+			Text:     result.Snippet,
+			Position: result.Position,
+			Links: []FeatureLink{{
+				Title: result.Title,
+				URL:   result.URL,
+			}},
+		}, ctx.Engine, sourceResultID, extractedAt))
+	}
+}
+
+// EnrichSerpFeature stamps a raw feature with stable public fields.
+func EnrichSerpFeature(raw SerpFeature, engine string, sourceResultID string, extractedAt time.Time) SerpFeature {
+	feature := raw
+	feature.Engine = engine
+	if feature.SourceResultIDs == nil {
+		feature.SourceResultIDs = []string{}
+	}
+	if sourceResultID != "" && !containsString(feature.SourceResultIDs, sourceResultID) {
+		feature.SourceResultIDs = append(feature.SourceResultIDs, sourceResultID)
+	}
+	for i := range feature.Links {
+		feature.Links[i].URL = normalizeURL(feature.Links[i].URL)
+	}
+	for i := range feature.Items {
+		feature.Items[i].Link = normalizeURL(feature.Items[i].Link)
+	}
+	if feature.ID == "" {
+		feature.ID = buildFeatureID(feature)
+	}
+	if feature.ExtractedAt == "" {
+		feature.ExtractedAt = extractedAt.UTC().Format(time.RFC3339)
+	}
+	return feature
+}
+
+func buildFeatureID(feature SerpFeature) string {
+	primaryLink := ""
+	if len(feature.Links) > 0 {
+		primaryLink = feature.Links[0].URL
+	}
+	if primaryLink == "" && len(feature.Items) > 0 {
+		primaryLink = feature.Items[0].Link
+	}
+	key := strings.Join([]string{
+		feature.Engine,
+		string(feature.Type),
+		strings.ToLower(strings.TrimSpace(feature.Title)),
+		strings.ToLower(strings.TrimSpace(feature.Text)),
+		primaryLink,
+	}, "|")
+	return "f_" + shortMD5(key)
+}
+
+func containsString(values []string, needle string) bool {
+	for _, value := range values {
+		if value == needle {
+			return true
+		}
+	}
+	return false
+}
+
+func shouldMirrorResultAsFeature(t ResultType) bool {
+	switch t {
+	case ResultTypeAnswerBox, ResultTypeFeaturedSnippet, ResultTypeKnowledgePanel,
+		ResultTypePeopleAlsoAsk, ResultTypeLocal:
+		return true
+	default:
+		return false
+	}
+}
+
+// EnrichImageResult converts a raw engine result into the v2 ImageResult shape.
 func EnrichImageResult(raw SearchResult, ctx EnrichContext) ImageResult {
 	imageURL := normalizeURL(raw.URL)
 	meta := parseImageDescription(raw.Description)
@@ -122,14 +222,22 @@ func shortMD5(value string) string {
 	return hex.EncodeToString(h[:responseIDBytes])
 }
 
-func computeResultPosition(rank, start int) (absolute, onPage int) {
-	if rank <= 0 {
-		return 0, 0
+func computeResultPosition(raw SearchResult, start int) int {
+	if raw.AbsoluteRank > 0 {
+		return raw.AbsoluteRank
+	}
+
+	rank := raw.Rank
+	if rank < 0 {
+		rank = -rank
+	}
+	if rank == 0 {
+		return 0
 	}
 	if start > 0 && rank > start {
-		return rank, rank - start
+		return rank
 	}
-	return start + rank, rank
+	return start + rank
 }
 
 // normalizeURL lowercases scheme+host, strips trailing slash, and removes

core/result.go

@@ -4,64 +4,109 @@ package core
 type ResultType string
 
 const (
-	ResultTypeOrganic         ResultType = "organic"
-	ResultTypeAd              ResultType = "ad"
-	ResultTypeFeaturedSnippet ResultType = "featured_snippet"
-	ResultTypeKnowledgePanel  ResultType = "knowledge_panel"
-	ResultTypePeopleAlsoAsk   ResultType = "people_also_ask"
-	ResultTypeVideo           ResultType = "video"
-	ResultTypeImage           ResultType = "image"
-	ResultTypeNews            ResultType = "news"
-	ResultTypeShopping        ResultType = "shopping"
-	ResultTypeLocal           ResultType = "local"
-	ResultTypeAnswerBox       ResultType = "answer_box"
+	ResultTypeOrganic          ResultType = "organic"
+	ResultTypeAd               ResultType = "ad"
+	ResultTypeFeaturedSnippet  ResultType = "featured_snippet"
+	ResultTypeKnowledgePanel   ResultType = "knowledge_panel"
+	ResultTypePeopleAlsoAsk    ResultType = "people_also_ask"
+	ResultTypeVideo            ResultType = "video"
+	ResultTypeImage            ResultType = "image"
+	ResultTypeNews             ResultType = "news"
+	ResultTypeShopping         ResultType = "shopping"
+	ResultTypeLocal            ResultType = "local"
+	ResultTypeAnswerBox        ResultType = "answer_box"
+	ResultTypeAISummary        ResultType = "ai_summary"
+	ResultTypeRelatedQuestions ResultType = "related_questions"
+	ResultTypeRelatedSearches  ResultType = "related_searches"
+	ResultTypeSitelinks        ResultType = "sitelinks"
+	ResultTypeVideos           ResultType = "videos"
+	ResultTypeImagesInline     ResultType = "images_inline"
+	ResultTypeCalculator       ResultType = "calculator"
+	ResultTypeWeather          ResultType = "weather"
+	ResultTypeDictionary       ResultType = "dictionary"
 )
 
 // Position describes where a result sits in the overall result stream.
 type Position struct {
-	// Absolute is the 1-based rank counting from the first result of the first page.
+	// Absolute is the 1-based rank counting from the first result of the first page,
+	// across both organic and ad blocks. Always emitted so SEO callers can plot
+	// rank vs. on-page position without inferring it from the result order.
 	Absolute int `json:"absolute"`
-	// Page is the 1-based page number derived from start/limit.
-	Page int `json:"page"`
-	// OnPage is the 1-based rank within this page.
-	OnPage int `json:"on_page"`
 }
 
 // DomainInfo carries TLD-derived category signals for a result domain.
 type DomainInfo struct {
-	TLD           string `json:"tld"`
-	SLD           string `json:"sld"`
-	IsGov         bool   `json:"is_gov"`
-	IsEdu         bool   `json:"is_edu"`
-	IsMil         bool   `json:"is_mil"`
-	IsNews        bool   `json:"is_news"`
-	IsForum       bool   `json:"is_forum"`
-	IsMarketplace bool   `json:"is_marketplace"`
-	IsSocial      bool   `json:"is_social"`
+	TLD string `json:"tld,omitempty"`
+	SLD string `json:"sld,omitempty"`
+	// Category is one of "gov", "edu", "mil", "news", "forum", "marketplace",
+	// "social", or "" when the domain does not match any known category.
+	Category string `json:"category"`
 }
 
 // Classification holds URL-path heuristic hints for downstream consumers.
 type Classification struct {
-	ContentType string `json:"content_type"`
-	SourceHint  string `json:"source_hint"`
+	ContentType string `json:"content_type,omitempty"`
+	SourceHint  string `json:"source_hint,omitempty"`
 }
 
-// Result is the v1 normalized result returned in every search response.
+// ExtractedContent carries one enriched target page. Content holds a single
+// representation chosen by the response format (markdown for json/ndjson/markdown,
+// plain text for text), with Format naming which it is — no duplicated blobs.
+type ExtractedContent struct {
+	Title     string `json:"title,omitempty"`
+	Format    string `json:"format,omitempty"`
+	Content   string `json:"content,omitempty"`
+	ModeUsed  string `json:"mode_used,omitempty"`
+	FetchedAt string `json:"fetched_at,omitempty"`
+	Error     string `json:"error,omitempty"`
+}
+
+// FeatureItem is one child entry inside a grouped SERP feature.
+type FeatureItem struct {
+	Title string `json:"title,omitempty"`
+	Text  string `json:"text,omitempty"`
+	Link  string `json:"link,omitempty"`
+}
+
+// FeatureLink is a source or citation associated with a SERP feature.
+type FeatureLink struct {
+	Title string `json:"title,omitempty"`
+	URL   string `json:"url,omitempty"`
+}
+
+// SerpFeature is a normalized non-organic SERP module surfaced separately
+// from rankable results.
+type SerpFeature struct {
+	ID              string        `json:"id"`
+	Engine          string        `json:"engine"`
+	Type            ResultType    `json:"type"`
+	Title           string        `json:"title,omitempty"`
+	Text            string        `json:"text,omitempty"`
+	Items           []FeatureItem `json:"items,omitempty"`
+	Links           []FeatureLink `json:"links,omitempty"`
+	SourceResultIDs []string      `json:"source_result_ids,omitempty"`
+	Position        *Position     `json:"position,omitempty"`
+	Confidence      float64       `json:"confidence,omitempty"`
+	ExtractedAt     string        `json:"extracted_at"`
+}
+
+// Result is the v2 normalized result returned in search responses. Optional
+// fields (Position, DomainInfo, Classification) are omitted when empty.
 type Result struct {
-	ID             string          `json:"id"`
-	Rank           int             `json:"rank"`
-	Type           ResultType      `json:"type"`
-	Title          string          `json:"title"`
-	URL            string          `json:"url"`
-	DisplayURL     string          `json:"display_url"`
-	Snippet        string          `json:"snippet"`
-	Domain         string          `json:"domain"`
-	Favicon        string          `json:"favicon"`
-	IsAd           bool            `json:"is_ad"`
-	Position       Position        `json:"position"`
-	Engine         string          `json:"engine"`
-	DomainInfo     *DomainInfo     `json:"domain_info,omitempty"`
-	Classification *Classification `json:"classification,omitempty"`
+	ID             string            `json:"id"`
+	Rank           int               `json:"rank"`
+	Type           ResultType        `json:"type"`
+	Title          string            `json:"title"`
+	URL            string            `json:"url"`
+	DisplayURL     string            `json:"display_url"`
+	Snippet        string            `json:"snippet"`
+	Domain         string            `json:"domain"`
+	Favicon        string            `json:"favicon"`
+	Position       *Position         `json:"position,omitempty"`
+	Engine         string            `json:"engine"`
+	DomainInfo     *DomainInfo       `json:"domain_info,omitempty"`
+	Classification *Classification   `json:"classification,omitempty"`
+	Extracted      *ExtractedContent `json:"extracted,omitempty"`
 }
 
 // ImageData holds image-specific URL and dimension fields.
@@ -78,7 +123,7 @@ type ImageSource struct {
 	Domain  string `json:"domain"`
 }
 
-// ImageResult is the v1 shape for image search results.
+// ImageResult is the v2 shape for image search results.
 type ImageResult struct {
 	ID     string      `json:"id"`
 	Rank   int         `json:"rank"`

core/result_rank_test.go

@@ -0,0 +1,119 @@
+package core
+
+import (
+	"testing"
+	"time"
+)
+
+func TestDeduplicateResultsOrdersByAbsoluteRank(t *testing.T) {
+	t.Parallel()
+
+	results := DeduplicateResults([]SearchResult{
+		{Rank: 1, AbsoluteRank: 3, URL: "https://organic.example.com/one"},
+		{Rank: 1, AbsoluteRank: 1, URL: "https://ads.example.com/one", Ad: true},
+		{Rank: 2, AbsoluteRank: 2, URL: "https://ads.example.com/two", Ad: true},
+		{Rank: 2, AbsoluteRank: 4, URL: "https://organic.example.com/two"},
+	})
+
+	wantRanks := []int{1, 2, 1, 2}
+	wantAbsoluteRanks := []int{1, 2, 3, 4}
+	if len(results) != len(wantRanks) {
+		t.Fatalf("len(results) = %d, want %d", len(results), len(wantRanks))
+	}
+	for i, want := range wantRanks {
+		if results[i].Rank != want {
+			t.Fatalf("result %d rank = %d, want %d", i, results[i].Rank, want)
+		}
+		if results[i].AbsoluteRank != wantAbsoluteRanks[i] {
+			t.Fatalf("result %d absolute rank = %d, want %d", i, results[i].AbsoluteRank, wantAbsoluteRanks[i])
+		}
+	}
+}
+
+func TestDeduplicateResultsKeepsAdAndOrganicForSameURL(t *testing.T) {
+	t.Parallel()
+
+	results := DeduplicateResults([]SearchResult{
+		{Rank: 1, AbsoluteRank: 1, URL: "https://example.com/page", Ad: true},
+		{Rank: 1, AbsoluteRank: 2, URL: "https://example.com/page"},
+	})
+
+	if len(results) != 2 {
+		t.Fatalf("len(results) = %d, want 2", len(results))
+	}
+	if !results[0].Ad || results[1].Ad {
+		t.Fatalf("expected ad and organic rows to be preserved separately: %+v", results)
+	}
+}
+
+func TestLimitOrganicResultsDoesNotCountAds(t *testing.T) {
+	t.Parallel()
+
+	results := LimitOrganicResults([]SearchResult{
+		{Rank: 1, URL: "https://ads.example.com/one", Ad: true},
+		{Rank: 1, URL: "https://organic.example.com/one"},
+		{Rank: 2, URL: "https://ads.example.com/two", Ad: true},
+		{Rank: 2, URL: "https://organic.example.com/two"},
+	}, 1)
+
+	if len(results) != 3 {
+		t.Fatalf("len(results) = %d, want 3", len(results))
+	}
+	if CountOrganicResults(results) != 1 {
+		t.Fatalf("organic count = %d, want 1", CountOrganicResults(results))
+	}
+}
+
+func TestEnrichResultUsesAdRankAndAbsolutePosition(t *testing.T) {
+	t.Parallel()
+
+	result := EnrichResult(SearchResult{
+		Rank:         1,
+		AbsoluteRank: 2,
+		URL:          "https://ads.example.com/",
+		Title:        "Ad",
+		Ad:           true,
+	}, EnrichContext{Engine: "google", Query: Query{Limit: 10}})
+
+	if result.Rank != 1 {
+		t.Fatalf("rank = %d, want 1", result.Rank)
+	}
+	if result.Position == nil || result.Position.Absolute != 2 {
+		t.Fatalf("unexpected position: %+v", result.Position)
+	}
+	if result.Type != ResultTypeAd {
+		t.Fatalf("unexpected result type: %q", result.Type)
+	}
+}
+
+func TestEnrichResultUsesExplicitResultType(t *testing.T) {
+	t.Parallel()
+
+	result := EnrichResult(SearchResult{
+		Rank:  1,
+		Type:  ResultTypePeopleAlsoAsk,
+		URL:   "https://example.com/question",
+		Title: "Question",
+	}, EnrichContext{Engine: "google", Query: Query{Limit: 10}})
+
+	if result.Type != ResultTypePeopleAlsoAsk {
+		t.Fatalf("unexpected result type: %q", result.Type)
+	}
+}
+
+func TestEnvelopePaginationCountsOrganicResults(t *testing.T) {
+	t.Parallel()
+
+	env := NewEnvelope(Query{Text: "ads", Limit: 2}, "req-1", time.Now(), []string{"bing"})
+	env.Results = []Result{
+		{Rank: 1, Type: ResultTypeAd},
+		{Rank: 2, Type: ResultTypeAd},
+		{Rank: 1, Type: ResultTypeOrganic},
+	}
+
+	env.Finalize(time.Now(), Query{Text: "ads", Limit: 2})
+
+	if env.Pagination.HasMore {
+		t.Fatalf("has_more = true, want false when organic count is below limit: %+v", env.Pagination)
+	}
+}