expose proxy-attempts + use-profile CORS headers; simplify DDG url builders

* feat: implement windowOrganicResults function and update DuckDuckGo search logic
* Add fallback ddg "more results" selector. Update test HTML + add test

---------

Co-authored-by: Rustem Kamalov <karust13@gmail.com>

.dockerignore

@@ -0,0 +1,14 @@
+.git
+.gitignore
+
+.release/
+docs/*.md
+*.md
+testdata/
+
+openserp
+openserp.exe
+bin/
+dist/
+*.tgz
+*.tar

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,76 @@
+name: Bug report
+description: Report a reproducible OpenSERP bug.
+title: "bug: "
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to report a bug. Reproducible reports are the fastest to fix.
+  - type: textarea
+    id: summary
+    attributes:
+      label: What happened?
+      description: Describe the unexpected behavior and what you expected instead.
+      placeholder: "Google search returns empty results for ..."
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: How do we reproduce it?
+      description: Include the exact command, request, config, and minimal input.
+      placeholder: |
+        1. Run `openserp serve ...`
+        2. Request `curl "..."`
+        3. See ...
+    validations:
+      required: true
+  - type: dropdown
+    id: mode
+    attributes:
+      label: Retrieval mode
+      options:
+        - browser
+        - raw
+        - both
+        - not sure
+    validations:
+      required: true
+  - type: dropdown
+    id: engine
+    attributes:
+      label: Engine
+      options:
+        - google
+        - yandex
+        - baidu
+        - bing
+        - duckduckgo
+        - ecosia
+        - megasearch
+        - not engine-specific
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: OpenSERP version
+      description: Paste `openserp --version`, the Docker tag, or the commit SHA.
+      placeholder: "0.8.0 / v0.7.15 / commit SHA"
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Environment
+      description: OS, Go version, Docker/browser version, proxy/captcha setup if relevant.
+      placeholder: "Windows 11, Go 1.24, Docker, Chromium ..."
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs or response body
+      description: Paste relevant logs, stack traces, or API responses. Remove secrets and proxy credentials.
+      render: shell

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Contributing guide
+    url: https://github.com/karust/openserp/blob/main/docs/CONTRIBUTING.md
+    about: Start here before opening a pull request.
+  - name: Add a new engine guide
+    url: https://github.com/karust/openserp/blob/main/docs/ADDING_AN_ENGINE.md
+    about: Implementation checklist for new search engines.

.github/ISSUE_TEMPLATE/engine_request.yml

@@ -0,0 +1,47 @@
+name: Engine request
+description: Request support for a new search engine or engine vertical.
+title: "engine: "
+labels: ["enhancement"]
+body:
+  - type: input
+    id: engine
+    attributes:
+      label: Engine name
+      placeholder: "Brave Search"
+    validations:
+      required: true
+  - type: input
+    id: search_url
+    attributes:
+      label: Public search URL
+      placeholder: "https://search.brave.com/search?q=openserp"
+    validations:
+      required: true
+  - type: dropdown
+    id: vertical
+    attributes:
+      label: Result type
+      options:
+        - web
+        - images
+        - web and images
+        - other
+    validations:
+      required: true
+  - type: textarea
+    id: selectors
+    attributes:
+      label: Known stable selectors or sample HTML
+      description: If you inspected the page, mention stable data attributes or attach a sanitized fixture.
+  - type: textarea
+    id: notes
+    attributes:
+      label: Notes
+      description: Region/language quirks, captcha behavior, pagination, or API constraints.
+  - type: checkboxes
+    id: contribution
+    attributes:
+      label: Contribution
+      options:
+        - label: I can help test this engine.
+        - label: I am willing to implement this engine.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,47 @@
+name: Feature request
+description: Suggest an OpenSERP improvement that is not engine-specific.
+title: "feat: "
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem
+      description: What user problem does this solve?
+      placeholder: "It is hard to ..."
+    validations:
+      required: true
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposed solution
+      description: Describe the behavior, API, CLI, or documentation change you want.
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives considered
+      description: What workarounds or smaller versions have you considered?
+  - type: dropdown
+    id: area
+    attributes:
+      label: Area
+      options:
+        - CLI
+        - API
+        - docs
+        - parser
+        - proxy
+        - captcha
+        - resilience
+        - tests
+        - other
+    validations:
+      required: true
+  - type: checkboxes
+    id: contribution
+    attributes:
+      label: Contribution
+      options:
+        - label: I am willing to open a PR for this.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,20 @@
+## Summary
+
+- 
+
+## Why
+
+- 
+
+## Testing
+
+- [ ] `make test`
+- [ ] `make lint`
+- [ ] `make test-integration` (only for browser, proxy, captcha, or live-engine changes)
+
+## Checklist
+
+- [ ] I linked the related issue or explained why there is none.
+- [ ] I updated docs or examples for changed user-facing behavior.
+- [ ] I kept unit tests deterministic and free of browser/network dependencies.
+- [ ] I removed secrets, proxy credentials, and private logs from examples.

.github/workflows/ci.yml

@@ -0,0 +1,47 @@
+name: CI
+
+on:
+  push:
+    branches: [main, tests]
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  test-build-lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.24"
+          cache: true
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Run unit tests
+        run: go test -race -count=1 ./...
+
+      - name: Run go vet
+        run: go vet ./...
+
+      - name: Lint OpenAPI spec
+        run: npx --yes @redocly/cli lint docs/openapi.yaml
+
+      - name: Build
+        run: go build .
+
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          version: v1.64.8
+          args: --config .golangci.yml

.github/workflows/docker.yml

@@ -0,0 +1,118 @@
+name: Docker
+
+on:
+  push:
+    branches: [main]
+    tags: ["v*.*.*"]
+  pull_request:
+    branches: [main]
+  # Manual republish (e.g. base-image refresh without a version bump).
+  # Must be dispatched FROM A TAG REF: metadata-action only derives semver
+  # image tags from the git tag, so a branch dispatch is rejected below.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  # Fail fast with a clear message instead of silently pushing a bare
+  # `latest` (or nothing) when someone dispatches from a branch.
+  guard-dispatch:
+    if: github.event_name == 'workflow_dispatch' && !startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          echo "::error::Manual runs must be dispatched from a v*.*.* tag ref (use the 'Run workflow' ref selector). Branch refs produce no semver image tags."
+          exit 1
+
+  # Catch tag/code drift before anything is published (root.go has drifted
+  # from tags before).
+  verify-version:
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check tag matches cmd/root.go version
+        run: |
+          TAG_VERSION="${GITHUB_REF_NAME#v}"
+          CODE_VERSION=$(sed -nE 's/^[[:space:]]*version[[:space:]]*=[[:space:]]*"([0-9]+\.[0-9]+\.[0-9]+)".*/\1/p' cmd/root.go)
+          echo "tag=v${TAG_VERSION} code=${CODE_VERSION}"
+          if [ "$TAG_VERSION" != "$CODE_VERSION" ]; then
+            echo "::error::Git tag v${TAG_VERSION} does not match version \"${CODE_VERSION}\" in cmd/root.go. Bump cmd/root.go before tagging."
+            exit 1
+          fi
+
+  # PR / main validation: build only, single platform (multi-arch under QEMU
+  # roughly doubles CI time for no extra signal; tag builds cover arm64).
+  build:
+    if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build (no push)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64
+          push: false
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  # Publish on version tags (and manual dispatch from a tag ref).
+  # No fork-secrets concern: this never runs for pull_request events.
+  publish:
+    if: startsWith(github.ref, 'refs/tags/v')
+    needs: verify-version
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # `latest` is applied automatically on semver tags (latest=auto).
+      # Caveat: it follows the most recently pushed tag, not the highest
+      # version — we only tag forward, never backport.
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: karust/openserp
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      # The pinned chromedp/headless-shell digest is a multi-arch OCI index
+      # (amd64 + arm64/v8), and the builder stage cross-compiles via
+      # $BUILDPLATFORM, so only the small final-stage apt layer runs under QEMU.
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/integration.yml

@@ -0,0 +1,27 @@
+name: Integration Tests
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  integration:
+    runs-on: ubuntu-latest
+
+    env:
+      OPENSERP_INTEGRATION_TESTS: "1"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.24"
+          cache: true
+
+      - name: Run integration tests
+        run: go test -race -count=1 -timeout=120s -tags=integration ./...

.github/workflows/release.yml

@@ -0,0 +1,100 @@
+name: Release
+
+# Publishes binary archives to the GitHub release for a version tag.
+# Docker images are published separately by docker.yml on the same tag.
+
+on:
+  push:
+    tags: ["v*.*.*"]
+  # Manual re-run (e.g. after a workflow fix) — must be dispatched from a tag ref.
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  binaries:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Guard non-tag dispatch
+        if: github.event_name == 'workflow_dispatch' && !startsWith(github.ref, 'refs/tags/v')
+        run: |
+          echo "::error::Manual runs must be dispatched from a v*.*.* tag ref (use the 'Run workflow' ref selector)."
+          exit 1
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Full history + tags, needed to diff against the previous release tag
+          fetch-depth: 0
+
+      - name: Check tag matches cmd/root.go version
+        run: |
+          TAG_VERSION="${GITHUB_REF_NAME#v}"
+          CODE_VERSION=$(sed -nE 's/^[[:space:]]*version[[:space:]]*=[[:space:]]*"([0-9]+\.[0-9]+\.[0-9]+)".*/\1/p' cmd/root.go)
+          echo "tag=v${TAG_VERSION} code=${CODE_VERSION}"
+          if [ "$TAG_VERSION" != "$CODE_VERSION" ]; then
+            echo "::error::Git tag v${TAG_VERSION} does not match version \"${CODE_VERSION}\" in cmd/root.go. Bump cmd/root.go before tagging."
+            exit 1
+          fi
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.24"
+          cache: true
+
+      - name: Run unit tests
+        run: go test -count=1 ./...
+
+      - name: Build release binaries
+        run: |
+          VERSION="${GITHUB_REF_NAME#v}"
+          mkdir -p dist
+          for GOOS in darwin linux windows; do
+            for GOARCH in amd64 386 arm64; do
+              if [ "$GOOS" = "darwin" ] && [ "$GOARCH" = "386" ]; then
+                continue
+              fi
+              BIN="openserp"
+              if [ "$GOOS" = "windows" ]; then
+                BIN="openserp.exe"
+              fi
+              rm -f "$BIN"
+              echo "Building ${GOOS}/${GOARCH}..."
+              CGO_ENABLED=0 GOOS="$GOOS" GOARCH="$GOARCH" \
+                go build -trimpath -ldflags="-s -w" -o "$BIN" .
+              tar --create --gzip --file="dist/openserp-${GOOS}-${GOARCH}-${VERSION}.tgz" "$BIN"
+              rm -f "$BIN"
+            done
+          done
+          ls -lh dist/
+
+      # Most changes land directly on main without PRs, and GitHub's
+      # auto-generated notes only list merged PRs — so build the changelog
+      # from commit subjects since the previous tag. Auto-notes stay enabled
+      # below for the compare link and occasional contributor PR credits.
+      - name: Generate commit changelog
+        run: |
+          PREV_TAG=$(git describe --tags --abbrev=0 "${GITHUB_REF_NAME}^" 2>/dev/null || true)
+          {
+            echo "## Changes"
+            echo ""
+            if [ -n "$PREV_TAG" ]; then
+              echo "Commits since ${PREV_TAG}:"
+              echo ""
+              git log --no-merges --pretty='- %s' "${PREV_TAG}..${GITHUB_REF_NAME}"
+            else
+              git log --no-merges --pretty='- %s' "${GITHUB_REF_NAME}"
+            fi
+          } > release_notes.md
+          cat release_notes.md
+
+      # Creates the release if it doesn't exist, or attaches the archives to
+      # an existing one (e.g. a release drafted manually in the GitHub UI).
+      - name: Create GitHub release and upload binaries
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*.tgz
+          body_path: release_notes.md
+          generate_release_notes: true

.gitignore

@@ -23,8 +23,20 @@ dev.md
 logrus.txt
 logs.txt
 .release
+!.release/
+.release/*
+!.release/site/
+!.release/site/**
 core/test/
 .aider*
 .gocache/
 openserp
 .gomodcache/
+AGENTS.md
+.claude
+sanitize_html_fixtures.py
+core/testdata/*
+.tmp-gocache
+.tmpcache/
+google/data/geotargets-2026-05-28.csv
+demo.yml

.golangci.yml

@@ -0,0 +1,20 @@
+run:
+  tests: true
+  timeout: 5m
+  go: "1.24"
+
+# Pinned set so CI and local runs agree
+linters:
+  disable-all: true
+  enable:
+    - govet
+    - staticcheck
+    - gosimple
+    - errcheck
+    - ineffassign
+    - unused
+    - misspell
+
+issues:
+  max-issues-per-linter: 0
+  max-same-issues: 0

Dockerfile

@@ -1,28 +1,41 @@
 # Build
-FROM golang:alpine as builder
-
-LABEL stage=gobuilder
-RUN apk update --no-cache && apk add --no-cache tzdata
+FROM --platform=$BUILDPLATFORM golang:1.24.6-bookworm@sha256:ab1d1823abb55a9504d2e3e003b75b36dbeb1cbcc4c92593d85a84ee46becc6c AS builder
 
 WORKDIR /build
 
-ADD go.mod .
-ADD go.sum .
+COPY go.mod go.sum ./
 RUN go mod download
 
 COPY . .
-RUN go build -o /app/openserp .
+ARG TARGETOS
+ARG TARGETARCH
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} go build -trimpath -ldflags="-s -w" -o /app/openserp .
 
-
-FROM zenika/alpine-chrome:with-chromedriver
+# `chromedp/headless-shell:stable` also works here
+FROM chromedp/headless-shell:stable@sha256:f7e7ac721b023cb8717f8108aef8b3e49995fb1e5a912f41e570c29e45d24961
 
 WORKDIR /usr/src/app
 
-COPY --from=builder /app/openserp /usr/local/bin/openserp
-COPY config.yaml ./config.yaml
+# wget: used by HEALTHCHECK (localhost, no TLS, so ca-certificates not required).
+# dumb-init: already provided by `docker run --init` / compose `init: true`, so we do NOT add tini here — the PID1 reaper is supplied by the runtime.
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends wget \
+  && rm -rf /var/lib/apt/lists/* \
+  && getent passwd chrome >/dev/null 2>&1 || useradd --create-home --uid 1001 --shell /bin/bash chrome \
+  && chown chrome:chrome /usr/src/app
 
-HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
-  CMD wget --no-verbose --tries=1 --spider http://localhost:7000/health || exit 1
+COPY --from=builder /app/openserp /usr/local/bin/openserp
+COPY --chown=chrome:chrome config.yaml ./config.yaml
+
+# Rod's launcher.LookPath does not know about /headless-shell/headless-shell.
+# Viper auto-binds OPENSERP_APP_BROWSER_PATH to app.browser_path, so this pins the binary and avoids Rod's runtime chromium auto-download (which would fail in this non-root, network-restricted image).
+ENV OPENSERP_APP_BROWSER_PATH=/headless-shell/headless-shell \
+  OPENSERP_SERVER_HOST=0.0.0.0 \
+  OPENSERP_SERVER_PORT=7000
+
+USER chrome
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
+  CMD wget --quiet --tries=1 --spider "http://127.0.0.1:${OPENSERP_SERVER_PORT}/health" || exit 1
 
 ENTRYPOINT ["openserp"]
-

Makefile

@@ -0,0 +1,25 @@
+GO ?= go
+BINARY ?= openserp
+PKGS ?= ./...
+GOFILES := $(shell git ls-files '*.go')
+
+.PHONY: build test test-integration lint run fmt
+
+build:
+	$(GO) build -o $(BINARY) .
+
+test:
+	$(GO) test -race -count=1 $(PKGS)
+
+test-integration:
+	OPENSERP_INTEGRATION_TESTS=1 $(GO) test -race -count=1 -timeout=120s -tags=integration $(PKGS)
+
+lint:
+	$(GO) vet $(PKGS)
+	golangci-lint run --config .golangci.yml
+
+run:
+	$(GO) run . serve
+
+fmt:
+	gofmt -w $(GOFILES)

README.md

@@ -1,142 +1,357 @@
-# OpenSERP (Search Engine Results)
+![OpenSERP](./logo.svg)
 
-![OpenSERP](/logo.svg)
+# OpenSERP
 
-[![Go Report Card](https://goreportcard.com/badge/github.com/karust/openserp)](https://goreportcard.com/report/github.com/karust/openserp)
-[![Go Reference](https://pkg.go.dev/badge/github/karust/openserp?style=for-the-badge)](https://pkg.go.dev/github.com/karust/openserp)
-[![release](https://img.shields.io/github/release/karust/openserp)](https://github.com/karust/openserp/releases)
+[![Go Reference](https://pkg.go.dev/badge/github.com/karust/openserp.svg)](https://pkg.go.dev/github.com/karust/openserp)
+[![release](https://img.shields.io/github/v/release/karust/openserp)](https://github.com/karust/openserp/releases)
+[![Docker Pulls](https://img.shields.io/docker/v/karust/openserp)](https://hub.docker.com/r/karust/openserp)
+[![CI](https://github.com/karust/openserp/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/karust/openserp/actions/workflows/ci.yml)
+[![Telegram](https://img.shields.io/badge/Telegram-openserp__cloud-26A5E4?logo=telegram&logoColor=white)](https://t.me/openserp_cloud)
 
-<!-- [![Docker Pulls](https://img.shields.io/docker/pulls/karust/openserp)](https://hub.docker.com/repository/docker/karust/openserp) -->
+**OpenSERP** is a free, open-source SERP API and CLI for Google, Yandex, Baidu, Bing, DuckDuckGo, and Ecosia.
 
-**OpenSERP** provides free API and CLI access to multiple search engines including **Google, Yandex, Baidu, Bing, and DuckDuckGo**. Get comprehensive search results without expensive API subscriptions!
+No API keys, no per-search billing: one command gives you live, structured search results on localhost - including engines the paid APIs don't cover. Use it as a search tool for LLMs and agents, or as a backend for SEO rank tracking. If you'd rather not run infrastructure, there is a [hosted version](https://openserp.org/cloud) with the same API.
+
+![OpenSERP CLI demo](./docs/demo.gif)
 
 ## Features
 
-- 🔍 **Multi-engine** - search with dedicated endpoints for each engine
-- 🌐 **Megasearch** - cross-engine aggregation with deduplication
-- 🖼 **Images** - image search is also available
-- 🎯 **Advanced filters** - language, date range, file type, and site queries
-- 🌍 **Configurable** - proxy, cache, and resilient mode support
-- 🐳 **Docker-ready** - local and container deployment
+- Dedicated endpoints for six engines, same JSON schema across all of them
+- **Megasearch** - one query across several engines at once, merged and deduped
+- **URL extraction** - search results plus clean markdown of the target pages in one call
+- SERP features: AI summaries, answer boxes, people-also-ask, related searches
+- Image search, filters (language, date range, file type, site), **JSON/Markdown/Text/NdJSON** output
+- Proxies, cache, resilient mode, prebuilt Docker images
 
-## Quick Start⚡️
+## Quick Start
 
-### Docker (Recommended)
+### Docker
 
-```bash
+Prebuilt images are published to [docker hub: `karust/openserp`](https://hub.docker.com/r/karust/openserp).
+
+```sh
 # Run the API server via prebuilt image
-docker run -p 127.0.0.1:7000:7000 -it karust/openserp serve -a 0.0.0.0 -p 7000
+docker run --rm -p 127.0.0.1:7000:7000 karust/openserp:latest serve -a 0.0.0.0 -p 7000
 
-# Or use docker-compose
-docker compose up --build
+# Or
+docker compose up
 ```
 
-### From source
+### Go install
 
-```bash
+```sh
+go install github.com/karust/openserp@latest
+openserp search duckduckgo "open source serp api" --format markdown
+```
+
+### From Source
+
+```sh
 git clone https://github.com/karust/openserp.git
 cd openserp
 go build -o openserp .
 ./openserp serve
 ```
 
-## 🌐 Megasearch & Megaimage
+### First request
 
-Search all engines at once:
-
-```bash
-curl "http://127.0.0.1:7000/mega/search?text=golang&limit=10"
+```sh
+# mode=any returns the first engine that responds
+curl "http://127.0.0.1:7000/mega/search?engines=bing,google&text=golang+vs+rust&extract=1&mode=any"
 ```
 
-Search only selected engines:
-
-```bash
-curl "http://127.0.0.1:7000/mega/search?text=golang&engines=duckduckgo,bing&limit=15"
-```
-
-Advanced filtering:
-
-```bash
-curl "http://127.0.0.1:7000/mega/search?text=Donald+Trump&engines=duckduckgo,bing&limit=20&date=20251005..20251005&lang=EN"
-```
-
-API response example:
+<details>
+<summary>Example JSON response</summary>
 
 ```json
-[
-  {
-    "rank": 1,
-    "url": "https://en.wikipedia.org/wiki/Golden_Retriever",
-    "title": "Golden Retriever - Wikipedia",
-    "description": "The Golden Retriever is a Scottish breed of retriever dog of medium size. It is characterised by a gentle and affectionate nature and a striking golden coat.",
-    "ad": false,
-    "engine": "duckduckgo"
+{
+  "query": {
+    "text": "golang vs rust",
+    "engines_requested": ["bing", "google"]
   },
-  {
-    "rank": 2,
-    "url": "https://www.bing.com/ck/a?!&&p=6f15ac4589858d0a104cd6f55cc8",
-    "title": "Golden Retriever Dog Forums",
-    "description": "Oct 20, 2024 · Back in the 1970s, Golden Retrievers routinely lived until 16 and 17 years old, they are now...",
-    "ad": false,
-    "engine": "bing"
+  "meta": {
+    "request_id": "019ecdc0-a66d-79a4-9d2b-9e9b480d495e",
+    "requested_at": "2026-06-16T00:06:55Z",
+    "took_ms": 720,
+    "engines_responded": ["bing"],
+    "engines_failed": [],
+    "version": "2.1"
   },
-  {
-    "rank": 3,
-    "url": "http://www.baidu.com/link?url==2544q3ugc68j0scVxdpWCSX-gl2AmuCy1l7uRR3loIfS1",
-    "title": "golden retrievers是什么意思",
-    "description": "2025年9月21日golden retrievers 读音:美英 golden retrievers基本解释 金毛猎犬 分词解释 golden金(黄)色的...",
-    "ad": false,
-    "engine": "baidu"
-  }
-]
+  "results": [
+    {
+      "id": "s_5a8273f16b19ab64",
+      "rank": 1,
+      "type": "organic",
+      "title": "The Go Programming Language",
+      "url": "https://go.dev/",
+      "display_url": "go.dev",
+      "snippet": "Get Started Playground Tour Stack Overflow Help Packages Standard Library …",
+      "domain": "go.dev",
+      "favicon": "https://go.dev/favicon.ico",
+      "position": {
+        "absolute": 1
+      },
+      "engine": "bing",
+      "domain_info": {
+        "tld": "dev",
+        "sld": "go",
+        "category": ""
+      },
+      "extracted": {
+        "title": "Build simple, secure, scalable systems with Go",
+        "format": "markdown",
+        "content": "## Build simple, secure, scalable systems with Go\n\n![Go Gopher climbing a ladder.](https://go.dev/images/gophers/ladder.svg)\n\n- “At the time, no single team member knew Go, but **within a month, everyone was writing in Go** and we were building out the endpoints. ........",
+        "mode_used": "fast",
+        "fetched_at": "2026-06-16T00:06:56Z"
+      }
+    },
+    {
+      "id": "s_1a364ebcb3035539",
+      "rank": 2,
+      "type": "organic",
+      "title": "Go (programming language) - Wikipedia",
+      "url": "https://en.wikipedia.org/wiki/Go_(programming_language)",
+      "display_url": "en.wikipedia.org › wiki › Go_(programming_language)",
+      "snippet": "In Go's package system, each package has a path (e.g., \"compress/bzip2\" or \"golang.org/x/net/html\") and a name (e.g., bzip2 or html). …",
+      "domain": "en.wikipedia.org",
+      "favicon": "https://en.wikipedia.org/favicon.ico",
+      "position": {
+        "absolute": 2
+      },
+      "engine": "bing",
+      "domain_info": {
+        "tld": "org",
+        "sld": "wikipedia",
+        "category": ""
+      },
+      "classification": {
+        "content_type": "article",
+        "source_hint": "encyclopedia"
+      }
+    },
+    ...
+  ],
+  "serp_features": [],
+  "pagination": {
+    "page": 1,
+    "has_more": false,
+    "next_start": 10
+  },
+  "clusters": [
+    {
+      "id": "c_f20b23a020101dce",
+      "canonical_url": "https://go.dev/",
+      "domain": "go.dev",
+      "title": "The Go Programming Language",
+      "occurrences": [
+        {
+          "engine": "bing",
+          "rank": 1,
+          "result_id": "s_5a8273f16b19ab64"
+        }
+      ],
+      "engines_count": 1,
+      "best_rank": 1,
+      "score": 0.5
+    },
+    ...
+  ]
+}
+```
+
+</details>
+
+## SDKs & Examples
+
+Official client packages. Each works against your self-hosted server (set `baseUrl`) or the [hosted API](https://openserp.org/cloud) (set `apiKey`):
+
+| Type                        | Package                                                                                      | Source                                                              | Install                         |
+| --------------------------- | -------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | ------------------------------- |
+| JavaScript / TypeScript SDK | [`@openserp/sdk`](https://www.npmjs.com/package/@openserp/sdk)                               | [openserpapi/sdk-js](https://github.com/openserpapi/sdk-js)         | `npm install @openserp/sdk`     |
+| Python SDK                  | [`openserp`](https://pypi.org/project/openserp/)                                             | [openserpapi/sdk-python](https://github.com/openserpapi/sdk-python) | `pip install openserp`          |
+| MCP server (AI agents)      | [`@openserp/mcp`](https://www.npmjs.com/package/@openserp/mcp)                               | [openserpapi/mcp](https://github.com/openserpapi/mcp)               | `npx @openserp/mcp`             |
+| n8n community node          | [`@openserp/n8n-nodes-openserp`](https://www.npmjs.com/package/@openserp/n8n-nodes-openserp) | [openserpapi/n8n](https://github.com/openserpapi/n8n)               | Install via n8n community nodes |
+
+See [**examples**](./examples) for small JavaScript and Python use cases covering search, AI grounding, SEO, content extraction, and image search.
+
+```js
+import { OpenSERP } from "@openserp/sdk";
+
+// Use your self-hosted server
+const client = new OpenSERP({ baseUrl: "http://localhost:7000" });
+const { results } = await client.search({ engine: "google", text: "openserp", limit: 5 });
+```
+
+## Search Endpoints
+
+Available engine names: `google`, `yandex`, `baidu`, `bing`, `duckduckgo`, `ecosia`.
+
+Dedicated engine endpoints:
+
+```bash
+curl "http://127.0.0.1:7000/google/search?text=golang&limit=10"
 ```
 
 Image search:
 
 ```bash
-curl "http://127.0.0.1:7000/mega/image?text=golang logo&limit=20"
+curl "http://127.0.0.1:7000/bing/image?text=golang+logo&limit=10"
 ```
 
-List available engines:
+Megasearch:
+
+```bash
+curl "http://127.0.0.1:7000/mega/search?text=golang&limit=10"
+```
+
+`/mega/search` returns the same envelope as engine endpoints plus `clusters`: results are deduplicated by normalized URL, and clusters keep the per-engine occurrences and ranks.
+
+| Mode       | Best for                             | Behavior                                       |
+| ---------- | ------------------------------------ | ---------------------------------------------- |
+| `balanced` | Most multi-engine SERP workflows     | Queries engines in parallel and merges results |
+| `fast`     | Lowest latency                       | Uses the fastest available engine              |
+| `any`      | Fallback-style availability checking | Tries engines sequentially until one responds  |
+
+<details>
+<summary>More megasearch examples</summary>
+
+```bash
+# Fast mode
+curl "http://127.0.0.1:7000/mega/search?text=golang&mode=fast&engines=google,bing,yandex"
+
+# Any mode
+curl "http://127.0.0.1:7000/mega/search?text=golang&mode=any&engines=google,yandex,bing"
+
+# Balanced mode with aggregation controls
+curl "http://127.0.0.1:7000/mega/search?text=golang&mode=balanced&dedupe=true&merge=true"
+
+# Advanced filtering
+curl "http://127.0.0.1:7000/mega/search?text=golang&engines=google,bing&limit=20&date=20250101..20251231&lang=EN&region=US"
+
+# Image megasearch
+curl "http://127.0.0.1:7000/mega/image?text=golang+logo&limit=20"
+```
+
+</details>
+
+List engines:
 
 ```bash
 curl "http://127.0.0.1:7000/mega/engines"
 ```
 
-**Available engines:** `google`, `yandex`, `baidu`, `bing`, `duckduckgo`
+URL extraction:
 
-## 🔍 Individual Engine APIs
+```bash
+# Extract one URL as JSON
+curl "http://127.0.0.1:7000/extract?url=https://example.com&mode=auto"
 
-Common query parameters:
+# Return clean page markdown
+curl "http://127.0.0.1:7000/extract?url=https://example.com&format=markdown"
 
-| Parameter | Description          | Example                           |
-| --------- | -------------------- | --------------------------------- |
-| `text`    | Search query         | `golang programming`              |
-| `lang`    | Language code        | `EN`, `DE`, `RU`, `ES`            |
-| `date`    | Date range           | `20230101..20231231`              |
-| `file`    | File extension       | `PDF`, `DOC`, `XLS`               |
-| `site`    | Site-specific search | `github.com`, `stackoverflow.com` |
-| `limit`   | Number of results    | `10`, `25`, `50`                  |
+# Extract several URLs at once - returns a bare [{page_content, metadata}] array
+# (Open WebUI external loader compatible); failed URLs become items with metadata.error
+curl -X POST "http://127.0.0.1:7000/extract/batch" \
+  -H "Content-Type: application/json" \
+  -d '{"urls":["https://example.com","https://go.dev"],"mode":"fast"}'
+
+# Embed extracted content under the top search results
+curl "http://127.0.0.1:7000/google/search?text=llm+observability&extract=2&format=markdown"
+```
+
+## CLI Search
+
+No server required - query an engine straight from the terminal. The CLI shares the same engines, formats, and filters as the API.
+
+```sh
+openserp search ecosia "weather in london" --format markdown
+```
+
+<details>
+<summary>CLI output and more examples</summary>
+
+```markdown
+# Search results for "weather in london"
+
+**Query:** weather in london - **Engines:** ecosia - **Took:** 866ms
+
+## Results
+
+### 1. London - BBC Weather
+
+**bbc.com › weather › 2643743** - organic
+
+Latest forecast for London ... Tonight will continue dry, and there will be mainly clear skies. Just a few patches of cloud drifting in from the north at times.
+
+-> https://www.bbc.com/weather/2643743
+
+### 2. London (Greater London) weather - Met Office
+
+**weather.metoffice.gov.uk › forecast › gcpvj0v07** - organic
+
+Remaining warm with light winds and dry. Possibly cloudy at times Monday and Tuesday, then Wednesday sunnier conditions are likely.
+
+-> https://weather.metoffice.gov.uk/forecast/gcpvj0v07
+
+### 3. London, London, United Kingdom Weather Forecast
+
+**accuweather.com › en › gb › london › ec4a-2 › wea…** - organic
+
+London, London, United Kingdom Weather Forecast, with current conditions, wind, air quality, and what to expect for the next 3 days.
+
+-> https://www.accuweather.com/en/gb/london/ec4a-2/weather-forecast/328328
+```
+
+More CLI examples:
+
+```sh
+# JSON is the default format
+openserp search google "golang generics" --limit 20
+
+# Plain text, German results
+openserp search yandex "wetter berlin" --format text --lang DE --region DE
+
+# Restrict to a site and stream NdJSON
+openserp search bing "release notes" --site github.com --format ndjson
+
+# Embed clean page content from the top 2 results
+openserp search google "llm observability" --extract 2 --format markdown
+
+# Browserless (raw HTTP) mode through a proxy
+# (raw mode: google, yandex, baidu, ecosia)
+openserp search ecosia "weather in london" --raw --proxy http://user:pass@127.0.0.1:8080
+```
+
+</details>
+
+Run `openserp search --help` for the full flag list. Engine names: `google`, `yandex`, `baidu`, `bing`, `duckduckgo`, `ecosia`.
+
+## Query Parameters
+
+Common parameters:
+
+| Parameter      | Description                                                                                                                                                                                             | Example                              |
+| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ |
+| `text`         | Search query                                                                                                                                                                                            | `golang programming`                 |
+| `lang`         | Language code                                                                                                                                                                                           | `EN`, `DE`, `RU`, `ES`               |
+| `region`       | Market/location hint. Countries/locales work across engines; Google also accepts city names via `uule`; Yandex accepts numeric `lr`.                                                                    | `DE`, `en-GB`, `Berlin`, `213`       |
+| `date`         | Date range                                                                                                                                                                                              | `20250101..20251231`                 |
+| `file`         | File extension                                                                                                                                                                                          | `pdf`, `doc`, `xls`                  |
+| `site`         | Site-specific search                                                                                                                                                                                    | `github.com`                         |
+| `limit`        | Number of organic results, max 100. When omitted or `<=10`, only the first SERP page is parsed.                                                                                                         | `25`, `50`                           |
+| `start`        | Pagination offset                                                                                                                                                                                       | `0`, `10`, `20`                      |
+| `format`       | Output format                                                                                                                                                                                           | `json`, `markdown`, `text`, `ndjson` |
+| `extract`      | Fetch and embed target-page content for top web results. Bool or int depth: `0`/`false` off, `true`/`1` top result, `N` top N (1-5). `extract_mode`/`min_runes` imply `extract=true` unless `extract=0` | `1`, `3`, `true`                     |
+| `extract_mode` | Extraction strategy: raw HTTP first, raw only, or browser-rendered                                                                                                                                      | `auto`, `fast`, `rendered`           |
 
 Engine-specific parameters:
 
-| Parameter | Supported engines                   | Notes                                                              |
-| --------- | ----------------------------------- | ------------------------------------------------------------------ |
-| `start`   | `google`, `bing`, `yandex`, `baidu` | Web search pagination offset.                                      |
-| `filter`  | `google`                            | Duplicate filter (`true` hides similar, `false` includes similar). |
-| `answers` | `google`                            | Include Google answer boxes in output with negative ranks.         |
+| Parameter  | Supported engines | Notes                                                                  |
+| ---------- | ----------------- | ---------------------------------------------------------------------- |
+| `filter`   | `google`          | Duplicate filter: `true` hides similar results, `false` includes them. |
+| `features` | browser `Search`  | Populate `serp_features[]` from the live page. Defaults to `true`.     |
 
-Examples:
-
-```bash
-curl "http://127.0.0.1:7000/duck/search?text=golang&limit=7"
-curl "http://127.0.0.1:7000/google/search?text=golang&lang=EN&limit=10"
-curl "http://127.0.0.1:7000/bing/search?text=golang&limit=10&start=20"
-curl "http://127.0.0.1:7000/yandex/search?text=golang&limit=10&start=10"
-curl "http://127.0.0.1:7000/bing/image?text=golang&limit=20"
-```
-
-## 🌍 Proxy Support
+## Proxy Support
 
 OpenSERP supports HTTP and SOCKS5 proxies.
 
@@ -147,33 +362,36 @@ Simple global proxy:
 ./openserp search bing "query" --proxy http://user:pass@127.0.0.1:8080
 ```
 
-Advanced proxy configuration is available in [config.yaml](./config.yaml).
-You can enable tagged proxy pools and per-request override via `X-Use-Proxy: <tag>` or `X-Use-Proxy: direct`.
+Advanced proxy configuration is available in [config.yaml](./config.yaml). You can enable tagged proxy pools and per-request override via `X-Use-Proxy: <tag>` or `X-Use-Proxy: direct`.
 
-## Health & Stats
+## API Docs
 
-```bash
-curl -i "http://127.0.0.1:7000/health"
-curl "http://127.0.0.1:7000/stats"
-curl "http://127.0.0.1:7000/stats/cache"
-curl "http://127.0.0.1:7000/stats/proxy"
-curl "http://127.0.0.1:7000/stats/cb"
-```
+Once the server is running, the interactive docs are available locally:
 
-Useful response headers in server mode: `X-Cache`, `X-Fallback-Engine`,`X-Proxy-Mode`, `X-Proxy-Tag`, `X-Proxy-Used`
+- Swagger UI: `http://127.0.0.1:7000/docs` - full schemas, error shapes, and the `/health`, `/ready`, `/stats/*` endpoints
+- OpenAPI YAML: `http://127.0.0.1:7000/openapi.yaml`
+
+To browse the spec without running the server, see [docs/openapi.yaml](./docs/openapi.yaml). For a higher-level overview of how OpenSERP works internally, see the [architecture docs](https://openserp.org/docs/architecture/).
+
+## Self-Hosted or Cloud
+
+- **Self-hosted (this repo)** - free, MIT-licensed, full control over runtime, proxies, cache, and scaling.
+- **[OpenSERP Cloud](https://openserp.org/cloud)** - same endpoints and response schema, no infrastructure to run.
+
+Client code migrates in either direction, so you are never locked in.
 
 ## License
 
 This project is licensed under the MIT License. See [LICENSE](LICENSE).
 
-## 🤝 Contributing
+## Contributing
 
-Contributions are welcome. Please feel free to submit a pull request.
+Contributions are welcome. See [docs/CONTRIBUTING.md](./docs/CONTRIBUTING.md).
 
-## 👾 Issues & Support
+## Feedback & Updates
 
-If you encounter issues or have questions:
+- [GitHub Issues](https://github.com/karust/openserp/issues) - bugs, feature ideas, and reproducible issues.
+- [feedback@openserp.org](mailto:feedback@openserp.org) - private notes, longer feedback, or anything that does not fit GitHub Issues.
+- [Telegram](https://t.me/openserp_cloud) - OpenSERP news, release notes, and project updates.
 
-- Open an issue on GitHub
-- Check existing issues for similar reports
-- Review the documentation and example config
+> OpenSERP is free and open-source. Only links listed in this repository and on [openserp.org](https://openserp.org) are associated with the project.

baidu/all_test.go

@@ -1,71 +1,187 @@
 package baidu
 
 import (
+	"net/url"
 	"testing"
-	"time"
 
 	"github.com/karust/openserp/core"
 )
 
-var browser *core.Browser
-var testQuery = core.Query{Text: "go", Site: "tutorialspoint.com", DateInterval: "20140101..20230101", Limit: 10}
-
-func init() {
-	core.InitLogger(true, true)
-
-	opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 10}
-	browser, _ = core.NewBrowser(opts)
-}
-
-func TestUrlBuild(t *testing.T) {
-	res, err := BuildURL(testQuery)
-	if err != nil {
-		t.Fatal(err)
+func TestBuildURL(t *testing.T) {
+	tests := []struct {
+		name    string
+		query   core.Query
+		wantErr bool
+		check   func(*testing.T, url.Values, string)
+	}{
+		{
+			name: "combined params with unicode and start zero",
+			query: core.Query{
+				Text:         "go 搜索",
+				Site:         "tutorialspoint.com",
+				Filetype:     "pdf",
+				DateInterval: "20140101..20230101",
+				Limit:        10,
+				Start:        0,
+			},
+			check: func(t *testing.T, params url.Values, host string) {
+				t.Helper()
+				if host != "www.baidu.com" {
+					t.Fatalf("unexpected host: %s", host)
+				}
+				if got := params.Get("wd"); got != "go 搜索 site:tutorialspoint.com filetype:pdf" {
+					t.Fatalf("unexpected wd value: %q", got)
+				}
+				if got := params.Get("gpc"); got != "stf=1388534400,1672531200|stftype=2" {
+					t.Fatalf("unexpected gpc value: %q", got)
+				}
+				if got := params.Get("rn"); got != "" {
+					t.Fatalf("unexpected rn value: %q", got)
+				}
+				if got := params.Get("pn"); got != "" {
+					t.Fatalf("pn should be omitted when Start=0, got %q", got)
+				}
+				if got := params.Get("f"); got != "8" {
+					t.Fatalf("unexpected f value: %q", got)
+				}
+				if got := params.Get("ie"); got != "utf-8" {
+					t.Fatalf("unexpected ie value: %q", got)
+				}
+			},
+		},
+		{
+			name: "very large start",
+			query: core.Query{
+				Text:  "golang",
+				Start: 2147483647,
+			},
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("pn"); got != "2147483647" {
+					t.Fatalf("unexpected pn value: %q", got)
+				}
+			},
+		},
+		{
+			name: "negative start returns error",
+			query: core.Query{
+				Text:  "golang",
+				Start: -1,
+			},
+			wantErr: true,
+		},
+		{
+			name:    "empty fields return error",
+			query:   core.Query{},
+			wantErr: true,
+		},
 	}
 
-	want := "https://www.baidu.com/s?f=8&gpc=stf%3D1388534400%2C1672531200%7Cstftype%3D2&ie=utf-8&rn=10&wd=go+site%3Atutorialspoint.com"
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := BuildURL(tt.query)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("BuildURL() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if tt.wantErr {
+				return
+			}
 
-	if want != res {
-		t.Fatalf("Wanted result `%s` doesn't match to resulted `%s`", want, res)
+			parsed, err := url.Parse(got)
+			if err != nil {
+				t.Fatalf("invalid URL returned: %v", err)
+			}
+			if tt.check != nil {
+				tt.check(t, parsed.Query(), parsed.Host)
+			}
+		})
 	}
 }
 
-func TestSearch(t *testing.T) {
-	baid := New(*browser, core.SearchEngineOptions{})
-	results, err := baid.Search(testQuery)
-	if err != nil {
-		t.Fatal(err)
+func TestBuildImageURL(t *testing.T) {
+	tests := []struct {
+		name    string
+		query   core.Query
+		pageNum int
+		wantErr bool
+		check   func(*testing.T, url.Values, string)
+	}{
+		{
+			name:    "unicode query",
+			query:   core.Query{Text: "金毛猎犬"},
+			pageNum: 0,
+			check: func(t *testing.T, params url.Values, host string) {
+				t.Helper()
+				if host != "image.baidu.com" {
+					t.Fatalf("unexpected host: %s", host)
+				}
+				if got := params.Get("word"); got != "金毛猎犬" {
+					t.Fatalf("expected unicode query, got %q", got)
+				}
+				if got := params.Get("tn"); got != "resultjson_com" {
+					t.Fatalf("unexpected tn value: %q", got)
+				}
+			},
+		},
+		{
+			name: "combined params with pagination",
+			query: core.Query{
+				Text:  "golang",
+				Limit: 25,
+			},
+			pageNum: 3,
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("rn"); got != "30" {
+					t.Fatalf("unexpected rn value: %q", got)
+				}
+				if got := params.Get("pn"); got != "90" {
+					t.Fatalf("unexpected pn value: %q", got)
+				}
+			},
+		},
+		{
+			name: "small limit omits result count param",
+			query: core.Query{
+				Text:  "golang",
+				Limit: 10,
+			},
+			pageNum: 2,
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("rn"); got != "" {
+					t.Fatalf("rn should be omitted when Limit<=10, got %q", got)
+				}
+				if got := params.Get("pn"); got != "60" {
+					t.Fatalf("unexpected pn value: %q", got)
+				}
+			},
+		},
+		{
+			name:    "empty fields return error",
+			query:   core.Query{},
+			pageNum: 0,
+			wantErr: true,
+		},
 	}
 
-	if len(results) == 0 {
-		t.Fatal("No results got from Baidu search")
-	}
-}
-
-func TestImageUrlBuild(t *testing.T) {
-	query := core.Query{Text: "金毛猎犬"}
-
-	got, err := BuildImageURL(query, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	want := "https://image.baidu.com/search/acjson?cl=2&fp=result&ie=utf-8&ipn=rj&oe=utf-8&pn=0&rn=30&tn=resultjson_com&word=%E9%87%91%E6%AF%9B%E7%8C%8E%E7%8A%AC"
-	if want != got {
-		t.Fatalf("Want: `%s`, Got `%s`", want, got)
-	}
-}
-
-func TestImageSearch(t *testing.T) {
-	baid := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "each each data", Limit: 60}
-	results, err := baid.SearchImage(query)
-	if err != nil {
-		t.Fatalf("Cannot [ImageBaidu]: %s", err)
-	}
-
-	if len(results) < 60 {
-		t.Fatalf("[ImageBaidu] returned not full result")
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := BuildImageURL(tt.query, tt.pageNum)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("BuildImageURL() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if tt.wantErr {
+				return
+			}
+
+			parsed, err := url.Parse(got)
+			if err != nil {
+				t.Fatalf("invalid URL returned: %v", err)
+			}
+			if tt.check != nil {
+				tt.check(t, parsed.Query(), parsed.Host)
+			}
+		})
 	}
 }

baidu/captcha_selector_test.go

@@ -0,0 +1,58 @@
+package baidu
+
+import (
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/karust/openserp/testutil"
+)
+
+// TestBaiduPageTypeSelectors verifies that the selectors defined in selectors.go
+// match (or don't match) real fixture HTML without needing a browser.
+func TestBaiduPageTypeSelectors(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		fixture  string
+		selector string
+		wantHit  bool
+	}{
+		{"search_captcha.html", Selectors.Captcha, true},
+		{"search_captcha.html", Selectors.Timeout, true},
+		{"search_captcha.html", Selectors.Results, false},
+
+		{"search_results.html", Selectors.Results, true},
+		{"search_results.html", Selectors.Captcha, false},
+		{"search_results.html", Selectors.Timeout, false},
+
+		{"search_no_results.html", Selectors.Captcha, false},
+		{"search_no_results.html", Selectors.Timeout, false},
+		{"search_no_results.html", Selectors.Results, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.fixture+"/"+tt.selector, func(t *testing.T) {
+			t.Parallel()
+			assertSelector(t, tt.fixture, tt.selector, tt.wantHit)
+		})
+	}
+}
+
+func assertSelector(t *testing.T, fixture, selector string, wantHit bool) {
+	t.Helper()
+
+	resp := testutil.ResponseFromFixture(t, fixture)
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		t.Fatalf("parse fixture: %v", err)
+	}
+
+	got := doc.Find(selector).Length() > 0
+	if got != wantHit {
+		if wantHit {
+			t.Fatalf("selector %q not found in %s — update selectors.go", selector, fixture)
+		} else {
+			t.Fatalf("selector %q unexpectedly present in %s", selector, fixture)
+		}
+	}
+}

baidu/features.go

@@ -0,0 +1,39 @@
+package baidu
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"github.com/karust/openserp/core"
+)
+
+func extractBaiduFeatures(doc *goquery.Document) []core.SerpFeature {
+	return core.ExtractSerpFeaturesBySelectors(doc, []core.SerpFeatureSelector{
+		{
+			Type:          core.ResultTypeAISummary,
+			Title:         "AI summary",
+			Container:     []string{"div[tpl='app/chat-input']", "div[tpl='ai_chat']", "div[tpl*='ai']", ".op-ai-answer", ".cosc-result", ".ai-answer"},
+			TitleSelector: []string{".c-title", "h2", "h3"},
+			TextSelector:  []string{".cosc-answer", ".op_ai_answer_content", ".ai-answer-content", ".c-abstract"},
+			LinkSelector:  []string{"a[href^='http']"},
+			Position:      1,
+			Confidence:    0.7,
+		},
+		{
+			Type:          core.ResultTypeAnswerBox,
+			Title:         "Answer",
+			Container:     []string{".op_exactqa_s_answer", ".op_dict_content", ".op_weather4_twoicon", "div[tpl='calculator']", "div[tpl='app/calc']"},
+			TitleSelector: []string{".c-title", "h2", "h3"},
+			TextSelector:  []string{".op_exactqa_s_answer", ".op_dict_content", ".op_weather4_twoicon", ".op_new_val_screen_result", ".c-abstract"},
+			LinkSelector:  []string{"a[href^='http']"},
+			Position:      1,
+			Confidence:    0.75,
+		},
+		{
+			Type:         core.ResultTypeRelatedSearches,
+			Title:        "Related searches",
+			Container:    []string{"div[tpl='app/rs']", "#rs_new", "#rs", ".opr-recommends-merge-content", ".c-recommend"},
+			ItemSelector: []string{"a"},
+			LinkSelector: []string{"a[href^='http']", "a"},
+			Confidence:   0.75,
+		},
+	})
+}

baidu/parse_html.go

@@ -0,0 +1,207 @@
+package baidu
+
+import (
+	"errors"
+	"io"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/karust/openserp/core"
+)
+
+// ParseHTML parses a Baidu SERP HTML document and returns search results.
+// No network I/O.
+func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(r)
+	if err != nil {
+		return nil, err
+	}
+	pageStatus := classifyBaiduDocument(doc)
+	if errors.Is(pageStatus, core.ErrEmptyResult) {
+		return []core.SearchResult{}, nil
+	}
+	if pageStatus != nil {
+		return nil, pageStatus
+	}
+	return parseBaiduDocument(doc), nil
+}
+
+func classifyBaiduDocument(doc *goquery.Document) error {
+	if doc.Find(Selectors.Captcha).Length() > 0 {
+		return core.ErrCaptcha
+	}
+	if doc.Find(Selectors.Timeout).Length() > 0 {
+		return core.ErrSearchTimeout
+	}
+	return core.ClassifyChallengeDocument(doc, core.DocSignals{
+		EmptySelectors: []string{Selectors.NoResults},
+	})
+}
+
+func parseBaiduDocument(doc *goquery.Document) []core.SearchResult {
+	features := extractBaiduFeatures(doc)
+	// Match all result-card variants in one pass so DOM order is preserved and
+	// every card type (organic www_index, baike/encyclopedia, op cards) is
+	// collected. Selecting one variant at a time and returning on the first hit
+	// dropped baike and other result-op cards that interleave with organic rows.
+	results := parseBaiduSelection(doc.Find(baiduResultSelector()))
+	return core.AttachFeaturesToFirstResult(results, features)
+}
+
+func baiduResultSelectors() []string {
+	selectors := make([]string, 0, len(Selectors.ResultsAlt)+1)
+	selectors = append(selectors, Selectors.Results)
+	selectors = append(selectors, Selectors.ResultsAlt...)
+	return selectors
+}
+
+func baiduResultSelector() string {
+	return strings.Join(baiduResultSelectors(), ", ")
+}
+
+func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult {
+	var results []core.SearchResult
+	rank := core.NewRankState(0)
+
+	sel.Each(func(_ int, item *goquery.Selection) {
+		isAd := baiduSelectionHasAdMarker(item)
+
+		// h3-first: organic results always carry a heading; this filters out
+		// non-result blocks that may share the wrapper class.
+		titleTag := item.Find("h3").First()
+		var (
+			title   string
+			linkTag *goquery.Selection
+		)
+		if titleTag.Length() > 0 {
+			title = strings.TrimSpace(titleTag.Text())
+			if child := titleTag.Find("a[href]").First(); child.Length() > 0 {
+				linkTag = child
+			} else if closest := titleTag.Closest("a[href]"); closest.Length() > 0 {
+				linkTag = closest
+			}
+		}
+		if linkTag == nil || linkTag.Length() == 0 {
+			first := item.Find(Selectors.Link).First()
+			if first.Length() == 0 {
+				return
+			}
+			linkTag = first
+		}
+		if title == "" {
+			title = strings.TrimSpace(linkTag.Text())
+		}
+		if title == "" {
+			return
+		}
+
+		href, exists := linkTag.Attr("href")
+		if !exists {
+			return
+		}
+		href = strings.TrimSpace(href)
+		if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") {
+			return
+		}
+		// Organic Baidu results link out through an absolute redirect
+		// (http://www.baidu.com/link?url=...). Op cards like "People also search"
+		// (tpl=recommend_list) instead carry relative on-site search links
+		// (/s?wd=...); treat those as related-search modules, not organic rows.
+		if strings.HasPrefix(href, "/") {
+			return
+		}
+		// Baidu result cards carry the canonical destination in the mu= attribute
+		// (e.g. baike.baidu.com, britannica.com), while the visible link is an
+		// opaque www.baidu.com/link?url= redirect. Prefer mu= so callers get the
+		// real URL, which also enables domain-based classification (encyclopedia,
+		// news, etc.) downstream.
+		if mu := canonicalBaiduURL(item); mu != "" {
+			href = mu
+		}
+
+		desc := ""
+		if descTag := item.Find(Selectors.Desc).First(); descTag.Length() > 0 {
+			desc = strings.TrimSpace(descTag.Text())
+		}
+		if desc == "" {
+			for _, alt := range Selectors.DescAlt {
+				if descTag := item.Find(alt).First(); descTag.Length() > 0 {
+					if t := strings.TrimSpace(descTag.Text()); t != "" {
+						desc = t
+						break
+					}
+				}
+			}
+		}
+		if desc == "" {
+			full := strings.TrimSpace(item.Text())
+			desc = strings.TrimSpace(strings.Replace(full, title, "", 1))
+		}
+
+		resultRank, absoluteRank := rank.Next(isAd)
+		results = append(results, core.SearchResult{
+			Rank:         resultRank,
+			AbsoluteRank: absoluteRank,
+			URL:          href,
+			Title:        title,
+			Description:  desc,
+			Ad:           isAd,
+		})
+	})
+
+	// Re-rank sequentially after dedup so callers get a clean 1..N sequence
+	// (dedup may drop intermediate ranks when the same URL appears in
+	// multiple Baidu result-card variants on the same SERP).
+	deduped := core.DeduplicateResults(results)
+	organicIdx := 0
+	for i := range deduped {
+		if deduped[i].Ad {
+			continue
+		}
+		organicIdx++
+		deduped[i].Rank = organicIdx
+	}
+	return deduped
+}
+
+// canonicalBaiduURL returns the card's mu= destination when it is an absolute
+// http(s) URL. The attribute lives on the result-card container; when the title
+// link is nested, walk up to the nearest ancestor that carries it.
+func canonicalBaiduURL(item *goquery.Selection) string {
+	mu := strings.TrimSpace(firstAttrValue(item, "mu"))
+	if mu == "" {
+		if host := item.Closest("[mu]"); host.Length() > 0 {
+			mu = strings.TrimSpace(firstAttrValue(host, "mu"))
+		}
+	}
+	if strings.HasPrefix(mu, "http://") || strings.HasPrefix(mu, "https://") {
+		return mu
+	}
+	return ""
+}
+
+func firstAttrValue(item *goquery.Selection, name string) string {
+	if value, ok := item.Attr(name); ok {
+		return value
+	}
+	return ""
+}
+
+func baiduSelectionHasAdMarker(item *goquery.Selection) bool {
+	for _, selector := range Selectors.AdMarkers {
+		if item.Is(selector) || item.Find(selector).Length() > 0 {
+			return true
+		}
+	}
+
+	isAd := false
+	item.Find("span, i, em").EachWithBreak(func(_ int, marker *goquery.Selection) bool {
+		text := strings.TrimSpace(marker.Text())
+		if text == "广告" || text == "推广" || text == "商业推广" {
+			isAd = true
+			return false
+		}
+		return true
+	})
+	return isAd
+}

baidu/parse_html_test.go

@@ -0,0 +1,192 @@
+package baidu
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestParseBaiduHTML(t *testing.T) {
+	t.Parallel()
+
+	data, err := os.ReadFile("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("read fixture: %v", err)
+	}
+
+	results, err := ParseHTML(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+
+	rank := 0
+	for i, r := range results {
+		if r.Ad {
+			continue
+		}
+		rank++
+		if r.Rank != rank {
+			t.Fatalf("rank sequence broken at index %d: got %d, want %d", i, r.Rank, rank)
+		}
+		if r.URL == "" {
+			t.Fatalf("result %d: empty URL", i)
+		}
+		if r.Title == "" {
+			t.Fatalf("result %d: empty Title", i)
+		}
+	}
+	if rank == 0 {
+		t.Fatal("expected at least one organic result")
+	}
+}
+
+// TestParseBaiduHTMLParsesBaike locks in two fixes: baike/encyclopedia cards
+// (div.result-op.c-container, tpl=bk_polysemy) are parsed alongside organic
+// www_index cards instead of being dropped by first-selector-wins, and op
+// "People also search" cards (relative /s? links) are excluded as non-organic.
+func TestParseBaiduHTMLParsesBaike(t *testing.T) {
+	t.Parallel()
+
+	data, err := os.ReadFile("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("read fixture: %v", err)
+	}
+	results, err := ParseHTML(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+
+	foundBaike := false
+	for i, r := range results {
+		if strings.Contains(strings.ToLower(r.Title), "baike") ||
+			strings.Contains(strings.ToLower(r.Title), "encyclopedia") {
+			foundBaike = true
+			if strings.TrimSpace(r.Description) == "" {
+				t.Fatalf("baike result %d has empty description", i)
+			}
+		}
+		// Op cards link to relative on-site search; organic results must not.
+		if strings.HasPrefix(r.URL, "/") {
+			t.Fatalf("result %d has a relative (non-organic) URL: %s", i, r.URL)
+		}
+	}
+	if !foundBaike {
+		t.Fatal("expected a baidu baike/encyclopedia result to be parsed")
+	}
+}
+
+func TestParseBaiduHTMLEmpty(t *testing.T) {
+	t.Parallel()
+
+	results, err := ParseHTML(bytes.NewReader([]byte("")))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 0 {
+		t.Fatalf("expected zero results for empty HTML, got %d", len(results))
+	}
+}
+
+func TestParseBaiduHTMLFallbackSelectors(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<div id="content_left">
+  <div class="result-op c-container">
+    <h3><a href="https://example.com/result">Fallback Title</a></h3>
+    <div class="summary-gap_3Jb4I">Fallback description</div>
+  </div>
+</div>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 1 {
+		t.Fatalf("expected 1 result, got %d", len(results))
+	}
+	if results[0].URL != "https://example.com/result" {
+		t.Fatalf("unexpected URL: %s", results[0].URL)
+	}
+	if results[0].Title != "Fallback Title" {
+		t.Fatalf("unexpected title: %s", results[0].Title)
+	}
+	if results[0].Description != "Fallback description" {
+		t.Fatalf("unexpected description: %s", results[0].Description)
+	}
+}
+
+func TestParseBaiduHTMLFallsBackWhenEarlierSelectorHasNoResult(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<div id="content_left">
+  <div class="result c-container"></div>
+  <div class="result-op c-container">
+    <h3><a href="https://example.com/parseable">Parseable Title</a></h3>
+    <div class="summary-gap_3Jb4I">Parseable description</div>
+  </div>
+</div>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 1 {
+		t.Fatalf("expected 1 result, got %d", len(results))
+	}
+	if results[0].URL != "https://example.com/parseable" {
+		t.Fatalf("unexpected URL: %s", results[0].URL)
+	}
+}
+
+func TestParseBaiduHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<div id="content_left">
+  <div class="result c-container" data-tuiguang="1">
+    <h3><a href="https://ads.example.com">Sponsored Result</a></h3>
+    <div class="c-abstract">Paid snippet</div>
+  </div>
+  <div class="result c-container">
+    <h3><a href="https://organic.example.com/one">Organic One</a></h3>
+    <div class="c-abstract">Organic snippet one</div>
+  </div>
+  <div class="result c-container">
+    <h3><a href="https://organic.example.com/two">Organic Two</a></h3>
+    <div class="c-abstract">Organic snippet two</div>
+  </div>
+</div>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 3 {
+		t.Fatalf("expected 3 results, got %d", len(results))
+	}
+
+	organicRank := 0
+	adRank := 0
+	for _, r := range results {
+		if r.Ad {
+			adRank++
+			if r.Rank != adRank {
+				t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
+			}
+			continue
+		}
+		organicRank++
+		if r.Rank != organicRank {
+			t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
+		}
+	}
+	if organicRank != 2 {
+		t.Fatalf("organic count = %d, want 2", organicRank)
+	}
+	if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
+		t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
+	}
+}

baidu/search.go

@@ -1,14 +1,16 @@
 package baidu
 
 import (
+	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/go-rod/rod"
 	"github.com/karust/openserp/core"
-	"golang.org/x/time/rate"
 )
 
 type imageDataJson struct {
@@ -31,12 +33,14 @@ type imageDataJson struct {
 	}
 }
 
+// Baidu implements core.SearchEngine for Baidu SERP pages.
 type Baidu struct {
 	core.Browser
 	core.SearchEngineOptions
 	logger *core.EngineLogger
 }
 
+// New creates a Baidu engine instance with browser/runtime options applied.
 func New(browser core.Browser, opts core.SearchEngineOptions) *Baidu {
 	baid := Baidu{Browser: browser}
 	opts.Init()
@@ -45,171 +49,202 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *Baidu {
 	return &baid
 }
 
+// Name returns the stable engine identifier.
 func (baid *Baidu) Name() string {
 	return "baidu"
 }
 
-func (baid *Baidu) GetRateLimiter() *rate.Limiter {
-	ratelimit := rate.Every(baid.GetRatelimit())
-	return rate.NewLimiter(ratelimit, baid.RateBurst)
+// classifyBlockPage runs the same captcha/timeout/no-results rules the raw
+// HTML path uses (classifyBaiduDocument), against a snapshot of the live
+// page, so both paths can't drift. ErrEmptyResult is not a block: callers
+// handle that themselves once they've also checked for parsed results.
+func (baid *Baidu) classifyBlockPage(page *rod.Page, url string) error {
+	err := core.ClassifyFromPage(page, classifyBaiduDocument)
+	if errors.Is(err, core.ErrEmptyResult) {
+		return nil
+	}
+	if err != nil {
+		baid.logger.Error("Page classified as %v: %s", err, url)
+	}
+	return err
 }
 
-func (baid *Baidu) isCaptcha(page *rod.Page) bool {
-	_, err := page.Timeout(baid.GetSelectorTimeout()).Search("div.passMod_dialog-body")
-	return err == nil
-}
+// Search executes a Baidu web search and returns normalized search results.
+// It may return core.ErrCaptcha or core.ErrSearchTimeout.
+func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
+	ctx = core.PrepareEngineContext(ctx, query, baid.Name())
+	scoped := *baid
+	scoped.logger = baid.logger.WithRequest(ctx)
+	if scoped.Browser.WaitLoadTime == 0 || scoped.Browser.WaitLoadTime > 250*time.Millisecond {
+		scoped.Browser.WaitLoadTime = 250 * time.Millisecond
+	}
+	baid = &scoped
 
-func (baid *Baidu) isTimeout(page *rod.Page) bool {
-	_, err := page.Timeout(baid.GetSelectorTimeout()).Search("button.timeout-button")
-	return err == nil
-}
-
-func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
 	baid.logger.Debug("Starting search, query: %+v", query)
 
-	searchResults := []core.SearchResult{}
-
 	// Build URL from query struct to open in browser
 	url, err := BuildURL(query)
 	if err != nil {
 		return nil, err
 	}
 
-	page, err := baid.Navigate(url)
+	page, err := baid.Navigate(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	defer core.DeferClosePage(ctx, page, &baid.Browser)()
+
+	searchResults, err := baid.waitForParsedSearchResults(ctx, page, url)
 	if err != nil {
 		return nil, err
 	}
 
-	results, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd")
-	if err != nil {
-		defer page.Close()
-		baid.logger.Error("Cannot parse search results: %s", err)
-		return nil, core.ErrSearchTimeout
-	}
-
-	// Check why no results, maybe captcha?
-	if results == nil {
-		defer page.Close()
-
-		if baid.isCaptcha(page) {
-			baid.logger.Error("Captcha detected: %s", url)
-			return nil, core.ErrCaptcha
-		} else if baid.isTimeout(page) {
-			baid.logger.Error("Timeout occurred: %s", url)
-			return nil, core.ErrCaptcha
+	for i := range searchResults {
+		if searchResults[i].AbsoluteRank > 0 {
+			searchResults[i].AbsoluteRank += query.Start
 		}
-		return nil, nil
-	}
-
-	resultElements, err := results.All()
-	if err != nil {
-		return nil, err
-	}
-
-	for i, r := range resultElements {
-		// Get URL
-		link, err := r.Element("a")
-		if err != nil {
+		if searchResults[i].Ad {
 			continue
 		}
-		linkText, err := link.Property("href")
-		if err != nil {
-			baid.logger.Error("Missing href tag")
-		}
-
-		// Get title
-		title, err := link.Text()
-		if err != nil {
-			baid.logger.Error("Failed to extract title")
-			title = "No title"
-		}
-
-		// Get description
-		desc, err := r.Text()
-		if err != nil {
-			desc = ""
-		}
-		desc = strings.ReplaceAll(desc, title, "")
-
-		gR := core.SearchResult{Rank: query.Start + i + 1, URL: linkText.String(), Title: title, Description: desc}
-		searchResults = append(searchResults, gR)
+		searchResults[i].Rank = query.Start + searchResults[i].Rank
 	}
-
-	if !baid.Browser.LeavePageOpen {
-		err = page.Close()
-		if err != nil {
-			baid.logger.Error("Page close error: %v", err)
-		}
-	}
-
-	return core.DeduplicateResults(searchResults), nil
+	searchResults = core.StripResultFeatures(searchResults, query.Features)
+	return searchResults, nil
 }
 
-func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
+func (baid *Baidu) waitForParsedSearchResults(ctx context.Context, page *rod.Page, url string) ([]core.SearchResult, error) {
+	timeout := baid.GetSelectorTimeout()
+	if timeout <= 0 {
+		timeout = 5 * time.Second
+	}
+	deadline := time.Now().Add(timeout)
+	var sawResultContainer bool
+	var lastErr error
+
+	for {
+		html, err := page.HTML()
+		if err == nil {
+			results, parseErr := ParseHTML(strings.NewReader(html))
+			if parseErr == nil && len(results) > 0 {
+				return results, nil
+			}
+			if parseErr != nil && !errors.Is(parseErr, core.ErrEmptyResult) {
+				baid.logger.Error("Page classified as %v: %s", parseErr, url)
+				return nil, parseErr
+			}
+			lastErr = parseErr
+		} else {
+			lastErr = err
+			if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
+				return nil, blockErr
+			}
+		}
+
+		if core.HasAnySelector(page, baiduResultSelectors()) {
+			sawResultContainer = true
+		}
+		if !time.Now().Before(deadline) {
+			break
+		}
+		if err := core.SleepContext(ctx, 120*time.Millisecond); err != nil {
+			return nil, err
+		}
+	}
+
+	if sawResultContainer {
+		if lastErr != nil {
+			baid.logger.Debug("Baidu result containers found but HTML parsing failed: %v", lastErr)
+		} else {
+			baid.logger.Debug("Baidu result containers found but no parseable organic results")
+		}
+		return nil, core.ErrParser
+	}
+	// The page never reached a recognizable state: no result containers, no
+	// captcha or timeout markers. Baidu hydrates result cards client-side and
+	// can exceed the selector deadline, so report a timeout rather than a
+	// successful empty SERP — callers must retry/skip, not trust 0 results.
+	baid.logger.Debug("No result containers or block markers within selector timeout")
+	return nil, core.ErrSearchTimeout
+}
+
+// SearchImage executes a Baidu image search and returns normalized image
+// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
+func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
+	ctx = core.PrepareEngineContext(ctx, query, baid.Name())
+	scoped := *baid
+	scoped.logger = baid.logger.WithRequest(ctx)
+	baid = &scoped
+
 	baid.logger.Debug("Starting image search, query: %+v", query)
 
 	searchResults := []core.SearchResult{}
 	searchPage := 0
 
-	for len(searchResults) < query.Limit {
+	// fetchPage loads one image-results page and appends parsed results.
+	// Returns (done, error): done=true ends the outer loop without error.
+	fetchPage := func() (bool, error) {
 		url, err := BuildImageURL(query, searchPage)
 		if err != nil {
-			return nil, err
+			return false, err
 		}
 
-		// Get anti-crawler cookies first, then reload page
-		page, err := baid.Navigate(url)
+		// First load often seeds anti-crawler cookies; results tend to become
+		// available after one explicit reload.
+		page, err := baid.Navigate(ctx, url)
 		if err != nil {
-			return nil, err
+			return false, err
+		}
+		defer core.DeferClosePage(ctx, page, &baid.Browser)()
+
+		jsonWaitTimeout := baid.GetSelectorTimeout()
+		if reloadErr := page.Reload(); reloadErr != nil {
+			return false, core.ErrSearchTimeout
 		}
 
-		if !baid.Browser.LeavePageOpen {
-			defer page.Close()
-		}
-		page.Reload()
-		page.WaitLoad()
-
-		result, err := page.Timeout(baid.Timeout).Search("body > pre")
+		preElements, _, err := core.WaitForElements(ctx, page, Selectors.ImageJSONRoot, jsonWaitTimeout)
 		if err != nil {
-			defer page.Close()
-			baid.logger.Error("Cannot parse search results: %s", err)
-			return nil, core.ErrSearchTimeout
-		}
-
-		// Check why no results, maybe captcha?
-		if result == nil {
-			defer page.Close()
-
-			if baid.isCaptcha(page) {
-				baid.logger.Error("Captcha detected: %s", url)
-				return nil, core.ErrCaptcha
-			} else if baid.isTimeout(page) {
-				baid.logger.Error("Timeout occurred: %s", url)
-				return nil, core.ErrCaptcha
+			if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
+				return false, blockErr
 			}
-			return nil, nil
+			baid.logger.Error("Cannot parse search results: %s", err)
+			return false, core.ErrSearchTimeout
 		}
 
-		jsonText, err := result.First.Text()
+		if len(preElements) == 0 {
+			if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
+				return false, blockErr
+			}
+			return true, nil
+		}
+
+		jsonText, err := preElements[0].Text()
 		if err != nil {
-			return nil, err
+			return false, err
 		}
 
 		var data imageDataJson
 
 		// Fix broken JSON
 		jsonText = strings.ReplaceAll(jsonText, `\'`, "'")
-		matchNewlines := regexp.MustCompile(`[\r\n\t]`)
+		matchNewlines, err := regexp.Compile(`[\r\n\t]`)
+		if err != nil {
+			return false, core.ErrParser
+		}
 		escapeNewlines := func(s string) string {
 			return matchNewlines.ReplaceAllString(s, "\\n")
 		}
-		re := regexp.MustCompile(`"[^"\\]*(?:\\[\s\S][^"\\]*)*"`)
+		re, err := regexp.Compile(`"[^"\\]*(?:\\[\s\S][^"\\]*)*"`)
+		if err != nil {
+			return false, core.ErrParser
+		}
 		fixedJson := re.ReplaceAllStringFunc(jsonText, escapeNewlines)
 
-		err = json.Unmarshal([]byte(fixedJson), &data)
-		if err != nil {
+		if err := json.Unmarshal([]byte(fixedJson), &data); err != nil {
 			baid.logger.Error("Failed to unmarshal JSON: %v", err)
-			return nil, err
+			return false, core.ErrParser
+		}
+		if len(data.Data) == 0 {
+			return true, nil
 		}
 
 		for i, img := range data.Data {
@@ -217,27 +252,44 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
 				continue
 			}
 			res := core.SearchResult{
-				Rank:        (searchPage * 30) + (i + 1),
-				URL:         img.URL[0].Original,
-				Title:       img.Title,
-				Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright),
-				Ad: func() bool {
-					if img.AdType != "0" {
-						return true
-					} else {
-						return false
-					}
-				}(),
+				Rank:  (searchPage * 30) + (i + 1),
+				URL:   img.URL[0].Original,
+				Title: img.Title,
+				Description: fmt.Sprintf(
+					"Source Page: %s, thumb_url:%s, %dx%d, date:%v, type:%v, copyright:%v",
+					img.URL[0].SourcePage,
+					img.ThumbURL,
+					img.Width,
+					img.Height,
+					img.PictureDate,
+					img.Type,
+					img.IsCopyright,
+				),
+				Ad: img.AdType != "0",
 			}
 			searchResults = append(searchResults, res)
+			if query.Limit > 0 && len(searchResults) >= query.Limit {
+				return true, nil
+			}
 		}
 
-		searchPage += 1
+		return false, nil
+	}
 
-		if !baid.Browser.LeavePageOpen {
-			page.Close()
+	for core.ShouldFetchResultPage(len(searchResults), query.Limit, searchPage) {
+		done, err := fetchPage()
+		if err != nil {
+			return nil, err
+		}
+		searchPage++
+		if done || !core.ShouldFetchResultPage(len(searchResults), query.Limit, searchPage) {
+			break
 		}
 	}
 
-	return core.DeduplicateResults(searchResults), nil
+	deduped := core.DeduplicateResults(searchResults)
+	if query.Limit > 0 && len(deduped) > query.Limit {
+		deduped = deduped[:query.Limit]
+	}
+	return deduped, nil
 }

baidu/search_integration_test.go

@@ -0,0 +1,17 @@
+//go:build integration
+// +build integration
+
+package baidu
+
+import (
+	"testing"
+
+	"github.com/karust/openserp/core"
+	"github.com/karust/openserp/testutil/ithelper"
+)
+
+func TestSearchBaidu(t *testing.T) {
+	ithelper.RunEngineTests(t, func(b *core.Browser) core.SearchEngine {
+		return New(*b, ithelper.EngineOptions())
+	})
+}

baidu/search_raw.go

@@ -1,104 +1,75 @@
 package baidu
 
 import (
+	"bytes"
+	"context"
+	"errors"
 	"fmt"
-	"net/http"
-	"strings"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/corpix/uarand"
 	"github.com/karust/openserp/core"
-	"github.com/sirupsen/logrus"
 )
 
-func baiduRequest(searchURL string, query core.Query) (*http.Response, error) {
-	baseClient, err := core.NewRawHTTPClient(query)
+func classifyBaiduRawHTML(body []byte) error {
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
 	if err != nil {
-		return nil, err
+		return err
 	}
-
-	req, err := http.NewRequest("GET", searchURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("User-Agent", uarand.GetRandom())
-
-	res, err := baseClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	return res, nil
+	return classifyBaiduDocument(doc)
 }
 
-func baiduResultParser(response *http.Response) ([]core.SearchResult, error) {
-	doc, err := goquery.NewDocumentFromReader(response.Body)
+func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
+	ctx = core.PrepareEngineContext(ctx, query, "baidu")
+
+	searchURL, err := BuildURL(query)
 	if err != nil {
 		return nil, err
 	}
+	core.WithRequest(ctx).WithField("url", searchURL).Debug(fmt.Sprintf("Baidu URL built: %s", searchURL))
 
-	results := []core.SearchResult{}
-	rank := 1
+	res, err := core.RawSearchRequest(ctx, searchURL, query)
+	if err != nil {
+		return nil, err
+	}
+	defer core.DrainAndCloseResponse(res)
+	core.WithRequest(ctx).WithField("status_code", res.StatusCode).Debug(
+		fmt.Sprintf("Baidu Raw response: code=%d", res.StatusCode),
+	)
 
-	// Get individual results
-	sel := doc.Find("div.c-container.new-pmd")
+	body, err := core.ReadRawSearchBody(res)
+	if err != nil {
+		return nil, err
+	}
+	htmlStatus := classifyBaiduRawHTML(body)
+	if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) {
+		return nil, htmlStatus
+	}
 
-	fmt.Println(sel.Length())
-
-	for i := range sel.Nodes {
-		item := sel.Eq(i)
-
-		// Find URL
-		linkTag := item.Find("a")
-		link, _ := linkTag.Attr("href")
-		link = strings.Trim(link, " ")
-
-		// Find title
-		title := linkTag.Text()
-
-		// Find description
-		desc := item.Text()
-		desc = strings.ReplaceAll(desc, title, "")
-
-		if link != "" && link != "#" {
-			result := core.SearchResult{
-				Rank:        rank,
-				URL:         link,
-				Title:       title,
-				Description: desc,
-			}
-
-			results = append(results, result)
-			rank++
+	parsedResults, err := ParseHTML(bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	if len(parsedResults) == 0 {
+		if errors.Is(htmlStatus, core.ErrEmptyResult) {
+			return []core.SearchResult{}, nil
 		}
-	}
-
-	logrus.Tracef("Baidu search document size: %d", len(doc.Text()))
-	return results, err
-}
-
-func Search(query core.Query) ([]core.SearchResult, error) {
-	googleURL, err := BuildURL(query)
-	if err != nil {
-		return nil, err
-	}
-	logrus.Debugf("Baidu URL built: %s", googleURL)
-
-	res, err := baiduRequest(googleURL, query)
-	if err != nil {
-		return nil, err
-	}
-	logrus.Debugf("Baidu Raw response: code=%d", res.StatusCode)
-
-	results, err := baiduResultParser(res)
-	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%w: baidu raw search returned no parseable results", core.ErrParser)
 	}
 	if query.Start > 0 {
-		for i := range results {
-			results[i].Rank = query.Start + i + 1
+		for i := range parsedResults {
+			if parsedResults[i].AbsoluteRank > 0 {
+				parsedResults[i].AbsoluteRank += query.Start
+			}
+			if parsedResults[i].Ad {
+				continue
+			}
+			parsedResults[i].Rank = query.Start + parsedResults[i].Rank
 		}
 	}
-	logrus.Debugf("Baidu Raw results : %v", results)
+	core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
+		fmt.Sprintf("Baidu Raw results : %v", parsedResults),
+	)
 
-	return core.DeduplicateResults(results), nil
+	deduped := core.StripResultFeatures(core.DeduplicateResults(parsedResults), query.Features)
+	return deduped, nil
 }

baidu/search_raw_test.go

@@ -0,0 +1,75 @@
+package baidu
+
+import (
+	"errors"
+	"io"
+	"testing"
+
+	"github.com/karust/openserp/core"
+	"github.com/karust/openserp/testutil"
+)
+
+// TestBaiduParseHTMLFixtures covers the no-results and captcha fixtures.
+// The happy path is covered in TestParseBaiduHTML; this file ensures the
+// shared parser (used by both raw mode and the /baidu/parse endpoint) does
+// not over-extract on captcha or empty SERPs.
+func TestBaiduParseHTMLFixtures(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		fixture string
+		wantErr error
+	}{
+		{name: "no results", fixture: "search_no_results.html"},
+		{name: "captcha page", fixture: "search_captcha.html", wantErr: core.ErrCaptcha},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body)
+			if tt.wantErr != nil {
+				if !errors.Is(err, tt.wantErr) {
+					t.Fatalf("expected %v for %s, got %v", tt.wantErr, tt.fixture, err)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("ParseHTML() error = %v", err)
+			}
+			if len(results) != 0 {
+				t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results))
+			}
+		})
+	}
+}
+
+func TestBaiduClassifyRawHTML(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		fixture string
+		want    error
+	}{
+		{name: "no results", fixture: "search_no_results.html", want: core.ErrEmptyResult},
+		{name: "captcha page", fixture: "search_captcha.html", want: core.ErrCaptcha},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			body, err := io.ReadAll(testutil.ResponseFromFixture(t, tt.fixture).Body)
+			if err != nil {
+				t.Fatalf("read fixture body: %v", err)
+			}
+			err = classifyBaiduRawHTML(body)
+			if !errors.Is(err, tt.want) {
+				t.Fatalf("expected %v for %s, got %v", tt.want, tt.fixture, err)
+			}
+		})
+	}
+}

baidu/selectors.go

@@ -0,0 +1,36 @@
+package baidu
+
+// Selectors is the single source of truth for Baidu SERP CSS selectors.
+var Selectors = struct {
+	Captcha       string
+	Timeout       string
+	NoResults     string
+	Results       string
+	ResultsAlt    []string
+	AdMarkers     []string
+	ImageJSONRoot []string
+	Link          string
+	Desc          string
+	// DescAlt are additional description containers tried when Desc misses.
+	// Baidu varies abstract markup across feature blocks (info cards, news rows).
+	DescAlt []string
+}{
+	Captcha:       "div.passMod_dialog-wrapper",
+	Timeout:       "button.timeout-button",
+	NoResults:     "div.content_none, div.nors",
+	Results:       "#content_left div.result.c-container",
+	ResultsAlt:    []string{"#content_left div.result-op.c-container", "div.c-container.new-pmd"},
+	AdMarkers:     []string{"[data-tuiguang]", "[data-click*='tuiguang']", ".ec-tuiguang", ".c-icon-bear-p"},
+	ImageJSONRoot: []string{"body > pre", "pre"},
+	Link:          "a",
+	Desc:          "div.c-abstract",
+	// DescAlt matches Baidu's hashed abstract containers by class *prefix*
+	// ([class*='summary-gap_']) rather than a frozen hash suffix
+	// (.summary-gap_3Jb4I): Baidu rotates the trailing hash per build (the same
+	// page already carries summary-gap_3Jb4I and summary-gap_68jXq), and the old
+	// content-right_8Zs40 suffix no longer appears at all. These two prefixes are
+	// specific enough to use as substrings. text_ is NOT: it is Baidu's generic
+	// text-styling class reused on dozens of nodes, so the baike abstract body is
+	// pinned to its exact .text_2NOr6 hash and tried last.
+	DescAlt: []string{"[class*='content-right_']", "[class*='summary-gap_']", "div.text_2NOr6"},
+}

baidu/serp_features_test.go

@@ -0,0 +1,94 @@
+package baidu
+
+import (
+	"bytes"
+	"os"
+	"testing"
+
+	"github.com/karust/openserp/core"
+)
+
+func TestParseHTMLFixtureExtractsRealFeatures(t *testing.T) {
+	t.Parallel()
+	f, err := os.Open("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("open fixture: %v", err)
+	}
+	defer f.Close()
+
+	results, err := ParseHTML(f)
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	assertFeatureType(t, results, core.ResultTypeRelatedSearches)
+}
+
+func TestParseHTMLExtractsSerpFeatures(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<div class="result-op c-container" tpl="ai_chat">
+  <div class="c-title">AI智能回答</div>
+  <div class="c-abstract">Baidu AI summary text.</div>
+  <a href="https://example.com/source">Source</a>
+</div>
+<div id="rs">
+  <table>
+    <tr>
+      <th><a href="https://example.com/related">baidu related search</a></th>
+    </tr>
+  </table>
+</div>
+<div id="content_left">
+  <div class="result c-container">
+    <h3><a href="https://example.com/result">Organic result</a></h3>
+    <div class="c-abstract">Snippet</div>
+  </div>
+</div>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	assertFeatureType(t, results, core.ResultTypeAISummary)
+	assertFeatureType(t, results, core.ResultTypeRelatedSearches)
+}
+
+func TestParseHTMLOrganicOnlyHasNoSerpFeatures(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<div id="content_left">
+  <div class="result c-container">
+    <h3><a href="https://example.com/result">Organic result</a></h3>
+    <div class="c-abstract">Snippet</div>
+  </div>
+</div>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	assertNoFeatures(t, results)
+}
+
+func assertFeatureType(t *testing.T, results []core.SearchResult, want core.ResultType) {
+	t.Helper()
+	for _, result := range results {
+		for _, feature := range result.Features {
+			if feature.Type == want {
+				return
+			}
+		}
+	}
+	t.Fatalf("expected feature type %q in %#v", want, results)
+}
+
+func assertNoFeatures(t *testing.T, results []core.SearchResult) {
+	t.Helper()
+	for _, result := range results {
+		if len(result.Features) > 0 {
+			t.Fatalf("expected no features, got %#v", result.Features)
+		}
+	}
+}

baidu/testdata/search_captcha.html

@@ -0,0 +1 @@
+<html lang="zh-CN"><head><title>百度安全验证</title></head><body style=""><div class="timeout hide-callback"><div class="timeout-img"></div><div class="timeout-title">网络不给力，请稍后重试</div><button type="button" class="timeout-button">返回首页</button></div><div class="timeout-feedback hide-callback"><div class="timeout-feedback-icon"></div><p class="timeout-feedback-title">问题反馈</p></div><div class="passMod_dialog-wrapper passMod_show"><div class="passMod_dialog-mask"></div><div class="passMod_dialog-container"><div class="passMod_dialog-header"><p>百度安全验证</p></div><div class="passMod_dialog-body"><div class="passMod_code-container"><div class="passMod_verify-item passMod_spin-wrapper" id="spin-0"><div class="passMod_spin-tip">请完成下方验证后继续操作</div><p class="passMod_spin-msg"></p><div class="passMod_spin-context-wrap"><div class="passMod_spin-context"><img class="passMod_spin-background" src="http://test.test" alt=""/><img class="passMod_spin-coordinate"/><div class="passMod_verify-container passMod_verify-container_uAMZ undefined" style=""><div class="passMod_verify-pending"><img alt=""/><span>正在验证...</span></div><div class="passMod_verify-success">验证通过</div><div class="passMod_verify-fail">图片未转正</div><div class="passMod_verify-network">网络不给力，请刷新重试</div><div class="passMod_verify-loading"><img alt=""/><span>加载中...</span></div></div><div class="passMod_spin-AI">图片由AI生成</div></div><div class="passMod_spin-footer"><div class="passMod_slide-control passMod_slide-control_FHGY"><div class="passMod_slide-grand passMod_slide-grand-loading"></div><p class="passMod_slide-tip slideShine">拖动左侧滑块使图片为正</p><div class="passMod_slide-btn passMod_slide-btn-loading"></div></div></div></div></div></div></div><div class="passMod_dialog-footer"><p><span class="passMod_dialog-footer-qrcode machine-hide">扫码验证<span class="passMod_dialog-footer-line">|</span></span><a class="passMod_dialog-footer-feedback" target="_blank" href="http://test.test">意见反馈</a><span class="passMod_dialog-footer-refresh"><span class="passMod_dialog-footer-line">|</span><span style=""></span>刷新</span></p></div></div></div></body></html>

baidu/url.go

@@ -22,6 +22,8 @@ func dateToTimestamp(date string) (int64, error) {
 	return t.Unix(), nil
 }
 
+// BuildURL builds a Baidu web search URL from Query fields.
+// It returns an error when query text, date, or pagination parameters are invalid.
 func BuildURL(q core.Query) (string, error) {
 	base, _ := url.Parse("https://www.baidu.com/")
 	base.Path += "s"
@@ -63,7 +65,7 @@ func BuildURL(q core.Query) (string, error) {
 		logrus.Warn("File search not supported")
 	}
 
-	if q.Limit != 0 {
+	if q.Limit > 10 {
 		params.Add("rn", strconv.Itoa(q.Limit))
 	}
 	if q.Start < 0 {
@@ -84,6 +86,8 @@ func BuildURL(q core.Query) (string, error) {
 	return base.String(), nil
 }
 
+// BuildImageURL builds a Baidu image search URL from Query fields and page
+// index. It returns an error when the query text is empty.
 func BuildImageURL(q core.Query, pageNum int) (string, error) {
 	base, _ := url.Parse("https://image.baidu.com/")
 	base.Path += "search/acjson"
@@ -100,8 +104,10 @@ func BuildImageURL(q core.Query, pageNum int) (string, error) {
 		return "", errors.New("Empty query built")
 	}
 
-	if q.Limit != 0 {
-		params.Add("rn", "30")                     // Results per page
+	if q.Limit > 10 {
+		params.Add("rn", "30") // Results per page
+	}
+	if pageNum > 0 {
 		params.Add("pn", strconv.Itoa(pageNum*30)) // Offset
 	}
 

bing/bing_test.go

@@ -2,160 +2,310 @@ package bing
 
 import (
 	"net/url"
-	"strings"
 	"testing"
-	"time"
 
 	"github.com/karust/openserp/core"
 )
 
-var browser *core.Browser
-
-func init() {
-	opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, UseStealth: true, Timeout: time.Second * 5, LeavePageOpen: true}
-	browser, _ = core.NewBrowser(opts)
-}
-
-func TestSearchBing(t *testing.T) {
-	bing := New(*browser, core.SearchEngineOptions{})
-
-	query := core.Query{Text: "golang programming", Limit: 10}
-	results, err := bing.Search(query)
-	if err != nil {
-		t.Fatalf("Cannot [SearchBing]: %s", err)
+func TestBuildURL(t *testing.T) {
+	tests := []struct {
+		name    string
+		query   core.Query
+		wantErr bool
+		check   func(*testing.T, url.Values, string)
+	}{
+		{
+			name: "combined params with unicode and start zero",
+			query: core.Query{
+				Text:         "golang тест",
+				Site:         "example.com",
+				Filetype:     "pdf",
+				DateInterval: "20240101..20240131",
+				LangCode:     "RU",
+				Limit:        30,
+				Start:        0,
+			},
+			check: func(t *testing.T, params url.Values, host string) {
+				t.Helper()
+				if host != "www.bing.com" {
+					t.Fatalf("unexpected host: %s", host)
+				}
+				if got := params.Get("q"); got != "golang тест site:example.com filetype:pdf" {
+					t.Fatalf("unexpected q value: %q", got)
+				}
+				if got := params.Get("setlang"); got != "ru" {
+					t.Fatalf("unexpected setlang value: %q", got)
+				}
+				if got := params.Get("mkt"); got != "ru-RU" {
+					t.Fatalf("unexpected mkt value: %q", got)
+				}
+				if got := params.Get("cc"); got != "RU" {
+					t.Fatalf("unexpected cc value: %q", got)
+				}
+				if got := params.Get("filters"); got != `ex1:"ez5_19723_19753"` {
+					t.Fatalf("unexpected filters value: %q", got)
+				}
+				if got := params.Get("count"); got != "30" {
+					t.Fatalf("unexpected count value: %q", got)
+				}
+				if got := params.Get("first"); got != "" {
+					t.Fatalf("first should be omitted when Start=0, got %q", got)
+				}
+				for _, key := range []string{"form", "qs", "sp", "pq"} {
+					if got := params.Get(key); got != "" {
+						t.Fatalf("expected %s to be omitted, got %q", key, got)
+					}
+				}
+			},
+		},
+		{
+			name: "date operators in text are converted to filters",
+			query: core.Query{
+				Text:  "megadeth tickets after:2026-01-01 before:2026-04-27",
+				Limit: 10,
+			},
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("q"); got != "megadeth tickets" {
+					t.Fatalf("unexpected q value: %q", got)
+				}
+				if got := params.Get("filters"); got != `ex1:"ez5_20454_20570"` {
+					t.Fatalf("unexpected filters value: %q", got)
+				}
+				if got := params.Get("count"); got != "" {
+					t.Fatalf("count should be omitted when Limit<=10, got %q", got)
+				}
+				// LangCode unset → no locale params; let Bing pick defaults
+				// from the request rather than biasing toward en-US.
+				for _, key := range []string{"mkt", "setlang", "cc"} {
+					if got := params.Get(key); got != "" {
+						t.Fatalf("expected %s to be empty, got %q", key, got)
+					}
+				}
+			},
+		},
+		{
+			name: "date param overrides date operators in text",
+			query: core.Query{
+				Text:         "megadeth tickets after:2026-01-01 before:2026-04-27",
+				DateInterval: "20240101..20240131",
+				LangCode:     "en-DE",
+			},
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("q"); got != "megadeth tickets" {
+					t.Fatalf("unexpected q value: %q", got)
+				}
+				if got := params.Get("filters"); got != `ex1:"ez5_19723_19753"` {
+					t.Fatalf("unexpected filters value: %q", got)
+				}
+				if got := params.Get("mkt"); got != "en-DE" {
+					t.Fatalf("unexpected mkt value: %q", got)
+				}
+				if got := params.Get("setlang"); got != "en" {
+					t.Fatalf("unexpected setlang value: %q", got)
+				}
+				if got := params.Get("cc"); got != "DE" {
+					t.Fatalf("unexpected cc value: %q", got)
+				}
+			},
+		},
+		{
+			name: "region overrides bing market country",
+			query: core.Query{
+				Text:     "weather",
+				LangCode: "en",
+				Region:   "DE",
+				Limit:    10,
+			},
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("mkt"); got != "en-DE" {
+					t.Fatalf("unexpected mkt value: %q", got)
+				}
+				if got := params.Get("setlang"); got != "en" {
+					t.Fatalf("unexpected setlang value: %q", got)
+				}
+				if got := params.Get("cc"); got != "DE" {
+					t.Fatalf("unexpected cc value: %q", got)
+				}
+			},
+		},
+		{
+			name: "region only sets bing country",
+			query: core.Query{
+				Text:   "weather",
+				Region: "RU",
+				Limit:  10,
+			},
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("cc"); got != "RU" {
+					t.Fatalf("unexpected cc value: %q", got)
+				}
+				if got := params.Get("mkt"); got != "" {
+					t.Fatalf("mkt should be omitted without language, got %q", got)
+				}
+				if got := params.Get("setlang"); got != "" {
+					t.Fatalf("setlang should be omitted without language, got %q", got)
+				}
+			},
+		},
+		{
+			name: "very large start",
+			query: core.Query{
+				Text:  "golang",
+				Start: 2147483647,
+				Limit: 20,
+			},
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("first"); got != "2147483648" {
+					t.Fatalf("unexpected first value: %q", got)
+				}
+				if got := params.Get("count"); got != "" {
+					t.Fatalf("count should be omitted when first is used, got %q", got)
+				}
+			},
+		},
+		{
+			name: "negative start returns error",
+			query: core.Query{
+				Text:  "golang",
+				Start: -1,
+			},
+			wantErr: true,
+		},
+		{
+			name: "reversed date interval returns error",
+			query: core.Query{
+				Text:         "golang",
+				DateInterval: "20240131..20240101",
+			},
+			wantErr: true,
+		},
+		{
+			name:    "empty fields return error",
+			query:   core.Query{},
+			wantErr: true,
+		},
 	}
 
-	if len(results) == 0 {
-		t.Fatalf("[SearchBing] returned empty result")
-	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := BuildURL(tt.query)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("BuildURL() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if tt.wantErr {
+				return
+			}
 
-	// Check that we have some basic fields populated
-	firstResult := results[0]
-	if firstResult.Title == "" {
-		t.Errorf("First result missing title: %+v", firstResult)
-	}
-	if firstResult.URL == "" {
-		t.Errorf("First result missing URL: %+v", firstResult)
-	}
-	if firstResult.Rank == 0 {
-		t.Errorf("First result missing rank: %+v", firstResult)
+			parsed, err := url.Parse(got)
+			if err != nil {
+				t.Fatalf("BuildURL() returned invalid URL: %v", err)
+			}
+			if tt.check != nil {
+				tt.check(t, parsed.Query(), parsed.Host)
+			}
+		})
 	}
 }
 
 func TestBuildImageURL(t *testing.T) {
 	tests := []struct {
-		name     string
-		query    core.Query
-		wantErr  bool
-		wantCont string
+		name    string
+		query   core.Query
+		wantErr bool
+		check   func(*testing.T, url.Values, string)
 	}{
 		{
-			name:     "basic image query",
-			query:    core.Query{Text: "test"},
-			wantErr:  false,
-			wantCont: "q=test",
+			name:  "basic image query",
+			query: core.Query{Text: "test"},
+			check: func(t *testing.T, params url.Values, host string) {
+				t.Helper()
+				if host != "www.bing.com" {
+					t.Fatalf("unexpected host: %s", host)
+				}
+				if got := params.Get("q"); got != "test" {
+					t.Fatalf("unexpected q value: %q", got)
+				}
+			},
 		},
 		{
-			name:     "image query with site",
-			query:    core.Query{Text: "cats", Site: "example.com"},
-			wantErr:  false,
-			wantCont: "q=cats+site%3Aexample.com",
+			name: "combined params with unicode",
+			query: core.Query{
+				Text:     "коты",
+				Site:     "example.com",
+				Filetype: "png",
+				LangCode: "EN",
+			},
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("q"); got != "коты site:example.com" {
+					t.Fatalf("unexpected q value: %q", got)
+				}
+				if got := params.Get("setlang"); got != "en" {
+					t.Fatalf("unexpected setlang value: %q", got)
+				}
+				if got := params.Get("mkt"); got != "en-US" {
+					t.Fatalf("unexpected mkt value: %q", got)
+				}
+				if got := params.Get("cc"); got != "US" {
+					t.Fatalf("unexpected cc value: %q", got)
+				}
+				if got := params.Get("form"); got != "HDRSC2" {
+					t.Fatalf("unexpected form value: %q", got)
+				}
+				if got := params.Get("first"); got != "1" {
+					t.Fatalf("unexpected first value: %q", got)
+				}
+				if got := params.Get("scenario"); got != "ImageBasicHover" {
+					t.Fatalf("unexpected scenario value: %q", got)
+				}
+			},
 		},
 		{
-			name:     "image query with filetype",
-			query:    core.Query{Text: "dogs", Filetype: "png"},
-			wantErr:  false,
-			wantCont: "q=dogs+filetype%3Apng",
+			name: "image region overrides market country",
+			query: core.Query{
+				Text:     "cats",
+				LangCode: "en",
+				Region:   "GB",
+			},
+			check: func(t *testing.T, params url.Values, _ string) {
+				t.Helper()
+				if got := params.Get("mkt"); got != "en-GB" {
+					t.Fatalf("unexpected mkt value: %q", got)
+				}
+				if got := params.Get("cc"); got != "GB" {
+					t.Fatalf("unexpected cc value: %q", got)
+				}
+			},
 		},
 		{
-			name:     "empty query",
-			query:    core.Query{Text: ""},
-			wantErr:  true,
-			wantCont: "",
+			name:    "empty fields return error",
+			query:   core.Query{},
+			wantErr: true,
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := BuildImageURL(tt.query)
-
 			if (err != nil) != tt.wantErr {
-				t.Errorf("BuildImageURL() error = %v, wantErr %v", err, tt.wantErr)
+				t.Fatalf("BuildImageURL() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if tt.wantErr {
 				return
 			}
 
-			if !tt.wantErr && got != "" {
-				if !strings.Contains(got, tt.wantCont) {
-					t.Errorf("BuildImageURL() = %v, should contain %v", got, tt.wantCont)
-				}
-
-				// Test that URL is valid
-				_, err := url.Parse(got)
-				if err != nil {
-					t.Errorf("BuildImageURL() returned invalid URL: %v", err)
-				}
-
-				// Should be a Bing images URL
-				if !strings.Contains(got, "bing.com/images/search") {
-					t.Errorf("BuildImageURL() should return Bing images URL, got: %v", got)
-				}
+			parsed, err := url.Parse(got)
+			if err != nil {
+				t.Fatalf("BuildImageURL() returned invalid URL: %v", err)
+			}
+			if tt.check != nil {
+				tt.check(t, parsed.Query(), parsed.Host)
 			}
 		})
 	}
 }
-
-func TestBingImageSearch(t *testing.T) {
-	bing := New(*browser, core.SearchEngineOptions{RateTime: 5})
-
-	query := core.Query{
-		Text:     "golden puppy",
-		Limit:    25,
-		Filetype: "jpg",
-	}
-
-	results, err := bing.SearchImage(query)
-	if err != nil {
-		t.Fatalf("Cannot search Bing images: %s", err)
-	}
-
-	if len(results) == 0 {
-		t.Fatalf("Bing image search returned empty result")
-	}
-
-	// Check that we have image results with proper fields
-	firstResult := results[0]
-	if firstResult.URL == "" {
-		t.Errorf("First result missing image URL: %+v", firstResult)
-	}
-	if firstResult.Title == "" {
-		t.Errorf("First result missing title: %+v", firstResult)
-	}
-
-	// Check that we have either image URL or source URL
-	hasImageURL := firstResult.URL != ""
-	hasSourceURL := firstResult.URL != ""
-	if !hasImageURL && !hasSourceURL {
-		t.Errorf("First result should have either image URL or source URL: %+v", firstResult)
-	}
-
-	// For image results, URL should typically point to an image file
-	if hasImageURL {
-		// Check if it looks like an image URL (common extensions)
-		imageExtensions := []string{".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp"}
-		hasImageExtension := false
-		for _, ext := range imageExtensions {
-			if strings.Contains(strings.ToLower(firstResult.URL), ext) {
-				hasImageExtension = true
-				break
-			}
-		}
-		if !hasImageExtension {
-			t.Logf("Image URL doesn't have common extension (might be valid): %s", firstResult.URL)
-		}
-	}
-
-	t.Logf("Found %d image results", len(results))
-	t.Logf("First result - Title: %s", firstResult.Title)
-	t.Logf("First result - Image URL: %s", firstResult.URL)
-	t.Logf("First result - Source URL: %s", firstResult.URL)
-}

bing/features.go

@@ -0,0 +1,55 @@
+package bing
+
+import (
+	"context"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/go-rod/rod"
+	"github.com/karust/openserp/core"
+)
+
+func extractBingFeatures(doc *goquery.Document) []core.SerpFeature {
+	return core.ExtractSerpFeaturesBySelectors(doc, []core.SerpFeatureSelector{
+		{
+			Type: core.ResultTypeAnswerBox,
+			// li.b_ans also wraps related modules; require answer payload.
+			Container:     []string{"li.b_ans:has(.b_focusTextLarge)", "li.b_ans:has(.b_focusLabel)", "li.b_ans:has(.b_xlText)", "li.b_ans:has(.b_factrow)"},
+			TitleSelector: []string{".b_focusLabel", "h2"},
+			TextSelector:  []string{".b_focusTextLarge", ".b_xlText", ".b_vPanel .b_factrow", ".b_caption p"},
+			LinkSelector:  []string{"a[href^='http']"},
+			Position:      1,
+			Confidence:    0.8,
+		},
+		{
+			Type:         core.ResultTypeRelatedQuestions,
+			Title:        "People also ask",
+			Container:    []string{".b_rrsr", ".rqnaacfacc", "li.b_ans:has(.df_alaskcr)"},
+			ItemSelector: []string{".df_qntext", ".rqnaacfacc a", "li a"},
+			LinkSelector: []string{"a[href^='http']"},
+			Confidence:   0.7,
+		},
+		{
+			Type:         core.ResultTypeRelatedSearches,
+			Title:        "Related searches",
+			Container:    []string{"#brsv3", "#rs_root", "#inline_rs", "#brs", "#b_rs", "ol#b_rs", "li.b_rs"},
+			ItemSelector: []string{"li.rslist a", "li a", "a"},
+			LinkSelector: []string{"a[href^='http']", "a"},
+			Confidence:   0.75,
+			SingleMatch:  true,
+		},
+		{
+			Type:          core.ResultTypeAISummary,
+			Title:         "AI answer",
+			Container:     []string{".developer_answercard_wrapper", "#ca_main", ".ca_container", "#b_sydConvCont", ".b_sydConvCont", "[data-testid='bing-chat-answer']"},
+			TitleSelector: []string{"h2.b_topTitle", ".b_sydAns"},
+			TextSelector:  []string{".devmag_card_content", ".rd_def_list", ".b_sydAns", "[data-testid='answer']", ".ca_div", "p"},
+			LinkSelector:  []string{".rd_cnt_srcs a[href^='http']", ".rd_gencon_attr a[href^='http']", "h2.b_topTitle a[href^='http']", "a[href^='http']"},
+			Position:      1,
+			Confidence:    0.6,
+		},
+	})
+}
+
+func extractBingFeaturesFromPage(ctx context.Context, page *rod.Page) []core.SerpFeature {
+	return core.FeaturesFromPageWithWait(ctx, page, extractBingFeatures)
+}

bing/parse_html.go

@@ -0,0 +1,143 @@
+package bing
+
+import (
+	"errors"
+	"io"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/karust/openserp/core"
+)
+
+// ParseHTML parses a Bing SERP HTML document and returns search results.
+// Mirrors the rod-based parser in search.go but operates on a goquery doc.
+// No network I/O.
+func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
+	doc, err := goquery.NewDocumentFromReader(r)
+	if err != nil {
+		return nil, err
+	}
+	pageStatus := classifyBingDocument(doc)
+	if errors.Is(pageStatus, core.ErrEmptyResult) {
+		return []core.SearchResult{}, nil
+	}
+	if pageStatus != nil {
+		return nil, pageStatus
+	}
+	return parseBingDocument(doc), nil
+}
+
+func classifyBingDocument(doc *goquery.Document) error {
+	return core.ClassifyChallengeDocument(doc, core.DocSignals{
+		CaptchaSelectors: Selectors.Captcha,
+		CaptchaMarkers:   Selectors.CaptchaMarkers,
+		EmptyMarkers:     Selectors.NoResultsMarkers,
+	})
+}
+
+func parseBingDocument(doc *goquery.Document) []core.SearchResult {
+	var results []core.SearchResult
+	rank := core.NewRankState(0)
+
+	doc.Find(Selectors.ResultItems).Each(func(_ int, item *goquery.Selection) {
+		isAd := item.Is(Selectors.Ads)
+		isOrganic := item.Is(Selectors.Results)
+		if !isAd && !isOrganic {
+			return
+		}
+
+		titleSelector := Selectors.Title
+		if isAd {
+			titleSelector = Selectors.AdTitle
+		}
+
+		titleTag := item.Find(titleSelector).First()
+		if titleTag.Length() == 0 {
+			return
+		}
+		href, _ := titleTag.Attr("href")
+		title := bingDocumentTitle(item, titleTag)
+		desc := bingDocumentDescription(item, title)
+
+		if res, ok := assembleBingRow(href, title, desc, isAd, rank); ok {
+			results = append(results, res)
+		}
+	})
+
+	return core.AttachFeaturesToFirstResult(core.DeduplicateResults(results), extractBingFeatures(doc))
+}
+
+// assembleBingRow validates an already-extracted Bing row and assigns ranks.
+// Shared by the rod (browser) and goquery (raw / parse) parsers, which differ
+// only in how they pull title/href/desc out of the DOM.
+func assembleBingRow(href, title, desc string, isAd bool, rank *core.RankState) (core.SearchResult, bool) {
+	url := strings.TrimSpace(href)
+	if url == "" || url == "#" || strings.HasPrefix(url, "javascript:") {
+		return core.SearchResult{}, false
+	}
+	if title == "" {
+		return core.SearchResult{}, false
+	}
+
+	resultRank, absoluteRank := rank.Next(isAd)
+	return core.SearchResult{
+		Rank:         resultRank,
+		AbsoluteRank: absoluteRank,
+		URL:          url,
+		Title:        title,
+		Description:  desc,
+		Ad:           isAd,
+	}, true
+}
+
+// bingDocumentTitle reproduces the rod path's title fallback for goquery: the
+// title anchor's aria-label/title attribute, then its text, then any fallback
+// selector's text or aria-label.
+func bingDocumentTitle(item, titleTag *goquery.Selection) string {
+	if title := firstNonEmptyAttr(titleTag, "aria-label", "title"); title != "" {
+		return title
+	}
+	if title := core.NormalizeWhitespace(titleTag.Text()); title != "" {
+		return title
+	}
+	for _, selector := range Selectors.TitleFallbacks {
+		tag := item.Find(selector).First()
+		if tag.Length() == 0 {
+			continue
+		}
+		if text := core.NormalizeWhitespace(tag.Text()); text != "" {
+			return text
+		}
+		if label := firstNonEmptyAttr(tag, "aria-label", "title"); label != "" {
+			return label
+		}
+	}
+	return ""
+}
+
+// bingDocumentDescription reproduces the rod path's 3-selector description
+// fallback plus the strip-title structural fallback. Bing renders snippet text
+// with heavy source-indentation whitespace, so each candidate is collapsed.
+func bingDocumentDescription(item *goquery.Selection, title string) string {
+	for _, selector := range []string{Selectors.DescPrimary, Selectors.DescFallback, Selectors.DescAny} {
+		if tag := item.Find(selector).First(); tag.Length() > 0 {
+			if text := core.NormalizeWhitespace(tag.Text()); text != "" {
+				return text
+			}
+		}
+	}
+	return core.NormalizeWhitespace(strings.Replace(item.Text(), title, "", 1))
+}
+
+func firstNonEmptyAttr(item *goquery.Selection, attrs ...string) string {
+	for _, attr := range attrs {
+		value, exists := item.Attr(attr)
+		if !exists {
+			continue
+		}
+		if value = core.NormalizeWhitespace(value); value != "" {
+			return value
+		}
+	}
+	return ""
+}

bing/parse_html_test.go

@@ -0,0 +1,219 @@
+package bing
+
+import (
+	"bytes"
+	"errors"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/karust/openserp/core"
+)
+
+func TestParseBingHTML(t *testing.T) {
+	t.Parallel()
+
+	data, err := os.ReadFile("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("read fixture: %v", err)
+	}
+
+	results, err := ParseHTML(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+
+	rank := 0
+	for i, r := range results {
+		if r.Ad {
+			continue
+		}
+		rank++
+		if r.Rank != rank {
+			t.Fatalf("rank sequence broken at index %d: got %d, want %d", i, r.Rank, rank)
+		}
+		if r.URL == "" {
+			t.Fatalf("result %d: empty URL", i)
+		}
+		if r.Title == "" {
+			t.Fatalf("result %d: empty Title", i)
+		}
+		if !strings.HasPrefix(r.URL, "http") {
+			t.Fatalf("result %d: URL not absolute: %s", i, r.URL)
+		}
+	}
+	if rank == 0 {
+		t.Fatal("expected at least one organic result")
+	}
+}
+
+func TestParseBingHTMLEmpty(t *testing.T) {
+	t.Parallel()
+
+	results, err := ParseHTML(bytes.NewReader([]byte("")))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 0 {
+		t.Fatalf("expected zero results for empty HTML, got %d", len(results))
+	}
+}
+
+func TestParseBingHTMLCaptcha(t *testing.T) {
+	t.Parallel()
+
+	html := `<html><body><div class="captcha">Enter the characters you see</div></body></html>`
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if !errors.Is(err, core.ErrCaptcha) {
+		t.Fatalf("expected ErrCaptcha, got results=%d err=%v", len(results), err)
+	}
+}
+
+func TestParseBingHTMLNoResults(t *testing.T) {
+	t.Parallel()
+
+	html := `<html><body><main>There are no results for this search.</main></body></html>`
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 0 {
+		t.Fatalf("expected zero results, got %d", len(results))
+	}
+}
+
+func TestParseBingHTMLAds(t *testing.T) {
+	t.Parallel()
+
+	data, err := os.ReadFile("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("read fixture: %v", err)
+	}
+
+	results, err := ParseHTML(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+
+	var ads []struct{ url, title string }
+	for _, r := range results {
+		if r.Ad {
+			ads = append(ads, struct{ url, title string }{r.URL, r.Title})
+		}
+	}
+
+	for i, ad := range ads {
+		if ad.url == "" {
+			t.Fatalf("ad result %d: empty URL", i)
+		}
+		if ad.title == "" {
+			t.Fatalf("ad result %d: empty Title", i)
+		}
+	}
+}
+
+func TestParseBingHTMLMixedAdsKeepAbsoluteOrder(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<ol id="b_results">
+  <li class="b_algo">
+    <h2><a href="https://organic.example.com/one">Organic One</a></h2>
+    <div class="b_caption"><p>Organic snippet one</p></div>
+  </li>
+  <li class="b_ad">
+    <h2><a href="https://ads.example.com">Sponsored Result</a></h2>
+    <p>Paid snippet</p>
+  </li>
+  <li class="b_algo">
+    <h2><a href="https://organic.example.com/two">Organic Two</a></h2>
+    <div class="b_caption"><p>Organic snippet two</p></div>
+  </li>
+</ol>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 3 {
+		t.Fatalf("expected 3 results, got %d", len(results))
+	}
+	if results[0].Ad || !results[1].Ad || results[2].Ad {
+		t.Fatalf("unexpected ad ordering: %+v", results)
+	}
+	if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
+		t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
+	}
+	if results[2].Rank != 2 {
+		t.Fatalf("second organic rank = %d, want 2", results[2].Rank)
+	}
+}
+
+func TestParseBingHTMLTitleFallback(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<ol id="b_results">
+  <li class="b_algo">
+    <h2><a aria-label="Fallback Title" href="https://example.com/fallback"></a></h2>
+    <div class="b_caption"><p>Snippet</p></div>
+  </li>
+</ol>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 1 {
+		t.Fatalf("expected 1 result, got %d", len(results))
+	}
+	if results[0].Title != "Fallback Title" {
+		t.Fatalf("title = %q, want fallback", results[0].Title)
+	}
+}
+
+func TestParseBingHTMLPrefersTitleAttribute(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<ol id="b_results">
+  <li class="b_algo">
+    <h2><a aria-label="Real SERP Title" href="https://example.com/result">example.com</a></h2>
+    <div class="b_caption"><p>Snippet</p></div>
+  </li>
+</ol>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 1 {
+		t.Fatalf("expected 1 result, got %d", len(results))
+	}
+	if results[0].Title != "Real SERP Title" {
+		t.Fatalf("title = %q, want attribute title", results[0].Title)
+	}
+}
+
+func TestParseBingHTMLDescriptionFallsThroughEmptyPrimary(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<ol id="b_results">
+  <li class="b_algo">
+    <h2><a href="https://example.com/result">Result title</a></h2>
+    <div class="b_caption"><p>   </p><div>Useful snippet text</div></div>
+  </li>
+</ol>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 1 {
+		t.Fatalf("expected 1 result, got %d", len(results))
+	}
+	if results[0].Description != "Useful snippet text" {
+		t.Fatalf("description = %q, want fallback snippet", results[0].Description)
+	}
+}

bing/search.go

@@ -1,6 +1,7 @@
 package bing
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -10,15 +11,16 @@ import (
 	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/proto"
 	"github.com/karust/openserp/core"
-	"golang.org/x/time/rate"
 )
 
+// Bing implements core.SearchEngine for Bing SERP pages.
 type Bing struct {
 	core.Browser
 	core.SearchEngineOptions
 	logger *core.EngineLogger
 }
 
+// New creates a Bing engine instance with browser/runtime options applied.
 func New(browser core.Browser, opts core.SearchEngineOptions) *Bing {
 	bing := Bing{Browser: browser}
 	opts.Init()
@@ -27,23 +29,21 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *Bing {
 	return &bing
 }
 
+// Name returns the stable engine identifier.
 func (bing *Bing) Name() string {
 	return "bing"
 }
 
-func (bing *Bing) GetRateLimiter() *rate.Limiter {
-	ratelimit := rate.Every(bing.GetRatelimit())
-	return rate.NewLimiter(ratelimit, bing.RateBurst)
-}
-
 func (bing *Bing) getTotalResults(page *rod.Page) (int, error) {
-	results, err := page.Timeout(bing.GetSelectorTimeout()).Elements("li.b_algo")
+	results, err := page.Timeout(bing.GetSelectorTimeout()).Elements(Selectors.Results)
 	if err != nil {
 		return 0, errors.New("Cannot find result elements: " + err.Error())
 	}
 	return len(results), nil
 }
 
+// checkCaptcha classifies the live page the same way classifyBingDocument
+// (parse_html.go) classifies raw HTML, so /bing/search and /bing/parse agree.
 func (bing *Bing) checkCaptcha(page *rod.Page) bool {
 	if page == nil {
 		return false
@@ -56,48 +56,110 @@ func (bing *Bing) checkCaptcha(page *rod.Page) bool {
 		}
 	}
 
-	timeout := bing.GetSelectorTimeout() / 2
-	if timeout <= 0 {
-		timeout = time.Second * 2
-	}
-
-	selectors := []string{
-		"div.captcha",
-		"div.captcha_header",
-	}
-
-	for _, selector := range selectors {
-		has, err, _ := page.Timeout(timeout).Has(selector)
-		if err == nil && has {
+	for _, selector := range Selectors.Captcha {
+		has, _, _ := page.Has(selector)
+		if has {
 			bing.logger.Debug("Captcha detected: %s", selector)
 			return true
 		}
 	}
 
+	return pageTextContainsAny(page, Selectors.CaptchaMarkers)
+}
+
+// checkNoResults reports whether the page text matches Bing's no-results
+// phrasing, mirroring classifyBingDocument's text-marker check.
+func (bing *Bing) checkNoResults(page *rod.Page) bool {
+	return pageTextContainsAny(page, Selectors.NoResultsMarkers)
+}
+
+func pageTextContainsAny(page *rod.Page, markers []string) bool {
+	if page == nil || len(markers) == 0 {
+		return false
+	}
+	html, err := page.HTML()
+	if err != nil {
+		return false
+	}
+	text := strings.ToLower(html)
+	for _, marker := range markers {
+		if strings.Contains(text, marker) {
+			return true
+		}
+	}
 	return false
 }
 
-func (bing *Bing) acceptCookies(page *rod.Page) {
-	consentBtn, err := page.Timeout(bing.Timeout / 10).Element("button#bnp_btn_accept")
+func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error {
+	// Probe first so a banner-less SERP returns immediately instead of blocking
+	// .Element for the full Timeout/10.
+	if has, _, err := page.Has(Selectors.CookieBtn); err != nil || !has {
+		return nil
+	}
+	consentBtn, err := page.Timeout(bing.Timeout / 10).Element(Selectors.CookieBtn)
 	if err != nil {
-		return
+		return nil
 	}
-	consentBtn.Click(proto.InputMouseButtonLeft, 1)
-	time.Sleep(time.Millisecond * 500)
+	if err := consentBtn.Click(proto.InputMouseButtonLeft, 1); err != nil {
+		bing.logger.Debug("Cookie consent click failed: %v", err)
+	}
+
+	return core.SleepContext(ctx, 500*time.Millisecond)
 }
 
-func (bing *Bing) close(page *rod.Page) {
-	if !bing.Browser.LeavePageOpen {
-		if page != nil {
-			err := page.Close()
-			if err != nil {
-				bing.logger.Debug("Page close error: %v", err)
-			}
-		}
+func bingElementMatches(el *rod.Element, selector string) bool {
+	if el == nil {
+		return false
 	}
+	matches, err := el.Matches(selector)
+	return err == nil && matches
 }
 
-func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) {
+func (bing *Bing) parseResultElement(el *rod.Element, isAd bool, rank *core.RankState) (core.SearchResult, bool) {
+	titleSelector := Selectors.Title
+	if isAd {
+		titleSelector = Selectors.AdTitle
+	}
+
+	titleElem, err := el.Element(titleSelector)
+	if err != nil {
+		bing.logger.Debug("Missing title")
+		return core.SearchResult{}, false
+	}
+	href, err := titleElem.Property("href")
+	if err != nil {
+		bing.logger.Debug("Missing URL")
+		return core.SearchResult{}, false
+	}
+
+	title := core.ElementAttribute(titleElem, "aria-label", "title")
+	if title == "" {
+		title = core.ElementText(titleElem)
+	}
+	if title == "" {
+		title = core.FirstNonEmptyText(el, Selectors.TitleFallbacks...)
+	}
+	if title == "" {
+		title = core.FirstNonEmptyAttribute(el, "aria-label", Selectors.TitleFallbacks...)
+	}
+
+	desc := core.FirstNonEmptyText(el, Selectors.DescPrimary, Selectors.DescFallback, Selectors.DescAny)
+	if desc == "" {
+		fullText, _ := el.Text()
+		desc = core.NormalizeWhitespace(strings.Replace(fullText, title, "", 1))
+	}
+
+	return assembleBingRow(href.String(), title, desc, isAd, rank)
+}
+
+// Search executes a Bing web search and returns normalized search results.
+// It may return core.ErrCaptcha or core.ErrSearchTimeout.
+func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
+	ctx = core.PrepareEngineContext(ctx, query, bing.Name())
+	scoped := *bing
+	scoped.logger = bing.logger.WithRequest(ctx)
+	bing = &scoped
+
 	bing.logger.Debug("Starting search, query: %+v", query)
 
 	searchResults := []core.SearchResult{}
@@ -107,132 +169,69 @@ func (bing *Bing) Search(query core.Query) ([]core.SearchResult, error) {
 		return nil, err
 	}
 
-	page, err := bing.Navigate(url)
+	page, err := bing.Navigate(ctx, url)
 	if err != nil {
 		return nil, err
 	}
-	defer bing.close(page)
-
-	page.WaitLoad()
+	defer core.DeferClosePage(ctx, page, &bing.Browser)()
 
 	if bing.checkCaptcha(page) {
 		bing.logger.Error("Captcha detected: %s", url)
 		return nil, core.ErrCaptcha
 	}
 
-	bing.acceptCookies(page)
-	page.WaitLoad()
-
-	organicElements, err := page.Timeout(bing.Timeout).Elements("li.b_algo")
-	if err != nil {
-		bing.logger.Error("Cannot parse organic results: %s", err)
-		return nil, core.ErrSearchTimeout
+	if err := bing.acceptCookies(ctx, page); err != nil {
+		return nil, err
 	}
 
-	adElements, err := page.Timeout(bing.Timeout).Elements("li.b_ad")
+	resultElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.ResultItems, Selectors.Results}, bing.GetSelectorTimeout())
 	if err != nil {
-		bing.logger.Debug("No ads found")
+		// Re-check captcha/no-results on timeout - Bing interstitials and
+		// no-results pages can both render after WaitLoad.
+		if bing.checkCaptcha(page) {
+			bing.logger.Error("Captcha detected: %s", url)
+			return nil, core.ErrCaptcha
+		}
+		if bing.checkNoResults(page) {
+			return []core.SearchResult{}, nil
+		}
+		bing.logger.Error("Cannot parse organic results: %s", err)
+		return nil, core.ErrSearchTimeout
 	}
 
 	totalResults, err := bing.getTotalResults(page)
 	if err != nil {
 		bing.logger.Debug("Failed to get total results: %v", err)
 	}
-	bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements))
+	bing.logger.Info("Found %d organic result containers", totalResults)
 
-	rank := query.Start
-	for _, result := range organicElements {
-		srchRes := core.SearchResult{}
-
-		titleElem, err := result.Element("a")
-		if err != nil {
-			bing.logger.Debug("Missing title")
+	rank := core.NewRankStateAt(query.Start, query.Start+1)
+	for _, result := range resultElements {
+		isAd := bingElementMatches(result, Selectors.Ads)
+		isOrganic := bingElementMatches(result, Selectors.Results)
+		if !isAd && !isOrganic {
 			continue
 		}
-		srchRes.Title, _ = titleElem.Text()
 
-		href, err := titleElem.Property("href")
-		if err != nil {
-			bing.logger.Debug("Missing URL")
+		srchRes, ok := bing.parseResultElement(result, isAd, rank)
+		if !ok {
 			continue
 		}
-		srchRes.URL = href.String()
-
-		var desc string
-		if descElem, err := result.Element("div.b_caption p"); err == nil {
-			desc, _ = descElem.Text()
-		} else if descElem, err := result.Element("div.b_caption div"); err == nil {
-			desc, _ = descElem.Text()
-		} else if descElem, err := result.Element("p"); err == nil {
-			desc, _ = descElem.Text()
-		} else {
-			fullText, _ := result.Text()
-			desc = strings.TrimSpace(strings.Replace(fullText, srchRes.Title, "", 1))
-		}
-		srchRes.Description = desc
-
-		rank++
-		srchRes.Rank = rank
-		srchRes.Ad = false
-
-		searchResults = append(searchResults, srchRes)
-	}
-
-	for _, adResult := range adElements {
-		srchRes := core.SearchResult{Ad: true}
-
-		titleElem, err := adResult.Element("h2 a")
-		if err != nil {
-			bing.logger.Debug("Ad missing title")
-			continue
-		}
-		srchRes.Title, _ = titleElem.Text()
-
-		href, err := titleElem.Property("href")
-		if err != nil {
-			bing.logger.Debug("Ad missing URL")
-			continue
-		}
-		srchRes.URL = href.String()
-
-		if descElem, err := adResult.Element("p"); err == nil {
-			srchRes.Description, _ = descElem.Text()
-		}
-
-		// Mark ads with negative rank
-		srchRes.Rank = -1
 		searchResults = append(searchResults, srchRes)
 	}
 
 	// Deduplicate results
 	deduped := core.DeduplicateResults(searchResults)
 
-	// Trim to exact limit if necessary (only organic results, not ads)
-	if query.Limit > 0 {
-		organicResults := []core.SearchResult{}
-		adResults := []core.SearchResult{}
+	deduped = core.LimitOrganicResults(deduped, query.Limit)
 
-		for _, result := range deduped {
-			if result.Ad {
-				adResults = append(adResults, result)
-			} else {
-				organicResults = append(organicResults, result)
-			}
-		}
-
-		// Trim organic results to limit
-		if len(organicResults) > query.Limit {
-			organicResults = organicResults[:query.Limit]
-		}
-
-		// Combine back: organic results + ads
-		deduped = append(organicResults, adResults...)
+	if query.Features {
+		deduped = core.AttachFeaturesToFirstResult(deduped, extractBingFeaturesFromPage(ctx, page))
 	}
-
 	return deduped, nil
 }
 
-// BingImageData represents the JSON structure in the m attribute of image elements
+// BingImageData represents metadata encoded in the image result `m` attribute.
 type BingImageData struct {
 	T      string `json:"t"`      // Title
 	Desc   string `json:"desc"`   // Description
@@ -244,8 +243,24 @@ type BingImageData struct {
 	MURL   string `json:"murl"`   // Image URL
 }
 
-// SearchImage performs Bing image search and returns results
-func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
+func resolveImageLinkElement(container *rod.Element) (*rod.Element, error) {
+	if container == nil {
+		return nil, errors.New("nil image container")
+	}
+	if core.HasAttribute(container, "m") {
+		return container, nil
+	}
+	return container.Element("a")
+}
+
+// SearchImage executes a Bing image search and returns normalized image
+// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
+func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
+	ctx = core.PrepareEngineContext(ctx, query, bing.Name())
+	scoped := *bing
+	scoped.logger = bing.logger.WithRequest(ctx)
+	bing = &scoped
+
 	bing.logger.Debug("Starting image search, query: %+v", query)
 
 	searchResults := []core.SearchResult{}
@@ -256,13 +271,11 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		return nil, err
 	}
 
-	page, err := bing.Navigate(url)
+	page, err := bing.Navigate(ctx, url)
 	if err != nil {
 		return nil, err
 	}
-	defer bing.close(page)
-
-	page.WaitLoad()
+	defer core.DeferClosePage(ctx, page, &bing.Browser)()
 
 	// Check for captcha
 	if bing.checkCaptcha(page) {
@@ -271,15 +284,20 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
 	}
 
 	// Accept cookies if present
-	bing.acceptCookies(page)
+	if err := bing.acceptCookies(ctx, page); err != nil {
+		return nil, err
+	}
 
-	// Wait for image results to load
-	page.WaitLoad()
-	time.Sleep(time.Second * 2)
-
-	// Find all image result containers using CSS selector
-	imageContainers, err := page.Timeout(bing.Timeout).Elements("div.iuscp, div.isv")
+	imageContainers, _, err := core.WaitForElements(
+		ctx,
+		page,
+		[]string{Selectors.ImageResults},
+		bing.GetSelectorTimeout(),
+	)
 	if err != nil {
+		if bing.checkCaptcha(page) {
+			return nil, core.ErrCaptcha
+		}
 		bing.logger.Error("Cannot parse image results: %s", err)
 		return nil, core.ErrSearchTimeout
 	}
@@ -294,10 +312,9 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
 	for _, c := range imageContainers {
 		srchRes := core.SearchResult{}
 
-		// Get the <a> element inside the div
-		linkElem, err := c.Element("a")
+		linkElem, err := resolveImageLinkElement(c)
 		if err != nil {
-			bing.logger.Debug("Missing <a> element")
+			bing.logger.Debug("Missing image link element")
 			continue
 		}
 
@@ -326,12 +343,14 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		// Extract information from the parsed data
 		srchRes.Title = imgData.T
 		srchRes.URL = imgData.IMGURL
-		srchRes.Description = imgData.Desc
-
-		// Add dimensions to description if available
-		if imgData.W > 0 && imgData.H > 0 {
-			srchRes.Description += fmt.Sprintf(" (%dx%d)", imgData.W, imgData.H)
-		}
+		srchRes.Description = fmt.Sprintf(
+			"%s Source Page: %s, thumb_url:%s, %dx%d",
+			imgData.Desc,
+			imgData.PURL,
+			imgData.TURL,
+			imgData.W,
+			imgData.H,
+		)
 
 		// Get the page URL
 		if imgData.MURL != "" {
@@ -342,6 +361,9 @@ func (bing *Bing) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		srchRes.Rank = rank
 
 		searchResults = append(searchResults, srchRes)
+		if query.Limit > 0 && len(searchResults) >= query.Limit {
+			break
+		}
 	}
 
 	return searchResults, nil

bing/search_integration_test.go

@@ -0,0 +1,17 @@
+//go:build integration
+// +build integration
+
+package bing
+
+import (
+	"testing"
+
+	"github.com/karust/openserp/core"
+	"github.com/karust/openserp/testutil/ithelper"
+)
+
+func TestSearchBing(t *testing.T) {
+	ithelper.RunEngineTests(t, func(b *core.Browser) core.SearchEngine {
+		return New(*b, ithelper.EngineOptions())
+	})
+}

bing/selectors.go

@@ -0,0 +1,47 @@
+package bing
+
+// Selectors is the single source of truth for Bing SERP CSS selectors.
+var Selectors = struct {
+	Captcha          []string
+	CaptchaMarkers   []string
+	NoResultsMarkers []string
+	CookieBtn        string
+	ResultItems      string
+	Results          string
+	Ads              string
+	ImageResults     string
+	Title            string
+	TitleFallbacks   []string
+	DescPrimary      string
+	DescFallback     string
+	DescAny          string
+	AdTitle          string
+}{
+	Captcha: []string{"div.captcha", "div.captcha_header"},
+	// CaptchaMarkers/NoResultsMarkers are checked against lowercased page text
+	// as a fallback when the CSS selectors above don't match a challenge page.
+	CaptchaMarkers: []string{
+		"verify that you are not a robot",
+		"enter the characters you see",
+	},
+	NoResultsMarkers: []string{
+		"there are no results for",
+		"no results found for",
+	},
+	CookieBtn: "button#bnp_btn_accept",
+	// ResultItems matches the main-column children only, so carousels and
+	// "related searches" cards that reuse b_algo-style markup are excluded.
+	ResultItems:  "#b_results > li.b_algo, #b_results > li.b_ad",
+	Results:      "li.b_algo",
+	Ads:          "li.b_ad",
+	ImageResults: "a.iusc, div.iuscp, div.isv",
+	Title:        "h2 a",
+	// TitleFallbacks are tried when the primary Title selector matches but
+	// yields empty text (Bing occasionally renders an empty <h2><a/></h2>
+	// while the visible label sits in aria-label or h2).
+	TitleFallbacks: []string{"h2", "a[aria-label]"},
+	DescPrimary:    "div.b_caption p",
+	DescFallback:   "div.b_caption div",
+	DescAny:        "p",
+	AdTitle:        "h2 a",
+}

bing/serp_features_test.go

@@ -0,0 +1,193 @@
+package bing
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/karust/openserp/core"
+)
+
+// TestParseHTMLFixtureExtractsRealFeatures guards the live SERP fixture.
+func TestParseHTMLFixtureExtractsRealFeatures(t *testing.T) {
+	t.Parallel()
+	f, err := os.Open("testdata/search_results.html")
+	if err != nil {
+		t.Fatalf("open fixture: %v", err)
+	}
+	defer f.Close()
+
+	results, err := ParseHTML(f)
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+
+	related := findFeature(results, core.ResultTypeRelatedSearches)
+	if related == nil {
+		t.Fatalf("expected related_searches feature in fixture")
+	}
+	if len(related.Items) == 0 {
+		t.Fatalf("related_searches feature has no items: %#v", related)
+	}
+	for _, r := range results {
+		for _, feature := range r.Features {
+			if feature.Type == core.ResultTypeAnswerBox && strings.Contains(feature.Title, "searches you might like") {
+				t.Fatalf("related-search module leaked into answer_box: %#v", feature)
+			}
+		}
+	}
+
+	for i, r := range results {
+		if strings.ContainsAny(r.Description, "\n\t") {
+			t.Fatalf("result %d description has raw whitespace: %q", i, r.Description)
+		}
+	}
+}
+
+func TestParseHTMLExtractsSerpFeatures(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<ol id="b_results">
+  <li class="b_ans">
+    <h2>Bing answer</h2>
+    <div class="b_focusTextLarge">Bing answer text.</div>
+    <div class="b_caption"><p>Source snippet</p></div>
+    <a href="https://example.com/source">Source</a>
+  </li>
+  <li class="b_rrsr">
+    <h2>People also ask</h2>
+    <ul>
+      <li><a href="https://example.com/question">What is OpenSERP?</a></li>
+    </ul>
+  </li>
+  <li class="b_algo">
+    <h2><a href="https://example.com/result">Organic result</a></h2>
+    <div class="b_caption"><p>Snippet</p></div>
+  </li>
+</ol>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	assertFeatureType(t, results, core.ResultTypeAnswerBox)
+	assertFeatureType(t, results, core.ResultTypeRelatedQuestions)
+}
+
+// TestParseHTMLExtractsRelatedSearchesFromBrsContainer guards the classic footer.
+func TestParseHTMLExtractsRelatedSearchesFromBrsContainer(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<ol id="b_results">
+  <li class="b_algo">
+    <h2><a href="https://example.com/result">Organic result</a></h2>
+    <div class="b_caption"><p>Snippet</p></div>
+  </li>
+</ol>
+<div id="brs">
+  <ul>
+    <li><a href="https://www.bing.com/search?q=best+languages+2026">best languages 2026</a></li>
+    <li><a href="https://www.bing.com/search?q=easiest+language+to+learn">easiest language to learn</a></li>
+  </ul>
+</div>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	assertFeatureType(t, results, core.ResultTypeRelatedSearches)
+}
+
+// TestParseHTMLExtractsSimilarSearchesFromInlineRail guards the inline rail.
+func TestParseHTMLExtractsSimilarSearchesFromInlineRail(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<ol id="b_results">
+  <li class="b_algo">
+    <h2><a href="https://example.com/result">Organic result</a></h2>
+    <div class="b_caption"><p>Snippet</p></div>
+  </li>
+  <li class="b_ans">
+    <div id="inline_rs" class="b_hide">
+      <div id="rs_root" class="rsExplr">
+        <h2><a>Users also search for</a><a>Close</a></h2>
+        <ul>
+          <li class="rslist"><a href="https://www.bing.com/ck/a?u=a1aHR0cA"><span class="b_suggestionText">learn coding free</span></a></li>
+          <li class="rslist"><a href="https://www.bing.com/ck/a?u=a1aHR0cB"><span class="b_suggestionText">where to start programming</span></a></li>
+        </ul>
+      </div>
+    </div>
+  </li>
+</ol>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	related := findFeature(results, core.ResultTypeRelatedSearches)
+	if related == nil {
+		t.Fatalf("expected related_searches from inline rail")
+	}
+	if len(related.Items) != 2 {
+		t.Fatalf("expected 2 rail items (header chrome excluded), got %d: %#v", len(related.Items), related.Items)
+	}
+	for _, it := range related.Items {
+		if strings.Contains(it.Text, "Users also search") || it.Text == "Close" {
+			t.Fatalf("rail header chrome leaked as item: %q", it.Text)
+		}
+	}
+}
+
+func TestParseHTMLOrganicOnlyHasNoSerpFeatures(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<ol id="b_results">
+  <li class="b_algo">
+    <h2><a href="https://example.com/result">Organic result</a></h2>
+    <div class="b_caption"><p>Snippet</p></div>
+  </li>
+</ol>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	assertNoFeatures(t, results)
+}
+
+func findFeature(results []core.SearchResult, want core.ResultType) *core.SerpFeature {
+	for _, result := range results {
+		for i := range result.Features {
+			if result.Features[i].Type == want {
+				return &result.Features[i]
+			}
+		}
+	}
+	return nil
+}
+
+func assertFeatureType(t *testing.T, results []core.SearchResult, want core.ResultType) {
+	t.Helper()
+	for _, result := range results {
+		for _, feature := range result.Features {
+			if feature.Type == want {
+				return
+			}
+		}
+	}
+	t.Fatalf("expected feature type %q in %#v", want, results)
+}
+
+func assertNoFeatures(t *testing.T, results []core.SearchResult) {
+	t.Helper()
+	for _, result := range results {
+		if len(result.Features) > 0 {
+			t.Fatalf("expected no features, got %#v", result.Features)
+		}
+	}
+}

bing/url.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -12,6 +13,31 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+var bingDateOperatorRE = regexp.MustCompile(`(?i)\b(after|before):(\d{4}-\d{2}-\d{2})\b`)
+
+// defaultBingCountryByLanguage maps a language subtag to the country Bing
+// pairs with it for the "mkt" parameter when the caller did not specify one.
+// Languages outside this map fall back to "US" rather than echoing the
+// language code, since Bing rejects unknown markets like "ja-JA".
+var defaultBingCountryByLanguage = map[string]string{
+	"en": "US",
+	"de": "DE",
+	"ru": "RU",
+	"fr": "FR",
+	"es": "ES",
+	"it": "IT",
+	"pt": "BR",
+	"zh": "CN",
+	"ja": "JP",
+	"ko": "KR",
+	"nl": "NL",
+	"pl": "PL",
+	"tr": "TR",
+	"ar": "SA",
+}
+
+// BuildURL builds a Bing web search URL from Query fields.
+// It returns an error when query text or date parameters are invalid.
 func BuildURL(q core.Query) (string, error) {
 	base, err := url.Parse("https://www.bing.com")
 	if err != nil {
@@ -23,7 +49,13 @@ func BuildURL(q core.Query) (string, error) {
 
 	// Set search query text with operators
 	if q.Text != "" || q.Site != "" || q.Filetype != "" {
-		text := q.Text
+		text, textDateInterval, err := normalizeBingQueryText(q.Text)
+		if err != nil {
+			return "", err
+		}
+		if q.DateInterval == "" {
+			q.DateInterval = textDateInterval
+		}
 		if q.Site != "" {
 			text += " site:" + q.Site
 		}
@@ -31,7 +63,7 @@ func BuildURL(q core.Query) (string, error) {
 			text += " filetype:" + q.Filetype
 		}
 
-		logrus.Tracef("Query text: %s", text)
+		logrus.WithField("query_hash", core.QueryHash(text)).Trace(fmt.Sprintf("Query text: %s", text))
 		params.Add("q", text)
 	}
 
@@ -39,8 +71,14 @@ func BuildURL(q core.Query) (string, error) {
 		return "", errors.New("empty query built")
 	}
 
-	if q.LangCode != "" {
-		params.Add("setlang", strings.ToLower(q.LangCode))
+	if locale, ok := bingLocale(q.LangCode, q.Region); ok {
+		if locale.market != "" {
+			params.Add("mkt", locale.market)
+		}
+		if locale.language != "" {
+			params.Add("setlang", locale.language)
+		}
+		params.Add("cc", locale.country)
 	}
 
 	// Set result offset (pagination) - Bing uses "first" parameter.
@@ -51,48 +89,127 @@ func BuildURL(q core.Query) (string, error) {
 	if q.Start > 0 {
 		// Bing uses 1-based first-result index for pagination.
 		params.Add("first", strconv.Itoa(q.Start+1))
-	} else if q.Limit > 0 {
+	} else if q.Limit > 10 {
 		params.Add("count", strconv.Itoa(q.Limit))
 	}
 
-	// Set search date range - Bing supports date filtering via query text
 	if q.DateInterval != "" {
-		intervals := strings.Split(q.DateInterval, "..")
-		if len(intervals) != 2 {
-			return "", errors.New("incorrect date interval provided, expected format: YYYYMMDD..YYYYMMDD")
-		}
-
-		// Convert YYYYMMDD to YYYY-MM-DD format for Bing
-		startDate, err := time.Parse("20060102", intervals[0])
+		filter, err := buildBingDateFilter(q.DateInterval)
 		if err != nil {
-			return "", errors.New("invalid start date format, expected YYYYMMDD")
+			return "", err
 		}
-
-		endDate, err := time.Parse("20060102", intervals[1])
-		if err != nil {
-			return "", errors.New("invalid end date format, expected YYYYMMDD")
-		}
-
-		// Add date range to the search query text (Bing supports this format)
-		dateRange := fmt.Sprintf(" after:%s before:%s",
-			startDate.Format("2006-01-02"),
-			endDate.Format("2006-01-02"))
-
-		// Update the query text to include date range
-		currentQuery := params.Get("q")
-		params.Set("q", currentQuery+dateRange)
+		params.Add("filters", filter)
 	}
 
-	// Bing-specific parameters for consistent results
-	params.Add("form", "QBLH")        // Standard search form
-	params.Add("qs", "HS")            // Query suggestions
-	params.Add("sp", "-1")            // Search provider
-	params.Add("pq", params.Get("q")) // Previous query
+	// Omit autosuggest state; fresh sessions can reuse stale q otherwise.
 
 	base.RawQuery = params.Encode()
 	return base.String(), nil
 }
 
+type bingLocaleParams struct {
+	language string
+	country  string
+	market   string
+}
+
+// bingLocale resolves a Bing market triplet (language, country, mkt) from a
+// caller-supplied language code. It returns ok=false when the input is empty
+// so callers can omit Bing's locale parameters entirely instead of forcing a
+// default market that biases results toward en-US.
+func bingLocale(langCode, region string) (bingLocaleParams, bool) {
+	parsed := core.ParseLocale(langCode)
+	country := core.CountryFromRegion(region)
+	if parsed.Language == "" {
+		if country != "" {
+			return bingLocaleParams{country: country}, true
+		}
+		return bingLocaleParams{}, false
+	}
+
+	if country == "" {
+		country = parsed.Country
+		if country == "" {
+			country = defaultBingCountry(parsed.Language)
+		}
+	}
+	return bingLocaleParams{
+		language: parsed.Language,
+		country:  country,
+		market:   parsed.Language + "-" + country,
+	}, true
+}
+
+func defaultBingCountry(language string) string {
+	if country, ok := defaultBingCountryByLanguage[language]; ok {
+		return country
+	}
+	return "US"
+}
+
+func normalizeBingQueryText(text string) (string, string, error) {
+	matches := bingDateOperatorRE.FindAllStringSubmatch(text, -1)
+	if len(matches) == 0 {
+		return text, "", nil
+	}
+
+	var after, before string
+	for _, match := range matches {
+		if len(match) != 3 {
+			continue
+		}
+		switch strings.ToLower(match[1]) {
+		case "after":
+			after = strings.ReplaceAll(match[2], "-", "")
+		case "before":
+			before = strings.ReplaceAll(match[2], "-", "")
+		}
+	}
+
+	cleaned := bingDateOperatorRE.ReplaceAllString(text, "")
+	cleaned = strings.Join(strings.Fields(cleaned), " ")
+
+	if after == "" && before == "" {
+		return cleaned, "", nil
+	}
+	if after == "" || before == "" {
+		return cleaned, "", nil
+	}
+	if _, err := buildBingDateFilter(after + ".." + before); err != nil {
+		return "", "", err
+	}
+	return cleaned, after + ".." + before, nil
+}
+
+func buildBingDateFilter(dateInterval string) (string, error) {
+	intervals := strings.Split(dateInterval, "..")
+	if len(intervals) != 2 {
+		return "", errors.New("incorrect date interval provided, expected format: YYYYMMDD..YYYYMMDD")
+	}
+
+	startDate, err := time.Parse("20060102", intervals[0])
+	if err != nil {
+		return "", errors.New("invalid start date format, expected YYYYMMDD")
+	}
+
+	endDate, err := time.Parse("20060102", intervals[1])
+	if err != nil {
+		return "", errors.New("invalid end date format, expected YYYYMMDD")
+	}
+
+	if startDate.After(endDate) {
+		return "", errors.New("start date must not be after end date")
+	}
+
+	const secondsPerDay = int64(24 * 60 * 60)
+	startDay := startDate.Unix() / secondsPerDay
+	endDay := endDate.Unix() / secondsPerDay
+
+	return fmt.Sprintf(`ex1:"ez5_%d_%d"`, startDay, endDay), nil
+}
+
+// BuildImageURL builds a Bing image search URL from Query fields.
+// It returns an error when the resulting query text is empty.
 func BuildImageURL(q core.Query) (string, error) {
 	base, err := url.Parse("https://www.bing.com")
 	if err != nil {
@@ -114,9 +231,14 @@ func BuildImageURL(q core.Query) (string, error) {
 		return "", errors.New("empty query built")
 	}
 
-	// Add common parameters
-	if q.LangCode != "" {
-		params.Add("setlang", strings.ToLower(q.LangCode))
+	if locale, ok := bingLocale(q.LangCode, q.Region); ok {
+		if locale.market != "" {
+			params.Add("mkt", locale.market)
+		}
+		if locale.language != "" {
+			params.Add("setlang", locale.language)
+		}
+		params.Add("cc", locale.country)
 	}
 
 	// Image-specific parameters

cmd/captcha_config.go

@@ -0,0 +1,24 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+)
+
+func resolveCaptchaSolverConfig() (bool, string, error) {
+	apiKey := strings.TrimSpace(config.Config2Capcha.ApiKey)
+	if !config.Captcha.SolverEnabled {
+		if apiKey != "" {
+			logrus.Warn("2captcha.apikey is set but captcha.solver_enabled=false; solver will not run")
+		}
+		return false, "", nil
+	}
+
+	if apiKey == "" {
+		return false, "", fmt.Errorf("captcha solver is enabled (captcha.solver_enabled=true) but 2captcha.apikey is empty")
+	}
+
+	return true, apiKey, nil
+}

cmd/engines.go

@@ -0,0 +1,78 @@
+package cmd
+
+import (
+	"context"
+	"io"
+
+	"github.com/karust/openserp/baidu"
+	"github.com/karust/openserp/bing"
+	"github.com/karust/openserp/core"
+	"github.com/karust/openserp/duckduckgo"
+	"github.com/karust/openserp/ecosia"
+	"github.com/karust/openserp/google"
+	"github.com/karust/openserp/yandex"
+)
+
+// engineSpec is the single registry row for a search engine, driving CLI search,
+// raw dispatch, serve's browserEngineSpecs, and the alias/validation strings.
+// cfg points into the live config global; rawSearchFn is nil when an engine has
+// no browserless mode.
+type engineSpec struct {
+	name        string
+	aliases     []string
+	factory     func(core.Browser, core.SearchEngineOptions) core.SearchEngine
+	rawSearchFn func(context.Context, core.Query) ([]core.SearchResult, error)
+	parseHTMLFn func(io.Reader) ([]core.SearchResult, error)
+	cfg         *EngineConfig
+}
+
+func (s engineSpec) opts() core.SearchEngineOptions {
+	return s.cfg.SearchEngineOptions
+}
+
+func engineSpecs() []engineSpec {
+	return []engineSpec{
+		{name: "google", factory: newEngine(google.New), rawSearchFn: google.Search, parseHTMLFn: google.ParseHTML, cfg: &config.GoogleConfig},
+		{name: "yandex", factory: newEngine(yandex.New), rawSearchFn: yandex.Search, parseHTMLFn: yandex.ParseHTML, cfg: &config.YandexConfig},
+		{name: "baidu", factory: newEngine(baidu.New), rawSearchFn: baidu.Search, parseHTMLFn: baidu.ParseHTML, cfg: &config.BaiduConfig},
+		{name: "bing", factory: newEngine(bing.New), parseHTMLFn: bing.ParseHTML, cfg: &config.BingConfig},
+		{name: "duckduckgo", aliases: []string{"duck", "ddg"}, factory: newEngine(duckduckgo.New), parseHTMLFn: duckduckgo.ParseHTML, cfg: &config.DuckDuckGoConfig},
+		{name: "ecosia", factory: newEngine(ecosia.New), rawSearchFn: ecosia.Search, parseHTMLFn: ecosia.ParseHTML, cfg: &config.EcosiaConfig},
+	}
+}
+
+// newEngine adapts a concrete pkg.New (returning *Engine) to the
+// core.SearchEngine-typed factory the registry stores.
+func newEngine[T core.SearchEngine](ctor func(core.Browser, core.SearchEngineOptions) T) func(core.Browser, core.SearchEngineOptions) core.SearchEngine {
+	return func(b core.Browser, o core.SearchEngineOptions) core.SearchEngine {
+		return ctor(b, o)
+	}
+}
+
+// engineValidArgs returns every accepted engine token (canonical names +
+// aliases) for cobra's OnlyValidArgs validation.
+func engineValidArgs() []string {
+	specs := engineSpecs()
+	args := make([]string, 0, len(specs))
+	for _, s := range specs {
+		args = append(args, s.name)
+		args = append(args, s.aliases...)
+	}
+	return args
+}
+
+// resolveEngineSpec returns the spec whose canonical name or alias matches raw
+// (case/space already normalized by the caller), or false when unknown.
+func resolveEngineSpec(raw string) (engineSpec, bool) {
+	for _, s := range engineSpecs() {
+		if s.name == raw {
+			return s, true
+		}
+		for _, alias := range s.aliases {
+			if alias == raw {
+				return s, true
+			}
+		}
+	}
+	return engineSpec{}, false
+}

cmd/proxy_policy.go

@@ -13,6 +13,7 @@ func buildEngineProxyPolicyMap() map[string]string {
 		"baidu":      config.BaiduConfig.Proxy,
 		"bing":       config.BingConfig.Proxy,
 		"duckduckgo": config.DuckDuckGoConfig.Proxy,
+		"ecosia":     config.EcosiaConfig.Proxy,
 	}
 }
 

cmd/root.go

@@ -5,8 +5,11 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/karust/openserp/core"
+	browserprofile "github.com/karust/openserp/core/browser"
+	extractpkg "github.com/karust/openserp/extract"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -14,7 +17,7 @@ import (
 )
 
 const (
-	version               = "0.6.0"
+	version               = "0.8.12"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )
@@ -24,15 +27,18 @@ type Config struct {
 	App              AppConfig            `mapstructure:"app"`
 	Proxies          core.ProxiesConfig   `mapstructure:"proxies"`
 	Cache            CacheConfig          `mapstructure:"cache"`
+	Extract          extractpkg.Config    `mapstructure:"extract"`
 	Resilience       ResilienceConfig     `mapstructure:"resilience"`
 	CircuitBreaker   CircuitBreakerConfig `mapstructure:"circuit_breaker"`
 	CORS             CORSConfig           `mapstructure:"cors"`
+	Captcha          CaptchaConfig        `mapstructure:"captcha"`
 	Config2Capcha    Config2Captcha       `mapstructure:"2captcha"`
 	GoogleConfig     EngineConfig         `mapstructure:"google"`
 	YandexConfig     EngineConfig         `mapstructure:"yandex"`
 	BaiduConfig      EngineConfig         `mapstructure:"baidu"`
 	BingConfig       EngineConfig         `mapstructure:"bing"`
 	DuckDuckGoConfig EngineConfig         `mapstructure:"duckduckgo"`
+	EcosiaConfig     EngineConfig         `mapstructure:"ecosia"`
 }
 
 type Config2Captcha struct {
@@ -45,17 +51,25 @@ type ServerConfig struct {
 	ConfigPath    string `mapstructure:"config_path"`
 	IsDebug       bool   `mapstructure:"debug"`
 	IsVerbose     bool   `mapstructure:"verbose"`
+	IsQuiet       bool   `mapstructure:"quiet"`
 	IsRawRequests bool   `mapstructure:"raw_requests"`
 	Insecure      bool   `mapstructure:"insecure"`
 }
 
 type AppConfig struct {
-	Timeout       int    `mapstructure:"timeout"`
-	BrowserPath   string `mapstructure:"browser_path"`
-	IsBrowserHead bool   `mapstructure:"head"`
-	IsLeaveHead   bool   `mapstructure:"leave_head"`
-	IsLeakless    bool   `mapstructure:"leakless"`
-	IsStealth     bool   `mapstructure:"stealth"`
+	Timeout        int           `mapstructure:"timeout"`
+	BrowserPath    string        `mapstructure:"browser_path"`
+	ProfilesJSON   string        `mapstructure:"profiles"`
+	IsBrowserHead  bool          `mapstructure:"head"`
+	IsLeaveHead    bool          `mapstructure:"leave_head"`
+	IsLeakless     bool          `mapstructure:"leakless"`
+	BlockResources string        `mapstructure:"block_resources"`
+	BlockTrackers  bool          `mapstructure:"block_trackers"`
+	DebugEndpoints bool          `mapstructure:"debug_endpoints"`
+	LogFormat      string        `mapstructure:"log_format"`
+	MaxProcesses   int           `mapstructure:"max_processes"`
+	IdleTTL        time.Duration `mapstructure:"idle_ttl"`
+	MegaTimeout    time.Duration `mapstructure:"mega_timeout"`
 }
 
 type EngineConfig struct {
@@ -87,6 +101,10 @@ type CORSConfig struct {
 	MaxAge       int    `mapstructure:"max_age"`
 }
 
+type CaptchaConfig struct {
+	SolverEnabled bool `mapstructure:"solver_enabled"`
+}
+
 var config = Config{}
 
 var flagToConfigKey = map[string]string{
@@ -95,15 +113,17 @@ var flagToConfigKey = map[string]string{
 	"timeout":                 "app.timeout",
 	"config":                  "server.config_path",
 	"browser-path":            "app.browser_path",
+	"profiles-json":           "app.profiles",
 	"verbose":                 "server.verbose",
 	"debug":                   "server.debug",
+	"quiet":                   "server.quiet",
 	"head":                    "app.head",
 	"leakless":                "app.leakless",
 	"raw":                     "server.raw_requests",
 	"leave":                   "app.leave_head",
 	"2captcha_key":            "2captcha.apikey",
 	"proxy":                   "proxies.global",
-	"stealth":                 "app.stealth",
+	"debug-endpoints":         "app.debug_endpoints",
 	"insecure":                "server.insecure",
 	"cache_ttl":               "cache.ttl_seconds",
 	"cache_max_size":          "cache.max_size",
@@ -112,6 +132,7 @@ var flagToConfigKey = map[string]string{
 	"cb_failures":             "circuit_breaker.failures",
 	"cb_recovery":             "circuit_breaker.recovery_seconds",
 	"cb_successes":            "circuit_breaker.successes",
+	"log_format":              "app.log_format",
 }
 
 var RootCmd = &cobra.Command{
@@ -125,13 +146,84 @@ var RootCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
+		if err := browserprofile.LoadProfilesFromJSON(config.App.ProfilesJSON); err != nil {
+			return fmt.Errorf("load app.profiles: %w", err)
+		}
 
-		core.InitLogger(config.Server.IsVerbose, config.Server.IsDebug)
-		logrus.Debugf("Final config: %+v", config)
+		logFormat, err := core.NormalizeLogFormat(config.App.LogFormat)
+		if err != nil {
+			return err
+		}
+		config.App.LogFormat = logFormat
+
+		// One-shot CLI commands default to quiet so stdout is payload-only.
+		// Server mode keeps request logs unless server.quiet is set.
+		quiet := config.Server.IsQuiet
+		if commandDefaultsToQuiet(cmd) && !cmd.Flags().Changed("quiet") {
+			quiet = true
+		}
+		config.Server.IsQuiet = quiet
+
+		core.InitLogger(config.Server.IsVerbose, config.Server.IsDebug, quiet, config.App.LogFormat)
+		logrus.WithField("config", sanitizedConfigForLog(config)).Debug("Final config")
 		return nil
 	},
 }
 
+func commandDefaultsToQuiet(cmd *cobra.Command) bool {
+	return cmd != nil && cmd.Name() != serveCMD.Name()
+}
+
+func sanitizedConfigForLog(cfg Config) map[string]interface{} {
+	return map[string]interface{}{
+		"server": cfg.Server,
+		"app": map[string]interface{}{
+			"timeout":         cfg.App.Timeout,
+			"browser_path":    cfg.App.BrowserPath != "",
+			"profiles":        cfg.App.ProfilesJSON != "",
+			"head":            cfg.App.IsBrowserHead,
+			"leave_head":      cfg.App.IsLeaveHead,
+			"leakless":        cfg.App.IsLeakless,
+			"block_resources": cfg.App.BlockResources,
+			"block_trackers":  cfg.App.BlockTrackers,
+			"debug_endpoints": cfg.App.DebugEndpoints,
+			"log_format":      cfg.App.LogFormat,
+			"max_processes":   cfg.App.MaxProcesses,
+			"idle_ttl":        cfg.App.IdleTTL.String(),
+			"mega_timeout":    cfg.App.MegaTimeout.String(),
+		},
+		"proxies": map[string]interface{}{
+			"global":                  maskedProxyForLog(cfg.Proxies.Global),
+			"entries":                 len(cfg.Proxies.Entries),
+			"allow_request_proxy_url": cfg.Proxies.AllowRequestProxyURL,
+			"health":                  cfg.Proxies.Health,
+			"lanes":                   cfg.Proxies.Lanes,
+		},
+		"cache":           cfg.Cache,
+		"extract":         cfg.Extract,
+		"resilience":      cfg.Resilience,
+		"circuit_breaker": cfg.CircuitBreaker,
+		"cors":            cfg.CORS,
+		"captcha":         cfg.Captcha,
+		"2captcha": map[string]interface{}{
+			"apikey_configured": strings.TrimSpace(cfg.Config2Capcha.ApiKey) != "",
+		},
+		"google":     cfg.GoogleConfig,
+		"yandex":     cfg.YandexConfig,
+		"baidu":      cfg.BaiduConfig,
+		"bing":       cfg.BingConfig,
+		"duckduckgo": cfg.DuckDuckGoConfig,
+		"ecosia":     cfg.EcosiaConfig,
+	}
+}
+
+func maskedProxyForLog(proxyURL string) string {
+	if strings.TrimSpace(proxyURL) == "" {
+		return ""
+	}
+	return core.MaskProxyURL(proxyURL)
+}
+
 // Bind each cobra flag to its associated viper configuration (config file and environment variable)
 func bindFlags(cmd *cobra.Command, vpr *viper.Viper) {
 	cmd.Flags().VisitAll(func(flg *pflag.Flag) {
@@ -141,13 +233,13 @@ func bindFlags(cmd *cobra.Command, vpr *viper.Viper) {
 		}
 
 		if err := vpr.BindPFlag(configName, flg); err != nil {
-			logrus.Errorf("Unable to bind flag %s: %v", flg.Name, err)
+			logrus.WithError(err).Error(fmt.Sprintf("Unable to bind flag %s: %v", flg.Name, err))
 		}
 
 		if flg.Changed {
 			val, err := parseFlagValue(flg)
 			if err != nil {
-				logrus.Errorf("Unable to parse flag %s: %v", flg.Name, err)
+				logrus.WithError(err).Error(fmt.Sprintf("Unable to parse flag %s: %v", flg.Name, err))
 				return
 			}
 			vpr.Set(configName, val)
@@ -201,13 +293,20 @@ func initializeConfig(cmd *cobra.Command) error {
 		envKey := envPrefix + "_" + strings.ToUpper(strings.ReplaceAll(key, ".", "_"))
 		err := v.BindEnv(key, envKey)
 		if err != nil {
-			logrus.Errorf("Unable to bind ENV valye: %v", err)
+			logrus.WithError(err).Error(fmt.Sprintf("Unable to bind ENV value: %v", err))
 		}
 	}
 
 	// 3. Command flags (highest priority). Bind the current command's flags to viper
 	bindFlags(cmd, v)
 
+	// Keep compatibility with historical typo in local configs. Runs after all
+	// sources are merged so CLI flags and env vars take precedence over the typo key.
+	if v.IsSet("app.block_resorces") && !v.IsSet("app.block_resources") {
+		v.Set("app.block_resources", v.Get("app.block_resorces"))
+		logrus.Warn(`config key "app.block_resorces" is deprecated, use "app.block_resources"`)
+	}
+
 	if err := validateRemovedConfigPaths(v); err != nil {
 		return err
 	}
@@ -222,20 +321,20 @@ func initializeConfig(cmd *cobra.Command) error {
 		return fmt.Errorf("cannot unmarshall config: %v", err)
 	}
 
+	if _, err := core.ParseBlockedResourceTypes(config.App.BlockResources); err != nil {
+		return fmt.Errorf("invalid app.block_resources: %w", err)
+	}
+
 	config.Proxies, err = core.NormalizeProxiesConfig(config.Proxies)
 	if err != nil {
 		return fmt.Errorf("invalid proxies config: %w", err)
 	}
 
-	if config.Server.IsDebug {
-		logrus.Debug("Viper config:")
-		v.Debug()
-	}
 	return nil
 }
 
 func validateEngineProxyTags(v *viper.Viper) error {
-	for _, engineName := range []string{"google", "yandex", "baidu", "bing", "duckduckgo"} {
+	for _, engineName := range []string{"google", "yandex", "baidu", "bing", "duckduckgo", "ecosia"} {
 		key := engineName + ".proxy"
 		if !v.IsSet(key) {
 			continue
@@ -295,22 +394,40 @@ func setConfigDefaults(v *viper.Viper) {
 	v.SetDefault("server.port", 7070)
 	v.SetDefault("server.debug", false)
 	v.SetDefault("server.verbose", false)
+	v.SetDefault("server.quiet", false)
 	v.SetDefault("server.raw_requests", false)
 	v.SetDefault("server.insecure", false)
+	v.SetDefault("app.log_format", "")
 
 	v.SetDefault("app.timeout", 30)
 	v.SetDefault("app.browser_path", "")
+	v.SetDefault("app.profiles", "")
 	v.SetDefault("app.head", false)
 	v.SetDefault("app.leave_head", false)
 	v.SetDefault("app.leakless", false)
-	v.SetDefault("app.stealth", false)
+	v.SetDefault("app.block_resources", "")
+	v.SetDefault("app.block_trackers", false)
+	v.SetDefault("app.debug_endpoints", false)
+	v.SetDefault("app.max_processes", 4)
+	v.SetDefault("app.idle_ttl", "10m")
+	v.SetDefault("app.mega_timeout", "90s")
 
 	v.SetDefault("proxies.entries", []interface{}{})
 	v.SetDefault("proxies.global", "")
+	v.SetDefault("proxies.allow_request_proxy_url", false)
 	v.SetDefault("proxies.health.failure_threshold", core.DefaultProxyFailureThreshold)
+	v.SetDefault("proxies.lanes.enabled", true)
+	v.SetDefault("proxies.lanes.max_lanes", core.DefaultProxyLaneMaxLanes)
+	v.SetDefault("proxies.lanes.drop_cookies_on_challenge", true)
 
 	v.SetDefault("cache.ttl_seconds", 300)
 	v.SetDefault("cache.max_size", 1000)
+	v.SetDefault("extract.enabled", true)
+	v.SetDefault("extract.default_mode", "auto")
+	v.SetDefault("extract.timeout", "20s")
+	v.SetDefault("extract.max_bytes", 2*1024*1024)
+	v.SetDefault("extract.max_concurrent", 2)
+	v.SetDefault("extract.allow_private_networks", false)
 	// Keep stage2 defaults stable even when config file is absent.
 	v.SetDefault("resilience.max_retries", 3)
 	v.SetDefault("resilience.allow_endpoint_fallback", false)
@@ -320,8 +437,9 @@ func setConfigDefaults(v *viper.Viper) {
 	v.SetDefault("cors.enabled", true)
 	v.SetDefault("cors.allow_origins", "*")
 	v.SetDefault("cors.allow_methods", "GET, POST, OPTIONS")
-	v.SetDefault("cors.allow_headers", "Origin, Content-Type, Accept, Authorization, X-Use-Proxy")
+	v.SetDefault("cors.allow_headers", "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Proxy-URL, X-Proxy-Country, X-Proxy-Class, X-Proxy-Provider, X-Proxy-Session-ID, X-Request-ID, X-Tenant")
 	v.SetDefault("cors.max_age", 86400)
+	v.SetDefault("captcha.solver_enabled", false)
 }
 
 func init() {
@@ -330,15 +448,17 @@ func init() {
 	RootCmd.PersistentFlags().IntVarP(&config.App.Timeout, "timeout", "t", 30, "Timeout to fail request")
 	RootCmd.PersistentFlags().StringVarP(&config.Server.ConfigPath, "config", "c", "", "Configuration file path")
 	RootCmd.PersistentFlags().StringVarP(&config.App.BrowserPath, "browser-path", "", "", "Custom browser binary path (Chrome/Chromium/Edge/Brave..)")
+	RootCmd.PersistentFlags().StringVar(&config.App.ProfilesJSON, "profiles", "", "Path to browser profile catalog JSON")
 	RootCmd.PersistentFlags().BoolVarP(&config.Server.IsVerbose, "verbose", "v", false, "Use verbose output")
 	RootCmd.PersistentFlags().BoolVarP(&config.Server.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser")
+	RootCmd.PersistentFlags().BoolVarP(&config.Server.IsQuiet, "quiet", "q", false, "Suppress info logs on stderr (default for CLI commands)")
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsBrowserHead, "head", "", false, "Enable browser UI")
-	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search")
+	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to ensure browser instances are closed after search")
 	RootCmd.PersistentFlags().BoolVarP(&config.Server.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
 	RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
 	RootCmd.PersistentFlags().StringVarP(&config.Proxies.Global, "proxy", "x", "", "Force a single proxy for all engines (same as proxies.global)")
-	RootCmd.PersistentFlags().BoolVarP(&config.App.IsStealth, "stealth", "s", false, "Use stealth browser plugin")
+	RootCmd.PersistentFlags().BoolVar(&config.App.DebugEndpoints, "debug-endpoints", false, "Enable debug-only HTTP endpoints")
 	RootCmd.PersistentFlags().BoolVarP(&config.Server.Insecure, "insecure", "k", false, "Allow insecure TLS connections")
 	RootCmd.PersistentFlags().IntVar(&config.Cache.TTLSeconds, "cache_ttl", 300, "Cache TTL in seconds (0 to disable)")
 	RootCmd.PersistentFlags().IntVar(&config.Cache.MaxSize, "cache_max_size", 1000, "Maximum number of cached responses")
@@ -347,4 +467,5 @@ func init() {
 	RootCmd.PersistentFlags().IntVar(&config.CircuitBreaker.Failures, "cb_failures", 5, "Consecutive failures before circuit breaker opens")
 	RootCmd.PersistentFlags().IntVar(&config.CircuitBreaker.RecoverySeconds, "cb_recovery", 60, "Seconds before retrying an engine with open circuit")
 	RootCmd.PersistentFlags().IntVar(&config.CircuitBreaker.Successes, "cb_successes", 2, "Consecutive successful half-open checks needed to close circuit")
+	RootCmd.PersistentFlags().StringVar(&config.App.LogFormat, "log_format", "", "Log format: json or text (default: json in production, text in debug)")
 }

cmd/search.go

@@ -1,37 +1,95 @@
 package cmd
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
+	"net/url"
 	"strings"
+	"sync"
 	"time"
 
-	"github.com/karust/openserp/baidu"
-	"github.com/karust/openserp/bing"
+	"github.com/google/uuid"
 	"github.com/karust/openserp/core"
-	"github.com/karust/openserp/duckduckgo"
-	"github.com/karust/openserp/google"
-	"github.com/karust/openserp/yandex"
+	extractpkg "github.com/karust/openserp/extract"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
-var searchCMD = &cobra.Command{
-	Use:     "search",
-	Aliases: []string{"find"},
-	Short:   "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo)",
-	Args:    cobra.MatchAll(cobra.OnlyValidArgs, cobra.ExactArgs(2)),
-	Run:     search,
+// searchFlags holds the per-invocation CLI flags for the search command.
+type searchFlags struct {
+	limit    int
+	lang     string
+	region   string
+	start    int
+	site     string
+	filetype string
+	format   string
+	full     bool
+	features bool
+	extract  int
+	timeout  int
 }
 
-func search(cmd *cobra.Command, args []string) {
+var searchOpts searchFlags
+
+var searchCMD = &cobra.Command{
+	Use:     "search [engine] [query]",
+	Aliases: []string{"find"},
+	Short:   "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo, ecosia)",
+	// Validate the engine ourselves; cobra.OnlyValidArgs would also reject the
+	// query arg. ValidArgs still feeds shell completion.
+	Args:      cobra.MatchAll(cobra.ExactArgs(2), validateEngineArg),
+	ValidArgs: engineValidArgs(),
+	RunE:      search,
+}
+
+// validateEngineArg checks args[0] against the registry with a clear error,
+// without rejecting the query arg.
+func validateEngineArg(cmd *cobra.Command, args []string) error {
+	if len(args) == 0 {
+		return nil
+	}
+	if _, ok := resolveEngineSpec(normalizeEngineArg(args[0])); !ok {
+		return fmt.Errorf("unknown engine %q; valid: %s", args[0], strings.Join(engineValidArgs(), ", "))
+	}
+	return nil
+}
+
+func search(cmd *cobra.Command, args []string) error {
+	// Already validated by validateEngineArg, so this can't miss.
 	engineType := normalizeEngineArg(args[0])
+	spec, _ := resolveEngineSpec(engineType)
+
+	format, err := normalizeSearchFormat(searchOpts.format)
+	if err != nil {
+		return err
+	}
+
+	limit := searchOpts.limit
+	if limit <= 0 {
+		limit = 10
+	}
 	query := core.Query{
 		Text:     args[1],
-		Limit:    10,
+		LangCode: searchOpts.lang,
+		Region:   searchOpts.region,
+		Site:     searchOpts.site,
+		Filetype: searchOpts.filetype,
+		Limit:    limit,
+		Start:    searchOpts.start,
 		Filter:   true,
+		Features: searchOpts.features,
 		Insecure: config.Server.Insecure,
 	}
+	if err := applyCLIExtractFlag(&query, searchOpts.extract); err != nil {
+		return err
+	}
+
+	captchaSolverEnabled, captchaSolverAPIKey, err := resolveCaptchaSolverConfig()
+	if err != nil {
+		return fmt.Errorf("validate captcha solver config: %w", err)
+	}
 
 	proxyRuntime := core.ProxyRuntimeBrowser
 	if config.Server.IsRawRequests {
@@ -40,51 +98,253 @@ func search(cmd *cobra.Command, args []string) {
 
 	proxyCfg, err := buildNormalizedProxyConfig(proxyRuntime)
 	if err != nil {
-		logrus.Errorf("Error validating proxy config: %v", err)
-		return
+		return fmt.Errorf("validate proxy config: %w", err)
 	}
 
 	policy := resolveEngineProxyPolicy(proxyCfg, engineType)
 
 	selectedProxy, err := selectCLIProxy(proxyCfg, policy)
 	if err != nil {
-		logrus.Errorf("Error selecting proxy for %s: %v", engineType, err)
-		return
+		return fmt.Errorf("select proxy for %s: %w", engineType, err)
 	}
 
 	if config.Server.IsRawRequests {
 		query.ProxyURL = selectedProxy
 	}
 
-	logrus.Infof("Starting SERP search request using %s engine for query: %s", engineType, query.Text)
+	// Bound the whole search so a wedged Chrome can't hang the CLI forever.
+	timeoutSec := searchOpts.timeout
+	if timeoutSec <= 0 {
+		timeoutSec = 60
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSec)*time.Second)
+	defer cancel()
 
+	logrus.WithFields(logrus.Fields{
+		"engine":     engineType,
+		"query_hash": core.QueryHashFromQuery(query),
+	}).Info(fmt.Sprintf("Starting SERP search request using %s engine for query: %s", engineType, query.Text))
+
+	startedAt := time.Now()
 	var results []core.SearchResult
 	if config.Server.IsRawRequests {
-		logrus.Infof("Using raw requests mode for %s search", engineType)
-		results, err = searchRaw(engineType, query)
+		logrus.WithField("engine", engineType).Info(fmt.Sprintf("Using raw requests mode for %s search", engineType))
+		results, err = searchRaw(ctx, spec, query)
 	} else {
-		logrus.Infof("Using browser mode for %s search", engineType)
-		results, err = searchBrowser(engineType, query, selectedProxy)
+		logrus.WithField("engine", engineType).Info(fmt.Sprintf("Using browser mode for %s search", engineType))
+		results, err = searchBrowser(ctx, spec, query, selectedProxy, captchaSolverEnabled, captchaSolverAPIKey)
 	}
 
 	if err != nil {
-		logrus.Errorf("Error during %s search: %s", engineType, err)
-		return
+		return fmt.Errorf("%s search: %w", engineType, err)
 	}
 
-	logrus.Infof("Successfully completed SERP search using %s engine, returned %d results", engineType, len(results))
+	logrus.WithFields(logrus.Fields{
+		"engine":        engineType,
+		"results_count": len(results),
+	}).Info(fmt.Sprintf("Successfully completed SERP search using %s engine, returned %d results", engineType, len(results)))
 
-	b, err := json.MarshalIndent(results, "", " ")
-	if err != nil {
-		logrus.Error(err)
-		return
+	env := buildCLIEnvelope(spec.name, query, results, startedAt)
+	if query.Extract {
+		if err := enrichCLIEnvelopeWithExtraction(ctx, env, query, format, selectedProxy, captchaSolverEnabled, captchaSolverAPIKey); err != nil {
+			return fmt.Errorf("extract search results: %w", err)
+		}
 	}
-
-	fmt.Println(string(b))
+	payload := renderCLIEnvelope(env, format, searchOpts.full)
+	fmt.Println(strings.TrimRight(string(payload), "\n"))
+	return nil
 }
 
-func searchBrowser(engineType string, query core.Query, browserProxyURL string) ([]core.SearchResult, error) {
-	var engine core.SearchEngine
+func buildCLIEnvelope(engineName string, query core.Query, results []core.SearchResult, startedAt time.Time) *core.Envelope {
+	env := core.NewEnvelope(query, uuid.NewString(), startedAt, []string{engineName})
+	ectx := core.EnrichContext{Engine: engineName, Query: query}
+	for _, r := range results {
+		core.AppendEnrichedSearchResult(env, r, ectx, startedAt)
+	}
+	env.Finalize(startedAt, query)
+	return env
+}
+
+// renderCLIEnvelope renders a v2.1 envelope. JSON/ndjson always carry the full
+// envelope; text/markdown omit serp_features unless --full.
+func renderCLIEnvelope(env *core.Envelope, format string, full bool) []byte {
+	if !full && format != "json" && format != "ndjson" {
+		env.SerpFeatures = nil
+	}
+
+	switch format {
+	case "text":
+		return core.RenderText(env)
+	case "markdown":
+		return core.RenderMarkdown(env)
+	case "ndjson":
+		return core.RenderNDJSON(env)
+	default: // json
+		b, err := json.MarshalIndent(env, "", "  ")
+		if err != nil {
+			logrus.WithError(err).Error("marshal envelope")
+			return nil
+		}
+		return b
+	}
+}
+
+const maxCLIExtractTop = 5
+
+func applyCLIExtractFlag(query *core.Query, extractTop int) error {
+	top, err := normalizeCLIExtractTop(extractTop)
+	if err != nil {
+		return err
+	}
+	if top == 0 {
+		return nil
+	}
+	if !config.Extract.Enabled {
+		return fmt.Errorf("extraction is disabled in config")
+	}
+	query.Extract = true
+	query.ExtractTop = top
+	query.ExtractMode = string(extractpkg.ModeAuto)
+	return nil
+}
+
+func normalizeCLIExtractTop(raw int) (int, error) {
+	if raw < 0 {
+		return 0, fmt.Errorf("--extract must be a non-negative integer")
+	}
+	if raw > maxCLIExtractTop {
+		return maxCLIExtractTop, nil
+	}
+	return raw, nil
+}
+
+func enrichCLIEnvelopeWithExtraction(ctx context.Context, env *core.Envelope, query core.Query, format string, proxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) error {
+	if env == nil || !query.Extract {
+		return nil
+	}
+	query.ProxyURL = proxyURL
+	extractor, closeExtractor, err := newCLIExtractor(captchaSolverEnabled, captchaSolverAPIKey)
+	if err != nil {
+		return err
+	}
+	defer closeExtractor()
+
+	// Same depth bounds, batch deadline, and candidate fill-in as the HTTP server.
+	core.EnrichEnvelopeWithExtraction(ctx, env, query, format, extractor, config.Extract)
+	return nil
+}
+
+// newCLIExtractor builds an Extractor backed by a lazily-created, single-use
+// browser. The raw path delegates to core.RawExtractFetch; the rendered path
+// validates the target, gates auth'd SOCKS, then reuses core.RenderExtractHTML.
+func newCLIExtractor(captchaSolverEnabled bool, captchaSolverAPIKey string) (extractpkg.Extractor, func(), error) {
+	cfg := config.Extract.Normalized()
+	var browserMu sync.Mutex
+	var browser *core.Browser
+
+	closeExtractor := func() {
+		browserMu.Lock()
+		defer browserMu.Unlock()
+		if browser == nil {
+			return
+		}
+		if err := browser.Close(); err != nil {
+			logrus.WithError(err).Debug("Extraction browser close error")
+		}
+		browser = nil
+	}
+
+	extractor := extractpkg.Extractor{
+		Cfg: cfg,
+		RawFetch: func(ctx context.Context, req extractpkg.ExtractRequest) (*extractpkg.FetchResponse, error) {
+			return core.RawExtractFetch(ctx, req, cfg, config.Server.Insecure)
+		},
+		RenderedFetch: func(ctx context.Context, req extractpkg.ExtractRequest) (*extractpkg.FetchResponse, error) {
+			if err := validateCLIExtractTargetURL(ctx, req.URL, cfg.AllowPrivateNetworks); err != nil {
+				return nil, err
+			}
+			if core.IsAuthenticatedSocksProxyURL(req.ProxyURL) {
+				return nil, fmt.Errorf(
+					"%w: browser runtime does not support authenticated SOCKS proxy %s",
+					core.ErrProxyUnavailable,
+					core.MaskProxyURL(req.ProxyURL),
+				)
+			}
+
+			browserMu.Lock()
+			if browser == nil {
+				created, err := newCLIExtractBrowser(cfg, req.ProxyURL, captchaSolverEnabled, captchaSolverAPIKey)
+				if err != nil {
+					browserMu.Unlock()
+					return nil, err
+				}
+				browser = created
+			}
+			current := browser
+			browserMu.Unlock()
+
+			return core.RenderExtractHTML(ctx, current, req)
+		},
+	}
+	return extractor, closeExtractor, nil
+}
+
+func newCLIExtractBrowser(cfg extractpkg.Config, proxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) (*core.Browser, error) {
+	blockedResourceTypes, err := core.ParseBlockedResourceTypes(config.App.BlockResources)
+	if err != nil {
+		return nil, fmt.Errorf("invalid block_resources config: %w", err)
+	}
+	opts := core.BrowserOpts{
+		IsHeadless:           !config.App.IsBrowserHead && !config.Server.IsDebug,
+		IsLeakless:           config.App.IsLeakless,
+		Timeout:              cfg.Timeout,
+		LeavePageOpen:        false,
+		CaptchaSolverEnabled: captchaSolverEnabled,
+		CaptchaSolverApiKey:  captchaSolverAPIKey,
+		BrowserPath:          config.App.BrowserPath,
+		ProxyURL:             proxyURL,
+		Insecure:             config.Server.Insecure,
+		BlockResourceTypes:   blockedResourceTypes,
+		BlockTrackers:        config.App.BlockTrackers,
+	}
+	return core.NewBrowser(opts)
+}
+
+func validateCLIExtractTargetURL(ctx context.Context, rawURL string, allowPrivateNetworks bool) error {
+	targetURL := extractpkg.NormalizeURL(strings.TrimSpace(rawURL))
+	if allowPrivateNetworks {
+		parsed, err := url.ParseRequestURI(targetURL)
+		if err != nil {
+			return fmt.Errorf("invalid url: %w", err)
+		}
+		if parsed.Scheme != "http" && parsed.Scheme != "https" {
+			return fmt.Errorf("url must use http or https")
+		}
+		return nil
+	}
+	return core.ValidatePublicHTTPURL(ctx, targetURL)
+}
+
+func normalizeSearchFormat(raw string) (string, error) {
+	switch strings.ToLower(strings.TrimSpace(raw)) {
+	case "", "json":
+		return "json", nil
+	case "text", "txt":
+		return "text", nil
+	case "markdown", "md":
+		return "markdown", nil
+	case "ndjson", "jsonl":
+		return "ndjson", nil
+	default:
+		return "", fmt.Errorf("invalid --format %q; valid: json, text, markdown, ndjson", raw)
+	}
+}
+
+func searchBrowser(ctx context.Context, spec engineSpec, query core.Query, browserProxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) ([]core.SearchResult, error) {
+	blockedResourceTypes, err := core.ParseBlockedResourceTypes(config.App.BlockResources)
+	if err != nil {
+		return nil, fmt.Errorf("invalid block_resources config: %w", err)
+	}
 	if core.IsAuthenticatedSocksProxyURL(browserProxyURL) {
 		return nil, fmt.Errorf(
 			"%w: browser runtime does not support authenticated SOCKS proxy %s",
@@ -94,15 +354,17 @@ func searchBrowser(engineType string, query core.Query, browserProxyURL string)
 	}
 
 	opts := core.BrowserOpts{
-		IsHeadless:          !config.App.IsBrowserHead,
-		IsLeakless:          config.App.IsLeakless,
-		Timeout:             time.Second * time.Duration(config.App.Timeout),
-		LeavePageOpen:       config.App.IsLeaveHead,
-		CaptchaSolverApiKey: config.Config2Capcha.ApiKey,
-		BrowserPath:         config.App.BrowserPath,
-		ProxyURL:            browserProxyURL,
-		Insecure:            config.Server.Insecure,
-		UseStealth:          config.App.IsStealth,
+		IsHeadless:           !config.App.IsBrowserHead,
+		IsLeakless:           config.App.IsLeakless,
+		Timeout:              time.Second * time.Duration(config.App.Timeout),
+		LeavePageOpen:        config.App.IsLeaveHead,
+		CaptchaSolverEnabled: captchaSolverEnabled,
+		CaptchaSolverApiKey:  captchaSolverAPIKey,
+		BrowserPath:          config.App.BrowserPath,
+		ProxyURL:             browserProxyURL,
+		Insecure:             config.Server.Insecure,
+		BlockResourceTypes:   blockedResourceTypes,
+		BlockTrackers:        config.App.BlockTrackers,
 	}
 
 	if config.Server.IsDebug {
@@ -113,44 +375,24 @@ func searchBrowser(engineType string, query core.Query, browserProxyURL string)
 	if err != nil {
 		return nil, err
 	}
+	// Close the browser so Chromium never outlives the CLI run.
+	defer func() {
+		if closeErr := browser.Close(); closeErr != nil {
+			logrus.WithError(closeErr).Debug("Browser close error")
+		}
+	}()
 
-	switch strings.ToLower(engineType) {
-	case "yandex":
-		engine = yandex.New(*browser, config.YandexConfig.SearchEngineOptions)
-	case "google":
-		engine = google.New(*browser, config.GoogleConfig.SearchEngineOptions)
-	case "baidu":
-		engine = baidu.New(*browser, config.BaiduConfig.SearchEngineOptions)
-	case "bing":
-		engine = bing.New(*browser, config.BingConfig.SearchEngineOptions)
-	case "duckduckgo":
-		engine = duckduckgo.New(*browser, config.DuckDuckGoConfig.SearchEngineOptions)
-	default:
-		return nil, fmt.Errorf("no %q search engine found", engineType)
-	}
-
-	return engine.Search(query)
+	engine := spec.factory(*browser, spec.opts())
+	return engine.Search(ctx, query)
 }
 
-func searchRaw(engineType string, query core.Query) ([]core.SearchResult, error) {
+func searchRaw(ctx context.Context, spec engineSpec, query core.Query) ([]core.SearchResult, error) {
 	logrus.Warn("Browserless results are very inconsistent or may not even work!")
-
-	switch strings.ToLower(engineType) {
-	case "yandex":
-		return yandex.Search(query)
-	case "google":
-		return google.Search(query)
-	case "baidu":
-		return baidu.Search(query)
-	case "bing":
-		logrus.Warn("Bing does not support raw HTTP requests mode. Please use browser mode instead.")
-		return nil, fmt.Errorf("bing does not support raw requests mode")
-	case "duckduckgo":
-		logrus.Warn("DuckDuckGo does not support raw HTTP requests mode. Please use browser mode instead.")
-		return nil, fmt.Errorf("duckduckgo does not support raw requests mode")
-	default:
-		return nil, fmt.Errorf("no %q search engine found", engineType)
+	if spec.rawSearchFn == nil {
+		logrus.Warnf("%s does not support raw HTTP requests mode. Please use browser mode instead.", spec.name)
+		return nil, fmt.Errorf("%s does not support raw requests mode", spec.name)
 	}
+	return spec.rawSearchFn(ctx, query)
 }
 
 func selectCLIProxy(proxyCfg core.ProxyConfig, policy core.ProxyPolicy) (string, error) {
@@ -175,14 +417,20 @@ func selectCLIProxy(proxyCfg core.ProxyConfig, policy core.ProxyPolicy) (string,
 }
 
 func normalizeEngineArg(raw string) string {
-	switch strings.ToLower(strings.TrimSpace(raw)) {
-	case "duck":
-		return "duckduckgo"
-	default:
-		return strings.ToLower(strings.TrimSpace(raw))
-	}
+	return strings.ToLower(strings.TrimSpace(raw))
 }
 
 func init() {
+	searchCMD.Flags().IntVar(&searchOpts.limit, "limit", 10, "Maximum number of results")
+	searchCMD.Flags().StringVar(&searchOpts.lang, "lang", "", "Language hint (e.g. EN, DE, RU)")
+	searchCMD.Flags().StringVar(&searchOpts.region, "region", "", "Region/market hint (e.g. RU, en-US)")
+	searchCMD.Flags().IntVar(&searchOpts.start, "start", 0, "Pagination start offset")
+	searchCMD.Flags().StringVar(&searchOpts.site, "site", "", "Restrict results to a domain (e.g. github.com)")
+	searchCMD.Flags().StringVar(&searchOpts.filetype, "file", "", "File type filter (e.g. pdf)")
+	searchCMD.Flags().StringVar(&searchOpts.format, "format", "json", "Output format: json, text, markdown, ndjson")
+	searchCMD.Flags().BoolVar(&searchOpts.full, "full", false, "Include SERP features in text/markdown output")
+	searchCMD.Flags().BoolVar(&searchOpts.features, "features", false, "Parse SERP feature modules (browser mode)")
+	searchCMD.Flags().IntVar(&searchOpts.extract, "extract", 0, "Extract clean content from the top N results using auto mode (1-5)")
+	searchCMD.Flags().IntVar(&searchOpts.timeout, "search-timeout", 60, "Overall search timeout in seconds")
 	RootCmd.AddCommand(searchCMD)
 }

cmd/search_test.go

@@ -0,0 +1,82 @@
+package cmd
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/karust/openserp/core"
+	extractpkg "github.com/karust/openserp/extract"
+)
+
+func TestNormalizeCLIExtractTop(t *testing.T) {
+	tests := []struct {
+		name    string
+		raw     int
+		want    int
+		wantErr bool
+	}{
+		{name: "disabled", raw: 0, want: 0},
+		{name: "one", raw: 1, want: 1},
+		{name: "clamped", raw: 20, want: maxCLIExtractTop},
+		{name: "negative", raw: -1, wantErr: true},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := normalizeCLIExtractTop(tc.raw)
+			if tc.wantErr {
+				if err == nil {
+					t.Fatal("expected error")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if got != tc.want {
+				t.Fatalf("normalizeCLIExtractTop(%d) = %d, want %d", tc.raw, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestApplyCLIExtractFlagSetsAutoMode(t *testing.T) {
+	previous := config
+	config.Extract = extractpkg.DefaultConfig()
+	defer func() { config = previous }()
+
+	query := core.Query{Text: "weather today"}
+	if err := applyCLIExtractFlag(&query, 2); err != nil {
+		t.Fatalf("applyCLIExtractFlag() error = %v", err)
+	}
+	if !query.Extract {
+		t.Fatal("expected query.Extract")
+	}
+	if query.ExtractTop != 2 {
+		t.Fatalf("ExtractTop = %d, want 2", query.ExtractTop)
+	}
+	if query.ExtractMode != string(extractpkg.ModeAuto) {
+		t.Fatalf("ExtractMode = %q, want auto", query.ExtractMode)
+	}
+}
+
+func TestApplyCLIExtractFlagRequiresEnabledConfig(t *testing.T) {
+	previous := config
+	config.Extract = extractpkg.Config{Enabled: false}
+	defer func() { config = previous }()
+
+	query := core.Query{Text: "weather today"}
+	err := applyCLIExtractFlag(&query, 1)
+	if err == nil {
+		t.Fatal("expected error")
+	}
+	if !strings.Contains(err.Error(), "disabled") {
+		t.Fatalf("error = %q, want disabled message", err.Error())
+	}
+}
+
+func TestSearchCommandHasExtractFlag(t *testing.T) {
+	if searchCMD.Flags().Lookup("extract") == nil {
+		t.Fatal("expected search command to expose --extract")
+	}
+}

cmd/serve.go

@@ -1,17 +1,21 @@
 package cmd
 
 import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
 	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"os/signal"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
-	"github.com/karust/openserp/baidu"
-	"github.com/karust/openserp/bing"
 	"github.com/karust/openserp/core"
-	"github.com/karust/openserp/duckduckgo"
-	"github.com/karust/openserp/google"
-	"github.com/karust/openserp/yandex"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"golang.org/x/time/rate"
@@ -19,25 +23,22 @@ import (
 
 // rawEngine implements SearchEngine interface for raw HTTP requests
 type rawEngine struct {
-	name string
+	name      string
+	limiterMu sync.Mutex
+	limiter   *rate.Limiter
 }
 
-func (r *rawEngine) Search(q core.Query) ([]core.SearchResult, error) {
+func (r *rawEngine) Search(ctx context.Context, q core.Query) ([]core.SearchResult, error) {
 	q.Insecure = config.Server.Insecure
 
-	switch r.name {
-	case "google":
-		return google.Search(q)
-	case "yandex":
-		return yandex.Search(q)
-	case "baidu":
-		return baidu.Search(q)
-	default:
+	spec, ok := resolveEngineSpec(r.name)
+	if !ok || spec.rawSearchFn == nil {
 		return nil, fmt.Errorf("unsupported engine: %s", r.name)
 	}
+	return spec.rawSearchFn(ctx, q)
 }
 
-func (r *rawEngine) SearchImage(q core.Query) ([]core.SearchResult, error) {
+func (r *rawEngine) SearchImage(_ context.Context, _ core.Query) ([]core.SearchResult, error) {
 	return nil, fmt.Errorf("image search is not supported in raw mode for %s", r.name)
 }
 
@@ -50,8 +51,13 @@ func (r *rawEngine) IsInitialized() bool {
 }
 
 func (r *rawEngine) GetRateLimiter() *rate.Limiter {
-	// Use default rate limiter for raw requests
-	return rate.NewLimiter(rate.Every(time.Second), 5)
+	r.limiterMu.Lock()
+	defer r.limiterMu.Unlock()
+	if r.limiter == nil {
+		// Use default rate limiter for raw requests.
+		r.limiter = rate.NewLimiter(rate.Every(time.Second), 5)
+	}
+	return r.limiter
 }
 
 var serveCMD = &cobra.Command{
@@ -69,6 +75,12 @@ func serve(cmd *cobra.Command, args []string) {
 	corsCfg.AllowHeaders = config.CORS.AllowHeaders
 	corsCfg.MaxAge = config.CORS.MaxAge
 
+	captchaSolverEnabled, captchaSolverAPIKey, err := resolveCaptchaSolverConfig()
+	if err != nil {
+		logrus.Error(err)
+		os.Exit(1)
+	}
+
 	proxyRuntime := core.ProxyRuntimeBrowser
 	if config.Server.IsRawRequests {
 		proxyRuntime = core.ProxyRuntimeRaw
@@ -76,65 +88,87 @@ func serve(cmd *cobra.Command, args []string) {
 
 	proxyCfg, err := buildNormalizedProxyConfig(proxyRuntime)
 	if err != nil {
-		logrus.Errorf("invalid proxy configuration: %v", err)
+		logrus.WithError(err).Error(fmt.Sprintf("invalid proxy configuration: %v", err))
 		return
 	}
 
+	fingerprintBrowserOpts := buildFingerprintBrowserOptions()
+
 	if config.Server.IsRawRequests {
 		logrus.Warn("Browserless results are very inconsistent or may not even work!")
-		serverOpts := buildServerOptions(corsCfg, proxyCfg)
+		serverOpts := buildServerOptions(corsCfg, proxyCfg, fingerprintBrowserOpts)
 		serv := core.NewServerWithOptions(config.Server.Host, config.Server.Port, serverOpts,
 			&rawEngine{name: "google"},
 			&rawEngine{name: "yandex"},
 			&rawEngine{name: "baidu"},
+			&rawEngine{name: "ecosia"},
 		)
-		if err := serv.Listen(); err != nil {
+		if err := listenWithGracefulShutdown(serv, nil); err != nil {
 			logrus.Error(err)
 		}
 		return
 	}
 
-	baseOpts := core.BrowserOpts{
-		IsHeadless:          !config.App.IsBrowserHead,
-		IsLeakless:          config.App.IsLeakless,
-		Timeout:             time.Second * time.Duration(config.App.Timeout),
-		LeavePageOpen:       config.App.IsLeaveHead,
-		CaptchaSolverApiKey: config.Config2Capcha.ApiKey,
-		BrowserPath:         config.App.BrowserPath,
-		Insecure:            config.Server.Insecure,
-		UseStealth:          config.App.IsStealth,
-	}
-	if config.Server.IsDebug {
-		baseOpts.IsHeadless = false
-	}
+	baseOpts := fingerprintBrowserOpts
+	baseOpts.LeavePageOpen = config.App.IsLeaveHead
+	baseOpts.CaptchaSolverEnabled = captchaSolverEnabled
+	baseOpts.CaptchaSolverApiKey = captchaSolverAPIKey
 
-	engines, err := buildBrowserEngines(baseOpts, proxyCfg)
+	engines, closeBrowsers, browserResolver, err := buildBrowserEngines(baseOpts, proxyCfg)
 	if err != nil {
 		logrus.Error(err)
 		return
 	}
 
-	serverOpts := buildServerOptions(corsCfg, proxyCfg)
+	serverOpts := buildServerOptions(corsCfg, proxyCfg, fingerprintBrowserOpts)
+	serverOpts.BrowserResolver = browserResolver
 	serv := core.NewServerWithOptions(config.Server.Host, config.Server.Port, serverOpts, engines...)
-	if err := serv.Listen(); err != nil {
+	if err := listenWithGracefulShutdown(serv, closeBrowsers); err != nil {
 		logrus.Error(err)
 	}
 }
 
-func buildServerOptions(corsCfg core.CORSConfig, proxyCfg core.ProxyConfig) core.ServerOptions {
+func buildFingerprintBrowserOptions() core.BrowserOpts {
+	blockedResourceTypes := core.MustParseBlockedResourceTypes(config.App.BlockResources)
+
+	opts := core.BrowserOpts{
+		IsHeadless:         !config.App.IsBrowserHead,
+		IsLeakless:         config.App.IsLeakless,
+		Timeout:            time.Second * time.Duration(config.App.Timeout),
+		BrowserPath:        config.App.BrowserPath,
+		Insecure:           config.Server.Insecure,
+		BlockResourceTypes: blockedResourceTypes,
+		BlockTrackers:      config.App.BlockTrackers,
+	}
+	if config.Server.IsDebug {
+		opts.IsHeadless = false
+	}
+	return opts
+}
+
+func buildServerOptions(corsCfg core.CORSConfig, proxyCfg core.ProxyConfig, fingerprintBrowserOpts core.BrowserOpts) core.ServerOptions {
+	retryCfg := core.RetryConfig{
+		MaxRetries:     config.Resilience.MaxRetries,
+		InitialBackoff: 1 * time.Second,
+		MaxBackoff:     30 * time.Second,
+		BackoffFactor:  2.0,
+	}
+	engineTimeout := time.Duration(config.App.Timeout) * time.Second
+
 	return core.ServerOptions{
-		CacheTTL:              time.Duration(config.Cache.TTLSeconds) * time.Second,
-		CacheMaxSize:          config.Cache.MaxSize,
-		EnableCORS:            config.CORS.Enabled,
-		CORS:                  corsCfg,
-		AllowEndpointFallback: config.Resilience.AllowEndpointFallback,
+		CacheTTL:               time.Duration(config.Cache.TTLSeconds) * time.Second,
+		CacheMaxSize:           config.Cache.MaxSize,
+		EnableCORS:             config.CORS.Enabled,
+		CORS:                   corsCfg,
+		AllowEndpointFallback:  config.Resilience.AllowEndpointFallback,
+		EnableDebugEndpoints:   config.App.DebugEndpoints,
+		FingerprintArtifactDir: core.DefaultFingerprintArtifactDir,
+		FingerprintBrowserOpts: fingerprintBrowserOpts,
+		MegaTimeout:            config.App.MegaTimeout,
+		RequestTimeout:         core.RequestTimeoutForRetries(engineTimeout, retryCfg),
+		Extract:                config.Extract,
 		Resilience: core.ResilientConfig{
-			Retry: core.RetryConfig{
-				MaxRetries:     config.Resilience.MaxRetries,
-				InitialBackoff: 1 * time.Second,
-				MaxBackoff:     30 * time.Second,
-				BackoffFactor:  2.0,
-			},
+			Retry: retryCfg,
 			CircuitBreaker: core.CircuitBreakerConfig{
 				FailureThreshold: config.CircuitBreaker.Failures,
 				RecoveryTimeout:  time.Duration(config.CircuitBreaker.RecoverySeconds) * time.Second,
@@ -145,70 +179,392 @@ func buildServerOptions(corsCfg core.CORSConfig, proxyCfg core.ProxyConfig) core
 	}
 }
 
-type browserPool struct {
-	mu      sync.Mutex
-	base    core.BrowserOpts
-	browser map[string]*core.Browser
+const gracefulShutdownTimeout = 30 * time.Second
+
+func listenWithGracefulShutdown(serv *core.Server, onShutdown func() error) error {
+	listenErrCh := make(chan error, 1)
+	go func() {
+		listenErrCh <- serv.Listen()
+	}()
+
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
+	defer signal.Stop(sigCh)
+
+	select {
+	case err := <-listenErrCh:
+		return err
+	case sig := <-sigCh:
+		logrus.WithField("signal", sig.String()).Info("Shutdown signal received, draining traffic")
+	}
+
+	serv.SetDraining(true)
+
+	shutdownErr := serv.ShutdownWithTimeout(gracefulShutdownTimeout)
+	if isServerNotRunningError(shutdownErr) {
+		shutdownErr = nil
+	}
+
+	if onShutdown != nil {
+		resourceErr := onShutdown()
+		if resourceErr != nil {
+			shutdownErr = errors.Join(shutdownErr, resourceErr)
+		}
+	}
+
+	if listenErr := waitForListenExit(listenErrCh); listenErr != nil && !isExpectedListenShutdownError(listenErr) {
+		shutdownErr = errors.Join(shutdownErr, listenErr)
+	}
+
+	return shutdownErr
 }
 
-func newBrowserPool(base core.BrowserOpts) *browserPool {
-	return &browserPool{
-		base:    base,
-		browser: map[string]*core.Browser{},
+func waitForListenExit(listenErrCh <-chan error) error {
+	select {
+	case err := <-listenErrCh:
+		return err
+	case <-time.After(time.Second):
+		return nil
 	}
 }
 
-func (p *browserPool) get(proxyURL string) (*core.Browser, error) {
-	key := strings.TrimSpace(proxyURL)
-	if key == "" {
-		key = "direct"
+func isExpectedListenShutdownError(err error) bool {
+	if err == nil {
+		return true
+	}
+	msg := strings.ToLower(err.Error())
+	return strings.Contains(msg, "server closed") ||
+		strings.Contains(msg, "closed network connection")
+}
+
+func isServerNotRunningError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(strings.ToLower(err.Error()), "server is not running")
+}
+
+// pooledBrowser is one Chrome process in the pool, dedicated to a single proxy
+// auth identity (or to the shared no-auth/unauth path when launchProxyURL=="").
+type pooledBrowser struct {
+	browser        *core.Browser
+	launchProxyURL string
+	lastUsedAt     time.Time
+}
+
+// browserPool keeps a bounded set of Chrome processes keyed by proxy auth
+// identity (scheme+host+port+username). Each entry was launched with its own
+// `l.Proxy(...)` so Chrome handles 407 natively for the main document AND all
+// subresources. Direct and unauthenticated proxies share one entry whose Chrome
+// was launched without a process-level proxy; per-BrowserContext ProxyServer is
+// applied at request time for unauthenticated request-URL proxies.
+type browserPool struct {
+	mu        sync.Mutex
+	base      core.BrowserOpts
+	laneStore *core.LaneStore
+
+	maxProcesses int
+	idleTTL      time.Duration
+
+	browsers map[string]*pooledBrowser
+
+	evictedLRU  int
+	evictedIdle int
+
+	stopSweeper chan struct{}
+	sweeperDone chan struct{}
+}
+
+const directBrowserKey = "direct"
+
+func newBrowserPool(base core.BrowserOpts, defaultLaunchProxyURL string, laneStore *core.LaneStore, maxProcesses int, idleTTL time.Duration) *browserPool {
+	base.ProxyLaneStore = laneStore
+	if maxProcesses <= 0 {
+		maxProcesses = 4
+	}
+	pool := &browserPool{
+		base:         base,
+		laneStore:    laneStore,
+		maxProcesses: maxProcesses,
+		idleTTL:      idleTTL,
+		browsers:     map[string]*pooledBrowser{},
+		stopSweeper:  make(chan struct{}),
+		sweeperDone:  make(chan struct{}),
+	}
+	if launchURL := strings.TrimSpace(defaultLaunchProxyURL); launchURL != "" {
+		if key := browserPoolKey(launchURL); key != directBrowserKey {
+			pool.browsers[key] = &pooledBrowser{
+				launchProxyURL: launchURL,
+				lastUsedAt:     time.Now(),
+			}
+		}
+	}
+	if idleTTL > 0 {
+		go pool.sweepIdle()
+	} else {
+		close(pool.sweeperDone)
+	}
+	return pool
+}
+
+// browserPoolKey derives the pool key from a request's proxy URL. Authenticated
+// HTTP/HTTPS proxies get their own Chrome keyed by scheme+host+port+username
+// plus a short auth hash, so provider session tokens do not share one Chrome.
+// Empty/unauthenticated/SOCKS request URLs fall through to the shared
+// "direct" Chrome.
+func browserPoolKey(requestProxyURL string) string {
+	requestProxyURL = strings.TrimSpace(requestProxyURL)
+	if requestProxyURL == "" {
+		return directBrowserKey
+	}
+	normalized, err := core.NormalizeProxyURL(requestProxyURL)
+	if err != nil || normalized == "" {
+		return directBrowserKey
+	}
+	parsed, err := url.Parse(normalized)
+	if err != nil {
+		return directBrowserKey
+	}
+	if parsed.Scheme != "http" && parsed.Scheme != "https" {
+		// Authenticated SOCKS is rejected upstream; unauthenticated SOCKS goes
+		// through the per-context proxy path on the shared Chrome.
+		return directBrowserKey
+	}
+	if parsed.User == nil {
+		return directBrowserKey
+	}
+	// Hash the full userinfo so a rotating password gets its own Chrome without
+	// leaking credentials into the key. Scheme+host+username stay readable.
+	sum := sha256.Sum256([]byte(parsed.User.String()))
+	return fmt.Sprintf("%s|%s|%s|%s", parsed.Scheme, parsed.Host, parsed.User.Username(), hex.EncodeToString(sum[:])[:16])
+}
+
+// browserLaunchURL returns the URL to pass to launcher.Proxy for a given
+// request URL, or "" when the launch should be unproxied (direct + unauth).
+func browserLaunchURL(requestProxyURL string) string {
+	requestProxyURL = strings.TrimSpace(requestProxyURL)
+	if requestProxyURL == "" {
+		return ""
+	}
+	normalized, err := core.NormalizeProxyURL(requestProxyURL)
+	if err != nil || normalized == "" {
+		return ""
+	}
+	parsed, err := url.Parse(normalized)
+	if err != nil {
+		return ""
+	}
+	if parsed.Scheme != "http" && parsed.Scheme != "https" {
+		return ""
+	}
+	if parsed.User == nil {
+		return ""
+	}
+	return normalized
+}
+
+// get returns a Chrome that can route the supplied request URL. For
+// authenticated HTTP(S) proxies it returns the dedicated Chrome (launching one
+// if needed). For everything else it returns the shared "direct" Chrome.
+func (p *browserPool) get(requestProxyURL string) (*core.Browser, error) {
+	key := browserPoolKey(requestProxyURL)
+	launchURL := ""
+	if key != directBrowserKey {
+		launchURL = browserLaunchURL(requestProxyURL)
 	}
 
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
-	if b, ok := p.browser[key]; ok {
-		return b, nil
+	if entry, ok := p.browsers[key]; ok && entry.browser != nil {
+		entry.lastUsedAt = time.Now()
+		return entry.browser, nil
+	}
+
+	// Use any pre-bound launchProxyURL on the existing entry (e.g. legacy
+	// global proxy) when the caller didn't supply one.
+	if entry, ok := p.browsers[key]; ok && entry.browser == nil {
+		if launchURL == "" {
+			launchURL = entry.launchProxyURL
+		}
 	}
 
 	opts := p.base
-	opts.ProxyURL = proxyURL
-	b, err := core.NewBrowser(opts)
+	opts.ProxyURL = launchURL
+	browser, err := core.NewBrowser(opts)
 	if err != nil {
 		return nil, err
 	}
 
-	// Reuse one launched browser per unique effective proxy so startup stays lazy
-	// and engines with identical proxy policy don't spawn duplicate browser processes.
-	p.browser[key] = b
-	return b, nil
+	p.browsers[key] = &pooledBrowser{
+		browser:        browser,
+		launchProxyURL: launchURL,
+		lastUsedAt:     time.Now(),
+	}
+	p.evictLRULocked()
+	return browser, nil
+}
+
+func (p *browserPool) evictLRULocked() {
+	for len(p.browsers) > p.maxProcesses {
+		var (
+			oldestKey string
+			oldest    time.Time
+			found     bool
+		)
+		for key, entry := range p.browsers {
+			if !found || entry.lastUsedAt.Before(oldest) {
+				oldestKey = key
+				oldest = entry.lastUsedAt
+				found = true
+			}
+		}
+		if !found {
+			return
+		}
+		entry := p.browsers[oldestKey]
+		delete(p.browsers, oldestKey)
+		p.evictedLRU++
+		go closePooledBrowser(entry, "lru")
+	}
+}
+
+func (p *browserPool) sweepIdle() {
+	defer close(p.sweeperDone)
+	interval := p.idleTTL / 4
+	if interval < time.Second {
+		interval = time.Second
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-p.stopSweeper:
+			return
+		case now := <-ticker.C:
+			p.mu.Lock()
+			for key, entry := range p.browsers {
+				if entry.browser == nil {
+					continue
+				}
+				if now.Sub(entry.lastUsedAt) < p.idleTTL {
+					continue
+				}
+				delete(p.browsers, key)
+				p.evictedIdle++
+				go closePooledBrowser(entry, "idle")
+			}
+			p.mu.Unlock()
+		}
+	}
+}
+
+func closePooledBrowser(entry *pooledBrowser, reason string) {
+	if entry == nil || entry.browser == nil {
+		return
+	}
+	if err := entry.browser.Close(); err != nil {
+		logrus.WithError(err).WithField("evict_reason", reason).Debug("Browser pool: close evicted browser failed")
+	}
+}
+
+func (p *browserPool) dropLaneCookies(ctx context.Context, engineName string, q core.Query) {
+	if p == nil || p.laneStore == nil {
+		return
+	}
+	laneKey := core.ProxyLaneKeyForTenant(engineName, core.TenantFromContext(ctx), q, q.ProxyURL)
+	p.laneStore.DropCookies(laneKey)
+}
+
+func (p *browserPool) laneStats() core.LaneStats {
+	if p == nil || p.laneStore == nil {
+		return core.LaneStats{}
+	}
+	return p.laneStore.Stats()
+}
+
+func (p *browserPool) browserStats() core.BrowserPoolStats {
+	if p == nil {
+		return core.BrowserPoolStats{}
+	}
+	p.mu.Lock()
+	active := 0
+	for _, entry := range p.browsers {
+		if entry.browser != nil {
+			active++
+		}
+	}
+	stats := core.BrowserPoolStats{
+		Active:      active,
+		Max:         p.maxProcesses,
+		EvictedLRU:  p.evictedLRU,
+		EvictedIdle: p.evictedIdle,
+	}
+	p.mu.Unlock()
+	return stats
+}
+
+func (p *browserPool) close() error {
+	if p == nil {
+		return nil
+	}
+	close(p.stopSweeper)
+	<-p.sweeperDone
+
+	p.mu.Lock()
+	entries := make([]*pooledBrowser, 0, len(p.browsers))
+	for key, entry := range p.browsers {
+		entries = append(entries, entry)
+		delete(p.browsers, key)
+	}
+	p.mu.Unlock()
+
+	var closeErr error
+	for _, entry := range entries {
+		if entry == nil || entry.browser == nil {
+			continue
+		}
+		if err := entry.browser.Close(); err != nil {
+			closeErr = errors.Join(closeErr, err)
+		}
+	}
+	return closeErr
 }
 
 type pooledBrowserEngine struct {
 	name    string
-	limiter *rate.Limiter
 	opts    core.SearchEngineOptions
 	factory func(core.Browser, core.SearchEngineOptions) core.SearchEngine
 	pool    *browserPool
 
-	mu      sync.Mutex
-	engines map[string]core.SearchEngine
+	reportLaneStats bool
 }
 
-func (e *pooledBrowserEngine) Search(q core.Query) ([]core.SearchResult, error) {
-	engine, err := e.getOrCreate(q.ProxyURL)
+// parsableEngine wraps pooledBrowserEngine and additionally satisfies
+// core.HTMLParser for engines that have a stateless HTML parse function.
+type parsableEngine struct {
+	*pooledBrowserEngine
+	parseHTMLFn func(io.Reader) ([]core.SearchResult, error)
+}
+
+func (e *parsableEngine) ParseHTML(r io.Reader) ([]core.SearchResult, error) {
+	return e.parseHTMLFn(r)
+}
+
+func (e *pooledBrowserEngine) Search(ctx context.Context, q core.Query) ([]core.SearchResult, error) {
+	engine, err := e.resolveEngine(q)
 	if err != nil {
 		return nil, err
 	}
-	return engine.Search(q)
+	return engine.Search(ctx, q)
 }
 
-func (e *pooledBrowserEngine) SearchImage(q core.Query) ([]core.SearchResult, error) {
-	engine, err := e.getOrCreate(q.ProxyURL)
+func (e *pooledBrowserEngine) SearchImage(ctx context.Context, q core.Query) ([]core.SearchResult, error) {
+	engine, err := e.resolveEngine(q)
 	if err != nil {
 		return nil, err
 	}
-	return engine.SearchImage(q)
+	return engine.SearchImage(ctx, q)
 }
 
 func (e *pooledBrowserEngine) IsInitialized() bool {
@@ -220,102 +576,105 @@ func (e *pooledBrowserEngine) Name() string {
 }
 
 func (e *pooledBrowserEngine) GetRateLimiter() *rate.Limiter {
-	return e.limiter
+	return e.opts.GetRateLimiter()
 }
 
-func (e *pooledBrowserEngine) getOrCreate(proxyURL string) (core.SearchEngine, error) {
-	key := strings.TrimSpace(proxyURL)
-	if key == "" {
-		key = "direct"
+func (e *pooledBrowserEngine) DropProxyLaneCookies(ctx context.Context, q core.Query) {
+	e.pool.dropLaneCookies(ctx, e.name, q)
+}
+
+func (e *pooledBrowserEngine) ProxyLaneStats() core.LaneStats {
+	if !e.reportLaneStats {
+		return core.LaneStats{}
 	}
+	return e.pool.laneStats()
+}
 
-	e.mu.Lock()
-	defer e.mu.Unlock()
-
-	if engine, ok := e.engines[key]; ok {
-		return engine, nil
+func (e *pooledBrowserEngine) BrowserPoolStats() core.BrowserPoolStats {
+	if !e.reportLaneStats {
+		return core.BrowserPoolStats{}
 	}
+	return e.pool.browserStats()
+}
 
-	browser, err := e.pool.get(proxyURL)
+// resolveEngine builds a fresh engine wrapper around the pool-resolved Browser.
+// The wrapper is intentionally not cached: pool eviction can replace the Chrome
+// behind a key, and a cached engine would carry a stale Browser value (closed
+// connection, dead browserAddr). Engines are thin wrappers, so per-call
+// construction is cheap.
+func (e *pooledBrowserEngine) resolveEngine(q core.Query) (core.SearchEngine, error) {
+	browser, err := e.pool.get(q.ProxyURL)
 	if err != nil {
 		return nil, err
 	}
-
-	engine := e.factory(*browser, e.opts)
-	e.engines[key] = engine
-	return engine, nil
+	return e.factory(*browser, e.opts), nil
 }
 
 type browserEngineSpec struct {
-	name    string
-	opts    core.SearchEngineOptions
-	factory func(core.Browser, core.SearchEngineOptions) core.SearchEngine
+	name        string
+	opts        core.SearchEngineOptions
+	factory     func(core.Browser, core.SearchEngineOptions) core.SearchEngine
+	parseHTMLFn func(io.Reader) ([]core.SearchResult, error)
 }
 
 func browserEngineSpecs() []browserEngineSpec {
-	return []browserEngineSpec{
-		{
-			name: "google",
-			opts: config.GoogleConfig.SearchEngineOptions,
-			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
-				return google.New(browser, opts)
-			},
-		},
-		{
-			name: "yandex",
-			opts: config.YandexConfig.SearchEngineOptions,
-			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
-				return yandex.New(browser, opts)
-			},
-		},
-		{
-			name: "baidu",
-			opts: config.BaiduConfig.SearchEngineOptions,
-			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
-				return baidu.New(browser, opts)
-			},
-		},
-		{
-			name: "bing",
-			opts: config.BingConfig.SearchEngineOptions,
-			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
-				return bing.New(browser, opts)
-			},
-		},
-		{
-			name: "duckduckgo",
-			opts: config.DuckDuckGoConfig.SearchEngineOptions,
-			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
-				return duckduckgo.New(browser, opts)
-			},
-		},
+	specs := engineSpecs()
+	out := make([]browserEngineSpec, 0, len(specs))
+	for _, s := range specs {
+		out = append(out, browserEngineSpec{
+			name:        s.name,
+			opts:        s.opts(),
+			factory:     s.factory,
+			parseHTMLFn: s.parseHTMLFn,
+		})
 	}
+	return out
 }
 
-func buildBrowserEngines(baseOpts core.BrowserOpts, proxyCfg core.ProxyConfig) ([]core.SearchEngine, error) {
-	pool := newBrowserPool(baseOpts)
+func buildBrowserEngines(baseOpts core.BrowserOpts, proxyCfg core.ProxyConfig) ([]core.SearchEngine, func() error, core.BrowserResolver, error) {
+	launchProxyURL := ""
+	if strings.TrimSpace(proxyCfg.Proxies.Global) != "" && !proxyCfg.Proxies.AllowRequestProxyURL {
+		launchProxyURL = proxyCfg.Proxies.Global
+	}
+	var laneStore *core.LaneStore
+	if proxyCfg.Proxies.Lanes.Enabled {
+		laneStore = core.NewLaneStore(proxyCfg.Proxies.Lanes.MaxLanes)
+	}
+	maxProcesses := config.App.MaxProcesses
+	if maxProcesses <= 0 {
+		maxProcesses = 4
+	}
+	idleTTL := config.App.IdleTTL
+	if idleTTL < 0 {
+		idleTTL = 0
+	}
+	pool := newBrowserPool(baseOpts, launchProxyURL, laneStore, maxProcesses, idleTTL)
 	specs := browserEngineSpecs()
 
 	engines := make([]core.SearchEngine, 0, len(specs))
-	for _, spec := range specs {
+	for idx, spec := range specs {
 		policy := resolveEngineProxyPolicy(proxyCfg, spec.name)
 		if err := validateBrowserProxyPolicy(proxyCfg, policy); err != nil {
-			return nil, fmt.Errorf("browser proxy validation failed for engine %s: %w", spec.name, err)
+			return nil, nil, nil, fmt.Errorf("browser proxy validation failed for engine %s: %w", spec.name, err)
 		}
 
 		opts := spec.opts
 		opts.Init()
-		engines = append(engines, &pooledBrowserEngine{
-			name:    spec.name,
-			limiter: rate.NewLimiter(rate.Every(opts.GetRatelimit()), opts.RateBurst),
-			opts:    opts,
-			factory: spec.factory,
-			pool:    pool,
-			engines: map[string]core.SearchEngine{},
-		})
+		base := &pooledBrowserEngine{
+			name:            spec.name,
+			opts:            opts,
+			factory:         spec.factory,
+			pool:            pool,
+			reportLaneStats: idx == 0,
+		}
+		if spec.parseHTMLFn != nil {
+			engines = append(engines, &parsableEngine{pooledBrowserEngine: base, parseHTMLFn: spec.parseHTMLFn})
+		} else {
+			engines = append(engines, base)
+		}
 	}
 
-	return engines, nil
+	return engines, pool.close, pool.get, nil
 }
 
 func validateBrowserProxyPolicy(proxyCfg core.ProxyConfig, policy core.ProxyPolicy) error {

cmd/serve_test.go

@@ -3,10 +3,177 @@ package cmd
 import (
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/karust/openserp/core"
 )
 
+func TestRawEngineCachesRateLimiter(t *testing.T) {
+	engine := &rawEngine{name: "google"}
+	first := engine.GetRateLimiter()
+	if first == nil {
+		t.Fatal("expected limiter")
+	}
+	if second := engine.GetRateLimiter(); second != first {
+		t.Fatal("expected rawEngine to return the cached limiter")
+	}
+}
+
+func TestCommandDefaultsToQuiet(t *testing.T) {
+	if !commandDefaultsToQuiet(searchCMD) {
+		t.Fatal("expected search command to default to quiet")
+	}
+	if commandDefaultsToQuiet(serveCMD) {
+		t.Fatal("expected serve command to keep server logging by default")
+	}
+}
+
+func TestBrowserPoolKey(t *testing.T) {
+	cases := []struct {
+		name       string
+		raw        string
+		wantDirect bool
+		wantPrefix string
+	}{
+		{"empty -> direct", "", true, ""},
+		{"unauth http -> direct", "http://proxy.example:8080", true, ""},
+		{"unauth socks -> direct", "socks5://proxy.example:1080", true, ""},
+		{"auth socks -> direct (rejected upstream)", "socks5://user:pass@proxy.example:1080", true, ""},
+		{"auth http", "http://user:pass@proxy.example:8080", false, "http|proxy.example:8080|user|"},
+		{"auth https different scheme", "https://user:pass@proxy.example:8443", false, "https|proxy.example:8443|user|"},
+		{"different user different key", "http://user2:pass@proxy.example:8080", false, "http|proxy.example:8080|user2|"},
+		{"different host different key", "http://user:pass@proxy2.example:8080", false, "http|proxy2.example:8080|user|"},
+		{"different port different key", "http://user:pass@proxy.example:9090", false, "http|proxy.example:9090|user|"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := browserPoolKey(tc.raw)
+			if tc.wantDirect {
+				if got != directBrowserKey {
+					t.Fatalf("browserPoolKey(%q) = %q, want %q", tc.raw, got, directBrowserKey)
+				}
+				return
+			}
+			if !strings.HasPrefix(got, tc.wantPrefix) {
+				t.Fatalf("browserPoolKey(%q) = %q, want prefix %q", tc.raw, got, tc.wantPrefix)
+			}
+			if strings.Contains(got, "pass") {
+				t.Fatalf("browser pool key leaked password material: %q", got)
+			}
+		})
+	}
+
+	first := browserPoolKey("http://user:pass@proxy.example:8080")
+	second := browserPoolKey("http://user:other-pass@proxy.example:8080")
+	if first == second {
+		t.Fatalf("expected different passwords to produce different browser pool keys: %q", first)
+	}
+}
+
+func TestBrowserLaunchURL(t *testing.T) {
+	cases := []struct {
+		name string
+		raw  string
+		want string
+	}{
+		{"empty -> empty", "", ""},
+		{"unauth http -> empty (per-context path)", "http://proxy.example:8080", ""},
+		{"unauth socks -> empty", "socks5://proxy.example:1080", ""},
+		{"auth http -> normalized", "http://user:pass@proxy.example:8080", "http://user:pass@proxy.example:8080"},
+		{"auth https -> normalized", "https://u:p@proxy.example:8443", "https://u:p@proxy.example:8443"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := browserLaunchURL(tc.raw)
+			if got != tc.want {
+				t.Fatalf("browserLaunchURL(%q) = %q, want %q", tc.raw, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestBrowserPoolGlobalProxyDoesNotOccupyDirectSlot(t *testing.T) {
+	globalProxy := "http://user:pass@proxy.example:8080"
+	pool := newBrowserPool(core.BrowserOpts{}, globalProxy, nil, 2, 0)
+	defer func() {
+		if err := pool.close(); err != nil {
+			t.Fatalf("close pool: %v", err)
+		}
+	}()
+
+	if _, ok := pool.browsers[directBrowserKey]; ok {
+		t.Fatalf("global proxy must not occupy %q browser slot", directBrowserKey)
+	}
+	key := browserPoolKey(globalProxy)
+	entry, ok := pool.browsers[key]
+	if !ok {
+		t.Fatalf("expected global proxy slot %q to be pre-bound", key)
+	}
+	if entry.launchProxyURL != globalProxy {
+		t.Fatalf("expected launch proxy %q, got %q", globalProxy, entry.launchProxyURL)
+	}
+}
+
+func TestBrowserPoolEvictLRU(t *testing.T) {
+	// Pre-populate with bare entries (browser=nil) so we exercise eviction
+	// without launching real Chrome. closePooledBrowser handles nil safely.
+	pool := &browserPool{
+		maxProcesses: 2,
+		browsers:     map[string]*pooledBrowser{},
+		stopSweeper:  make(chan struct{}),
+		sweeperDone:  make(chan struct{}),
+	}
+	close(pool.sweeperDone)
+
+	now := time.Now()
+	pool.browsers["a"] = &pooledBrowser{lastUsedAt: now.Add(-3 * time.Second)}
+	pool.browsers["b"] = &pooledBrowser{lastUsedAt: now.Add(-2 * time.Second)}
+	pool.browsers["c"] = &pooledBrowser{lastUsedAt: now.Add(-1 * time.Second)}
+
+	pool.mu.Lock()
+	pool.evictLRULocked()
+	pool.mu.Unlock()
+
+	if _, ok := pool.browsers["a"]; ok {
+		t.Fatal("expected oldest entry 'a' to be evicted")
+	}
+	if _, ok := pool.browsers["b"]; !ok {
+		t.Fatal("expected entry 'b' to remain")
+	}
+	if _, ok := pool.browsers["c"]; !ok {
+		t.Fatal("expected entry 'c' to remain")
+	}
+	if pool.evictedLRU != 1 {
+		t.Fatalf("expected 1 LRU eviction, got %d", pool.evictedLRU)
+	}
+}
+
+func TestBrowserPoolBrowserStats(t *testing.T) {
+	pool := &browserPool{
+		maxProcesses: 4,
+		browsers:     map[string]*pooledBrowser{},
+		stopSweeper:  make(chan struct{}),
+		sweeperDone:  make(chan struct{}),
+	}
+	close(pool.sweeperDone)
+
+	// Pre-bound entry without a launched browser should not count as active.
+	pool.browsers["pre-bound"] = &pooledBrowser{launchProxyURL: "http://u:p@proxy.example:8080", lastUsedAt: time.Now()}
+	pool.evictedLRU = 2
+	pool.evictedIdle = 5
+
+	stats := pool.browserStats()
+	if stats.Max != 4 {
+		t.Fatalf("expected max=4, got %d", stats.Max)
+	}
+	if stats.Active != 0 {
+		t.Fatalf("expected active=0 (entry has no live browser), got %d", stats.Active)
+	}
+	if stats.EvictedLRU != 2 || stats.EvictedIdle != 5 {
+		t.Fatalf("unexpected stats: %#v", stats)
+	}
+}
+
 func TestValidateBrowserProxyPolicyRejectsAuthenticatedSocks(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -103,3 +270,108 @@ func TestValidateBrowserProxyPolicyRejectsTaggedAuthenticatedSocksInPool(t *test
 		t.Fatalf("expected explicit authenticated SOCKS error, got %v", err)
 	}
 }
+
+func TestResolveCaptchaSolverConfigDisabledWithoutKey(t *testing.T) {
+	origEnabled := config.Captcha.SolverEnabled
+	origKey := config.Config2Capcha.ApiKey
+	defer func() {
+		config.Captcha.SolverEnabled = origEnabled
+		config.Config2Capcha.ApiKey = origKey
+	}()
+
+	config.Captcha.SolverEnabled = false
+	config.Config2Capcha.ApiKey = ""
+
+	enabled, key, err := resolveCaptchaSolverConfig()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	if enabled {
+		t.Fatal("expected solver to be disabled")
+	}
+	if key != "" {
+		t.Fatalf("expected empty solver key when disabled, got %q", key)
+	}
+}
+
+func TestResolveCaptchaSolverConfigEnabledWithoutKeyFails(t *testing.T) {
+	origEnabled := config.Captcha.SolverEnabled
+	origKey := config.Config2Capcha.ApiKey
+	defer func() {
+		config.Captcha.SolverEnabled = origEnabled
+		config.Config2Capcha.ApiKey = origKey
+	}()
+
+	config.Captcha.SolverEnabled = true
+	config.Config2Capcha.ApiKey = ""
+
+	_, _, err := resolveCaptchaSolverConfig()
+	if err == nil {
+		t.Fatal("expected missing API key error")
+	}
+	if !strings.Contains(err.Error(), "captcha solver is enabled") {
+		t.Fatalf("expected clear startup error, got %v", err)
+	}
+}
+
+func TestResolveCaptchaSolverConfigEnabledWithKey(t *testing.T) {
+	origEnabled := config.Captcha.SolverEnabled
+	origKey := config.Config2Capcha.ApiKey
+	defer func() {
+		config.Captcha.SolverEnabled = origEnabled
+		config.Config2Capcha.ApiKey = origKey
+	}()
+
+	config.Captcha.SolverEnabled = true
+	config.Config2Capcha.ApiKey = "api-key"
+
+	enabled, key, err := resolveCaptchaSolverConfig()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	if !enabled {
+		t.Fatal("expected solver to be enabled")
+	}
+	if key != "api-key" {
+		t.Fatalf("expected configured API key, got %q", key)
+	}
+}
+
+func TestBuildFingerprintBrowserOptionsRespectsBlockConfig(t *testing.T) {
+	origBlockResources := config.App.BlockResources
+	origBlockTrackers := config.App.BlockTrackers
+	origHead := config.App.IsBrowserHead
+	origLeakless := config.App.IsLeakless
+	origTimeout := config.App.Timeout
+	origBrowserPath := config.App.BrowserPath
+	origInsecure := config.Server.Insecure
+	origDebug := config.Server.IsDebug
+	defer func() {
+		config.App.BlockResources = origBlockResources
+		config.App.BlockTrackers = origBlockTrackers
+		config.App.IsBrowserHead = origHead
+		config.App.IsLeakless = origLeakless
+		config.App.Timeout = origTimeout
+		config.App.BrowserPath = origBrowserPath
+		config.Server.Insecure = origInsecure
+		config.Server.IsDebug = origDebug
+	}()
+
+	config.App.BlockResources = "image,font,css,media"
+	config.App.BlockTrackers = true
+	config.App.IsBrowserHead = false
+	config.App.IsLeakless = false
+	config.App.Timeout = 15
+	config.App.BrowserPath = ""
+	config.Server.Insecure = false
+	config.Server.IsDebug = false
+
+	opts := buildFingerprintBrowserOptions()
+	if len(opts.BlockResourceTypes) != 4 {
+		t.Fatalf("expected 4 blocked resource types, got %d", len(opts.BlockResourceTypes))
+	}
+	if !opts.BlockTrackers {
+		t.Fatal("expected tracker blocking to be enabled")
+	}
+
+}

config.yaml

@@ -2,19 +2,34 @@ server:
   host: 0.0.0.0 # API host to bind
   port: 7000 # API port to bind
   debug: false # Enable debug logs and force browser UI mode
-  verbose: true # Enable info-level request logs
+  verbose: false # Enable debug-level request logs
   raw_requests: false # true = raw HTTP mode, false = browser mode
   insecure: true # Allow insecure TLS connections
 
 app:
-  timeout: 15 # Browser/search timeout in seconds
+  log_format: "text" # json|text
+  timeout: 15 # Browser/search timeout in seconds, per attempt; the request deadline is derived from this x retries
   browser_path: "" # Custom browser binary path (chrome/chromium/edge..)
-  head: false # Show browser UI (headful mode)
+  profiles: "" # Overriding built-in browser profiles
+  head: false # Headful mode
   leakless: false # Force browser process cleanup after request
-  leave_head: false # Keep tabs open after request for debugging
-  stealth: false # Enable stealth browser plugin
+  leave_head: false # Keep tabs open after request
+  block_resources: "image,font,css,media" # Block heavy subresources in browser mode
+  block_trackers: true # Block known tracker domains
+  max_processes: 6 # Concurrent Chrome processes
+  idle_ttl: 5m # close a Chrome that has not served traffic for this long
+
+  mega_timeout: 90s # max total wait for /mega/* requests; slow engines return partial results
+
+extract:
+  enabled: true
+  default_mode: auto # auto|fast|rendered
+  timeout: 20s
+  max_bytes: 2000000
+  max_concurrent: 2
 
 proxies:
+  allow_request_proxy_url: true
   # Force a single proxy for all engines.
   # Same behavior as passing --proxy on the CLI.
   #global: http://127.0.0.1:8080
@@ -26,14 +41,18 @@ proxies:
   #  - url: socks5h://127.0.0.1:1080
   #    tags: [eu]
   health:
-    failure_threshold: 3 # Disable proxy after this many consecutive failures
+    failure_threshold: 2 # Disable proxy after this many consecutive failures
+  lanes:
+    enabled: true # Reuse browser profile/cookies per engine + proxy session ID
+    max_lanes: 100 # LRU cap for sticky lanes kept in worker memory
+    drop_cookies_on_challenge: true # Clear lane cookies on captcha/challenge only
 
 cache:
-  ttl_seconds: 60 # Dedicated endpoint cache TTL in seconds (0 disables cache)
+  ttl_seconds: 120 # Dedicated endpoint cache TTL in seconds (0 disables cache)
   max_size: 1000 # Maximum cached dedicated responses before oldest-entry eviction
 
 resilience:
-  max_retries: 2 # Retry attempts per engine request (0 disables retries)
+  max_retries: 1 # Retry attempts per engine request (0 disables retries)
   allow_endpoint_fallback: false # Keep dedicated endpoints engine-pure by default
 
 # circuit_breaker:
@@ -45,31 +64,39 @@ cors:
   enabled: true
   allow_origins: "*"
   allow_methods: "GET, POST, OPTIONS"
-  allow_headers: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy"
+  allow_headers: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Proxy-URL, X-Proxy-Country, X-Proxy-Class, X-Proxy-Provider, X-Proxy-Session-ID, X-Request-ID, X-Tenant, X-Use-Profile"
   max_age: 86400
 
 # 2captcha:
 #   apikey: "123123123123123"
 
+captcha:
+  solver_enabled: false # Global captcha solver gate (requires 2captcha.apikey)
+
 google:
-  rate_requests: 4 # Allowed average requests per minute
-  rate_burst: 2 # Burst requests before limiter applies
-  captcha: true # Enable captcha solver path
+  rate_requests: 60 # Allowed average requests per minute
+  rate_burst: 3 # Burst requests before limiter applies
+  #captcha: true # Engine-level solver flag (also requires captcha.solver_enabled=true)
 
 yandex:
-  rate_requests: 4
-  rate_burst: 2
+  rate_requests: 60
+  rate_burst: 3
 
 baidu:
-  rate_requests: 4
-  rate_burst: 2
+  rate_requests: 60
+  rate_burst: 3
   # No proxy tag means direct traffic
 
 bing:
-  rate_requests: 4
-  rate_burst: 2
+  rate_requests: 60
+  rate_burst: 3
   # No proxy tag means direct traffic
 
 duckduckgo:
-  rate_requests: 4
-  rate_burst: 2
+  rate_requests: 60
+  rate_burst: 3
+
+ecosia:
+  rate_requests: 60
+  rate_burst: 3
+  # No proxy tag means direct traffic

core/browser/profile.go

@@ -0,0 +1,409 @@
+package browser
+
+import (
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"hash/fnv"
+	"os"
+	"runtime"
+	"slices"
+	"strings"
+	"sync"
+)
+
+type BrandVersion struct {
+	Brand   string `json:"brand"`
+	Version string `json:"version"`
+}
+
+type Viewport struct {
+	Width  int `json:"width"`
+	Height int `json:"height"`
+}
+
+type Profile struct {
+	ID                string         `json:"id"`
+	UserAgentTemplate string         `json:"user_agent_template"`
+	UserAgent         string         `json:"user_agent,omitempty"`
+	UACHBrands        []BrandVersion `json:"uach_brands,omitempty"`
+	UACHFullVerList   []BrandVersion `json:"uach_full_version_list,omitempty"`
+	Platform          string         `json:"platform"`
+	PlatformVersion   string         `json:"platform_version"`
+	Architecture      string         `json:"architecture"`
+	Bitness           string         `json:"bitness"`
+	Mobile            bool           `json:"mobile"`
+	AcceptLanguage    string         `json:"accept_language"`
+	NavigatorLangs    []string       `json:"navigator_langs"`
+	Locale            string         `json:"locale"`
+	Timezone          string         `json:"timezone"`
+	Viewport          Viewport       `json:"viewport"`
+	Tags              []string       `json:"tags"`
+	Weight            int            `json:"weight"`
+}
+
+type catalogConfig struct {
+	Profiles              []Profile         `json:"profiles"`
+	LaneProfileIDs        map[string]string `json:"lane_profile_ids"`
+	DefaultRegionByEngine map[string]string `json:"default_region_by_engine"`
+}
+
+const (
+	ProfileChromeWinUS   = "chrome-win-uhd620"
+	ProfileChromeMacUS   = "chrome-macos-intel-iris"
+	ProfileChromeLinuxUS = "chrome-linux-mesa-uhd620"
+)
+
+//go:embed profiles.json
+var defaultProfilesJSON []byte
+
+var profileCatalogMu sync.RWMutex
+
+var catalog = map[string]Profile{}
+
+var laneProfileIDs = map[string]string{}
+
+var defaultRegionByEngine = map[string]string{}
+
+func init() {
+	if err := loadProfilesFromJSONBytes(defaultProfilesJSON); err != nil {
+		panic(fmt.Sprintf("load embedded browser profiles: %v", err))
+	}
+}
+
+func LoadProfilesFromJSON(path string) error {
+	path = strings.TrimSpace(path)
+	if path == "" {
+		return nil
+	}
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("read profiles json %q: %w", path, err)
+	}
+
+	if err := loadProfilesFromJSONBytes(data); err != nil {
+		return fmt.Errorf("parse profiles json %q: %w", path, err)
+	}
+	return nil
+}
+
+func loadProfilesFromJSONBytes(data []byte) error {
+	var cfg catalogConfig
+	if err := json.Unmarshal(data, &cfg); err != nil {
+		return err
+	}
+
+	if len(cfg.Profiles) == 0 {
+		return fmt.Errorf("profiles list is empty")
+	}
+
+	nextCatalog := make(map[string]Profile, len(cfg.Profiles))
+	for i, profile := range cfg.Profiles {
+		profile.ID = strings.TrimSpace(profile.ID)
+		if profile.ID == "" {
+			return fmt.Errorf("profiles[%d].id is empty", i)
+		}
+		if strings.TrimSpace(profile.UserAgentTemplate) == "" && strings.TrimSpace(profile.UserAgent) == "" {
+			return fmt.Errorf("profiles[%d] has no user_agent_template or user_agent", i)
+		}
+		if _, exists := nextCatalog[profile.ID]; exists {
+			return fmt.Errorf("duplicate profile id %q", profile.ID)
+		}
+		nextCatalog[profile.ID] = profile
+	}
+
+	nextLaneProfileIDs := make(map[string]string, len(cfg.LaneProfileIDs))
+	for rawLaneKey, profileID := range cfg.LaneProfileIDs {
+		laneKey, err := normalizeLaneKey(rawLaneKey)
+		if err != nil {
+			return err
+		}
+		profileID = strings.TrimSpace(profileID)
+		if profileID == "" {
+			return fmt.Errorf("lane profile id for %q is empty", laneKey)
+		}
+		if _, exists := nextCatalog[profileID]; !exists {
+			return fmt.Errorf("lane %q references unknown profile id %q", laneKey, profileID)
+		}
+		nextLaneProfileIDs[laneKey] = profileID
+	}
+
+	nextDefaultRegionByEngine := map[string]string{
+		"yandex": "ru",
+	}
+	for engine, region := range cfg.DefaultRegionByEngine {
+		engine = NormalizeEngine(engine)
+		if engine == "" {
+			return fmt.Errorf("default_region_by_engine contains empty engine key")
+		}
+		nextDefaultRegionByEngine[engine] = normalizeConfiguredRegion(region)
+	}
+
+	profileCatalogMu.Lock()
+	catalog = nextCatalog
+	laneProfileIDs = nextLaneProfileIDs
+	defaultRegionByEngine = nextDefaultRegionByEngine
+	profileCatalogMu.Unlock()
+
+	return nil
+}
+
+func normalizeLaneKey(value string) (string, error) {
+	value = strings.TrimSpace(value)
+	parts := strings.SplitN(value, ":", 2)
+	if len(parts) != 2 {
+		return "", fmt.Errorf("invalid lane key %q, expected engine:region", value)
+	}
+	engine := NormalizeEngine(parts[0])
+	if engine == "" {
+		return "", fmt.Errorf("invalid lane key %q, engine is empty", value)
+	}
+	region := normalizeConfiguredRegion(parts[1])
+	return engine + ":" + region, nil
+}
+
+func normalizeConfiguredRegion(region string) string {
+	region = strings.TrimSpace(region)
+	if region == "" {
+		return "us"
+	}
+	return NormalizeRegion(region)
+}
+
+func Catalog() []Profile {
+	profileCatalogMu.RLock()
+	defer profileCatalogMu.RUnlock()
+
+	out := make([]Profile, 0, len(catalog))
+	for _, profile := range catalog {
+		out = append(out, profile)
+	}
+	return out
+}
+
+// SelectProfile returns a deterministic profile for the given engine and region.
+// It respects lane_profile_ids overrides and falls back to the OS-preferred default.
+// Used by tests and single-instance callers; internally delegates to SelectProfileForSession with empty salt.
+func SelectProfile(engine string, region string) Profile {
+	return SelectProfileForSession(engine, region, "")
+}
+
+// SelectProfileForSession picks a profile for (engine, region, salt).
+// If a lane_profile_ids override exists it is always honoured.
+// Empty salt picks the first eligible profile (same as SelectProfile).
+// Non-empty salt uses weighted selection seeded by FNV-1a hash of salt,
+// giving each session a stable but varied profile.
+func SelectProfileForSession(engine, region, salt string) Profile {
+	return SelectProfileForSessionHeadless(engine, region, salt, false)
+}
+
+// SelectProfileForSessionHeadless is SelectProfileForSession, but headless Linux
+// (the Docker deployment) renders WebGL via SwiftShader, so it only picks
+// swiftshader-tagged profiles. Runtimes with a real GPU exclude them.
+func SelectProfileForSessionHeadless(engine, region, salt string, headless bool) Profile {
+	engine = NormalizeEngine(engine)
+	region = NormalizeRegion(region)
+	if region == "" {
+		region = DefaultRegionForEngine(engine)
+	}
+	laneKey := engine + ":" + region
+
+	profileCatalogMu.RLock()
+	profileID, ok := laneProfileIDs[laneKey]
+	profileCatalogMu.RUnlock()
+	if ok {
+		return profileByID(profileID)
+	}
+
+	pool := eligibleProfiles(runtime.GOOS, headless)
+	return pickWeighted(pool, salt)
+}
+
+type weightedProfile struct {
+	profile Profile
+	weight  int
+}
+
+// eligibleProfiles builds the pool for the runtime platform. Headless Linux
+// keeps only swiftshader-tagged profiles (SwiftShader WebGL); a real GPU
+// excludes them. We no longer spoof WebGL, so the GPU sub-tag (nvidia/amd/mesa)
+// only steers selection - it matches reality on Docker, cosmetic on a headful box.
+func eligibleProfiles(goos string, headless bool) []weightedProfile {
+	profileCatalogMu.RLock()
+	snap := make([]Profile, 0, len(catalog))
+	for _, p := range catalog {
+		snap = append(snap, p)
+	}
+	profileCatalogMu.RUnlock()
+
+	// Stable ordering so empty-salt picks are deterministic across map iterations.
+	slices.SortFunc(snap, func(a, b Profile) int {
+		return strings.Compare(a.ID, b.ID)
+	})
+
+	platformTag := runtimePlatformTag(goos)
+	wantSwiftShader := headless && goos == "linux"
+
+	var pool []weightedProfile
+	for _, p := range snap {
+		if platformTag != "" && !slices.Contains(p.Tags, platformTag) {
+			continue
+		}
+		if slices.Contains(p.Tags, "swiftshader") != wantSwiftShader {
+			continue
+		}
+		w := p.Weight
+		if w <= 0 {
+			w = 1
+		}
+
+		pool = append(pool, weightedProfile{profile: p, weight: w})
+	}
+
+	// Fall back to the platform pool if no swiftshader profile exists yet, so
+	// selection never returns empty.
+	if len(pool) == 0 && wantSwiftShader {
+		return eligibleProfiles(goos, false)
+	}
+
+	return pool
+}
+
+func runtimePlatformTag(goos string) string {
+	switch strings.ToLower(strings.TrimSpace(goos)) {
+	case "windows":
+		return "windows"
+	case "darwin":
+		return "macos"
+	case "linux":
+		return "linux"
+	default:
+		return ""
+	}
+}
+
+// pickWeighted selects a profile from pool using FNV-1a hash of salt modulo total weight.
+// Empty salt returns the first profile in the pool (deterministic for tests).
+func pickWeighted(pool []weightedProfile, salt string) Profile {
+	if len(pool) == 0 {
+		return profileByID(defaultProfileID())
+	}
+	if salt == "" {
+		return pool[0].profile
+	}
+
+	total := 0
+	for _, wp := range pool {
+		total += wp.weight
+	}
+	if total <= 0 {
+		return pool[0].profile
+	}
+
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(salt))
+	idx := int(h.Sum32()) % total
+
+	cumulative := 0
+	for _, wp := range pool {
+		cumulative += wp.weight
+		if idx < cumulative {
+			return wp.profile
+		}
+	}
+	return pool[len(pool)-1].profile
+}
+
+func LaneKey(engine string, region string) string {
+	engine = NormalizeEngine(engine)
+	if engine == "" {
+		engine = "unknown"
+	}
+
+	region = NormalizeRegion(region)
+	if region == "" {
+		region = DefaultRegionForEngine(engine)
+	}
+
+	return engine + ":" + region
+}
+
+func DefaultRegionForEngine(engine string) string {
+	engine = NormalizeEngine(engine)
+
+	profileCatalogMu.RLock()
+	defer profileCatalogMu.RUnlock()
+
+	if region, ok := defaultRegionByEngine[engine]; ok {
+		return region
+	}
+	return "us"
+}
+
+func NormalizeEngine(engine string) string {
+	return strings.ToLower(strings.TrimSpace(engine))
+}
+
+func NormalizeRegion(region string) string {
+	region = strings.TrimSpace(strings.ToLower(region))
+	if region == "" {
+		return ""
+	}
+
+	if idx := strings.Index(region, ","); idx >= 0 {
+		region = region[:idx]
+	}
+	if idx := strings.Index(region, ";"); idx >= 0 {
+		region = region[:idx]
+	}
+
+	region = strings.ReplaceAll(region, "_", "-")
+	if idx := strings.Index(region, "-"); idx >= 0 {
+		region = region[:idx]
+	}
+
+	switch region {
+	case "ru", "be", "kz", "ky":
+		return "ru"
+	default:
+		return "us"
+	}
+}
+
+// ProfileByID looks up a profile by exact ID. Returns (profile, true) when found,
+// (zero, false) when the ID is not in the catalog. Unlike the internal profileByID,
+// it does not fall back to a default; the caller decides what to do on miss.
+func ProfileByID(profileID string) (Profile, bool) {
+	profileCatalogMu.RLock()
+	defer profileCatalogMu.RUnlock()
+	profile, ok := catalog[strings.TrimSpace(profileID)]
+	return profile, ok
+}
+
+func profileByID(profileID string) Profile {
+	profileCatalogMu.RLock()
+	defer profileCatalogMu.RUnlock()
+
+	if profile, ok := catalog[profileID]; ok {
+		return profile
+	}
+	if fallback, ok := catalog[defaultProfileID()]; ok {
+		return fallback
+	}
+	for _, profile := range catalog {
+		return profile
+	}
+	return Profile{}
+}
+
+func defaultProfileID() string {
+	switch runtime.GOOS {
+	case "windows":
+		return ProfileChromeWinUS
+	case "darwin":
+		return ProfileChromeMacUS
+	default:
+		return ProfileChromeLinuxUS
+	}
+}

core/browser/profile_coherence_test.go

@@ -0,0 +1,270 @@
+//go:build integration
+// +build integration
+
+package browser_test
+
+import (
+	"context"
+	_ "embed"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/go-rod/rod"
+	"github.com/karust/openserp/core"
+	browserprofile "github.com/karust/openserp/core/browser"
+	"github.com/karust/openserp/testutil"
+)
+
+//go:embed profile_surface_test.js
+var profileSurfaceScript string
+
+func TestProfileCoherence(t *testing.T) {
+	testutil.RequireIntegration(t)
+
+	fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = w.Write([]byte(`<!doctype html><html><head><meta charset="utf-8"><title>coherence</title></head><body>ok</body></html>`))
+	}))
+	defer fixture.Close()
+
+	browser, err := core.NewBrowser(core.BrowserOpts{
+		IsHeadless: true,
+		IsLeakless: false,
+		Timeout:    20 * time.Second,
+	})
+	if err != nil {
+		t.Fatalf("create browser: %v", err)
+	}
+	defer func() {
+		if closeErr := browser.Close(); closeErr != nil {
+			t.Fatalf("close browser: %v", closeErr)
+		}
+	}()
+
+	cases := []struct {
+		name   string
+		engine string
+		region string
+	}{
+		{
+			name:   "ru lane",
+			engine: "google",
+			region: "ru",
+		},
+		{
+			name:   "us lane",
+			engine: "bing",
+			region: "en-US",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := core.WithEngine(context.Background(), tc.engine)
+			ctx = core.WithProfileRegion(ctx, tc.region)
+			ctx = core.WithBrowserProfileUsage(ctx)
+
+			page, err := browser.Navigate(ctx, fixture.URL)
+			if err != nil {
+				t.Fatalf("navigate fixture: %v", err)
+			}
+			defer func() {
+				if closeErr := core.ClosePageWithTimeout(context.Background(), page, time.Second); closeErr != nil {
+					t.Fatalf("close page: %v", closeErr)
+				}
+			}()
+
+			got, err := browserProfileSurface(page)
+			if err != nil {
+				t.Fatalf("collect profile surfaces: %v", err)
+			}
+
+			expected := selectedProfileFromContext(t, ctx)
+			expected.UserAgent = expectedUserAgentForRuntime(expectedProfileUserAgent(expected, got.UserAgent), got.UserAgent)
+			if got.UserAgent != expected.UserAgent {
+				t.Fatalf("navigator.userAgent mismatch:\nexpected: %s\nactual:   %s", expected.UserAgent, got.UserAgent)
+			}
+			if got.Platform != expected.Platform {
+				t.Fatalf("navigator.userAgentData.platform mismatch: expected %q got %q", expected.Platform, got.Platform)
+			}
+			if got.NavigatorPlatform != expectedNavigatorPlatform(expected.Platform) {
+				t.Fatalf("navigator.platform mismatch: expected %q got %q", expectedNavigatorPlatform(expected.Platform), got.NavigatorPlatform)
+			}
+			if got.Locale != expected.Locale {
+				t.Fatalf("Intl locale mismatch: expected %q got %q", expected.Locale, got.Locale)
+			}
+			if got.Timezone != expected.Timezone {
+				t.Fatalf("Intl timezone mismatch: expected %q got %q", expected.Timezone, got.Timezone)
+			}
+			if len(got.NavigatorLanguages) == 0 {
+				t.Fatal("navigator.languages is empty")
+			}
+			if got.NavigatorLanguages[0] != expected.NavigatorLangs[0] {
+				t.Fatalf("navigator.languages[0] mismatch: expected %q got %q", expected.NavigatorLangs[0], got.NavigatorLanguages[0])
+			}
+			// Real Chrome exposes navigator.webdriver as an inherited boolean
+			// that reads false; --disable-blink-features=AutomationControlled
+			// gives us that. Deleting the property (typeof undefined) is itself a
+			// bot tell, so we assert the genuine-browser shape instead.
+			if got.WebdriverType != "boolean" {
+				t.Fatalf("navigator.webdriver expected boolean, got %q", got.WebdriverType)
+			}
+			if got.WebdriverValue {
+				t.Fatal("navigator.webdriver should read false")
+			}
+			if got.WebdriverOwnPropPresent {
+				t.Fatal("navigator.webdriver should be inherited, not an own property")
+			}
+			// hardwareConcurrency is left native (no override), so main and
+			// worker must simply agree on the machine's real core count.
+			if got.HardwareConcurrency <= 0 {
+				t.Fatal("navigator.hardwareConcurrency should be reported")
+			}
+			if got.WorkerHardwareConcurrency != got.HardwareConcurrency {
+				t.Fatalf("worker hardwareConcurrency mismatch: main %d worker %d", got.HardwareConcurrency, got.WorkerHardwareConcurrency)
+			}
+			if got.WorkerUserAgent != got.UserAgent {
+				t.Fatalf("worker userAgent mismatch: main %q worker %q", got.UserAgent, got.WorkerUserAgent)
+			}
+			if got.WorkerPlatform != got.NavigatorPlatform {
+				t.Fatalf("worker platform mismatch: main %q worker %q", got.NavigatorPlatform, got.WorkerPlatform)
+			}
+			if len(got.WorkerNavigatorLangs) == 0 {
+				t.Fatal("worker navigator.languages is empty")
+			}
+			if got.WorkerNavigatorLangs[0] != got.NavigatorLanguages[0] {
+				t.Fatalf("worker navigator.languages[0] mismatch: main %q worker %q", got.NavigatorLanguages[0], got.WorkerNavigatorLangs[0])
+			}
+			if got.WorkerTimezone != got.Timezone {
+				t.Fatalf("worker timezone mismatch: main %q worker %q", got.Timezone, got.WorkerTimezone)
+			}
+			if got.WorkerWebGLVendor != got.WebGLVendor {
+				t.Fatalf("worker WebGL vendor mismatch: main %q worker %q", got.WebGLVendor, got.WorkerWebGLVendor)
+			}
+			if got.WorkerWebGLRenderer != got.WebGLRenderer {
+				t.Fatalf("worker WebGL renderer mismatch: main %q worker %q", got.WebGLRenderer, got.WorkerWebGLRenderer)
+			}
+			if got.InnerHeight >= got.OuterHeight {
+				t.Fatalf("innerHeight should be smaller than outerHeight, got inner=%d outer=%d", got.InnerHeight, got.OuterHeight)
+			}
+			if got.OuterHeight > got.ScreenAvailHeight {
+				t.Fatalf("outerHeight should fit in screen.availHeight, got outer=%d avail=%d", got.OuterHeight, got.ScreenAvailHeight)
+			}
+			if got.ScreenAvailHeight > got.ScreenHeight {
+				t.Fatalf("screen.availHeight should fit in screen.height, got avail=%d screen=%d", got.ScreenAvailHeight, got.ScreenHeight)
+			}
+		})
+	}
+}
+
+type profileSurface struct {
+	UserAgent                 string   `json:"userAgent"`
+	Platform                  string   `json:"platform"`
+	NavigatorPlatform         string   `json:"navigatorPlatform"`
+	NavigatorLanguages        []string `json:"navigatorLanguages"`
+	Timezone                  string   `json:"timezone"`
+	Locale                    string   `json:"locale"`
+	WebdriverType             string   `json:"webdriverType"`
+	WebdriverValue            bool     `json:"webdriverValue"`
+	WebdriverOwnPropPresent   bool     `json:"webdriverOwnPropPresent"`
+	HardwareConcurrency       int      `json:"hardwareConcurrency"`
+	WorkerHardwareConcurrency int      `json:"workerHardwareConcurrency"`
+	WorkerUserAgent           string   `json:"workerUserAgent"`
+	WorkerPlatform            string   `json:"workerPlatform"`
+	WorkerNavigatorLangs      []string `json:"workerNavigatorLangs"`
+	WorkerTimezone            string   `json:"workerTimezone"`
+	WebGLVendor               string   `json:"webGLVendor"`
+	WebGLRenderer             string   `json:"webGLRenderer"`
+	WorkerWebGLVendor         string   `json:"workerWebGLVendor"`
+	WorkerWebGLRenderer       string   `json:"workerWebGLRenderer"`
+	InnerHeight               int      `json:"innerHeight"`
+	OuterHeight               int      `json:"outerHeight"`
+	ScreenHeight              int      `json:"screenHeight"`
+	ScreenAvailHeight         int      `json:"screenAvailHeight"`
+}
+
+func browserProfileSurface(page *rod.Page) (profileSurface, error) {
+	result, err := page.Eval(profileSurfaceScript)
+	if err != nil {
+		return profileSurface{}, err
+	}
+
+	var out profileSurface
+	if err := result.Value.Unmarshal(&out); err != nil {
+		return profileSurface{}, fmt.Errorf("decode eval payload: %w", err)
+	}
+	return out, nil
+}
+
+func expectedNavigatorPlatform(platform string) string {
+	switch platform {
+	case "Windows":
+		return "Win32"
+	case "macOS":
+		return "MacIntel"
+	default:
+		return "Linux x86_64"
+	}
+}
+
+func selectedProfileFromContext(t *testing.T, ctx context.Context) browserprofile.Profile {
+	t.Helper()
+
+	ids := core.BrowserProfileIDsFromContext(ctx)
+	if len(ids) == 0 {
+		t.Fatal("expected selected browser profile id")
+	}
+	profile, ok := browserprofile.ProfileByID(ids[0])
+	if !ok {
+		t.Fatalf("selected browser profile %q not found", ids[0])
+	}
+	return profile
+}
+
+// expectedProfileUserAgent resolves the profile's user agent, expanding the
+// {chrome_major} template with the runtime's major when the profile carries a
+// template rather than a literal UA.
+func expectedProfileUserAgent(profile browserprofile.Profile, runtimeUserAgent string) string {
+	if ua := strings.TrimSpace(profile.UserAgent); ua != "" {
+		return ua
+	}
+	template := strings.TrimSpace(profile.UserAgentTemplate)
+	if template == "" {
+		return ""
+	}
+	major := chromeToken(runtimeUserAgent)
+	major = strings.TrimPrefix(major, "Chrome/")
+	if idx := strings.IndexByte(major, '.'); idx >= 0 {
+		major = major[:idx]
+	}
+	return strings.ReplaceAll(template, "{chrome_major}", major)
+}
+
+func expectedUserAgentForRuntime(profileUserAgent, runtimeUserAgent string) string {
+	runtimeChrome := chromeToken(runtimeUserAgent)
+	if runtimeChrome == "" {
+		return profileUserAgent
+	}
+	profileChrome := chromeToken(profileUserAgent)
+	if profileChrome == "" {
+		return profileUserAgent
+	}
+	return strings.Replace(profileUserAgent, profileChrome, runtimeChrome, 1)
+}
+
+func chromeToken(userAgent string) string {
+	const prefix = "Chrome/"
+	start := strings.Index(userAgent, prefix)
+	if start < 0 {
+		return ""
+	}
+	end := strings.IndexByte(userAgent[start:], ' ')
+	if end < 0 {
+		return userAgent[start:]
+	}
+	return userAgent[start : start+end]
+}

core/browser/profile_surface_test.js

@@ -0,0 +1,85 @@
+async () => {
+  const readWebGL = (canvas) => {
+    try {
+      const gl = canvas ? (canvas.getContext('webgl') || canvas.getContext('experimental-webgl') || canvas.getContext('webgl2')) : null;
+      const debugInfo = gl && gl.getExtension('WEBGL_debug_renderer_info');
+      return {
+        vendor: gl && debugInfo ? (gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL) || '') : '',
+        renderer: gl && debugInfo ? (gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL) || '') : '',
+      };
+    } catch (_) {
+      return { vendor: '', renderer: '' };
+    }
+  };
+  const webGL = readWebGL(document.createElement('canvas'));
+  const workerData = await new Promise((resolve) => {
+    try {
+      const source = [
+        "self.onmessage = () => {",
+        "let webGLVendor = '';",
+        "let webGLRenderer = '';",
+        "try {",
+        "const canvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : null;",
+        "const gl = canvas ? (canvas.getContext('webgl') || canvas.getContext('experimental-webgl') || canvas.getContext('webgl2')) : null;",
+        "const debugInfo = gl && gl.getExtension('WEBGL_debug_renderer_info');",
+        "if (gl && debugInfo) {",
+        "webGLVendor = gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL) || '';",
+        "webGLRenderer = gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL) || '';",
+        "}",
+        "} catch (_) {}",
+        "self.postMessage({",
+        "userAgent: self.navigator.userAgent || '',",
+        "platform: self.navigator.platform || '',",
+        "navigatorLanguages: Array.from(self.navigator.languages || []),",
+        "timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || '',",
+        "hardwareConcurrency: self.navigator.hardwareConcurrency || 0,",
+        "webGLVendor,",
+        "webGLRenderer,",
+        "});",
+        "};",
+      ].join("\n");
+      const blob = new Blob([source], { type: "application/javascript" });
+      const url = URL.createObjectURL(blob);
+      const worker = new Worker(url);
+      worker.onmessage = (event) => {
+        resolve(event.data || {});
+        worker.terminate();
+        URL.revokeObjectURL(url);
+      };
+      worker.onerror = () => {
+        resolve({});
+        worker.terminate();
+        URL.revokeObjectURL(url);
+      };
+      worker.postMessage("run");
+    } catch (_) {
+      resolve({});
+    }
+  });
+
+  return {
+    userAgent: navigator.userAgent || "",
+    platform: navigator.userAgentData ? (navigator.userAgentData.platform || "") : "",
+    navigatorPlatform: navigator.platform || "",
+    navigatorLanguages: Array.from(navigator.languages || []),
+    timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || "",
+    locale: Intl.DateTimeFormat().resolvedOptions().locale || "",
+    webdriverType: typeof navigator.webdriver,
+    webdriverValue: navigator.webdriver === true,
+    webdriverOwnPropPresent: Object.getOwnPropertyNames(navigator).includes("webdriver"),
+    hardwareConcurrency: navigator.hardwareConcurrency || 0,
+    workerHardwareConcurrency: workerData.hardwareConcurrency || 0,
+    workerUserAgent: workerData.userAgent || "",
+    workerPlatform: workerData.platform || "",
+    workerNavigatorLangs: Array.from(workerData.navigatorLanguages || []),
+    workerTimezone: workerData.timezone || "",
+    webGLVendor: webGL.vendor,
+    webGLRenderer: webGL.renderer,
+    workerWebGLVendor: workerData.webGLVendor || "",
+    workerWebGLRenderer: workerData.webGLRenderer || "",
+    innerHeight: window.innerHeight || 0,
+    outerHeight: window.outerHeight || 0,
+    screenHeight: window.screen ? (window.screen.height || 0) : 0,
+    screenAvailHeight: window.screen ? (window.screen.availHeight || 0) : 0,
+  };
+}

core/browser/profile_test.go

@@ -0,0 +1,257 @@
+package browser
+
+import (
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+)
+
+func TestSelectProfile(t *testing.T) {
+	tests := []struct {
+		name   string
+		engine string
+		region string
+	}{
+		{
+			name:   "google ru lane",
+			engine: "google",
+			region: "ru",
+		},
+		{
+			name:   "yandex defaults to ru",
+			engine: "yandex",
+			region: "",
+		},
+		{
+			name:   "google default lane uses us profile",
+			engine: "google",
+			region: "en",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			profile := SelectProfile(tt.engine, tt.region)
+			if profile.ID == "" {
+				t.Fatal("expected non-empty profile ID")
+			}
+			if profile.Platform == "" {
+				t.Fatal("expected non-empty platform")
+			}
+			if profile.Viewport.Width <= 0 || profile.Viewport.Height <= 0 {
+				t.Fatalf("expected positive viewport, got %+v", profile.Viewport)
+			}
+		})
+	}
+}
+
+func TestSelectProfileForSession(t *testing.T) {
+	t.Run("same salt returns same profile", func(t *testing.T) {
+		first := SelectProfileForSession("google", "us", "session-abc")
+		for i := 0; i < 10; i++ {
+			got := SelectProfileForSession("google", "us", "session-abc")
+			if got.ID != first.ID {
+				t.Fatalf("iteration %d: expected %q, got %q", i, first.ID, got.ID)
+			}
+		}
+	})
+
+	t.Run("rotation produces multiple distinct profiles", func(t *testing.T) {
+		seen := map[string]bool{}
+		for i := 0; i < 100; i++ {
+			salt := "session-" + string(rune('a'+i%26)) + string(rune('0'+i/26%10))
+			p := SelectProfileForSession("google", "us", salt)
+			seen[p.ID] = true
+		}
+		if len(seen) < 3 {
+			t.Fatalf("expected at least 3 distinct profiles, got %d: %v", len(seen), seen)
+		}
+	})
+
+	t.Run("locale does not change hardware profile pool", func(t *testing.T) {
+		p := SelectProfileForSession("yandex", "ru", "some-session")
+		if slices.Contains(p.Tags, "ru") {
+			t.Fatalf("expected locale-neutral profile, got ID=%q tags=%v", p.ID, p.Tags)
+		}
+	})
+}
+
+func TestEligibleProfilesMatchRuntimePlatform(t *testing.T) {
+	tests := []struct {
+		goos string
+		tag  string
+	}{
+		{goos: "linux", tag: "linux"},
+		{goos: "windows", tag: "windows"},
+		{goos: "darwin", tag: "macos"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.goos, func(t *testing.T) {
+			pool := eligibleProfiles(tt.goos, false)
+			if len(pool) == 0 {
+				t.Fatal("expected eligible profiles")
+			}
+			for _, candidate := range pool {
+				if !slices.Contains(candidate.profile.Tags, tt.tag) {
+					t.Fatalf("profile %q does not match %s", candidate.profile.ID, tt.goos)
+				}
+			}
+		})
+	}
+}
+
+// Headless Linux (the Docker deployment) has no real GPU, so only
+// swiftshader-tagged profiles are eligible; headful Linux excludes them.
+func TestEligibleProfilesHeadlessLinuxUsesSwiftShader(t *testing.T) {
+	headless := eligibleProfiles("linux", true)
+	if len(headless) == 0 {
+		t.Fatal("expected a Linux SwiftShader profile")
+	}
+	for _, candidate := range headless {
+		if !slices.Contains(candidate.profile.Tags, "swiftshader") {
+			t.Fatalf("headless profile %q is not a SwiftShader profile", candidate.profile.ID)
+		}
+	}
+	if len(headless) < 2 {
+		t.Fatalf("expected multiple SwiftShader profiles so Docker is not a single fingerprint, got %d", len(headless))
+	}
+	timezones := make(map[string]struct{}, len(headless))
+	for _, candidate := range headless {
+		timezones[candidate.profile.Timezone] = struct{}{}
+	}
+	if len(timezones) < 2 {
+		t.Fatalf("expected distinct CDP-visible SwiftShader profiles, got timezones %v", timezones)
+	}
+
+	headful := eligibleProfiles("linux", false)
+	if len(headful) == 0 {
+		t.Fatal("expected headful Linux profiles")
+	}
+	for _, candidate := range headful {
+		if slices.Contains(candidate.profile.Tags, "swiftshader") {
+			t.Fatalf("headful profile %q should not be a SwiftShader profile", candidate.profile.ID)
+		}
+	}
+}
+
+func TestNormalizeRegion(t *testing.T) {
+	tests := []struct {
+		input string
+		want  string
+	}{
+		{input: "", want: ""},
+		{input: "ru", want: "ru"},
+		{input: "RU", want: "ru"},
+		{input: "ru-RU", want: "ru"},
+		{input: "ru_RU", want: "ru"},
+		{input: "ru-RU,ru;q=0.9", want: "ru"},
+		{input: "en-US,en;q=0.9", want: "us"},
+		{input: "de", want: "us"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			if got := NormalizeRegion(tt.input); got != tt.want {
+				t.Fatalf("NormalizeRegion(%q) = %q, want %q", tt.input, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestLoadProfilesFromJSON(t *testing.T) {
+	originalCatalog, originalLaneProfiles, originalDefaultRegions := snapshotProfileState()
+	t.Cleanup(func() {
+		restoreProfileState(originalCatalog, originalLaneProfiles, originalDefaultRegions)
+	})
+
+	path := filepath.Join(t.TempDir(), "profiles.json")
+	payload := `{
+		"profiles": [
+			{
+				"id": "custom-ru",
+				"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+				"uach_brands": [
+					{"brand": "Chromium", "version": "136"}
+				],
+				"uach_full_version_list": [
+					{"brand": "Chromium", "version": "136.0.0.0"}
+				],
+				"platform": "Linux",
+				"platform_version": "6.0.0",
+				"architecture": "x86",
+				"mobile": false,
+				"accept_language": "ru-RU,ru;q=0.9",
+				"navigator_langs": ["ru-RU"],
+				"locale": "ru-RU",
+				"timezone": "Europe/Moscow",
+				"viewport": {"width": 1920, "height": 1080},
+				"tags": ["linux", "ru"],
+				"weight": 2
+			}
+		],
+		"lane_profile_ids": {
+			"google:ru-RU": "custom-ru"
+		},
+		"default_region_by_engine": {
+			"google": "ru-RU"
+		}
+	}`
+	if err := os.WriteFile(path, []byte(payload), 0o644); err != nil {
+		t.Fatalf("write profiles json: %v", err)
+	}
+
+	if err := LoadProfilesFromJSON(path); err != nil {
+		t.Fatalf("load profiles json: %v", err)
+	}
+
+	if got := DefaultRegionForEngine("google"); got != "ru" {
+		t.Fatalf("expected google default region ru, got %q", got)
+	}
+
+	profile := SelectProfile("google", "")
+	if profile.ID != "custom-ru" {
+		t.Fatalf("expected custom profile id custom-ru, got %q", profile.ID)
+	}
+	if profile.Timezone != "Europe/Moscow" {
+		t.Fatalf("expected timezone Europe/Moscow, got %q", profile.Timezone)
+	}
+	if !slices.Contains(profile.Tags, "ru") {
+		t.Fatalf("expected tags to contain 'ru', got %v", profile.Tags)
+	}
+	if profile.Weight != 2 {
+		t.Fatalf("expected weight 2, got %d", profile.Weight)
+	}
+}
+
+func snapshotProfileState() (map[string]Profile, map[string]string, map[string]string) {
+	profileCatalogMu.RLock()
+	defer profileCatalogMu.RUnlock()
+
+	catalogCopy := make(map[string]Profile, len(catalog))
+	for k, v := range catalog {
+		catalogCopy[k] = v
+	}
+
+	laneProfilesCopy := make(map[string]string, len(laneProfileIDs))
+	for k, v := range laneProfileIDs {
+		laneProfilesCopy[k] = v
+	}
+
+	defaultRegionsCopy := make(map[string]string, len(defaultRegionByEngine))
+	for k, v := range defaultRegionByEngine {
+		defaultRegionsCopy[k] = v
+	}
+
+	return catalogCopy, laneProfilesCopy, defaultRegionsCopy
+}
+
+func restoreProfileState(catalogState map[string]Profile, laneProfiles map[string]string, defaultRegions map[string]string) {
+	profileCatalogMu.Lock()
+	defer profileCatalogMu.Unlock()
+
+	catalog = catalogState
+	laneProfileIDs = laneProfiles
+	defaultRegionByEngine = defaultRegions
+}

core/browser/profiles.json

@@ -0,0 +1,97 @@
+{
+  "profiles": [
+    {
+      "id": "chrome-linux-mesa-uhd620",
+      "user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "mesa", "integrated"], "weight": 3
+    },
+    {
+      "id": "chrome-linux-mesa-iris-xe",
+      "user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "mesa", "integrated"], "weight": 2
+    },
+    {
+      "id": "chrome-linux-nvidia",
+      "user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "nvidia"], "weight": 2
+    },
+    {
+      "id": "chrome-linux-amd",
+      "user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "amd"], "weight": 1
+    },
+    {
+      "id": "chrome-linux-swiftshader",
+      "user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "swiftshader", "headless"], "weight": 1
+    },
+    {
+      "id": "chrome-linux-swiftshader-fhd",
+      "user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/Chicago",
+      "viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "swiftshader", "headless"], "weight": 1
+    },
+    {
+      "id": "chrome-win-uhd620",
+      "user_agent_template": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "Windows", "platform_version": "15.0.0", "architecture": "x86", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080}, "tags": ["windows"], "weight": 2
+    },
+    {
+      "id": "chrome-win-nvidia",
+      "user_agent_template": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "Windows", "platform_version": "15.0.0", "architecture": "x86", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080}, "tags": ["windows", "nvidia"], "weight": 2
+    },
+    {
+      "id": "chrome-win-iris-xe",
+      "user_agent_template": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "Windows", "platform_version": "15.0.0", "architecture": "x86", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080}, "tags": ["windows", "integrated"], "weight": 1
+    },
+    {
+      "id": "chrome-win-amd",
+      "user_agent_template": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "Windows", "platform_version": "15.0.0", "architecture": "x86", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
+      "viewport": {"width": 1920, "height": 1080}, "tags": ["windows", "amd"], "weight": 1
+    },
+    {
+      "id": "chrome-macos-m1",
+      "user_agent_template": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "macOS", "platform_version": "13.0.0", "architecture": "arm", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
+      "viewport": {"width": 1440, "height": 900}, "tags": ["macos", "apple-silicon"], "weight": 2
+    },
+    {
+      "id": "chrome-macos-m2",
+      "user_agent_template": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "macOS", "platform_version": "14.0.0", "architecture": "arm", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
+      "viewport": {"width": 1512, "height": 982}, "tags": ["macos", "apple-silicon"], "weight": 2
+    },
+    {
+      "id": "chrome-macos-intel-iris",
+      "user_agent_template": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
+      "platform": "macOS", "platform_version": "14.0.0", "architecture": "x86", "bitness": "64", "mobile": false,
+      "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
+      "viewport": {"width": 1680, "height": 1050}, "tags": ["macos", "integrated"], "weight": 1
+    }
+  ],
+  "lane_profile_ids": {},
+  "default_region_by_engine": {"yandex": "ru"}
+}

core/browser_panic_test.go

@@ -0,0 +1,13 @@
+package core
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestRecoverEnginePanic_ReturnsTypedError(t *testing.T) {
+	err := RecoverEnginePanic("test-engine", "boom", nil)
+	if !errors.Is(err, ErrEngineInternal) {
+		t.Fatalf("expected ErrEngineInternal, got: %v", err)
+	}
+}

core/browser_profile_usage.go

@@ -0,0 +1,62 @@
+package core
+
+import (
+	"context"
+	"slices"
+	"strings"
+	"sync"
+)
+
+type browserProfileUsageContextKey struct{}
+
+type browserProfileUsageTracker struct {
+	mu  sync.Mutex
+	ids []string
+}
+
+func WithBrowserProfileUsage(ctx context.Context) context.Context {
+	ctx = EnsureContext(ctx)
+	if browserProfileUsageFromContext(ctx) != nil {
+		return ctx
+	}
+	return context.WithValue(ctx, browserProfileUsageContextKey{}, &browserProfileUsageTracker{})
+}
+
+func SetBrowserProfileID(ctx context.Context, profileID string) {
+	profileID = strings.TrimSpace(profileID)
+	if profileID == "" {
+		return
+	}
+	tracker := browserProfileUsageFromContext(ctx)
+	if tracker == nil {
+		return
+	}
+
+	tracker.mu.Lock()
+	defer tracker.mu.Unlock()
+	if slices.Contains(tracker.ids, profileID) {
+		return
+	}
+	tracker.ids = append(tracker.ids, profileID)
+}
+
+func BrowserProfileIDsFromContext(ctx context.Context) []string {
+	tracker := browserProfileUsageFromContext(ctx)
+	if tracker == nil {
+		return nil
+	}
+
+	tracker.mu.Lock()
+	defer tracker.mu.Unlock()
+	out := make([]string, len(tracker.ids))
+	copy(out, tracker.ids)
+	return out
+}
+
+func browserProfileUsageFromContext(ctx context.Context) *browserProfileUsageTracker {
+	if ctx == nil {
+		return nil
+	}
+	tracker, _ := ctx.Value(browserProfileUsageContextKey{}).(*browserProfileUsageTracker)
+	return tracker
+}

core/browser_resource_blocking_test.go

@@ -0,0 +1,101 @@
+package core
+
+import (
+	"testing"
+
+	"github.com/go-rod/rod/lib/proto"
+)
+
+func TestBuildTrackingDomainURLPatterns(t *testing.T) {
+	patterns := buildTrackingDomainURLPatterns([]string{"google-analytics.com"})
+	if len(patterns) != 2 {
+		t.Fatalf("expected 2 URL patterns, got %d", len(patterns))
+	}
+	if patterns[0] != "*://google-analytics.com/*" {
+		t.Fatalf("unexpected root pattern: %q", patterns[0])
+	}
+	if patterns[1] != "*://*.google-analytics.com/*" {
+		t.Fatalf("unexpected subdomain pattern: %q", patterns[1])
+	}
+}
+
+func TestShouldBlockResourceType(t *testing.T) {
+	blockedTypes := blockedResourceTypeSet([]proto.NetworkResourceType{
+		proto.NetworkResourceTypeImage,
+		proto.NetworkResourceTypeFont,
+		proto.NetworkResourceTypeMedia,
+		proto.NetworkResourceTypeStylesheet,
+		proto.NetworkResourceTypeScript,
+	})
+
+	tests := []struct {
+		resourceType proto.NetworkResourceType
+		wantBlocked  bool
+	}{
+		{resourceType: proto.NetworkResourceTypeImage, wantBlocked: true},
+		{resourceType: proto.NetworkResourceTypeFont, wantBlocked: true},
+		{resourceType: proto.NetworkResourceTypeMedia, wantBlocked: true},
+		{resourceType: proto.NetworkResourceTypeStylesheet, wantBlocked: true},
+		{resourceType: proto.NetworkResourceTypeScript, wantBlocked: true},
+		{resourceType: proto.NetworkResourceTypeDocument, wantBlocked: false},
+		{resourceType: proto.NetworkResourceTypeXHR, wantBlocked: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(string(tt.resourceType), func(t *testing.T) {
+			_, got := blockedTypes[tt.resourceType]
+			if got != tt.wantBlocked {
+				t.Fatalf("resource type %s: got blocked=%t want %t", tt.resourceType, got, tt.wantBlocked)
+			}
+		})
+	}
+}
+
+func TestProxyAuthFetchPatternsOnlyInterceptDocuments(t *testing.T) {
+	patterns := proxyAuthFetchPatterns()
+	if len(patterns) != 2 {
+		t.Fatalf("expected 2 proxy auth fetch patterns, got %d", len(patterns))
+	}
+
+	for _, pattern := range patterns {
+		if pattern.URLPattern != "http://*/*" && pattern.URLPattern != "https://*/*" {
+			t.Fatalf("unexpected proxy auth URL pattern: %q", pattern.URLPattern)
+		}
+		if pattern.ResourceType != proto.NetworkResourceTypeDocument {
+			t.Fatalf("expected document-only proxy auth interception, got %s", pattern.ResourceType)
+		}
+		if pattern.RequestStage != proto.FetchRequestStageRequest {
+			t.Fatalf("expected request-stage proxy auth interception, got %s", pattern.RequestStage)
+		}
+	}
+}
+
+func TestParseBlockedResourceTypes(t *testing.T) {
+	got, err := ParseBlockedResourceTypes("image,font,css,js,media")
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	expectedSet := map[proto.NetworkResourceType]struct{}{
+		proto.NetworkResourceTypeImage:      {},
+		proto.NetworkResourceTypeFont:       {},
+		proto.NetworkResourceTypeStylesheet: {},
+		proto.NetworkResourceTypeScript:     {},
+		proto.NetworkResourceTypeMedia:      {},
+	}
+	gotSet := blockedResourceTypeSet(got)
+	if len(gotSet) != len(expectedSet) {
+		t.Fatalf("expected %d unique resource types, got %d", len(expectedSet), len(gotSet))
+	}
+	for resourceType := range expectedSet {
+		if _, ok := gotSet[resourceType]; !ok {
+			t.Fatalf("expected resource type %s to be present", resourceType)
+		}
+	}
+}
+
+func TestParseBlockedResourceTypesInvalid(t *testing.T) {
+	if _, err := ParseBlockedResourceTypes("image,unknown"); err == nil {
+		t.Fatal("expected invalid token to return error")
+	}
+}

core/browser_test.go

@@ -1,49 +1,354 @@
+//go:build integration
+// +build integration
+
 package core
 
 import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
 	"testing"
+	"time"
+
+	"github.com/karust/openserp/core/fpcheck"
+	"github.com/karust/openserp/core/fpcheck/detectors"
+	"github.com/karust/openserp/testutil"
 )
 
-var browser *Browser
+const botFingerprintTestsEnv = "OPENSERP_BOT_TESTS"
+const botFingerprintArtifactDir = "testdata"
 
 func TestCreateBrowser(t *testing.T) {
-	// if browser != nil && browser.IsInitialized() {
-	// 	return
-	// }
+	testutil.RequireIntegration(t)
 
-	var err error
 	opts := BrowserOpts{IsHeadless: true, IsLeakless: false}
-	browser, err = NewBrowser(opts)
-	if err != nil {
-		t.Fatalf("Error failed initializing browser: %s", err)
-	}
-}
-
-// func TestCreateLeaklessBrowser(t *testing.T) {
-// 	var err error
-// 	opts := BrowserOpts{IsHeadless: true, IsLeakless: true}
-// 	browser, err = NewBrowser(opts)
-// 	if err != nil {
-// 		t.Fatalf("Error failed initializing leakless browser: %s", err)
-// 	}
-// }
-
-// Manually observe test results for now
-func TestBot(t *testing.T) {
-
-	var err error
-	opts := BrowserOpts{IsHeadless: false, IsLeakless: true, LeavePageOpen: true}
-	browser, err = NewBrowser(opts)
+	browser, err := NewBrowser(opts)
 	if err != nil {
 		t.Fatalf("Error failed initializing browser: %s", err)
 	}
 
-	page, _ := browser.Navigate("https://bot.sannysoft.com")
-	page.MustScreenshotFullPage("./test/screenshot_bot.png")
-
-	page, _ = browser.Navigate("https://www.whatismybrowser.com/")
-	page.MustScreenshotFullPage("./test/screenshot_browser.png")
-
-	page, _ = browser.Navigate("https://abrahamjuliot.github.io/creepjs/")
-	page.MustScreenshotFullPage("./test/screenshot_creep.png")
+	page, err := browser.Navigate(context.Background(), "about:blank")
+	if err != nil {
+		t.Fatalf("navigate about:blank: %v", err)
+	}
+	defer closeTestBrowser(t, browser)
+	defer func() {
+		if err := ClosePageWithTimeout(context.Background(), page, time.Second); err != nil {
+			t.Logf("close page: %v", err)
+		}
+	}()
+}
+
+func TestNavigateUsesIsolatedBrowserContext(t *testing.T) {
+	testutil.RequireIntegration(t)
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/cookies/set":
+			http.SetCookie(w, &http.Cookie{
+				Name:  "openserp_session",
+				Value: "request-a",
+				Path:  "/",
+			})
+			_, _ = w.Write([]byte("cookie-set"))
+		case "/cookies":
+			_, _ = w.Write([]byte(r.Header.Get("Cookie")))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	opts := BrowserOpts{IsHeadless: true, IsLeakless: false, Timeout: 15 * time.Second}
+	browser, err := NewBrowser(opts)
+	if err != nil {
+		t.Fatalf("failed initializing browser: %s", err)
+	}
+	defer closeTestBrowser(t, browser)
+
+	pageA, err := browser.Navigate(context.Background(), srv.URL+"/cookies/set")
+	if err != nil {
+		t.Fatalf("navigate cookie setter: %v", err)
+	}
+	if err := ClosePageWithTimeout(context.Background(), pageA, time.Second); err != nil {
+		t.Fatalf("close setter page: %v", err)
+	}
+
+	pageB, err := browser.Navigate(context.Background(), srv.URL+"/cookies")
+	if err != nil {
+		t.Fatalf("navigate cookie reader: %v", err)
+	}
+	defer func() {
+		if err := ClosePageWithTimeout(context.Background(), pageB, time.Second); err != nil {
+			t.Logf("close reader page: %v", err)
+		}
+	}()
+
+	body, err := pageB.Timeout(5 * time.Second).Element("body")
+	if err != nil {
+		t.Fatalf("read response body: %v", err)
+	}
+	cookieHeader, err := body.Text()
+	if err != nil {
+		t.Fatalf("extract response text: %v", err)
+	}
+
+	if strings.Contains(cookieHeader, "openserp_session=request-a") {
+		t.Fatalf("cookie leaked between requests; got header %q", cookieHeader)
+	}
+}
+
+func TestNavigateReusesCookiesForSameProxyLane(t *testing.T) {
+	testutil.RequireIntegration(t)
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/cookies/set":
+			http.SetCookie(w, &http.Cookie{Name: "openserp_lane", Value: "same-lane", Path: "/"})
+			_, _ = w.Write([]byte("cookie-set"))
+		case "/cookies":
+			_, _ = w.Write([]byte(r.Header.Get("Cookie")))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	browser, err := NewBrowser(BrowserOpts{
+		IsHeadless:     true,
+		IsLeakless:     false,
+		Timeout:        15 * time.Second,
+		ProxyLaneStore: NewLaneStore(10),
+	})
+	if err != nil {
+		t.Fatalf("failed initializing browser: %s", err)
+	}
+	defer closeTestBrowser(t, browser)
+
+	laneCtx := WithProxyLaneKey(WithEngine(context.Background(), "google"), ProxyLaneKey{Engine: "google", SessionID: "sid-a"})
+	pageA, err := browser.Navigate(laneCtx, srv.URL+"/cookies/set")
+	if err != nil {
+		t.Fatalf("navigate cookie setter: %v", err)
+	}
+	if err := ClosePageWithTimeout(context.Background(), pageA, time.Second); err != nil {
+		t.Fatalf("close setter page: %v", err)
+	}
+
+	pageB, err := browser.Navigate(laneCtx, srv.URL+"/cookies")
+	if err != nil {
+		t.Fatalf("navigate cookie reader: %v", err)
+	}
+	defer func() {
+		if err := ClosePageWithTimeout(context.Background(), pageB, time.Second); err != nil {
+			t.Logf("close reader page: %v", err)
+		}
+	}()
+
+	body, err := pageB.Timeout(5 * time.Second).Element("body")
+	if err != nil {
+		t.Fatalf("read response body: %v", err)
+	}
+	cookieHeader, err := body.Text()
+	if err != nil {
+		t.Fatalf("extract response text: %v", err)
+	}
+	if !strings.Contains(cookieHeader, "openserp_lane=same-lane") {
+		t.Fatalf("expected same proxy lane to restore cookie, got %q", cookieHeader)
+	}
+}
+
+func TestNavigateDoesNotShareCookiesAcrossProxyLanes(t *testing.T) {
+	testutil.RequireIntegration(t)
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/cookies/set":
+			http.SetCookie(w, &http.Cookie{Name: "openserp_lane", Value: "lane-a", Path: "/"})
+			_, _ = w.Write([]byte("cookie-set"))
+		case "/cookies":
+			_, _ = w.Write([]byte(r.Header.Get("Cookie")))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	browser, err := NewBrowser(BrowserOpts{
+		IsHeadless:     true,
+		IsLeakless:     false,
+		Timeout:        15 * time.Second,
+		ProxyLaneStore: NewLaneStore(10),
+	})
+	if err != nil {
+		t.Fatalf("failed initializing browser: %s", err)
+	}
+	defer closeTestBrowser(t, browser)
+
+	laneA := WithProxyLaneKey(WithEngine(context.Background(), "google"), ProxyLaneKey{Engine: "google", SessionID: "sid-a"})
+	laneB := WithProxyLaneKey(WithEngine(context.Background(), "google"), ProxyLaneKey{Engine: "google", SessionID: "sid-b"})
+	pageA, err := browser.Navigate(laneA, srv.URL+"/cookies/set")
+	if err != nil {
+		t.Fatalf("navigate cookie setter: %v", err)
+	}
+	if err := ClosePageWithTimeout(context.Background(), pageA, time.Second); err != nil {
+		t.Fatalf("close setter page: %v", err)
+	}
+
+	pageB, err := browser.Navigate(laneB, srv.URL+"/cookies")
+	if err != nil {
+		t.Fatalf("navigate cookie reader: %v", err)
+	}
+	defer func() {
+		if err := ClosePageWithTimeout(context.Background(), pageB, time.Second); err != nil {
+			t.Logf("close reader page: %v", err)
+		}
+	}()
+
+	body, err := pageB.Timeout(5 * time.Second).Element("body")
+	if err != nil {
+		t.Fatalf("read response body: %v", err)
+	}
+	cookieHeader, err := body.Text()
+	if err != nil {
+		t.Fatalf("extract response text: %v", err)
+	}
+	if strings.Contains(cookieHeader, "openserp_lane=lane-a") {
+		t.Fatalf("cookie leaked across proxy lanes; got header %q", cookieHeader)
+	}
+}
+
+func TestNavigateClassifiesMainDocumentStatus(t *testing.T) {
+	testutil.RequireIntegration(t)
+
+	tests := []struct {
+		name   string
+		status int
+		want   error
+	}{
+		{name: "blocked", status: http.StatusForbidden, want: ErrBlocked},
+		{name: "rate limited", status: http.StatusTooManyRequests, want: ErrRateLimited},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				w.WriteHeader(tt.status)
+				_, _ = w.Write([]byte("classified"))
+			}))
+			defer srv.Close()
+
+			browser, err := NewBrowser(BrowserOpts{IsHeadless: true, IsLeakless: false, Timeout: 15 * time.Second})
+			if err != nil {
+				t.Fatalf("failed initializing browser: %s", err)
+			}
+			defer closeTestBrowser(t, browser)
+
+			page, err := browser.Navigate(context.Background(), srv.URL)
+			if page != nil {
+				_ = ClosePageWithTimeout(context.Background(), page, time.Second)
+			}
+			if !errors.Is(err, tt.want) {
+				t.Fatalf("expected %v, got %v", tt.want, err)
+			}
+		})
+	}
+}
+
+func TestFingerprintDetectors(t *testing.T) {
+	testutil.RequireIntegration(t)
+	if strings.TrimSpace(os.Getenv(botFingerprintTestsEnv)) != "1" {
+		t.Skipf("set %s=1 to run fingerprint tests", botFingerprintTestsEnv)
+	}
+
+	reports := make(map[string]fpcheck.Report)
+	criticalFailures := make([]string, 0)
+
+	for _, detector := range detectors.All() {
+		report := runFingerprintDetector(t, detector)
+		key := detector.Name()
+		reports[key] = report
+
+		if len(report.Summary.Critical) > 0 {
+			for _, critical := range report.Summary.Critical {
+				criticalFailures = append(criticalFailures, fmt.Sprintf("%s:%s", key, critical))
+			}
+		}
+	}
+
+	reportPath := filepath.Join(botFingerprintArtifactDir, "fingerprint_report.json")
+	if err := writeFingerprintReport(reportPath, reports); err != nil {
+		t.Fatalf("write fingerprint report: %v", err)
+	}
+	absReportPath, err := filepath.Abs(reportPath)
+	if err == nil {
+		t.Logf("Fingerprint report artifact: %s", absReportPath)
+	} else {
+		t.Logf("Fingerprint report artifact: core/%s", filepath.ToSlash(reportPath))
+	}
+
+	if len(criticalFailures) > 0 {
+		t.Fatalf("critical fingerprint detections found: %s", strings.Join(criticalFailures, ", "))
+	}
+}
+
+func runFingerprintDetector(t *testing.T, detector fpcheck.Detector) fpcheck.Report {
+	t.Helper()
+
+	opts := BrowserOpts{
+		IsHeadless: true,
+		IsLeakless: false,
+		Timeout:    20 * time.Second,
+	}
+	browser, err := NewBrowser(opts)
+	if err != nil {
+		t.Fatalf("create browser: %v", err)
+	}
+	defer closeTestBrowser(t, browser)
+
+	report, err := fpcheck.Run(context.Background(), browser, detector, botFingerprintArtifactDir)
+	if err != nil {
+		t.Fatalf("run detector %s: %v", detector.Name(), err)
+	}
+
+	t.Logf(
+		"Fingerprint %s: passed=%d failed=%d critical=%d",
+		detector.Name(),
+		report.Summary.Passed,
+		report.Summary.Failed,
+		len(report.Summary.Critical),
+	)
+	return report
+}
+
+func writeFingerprintReport(path string, reports map[string]fpcheck.Report) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("create artifact directory: %w", err)
+	}
+
+	data, err := json.MarshalIndent(reports, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal report: %w", err)
+	}
+	data = append(data, '\n')
+
+	if err := os.WriteFile(path, data, 0o644); err != nil {
+		return fmt.Errorf("write report file %s: %w", path, err)
+	}
+	return nil
+}
+
+func closeTestBrowser(t *testing.T, browser *Browser) {
+	t.Helper()
+	if browser == nil {
+		return
+	}
+	if err := browser.Close(); err != nil {
+		t.Logf("close browser: %v", err)
+	}
 }

core/browser_unit_test.go

@@ -0,0 +1,235 @@
+package core
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	browserprofile "github.com/karust/openserp/core/browser"
+)
+
+func TestResolveBrowserBinaryPathPrefersExplicit(t *testing.T) {
+	dir := t.TempDir()
+	bin := filepath.Join(dir, "chromium")
+	if err := os.WriteFile(bin, []byte("test"), 0o755); err != nil {
+		t.Fatalf("write temp browser binary: %v", err)
+	}
+
+	path, err := resolveBrowserBinaryPath(bin, func() (string, bool) {
+		return "/should/not/be/used", true
+	})
+	if err != nil {
+		t.Fatalf("resolve browser path: %v", err)
+	}
+	if path != bin {
+		t.Fatalf("expected explicit browser path %q, got %q", bin, path)
+	}
+}
+
+func TestResolveBrowserBinaryPathFallsBackToLookPath(t *testing.T) {
+	want := "/usr/bin/chromium"
+	path, err := resolveBrowserBinaryPath("", func() (string, bool) {
+		return want, true
+	})
+	if err != nil {
+		t.Fatalf("resolve browser path: %v", err)
+	}
+	if path != want {
+		t.Fatalf("expected lookPath result %q, got %q", want, path)
+	}
+}
+
+func TestResolveBrowserBinaryPathReturnsEmptyWhenNothingResolved(t *testing.T) {
+	path, err := resolveBrowserBinaryPath("", func() (string, bool) {
+		return "", false
+	})
+	if err != nil {
+		t.Fatalf("resolve browser path: %v", err)
+	}
+	if path != "" {
+		t.Fatalf("expected empty path, got %q", path)
+	}
+}
+
+func TestResolveBrowserBinaryPathRejectsInvalidExplicit(t *testing.T) {
+	dir := t.TempDir()
+	if _, err := resolveBrowserBinaryPath(dir, func() (string, bool) {
+		return "", false
+	}); err == nil {
+		t.Fatalf("expected error when explicit browser_path points to a directory")
+	}
+}
+
+func TestBrowserLaunchLanguageIsProcessStable(t *testing.T) {
+	tests := []struct {
+		name string
+		opts BrowserOpts
+		want string
+	}{
+		{name: "default locale", opts: BrowserOpts{}, want: "en-US"},
+		{name: "request hint does not change process locale", opts: BrowserOpts{LanguageCode: "de"}, want: "en-US"},
+		{name: "regional hint does not change process locale", opts: BrowserOpts{LanguageCode: "en-GB"}, want: "en-US"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := browserLaunchLanguage(tt.opts); got != tt.want {
+				t.Fatalf("browserLaunchLanguage() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestProfileNavigatorLanguagesStripsHeaderWeights(t *testing.T) {
+	profile := browserprofile.Profile{
+		AcceptLanguage: "en-US,en;q=0.9",
+		NavigatorLangs: []string{"en-US"},
+	}
+
+	got := profileNavigatorLanguages(profile)
+	want := []string{"en-US", "en"}
+	if len(got) != len(want) {
+		t.Fatalf("profileNavigatorLanguages() = %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("profileNavigatorLanguages() = %v, want %v", got, want)
+		}
+	}
+}
+
+func TestProfileNavigatorLanguagesForRuntime(t *testing.T) {
+	profile := browserprofile.Profile{
+		AcceptLanguage: "en-US,en;q=0.9",
+		NavigatorLangs: []string{"en-US"},
+	}
+
+	linuxHeadless := profileNavigatorLanguagesForRuntime(profile, "linux", true)
+	if len(linuxHeadless) != 1 || linuxHeadless[0] != "en-US" {
+		t.Fatalf("linux headless languages = %v, want [en-US]", linuxHeadless)
+	}
+
+	windowsHeadless := profileNavigatorLanguagesForRuntime(profile, "windows", true)
+	if len(windowsHeadless) != 2 || windowsHeadless[0] != "en-US" || windowsHeadless[1] != "en" {
+		t.Fatalf("windows headless languages = %v, want [en-US en]", windowsHeadless)
+	}
+}
+
+func TestMinPositiveDuration(t *testing.T) {
+	tests := []struct {
+		name string
+		a    time.Duration
+		b    time.Duration
+		want time.Duration
+	}{
+		{name: "both positive", a: 30 * time.Second, b: 2 * time.Second, want: 2 * time.Second},
+		{name: "first unset", a: 0, b: 2 * time.Second, want: 2 * time.Second},
+		{name: "second unset", a: 30 * time.Second, b: 0, want: 30 * time.Second},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := minPositiveDuration(tt.a, tt.b); got != tt.want {
+				t.Fatalf("minPositiveDuration() = %s, want %s", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestApplyProfileLanguageHintRewritesTimezone(t *testing.T) {
+	profile := browserprofile.Profile{
+		AcceptLanguage: "ru-RU,ru;q=0.9",
+		NavigatorLangs: []string{"ru-RU"},
+		Locale:         "ru-RU",
+		Timezone:       "Europe/Moscow",
+	}
+	got := applyProfileLanguageHint(profile, "de-DE")
+	if got.Timezone != "Europe/Berlin" {
+		t.Fatalf("expected timezone Europe/Berlin, got %q", got.Timezone)
+	}
+}
+
+func TestRemoveChromeBrand(t *testing.T) {
+	profile := browserprofile.Profile{
+		UACHBrands: []browserprofile.BrandVersion{
+			{Brand: "Not_A Brand", Version: "24"},
+			{Brand: "Chromium", Version: "136"},
+			{Brand: "Google Chrome", Version: "136"},
+		},
+		UACHFullVerList: []browserprofile.BrandVersion{
+			{Brand: "Chromium", Version: "136.0.0.0"},
+			{Brand: "Google Chrome", Version: "136.0.0.0"},
+		},
+	}
+
+	got := removeChromeBrand(profile)
+	for _, brand := range got.UACHBrands {
+		if brand.Brand == "Google Chrome" {
+			t.Fatal("expected Google Chrome brand to be removed from UACHBrands")
+		}
+	}
+	for _, brand := range got.UACHFullVerList {
+		if brand.Brand == "Google Chrome" {
+			t.Fatal("expected Google Chrome brand to be removed from UACHFullVerList")
+		}
+	}
+	if len(got.UACHBrands) != 2 || len(got.UACHFullVerList) != 1 {
+		t.Fatalf("unexpected brand counts: brands=%d fullList=%d", len(got.UACHBrands), len(got.UACHFullVerList))
+	}
+}
+
+func TestApplyProfileLanguageHint(t *testing.T) {
+	base := browserprofile.Profile{
+		AcceptLanguage: "en-US,en;q=0.9",
+		NavigatorLangs: []string{"en-US"},
+		Locale:         "en-US",
+	}
+
+	tests := []struct {
+		name   string
+		lang   string
+		wantAL string
+		wantL  string
+	}{
+		{
+			name:   "empty hint keeps profile",
+			lang:   "",
+			wantAL: "en-US,en;q=0.9",
+			wantL:  "en-US",
+		},
+		{
+			name:   "same language without region keeps profile",
+			lang:   "en",
+			wantAL: "en-US,en;q=0.9",
+			wantL:  "en-US",
+		},
+		{
+			name:   "new language overrides locale headers",
+			lang:   "de",
+			wantAL: "de-DE,de;q=0.9",
+			wantL:  "de-DE",
+		},
+		{
+			name:   "explicit region overrides locale headers",
+			lang:   "en-GB",
+			wantAL: "en-GB,en;q=0.9",
+			wantL:  "en-GB",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := applyProfileLanguageHint(base, tt.lang)
+			if got.AcceptLanguage != tt.wantAL {
+				t.Fatalf("AcceptLanguage = %q, want %q", got.AcceptLanguage, tt.wantAL)
+			}
+			if got.Locale != tt.wantL {
+				t.Fatalf("Locale = %q, want %q", got.Locale, tt.wantL)
+			}
+			if len(got.NavigatorLangs) != 1 || got.NavigatorLangs[0] != tt.wantL {
+				t.Fatalf("NavigatorLangs = %v, want [%q]", got.NavigatorLangs, tt.wantL)
+			}
+		})
+	}
+}

core/cache.go

@@ -4,6 +4,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 )
@@ -35,25 +36,58 @@ func NewResponseCache(ttl time.Duration, maxSize int) *ResponseCache {
 }
 
 func BuildCacheKey(engine string, action string, q Query) string {
+	country, class, provider := cacheProxyMarket(q)
 	raw := fmt.Sprintf(
-		"%s|%s|%s|%s|%s|%s|%s|%d|%d|%t|%t|%s",
-		engine,
-		action,
-		q.Text,
-		q.LangCode,
-		q.DateInterval,
-		q.Filetype,
-		q.Site,
+		"%s|%s|%s|%s|%s|%s|%s|%s|%d|%d|%t|%t|%s|%s|%s",
+		cacheToken(engine),
+		cacheToken(action),
+		strings.TrimSpace(q.Text),
+		cacheToken(q.LangCode),
+		cacheToken(q.Region),
+		strings.TrimSpace(q.DateInterval),
+		cacheToken(q.Filetype),
+		cacheToken(q.Site),
 		q.Limit,
 		q.Start,
 		q.Filter,
-		q.Answers,
-		q.ProxyOverride,
+		q.Features,
+		country,
+		class,
+		provider,
 	)
 	hash := sha256.Sum256([]byte(raw))
 	return hex.EncodeToString(hash[:])
 }
 
+func cacheToken(value string) string {
+	return strings.ToLower(strings.TrimSpace(value))
+}
+
+func cacheProxyMarket(q Query) (country string, class string, provider string) {
+	country = cacheToken(q.ProxyCountry)
+	if country == "" {
+		// Region is a stronger market signal than LangCode; LangCode is the last
+		// fallback. TODO: Use explicit balancer market metadata everywhere.
+		if region := CountryFromRegion(q.Region); region != "" {
+			country = strings.ToLower(region)
+		} else {
+			country = cacheToken(q.LangCode)
+		}
+	}
+	return country,
+		cacheToken(q.ProxyClass),
+		cacheToken(q.ProxyProvider)
+}
+
+func ShouldBypassCacheForProxyMarket(q Query) bool {
+	if strings.TrimSpace(q.ProxyURL) == "" && strings.TrimSpace(q.ProxyOverride) == "" {
+		return false
+	}
+	return strings.TrimSpace(q.ProxyCountry) == "" &&
+		strings.TrimSpace(q.ProxyClass) == "" &&
+		strings.TrimSpace(q.ProxyProvider) == ""
+}
+
 func (c *ResponseCache) Get(key string) ([]byte, bool) {
 	c.mu.Lock()
 	defer c.mu.Unlock()

core/cache_test.go

@@ -5,6 +5,28 @@ import (
 	"time"
 )
 
+func TestCacheProxyMarketFallback(t *testing.T) {
+	tests := []struct {
+		name    string
+		q       Query
+		country string
+	}{
+		{name: "proxy country wins", q: Query{ProxyCountry: "DE", Region: "RU", LangCode: "EN"}, country: "de"},
+		{name: "region country fallback", q: Query{Region: "RU", LangCode: "EN"}, country: "ru"},
+		{name: "region locale fallback", q: Query{Region: "en-GB", LangCode: "EN"}, country: "gb"},
+		{name: "lang code last resort", q: Query{LangCode: "EN"}, country: "en"},
+		{name: "yandex numeric region ignored for market", q: Query{Region: "213", LangCode: "RU"}, country: "ru"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			country, _, _ := cacheProxyMarket(tt.q)
+			if country != tt.country {
+				t.Fatalf("cacheProxyMarket country = %q, want %q", country, tt.country)
+			}
+		})
+	}
+}
+
 func TestResponseCacheSetAndGet(t *testing.T) {
 	cache := NewResponseCache(5*time.Second, 10)
 	key := BuildCacheKey("google", "search", Query{Text: "golang", Limit: 10})
@@ -104,7 +126,7 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) {
 		Limit:    10,
 		Start:    0,
 		Filter:   true,
-		Answers:  false,
+		Features: false,
 	}
 
 	baseKey := BuildCacheKey("google", "search", base)
@@ -112,13 +134,24 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) {
 	if same := BuildCacheKey("google", "search", base); same != baseKey {
 		t.Fatal("expected deterministic key for same query")
 	}
+	if changed := BuildCacheKey("google", "search", Query{
+		Text:     "golang",
+		LangCode: "EN",
+		Region:   "US",
+		Limit:    10,
+		Start:    0,
+		Filter:   true,
+		Features: false,
+	}); changed == baseKey {
+		t.Fatal("expected region to affect cache key")
+	}
 	if changed := BuildCacheKey("google", "search", Query{
 		Text:     "golang",
 		LangCode: "EN",
 		Limit:    20,
 		Start:    0,
 		Filter:   true,
-		Answers:  false,
+		Features: false,
 	}); changed == baseKey {
 		t.Fatal("expected limit to affect cache key")
 	}
@@ -128,7 +161,7 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) {
 		Limit:    10,
 		Start:    10,
 		Filter:   true,
-		Answers:  false,
+		Features: false,
 	}); changed == baseKey {
 		t.Fatal("expected start to affect cache key")
 	}
@@ -138,7 +171,7 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) {
 		Limit:    10,
 		Start:    0,
 		Filter:   false,
-		Answers:  false,
+		Features: false,
 	}); changed == baseKey {
 		t.Fatal("expected filter to affect cache key")
 	}
@@ -148,8 +181,107 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) {
 		Limit:    10,
 		Start:    0,
 		Filter:   true,
-		Answers:  true,
+		Features: true,
 	}); changed == baseKey {
-		t.Fatal("expected answers to affect cache key")
+		t.Fatal("expected features to affect cache key")
+	}
+}
+
+func TestBuildCacheKeyNormalizesStableFields(t *testing.T) {
+	base := BuildCacheKey("google", "search", Query{
+		Text:         " golang ",
+		LangCode:     "EN",
+		Region:       " us ",
+		Filetype:     "PDF",
+		Site:         "EXAMPLE.COM",
+		Limit:        10,
+		ProxyCountry: " US ",
+		ProxyClass:   " Residential ",
+	})
+	same := BuildCacheKey("Google", "Search", Query{
+		Text:         "golang",
+		LangCode:     "en",
+		Region:       "US",
+		Filetype:     "pdf",
+		Site:         "example.com",
+		Limit:        10,
+		ProxyCountry: "us",
+		ProxyClass:   "residential",
+	})
+	if same != base {
+		t.Fatal("expected cache key to normalize engine/action, locale, filters, and proxy market fields")
+	}
+}
+
+func TestBuildCacheKeyUsesProxyMarketNotSessionOrURL(t *testing.T) {
+	base := Query{
+		Text:           "golang",
+		LangCode:       "EN",
+		Limit:          10,
+		ProxyURL:       "http://user:password-a@proxy-a:8080",
+		ProxyCountry:   " US ",
+		ProxyClass:     " Residential ",
+		ProxyProvider:  " WebShare ",
+		ProxySessionID: "sid-a",
+	}
+	baseKey := BuildCacheKey("google", "search", base)
+
+	sameMarket := base
+	sameMarket.ProxyURL = "http://user:password-b@proxy-b:8080"
+	sameMarket.ProxySessionID = "sid-b"
+	if got := BuildCacheKey("google", "search", sameMarket); got != baseKey {
+		t.Fatal("expected proxy URL and session id not to affect cache key")
+	}
+
+	differentCountry := base
+	differentCountry.ProxyCountry = "de"
+	if got := BuildCacheKey("google", "search", differentCountry); got == baseKey {
+		t.Fatal("expected proxy country to affect cache key")
+	}
+
+	differentClass := base
+	differentClass.ProxyClass = "datacenter"
+	if got := BuildCacheKey("google", "search", differentClass); got == baseKey {
+		t.Fatal("expected proxy class to affect cache key")
+	}
+
+	differentProvider := base
+	differentProvider.ProxyProvider = "brightdata"
+	if got := BuildCacheKey("google", "search", differentProvider); got == baseKey {
+		t.Fatal("expected proxy provider to affect cache key")
+	}
+}
+
+func TestBuildCacheKeyFallsBackToLanguageWhenCountryAbsent(t *testing.T) {
+	base := Query{Text: "golang", LangCode: "en", Limit: 10}
+	baseKey := BuildCacheKey("google", "search", base)
+
+	changed := base
+	changed.LangCode = "de"
+	if got := BuildCacheKey("google", "search", changed); got == baseKey {
+		t.Fatal("expected language fallback to affect cache key when proxy country is absent")
+	}
+
+	withCountry := base
+	withCountry.ProxyCountry = "us"
+	changedWithCountry := withCountry
+	changedWithCountry.LangCode = "de"
+	if got := BuildCacheKey("google", "search", changedWithCountry); got == BuildCacheKey("google", "search", withCountry) {
+		t.Fatal("expected language itself to remain part of the cache key")
+	}
+}
+
+func TestShouldBypassCacheForProxyMarket(t *testing.T) {
+	if !ShouldBypassCacheForProxyMarket(Query{ProxyURL: "http://proxy.example:8080"}) {
+		t.Fatal("expected request proxy without market metadata to bypass cache")
+	}
+	if !ShouldBypassCacheForProxyMarket(Query{ProxyOverride: "us"}) {
+		t.Fatal("expected tag override without market metadata to bypass cache")
+	}
+	if ShouldBypassCacheForProxyMarket(Query{ProxyURL: "http://proxy.example:8080", ProxyCountry: "us"}) {
+		t.Fatal("expected explicit country market metadata to allow cache")
+	}
+	if ShouldBypassCacheForProxyMarket(Query{Text: "golang"}) {
+		t.Fatal("expected direct query without proxy override to allow cache")
 	}
 }

core/captcha.go

@@ -1,28 +1,106 @@
 package core
 
 import (
+	"net/url"
+	"strings"
+	"sync/atomic"
+
 	api2captcha "github.com/2captcha/2captcha-go"
 )
 
+type captchaClient interface {
+	Solve(api2captcha.Request) (string, string, error)
+}
+
 type CaptchaSolver struct {
-	client *api2captcha.Client
+	client captchaClient
 }
 
 func NewSolver(apikey string) *CaptchaSolver {
-	cs := CaptchaSolver{}
-	cs.client = api2captcha.NewClient(apikey)
-	return &cs
+	return &CaptchaSolver{
+		client: api2captcha.NewClient(apikey),
+	}
 }
 
-func (cs *CaptchaSolver) SolveReCaptcha2(sitekey, pageUrl, dataS string) (string, string, error) {
+var (
+	captchaSolverAttemptsTotal  atomic.Uint64
+	captchaSolverSuccessesTotal atomic.Uint64
+	captchaSolverFailuresTotal  atomic.Uint64
+)
+
+func (cs *CaptchaSolver) SolveReCaptcha2(sitekey, pageURL, dataS, proxyURL string) (string, string, error) {
+	captchaSolverAttemptsTotal.Add(1)
+
 	cap := api2captcha.ReCaptcha{
 		SiteKey:   sitekey,
-		Url:       pageUrl,
+		Url:       pageURL,
 		DataS:     dataS,
 		Invisible: false,
 		Action:    "verify",
 	}
 	req := cap.ToRequest()
-	req.SetProxy("HTTPS", "login:password@IP_address:PORT")
-	return cs.client.Solve(req)
+
+	if proxyType, proxyAddr, ok := toCaptchaProxy(proxyURL); ok {
+		req.SetProxy(proxyType, proxyAddr)
+	}
+
+	resp, id, err := cs.client.Solve(req)
+	if err != nil {
+		captchaSolverFailuresTotal.Add(1)
+		return resp, id, err
+	}
+
+	captchaSolverSuccessesTotal.Add(1)
+	return resp, id, nil
+}
+
+func toCaptchaProxy(raw string) (string, string, bool) {
+	normalized, err := NormalizeProxyURL(raw)
+	if err != nil || normalized == "" {
+		return "", "", false
+	}
+
+	parsed, err := url.Parse(normalized)
+	if err != nil || parsed.Host == "" {
+		return "", "", false
+	}
+
+	var proxyType string
+	switch strings.ToLower(parsed.Scheme) {
+	case "http":
+		proxyType = "HTTP"
+	case "https":
+		proxyType = "HTTPS"
+	case "socks5", "socks5h":
+		proxyType = "SOCKS5"
+	default:
+		return "", "", false
+	}
+
+	proxyAddr := parsed.Host
+	if parsed.User != nil {
+		user := parsed.User.Username()
+		password, _ := parsed.User.Password()
+		if user != "" && password != "" {
+			proxyAddr = user + ":" + password + "@" + parsed.Host
+		} else if user != "" {
+			proxyAddr = user + "@" + parsed.Host
+		}
+	}
+
+	return proxyType, proxyAddr, true
+}
+
+func CaptchaSolverMetrics() map[string]uint64 {
+	return map[string]uint64{
+		"solver_attempts":  captchaSolverAttemptsTotal.Load(),
+		"solver_successes": captchaSolverSuccessesTotal.Load(),
+		"solver_failures":  captchaSolverFailuresTotal.Load(),
+	}
+}
+
+func resetCaptchaSolverMetrics() {
+	captchaSolverAttemptsTotal.Store(0)
+	captchaSolverSuccessesTotal.Store(0)
+	captchaSolverFailuresTotal.Store(0)
 }

core/captcha_test.go

@@ -1,19 +1,25 @@
+//go:build integration
+// +build integration
+
 package core
 
 import (
 	"testing"
+
+	"github.com/karust/openserp/testutil"
 )
 
-var (
-	API_KEY = ""
-)
+const captchaAPIKeyEnv = "OPENSERP_2CAPTCHA_API_KEY"
 
 func Test2Captcha(t *testing.T) {
-	solver := NewSolver(API_KEY)
+	testutil.RequireIntegration(t)
+	apiKey := testutil.RequireEnv(t, captchaAPIKeyEnv)
+
+	solver := NewSolver(apiKey)
 	sitekey := "6LfwuyUTAAAAAOAmoS0fdqijC2PbbdH4kjq62Y1b"
 	url := "https://www.google.com/sorry/index?continue=https://www.google.de/search%3Fhl%3DDE%26lr%3Dlang_de%26nfpr%3D1%26num%3D500%26pws%3D0%26q%3Dwhere%2Bwhy%2Beach&hl=DE&q=EgRegw55GObHiq4GIjDqmzFKayGXrS2-s9ooWfcskhpK8-6tIjWSaSvhxd3f5eAyUXj7lYq2DYLDXB8ASz0yAXJaAUM"
 	datas := "Ghk0n7ZQNDS0c7ES53eef_YBfSdfeXnyRD0p2OR0R4Dg91CUXKS_hio5Do6TpJ8sHhhOat_NymTASZGe1gqAjP7w9dSvhvRT7QXsrdziO3JPngLDSRzDdjT42GDcSbO0kzInlDPxe1yy2t4yifo9xHpMnlZU7pTVNTQUIXqOMLHAR-iERi6aoSQDQ4d-88-jW3LEinquxEut0OhHG2l2stwG9AnCmNvCsUNJda-H24saFlOh5csK9KNXeeQmpr6at52_skMIMiLXSlY56vYFVCRMkXLQdAM"
-	resp, _, err := solver.SolveReCaptcha2(sitekey, url, datas)
+	resp, _, err := solver.SolveReCaptcha2(sitekey, url, datas, "")
 	if err != nil || resp == "" {
 		t.Fatalf("Failed to solve recaptchaV2: %s", err)
 	}

core/captcha_unit_test.go

@@ -0,0 +1,200 @@
+package core
+
+import (
+	"errors"
+	"testing"
+
+	api2captcha "github.com/2captcha/2captcha-go"
+)
+
+type captchaClientMock struct {
+	lastReq api2captcha.Request
+	calls   int
+	err     error
+}
+
+func (m *captchaClientMock) Solve(req api2captcha.Request) (string, string, error) {
+	m.calls++
+	m.lastReq = req
+	if m.err != nil {
+		return "", "", m.err
+	}
+	return "token", "captcha-id", nil
+}
+
+func TestToCaptchaProxy(t *testing.T) {
+	tests := []struct {
+		name      string
+		raw       string
+		wantType  string
+		wantAddr  string
+		wantValid bool
+	}{
+		{
+			name:      "http without auth",
+			raw:       "http://127.0.0.1:8080",
+			wantType:  "HTTP",
+			wantAddr:  "127.0.0.1:8080",
+			wantValid: true,
+		},
+		{
+			name:      "https with auth",
+			raw:       "https://user:pass@127.0.0.1:8443",
+			wantType:  "HTTPS",
+			wantAddr:  "user:pass@127.0.0.1:8443",
+			wantValid: true,
+		},
+		{
+			name:      "socks5h with user only",
+			raw:       "socks5h://user@127.0.0.1:1080",
+			wantType:  "SOCKS5",
+			wantAddr:  "user@127.0.0.1:1080",
+			wantValid: true,
+		},
+		{
+			name:      "invalid url",
+			raw:       "://bad",
+			wantValid: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotType, gotAddr, ok := toCaptchaProxy(tt.raw)
+			if ok != tt.wantValid {
+				t.Fatalf("expected valid=%v, got %v", tt.wantValid, ok)
+			}
+			if !tt.wantValid {
+				return
+			}
+			if gotType != tt.wantType {
+				t.Fatalf("expected type %q, got %q", tt.wantType, gotType)
+			}
+			if gotAddr != tt.wantAddr {
+				t.Fatalf("expected addr %q, got %q", tt.wantAddr, gotAddr)
+			}
+		})
+	}
+}
+
+func TestSolveReCaptcha2RecordsMetricsAndProxy(t *testing.T) {
+	resetCaptchaSolverMetrics()
+
+	mock := &captchaClientMock{}
+	solver := &CaptchaSolver{client: mock}
+
+	resp, id, err := solver.SolveReCaptcha2("sitekey", "https://example.com", "datas", "https://user:pass@127.0.0.1:8443")
+	if err != nil {
+		t.Fatalf("expected success, got %v", err)
+	}
+	if resp != "token" || id != "captcha-id" {
+		t.Fatalf("unexpected solver response: resp=%q id=%q", resp, id)
+	}
+
+	if got := mock.lastReq.Params["proxytype"]; got != "HTTPS" {
+		t.Fatalf("expected proxytype HTTPS, got %q", got)
+	}
+	if got := mock.lastReq.Params["proxy"]; got != "user:pass@127.0.0.1:8443" {
+		t.Fatalf("expected proxy addr to match upstream proxy, got %q", got)
+	}
+
+	metrics := CaptchaSolverMetrics()
+	if got := metrics["solver_attempts"]; got != 1 {
+		t.Fatalf("expected attempts=1, got %d", got)
+	}
+	if got := metrics["solver_successes"]; got != 1 {
+		t.Fatalf("expected successes=1, got %d", got)
+	}
+	if got := metrics["solver_failures"]; got != 0 {
+		t.Fatalf("expected failures=0, got %d", got)
+	}
+}
+
+func TestSolveReCaptcha2RecordsFailureMetric(t *testing.T) {
+	resetCaptchaSolverMetrics()
+
+	mock := &captchaClientMock{err: errors.New("boom")}
+	solver := &CaptchaSolver{client: mock}
+
+	if _, _, err := solver.SolveReCaptcha2("sitekey", "https://example.com", "datas", ""); err == nil {
+		t.Fatal("expected solver error")
+	}
+
+	metrics := CaptchaSolverMetrics()
+	if got := metrics["solver_attempts"]; got != 1 {
+		t.Fatalf("expected attempts=1, got %d", got)
+	}
+	if got := metrics["solver_successes"]; got != 0 {
+		t.Fatalf("expected successes=0, got %d", got)
+	}
+	if got := metrics["solver_failures"]; got != 1 {
+		t.Fatalf("expected failures=1, got %d", got)
+	}
+}
+
+// Acceptance for milestone 1 task 1.4: with the solver gated off, no 2captcha
+// API calls are made. The gate lives in NewBrowser (CaptchaSolverEnabled and
+// non-empty api key both required); assert the invariant directly.
+func TestCaptchaSolverConstructionGate(t *testing.T) {
+	cases := []struct {
+		name    string
+		enabled bool
+		apiKey  string
+		want    bool
+	}{
+		{"disabled with key", false, "abc", false},
+		{"disabled without key", false, "", false},
+		{"enabled without key", true, "", false},
+		{"enabled with key", true, "abc", true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := tc.enabled && tc.apiKey != ""
+			if got != tc.want {
+				t.Fatalf("gate result mismatch: want %v, got %v", tc.want, got)
+			}
+		})
+	}
+}
+
+// When a solver instance exists but the engine's gate (IsSolveCaptcha and
+// CaptchaSolverEnabled) evaluates false, the 2captcha client must not receive
+// a request. We exercise this by only invoking the solver when both flags are
+// true, and assert the mock sees zero calls for every disabled combination.
+func TestCaptchaSolverEngineGateSkipsClient(t *testing.T) {
+	cases := []struct {
+		name             string
+		isSolveCaptcha   bool
+		solverEnabled    bool
+		expectInvocation bool
+	}{
+		{"engine off, solver off", false, false, false},
+		{"engine on, solver off", true, false, false},
+		{"engine off, solver on", false, true, false},
+		{"engine on, solver on", true, true, true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			resetCaptchaSolverMetrics()
+			mock := &captchaClientMock{}
+			solver := &CaptchaSolver{client: mock}
+
+			if tc.isSolveCaptcha && tc.solverEnabled {
+				if _, _, err := solver.SolveReCaptcha2("sitekey", "https://example.com", "datas", ""); err != nil {
+					t.Fatalf("unexpected solver error: %v", err)
+				}
+			}
+
+			wantCalls := 0
+			if tc.expectInvocation {
+				wantCalls = 1
+			}
+			if mock.calls != wantCalls {
+				t.Fatalf("expected %d client calls, got %d", wantCalls, mock.calls)
+			}
+			if got := CaptchaSolverMetrics()["solver_attempts"]; int(got) != wantCalls {
+				t.Fatalf("expected %d attempts, got %d", wantCalls, got)
+			}
+		})
+	}
+}

core/circuit_breaker.go

@@ -1,11 +1,10 @@
 package core
 
 import (
+	"context"
 	"fmt"
 	"sync"
 	"time"
-
-	"github.com/sirupsen/logrus"
 )
 
 type CircuitState int
@@ -51,6 +50,8 @@ type CircuitBreaker struct {
 	config          CircuitBreakerConfig
 	failureCount    int
 	successCount    int
+	successLatency  time.Duration
+	successSamples  int64
 	lastFailureTime time.Time
 	lastStateChange time.Time
 }
@@ -64,7 +65,7 @@ func NewCircuitBreaker(name string, cfg CircuitBreakerConfig) *CircuitBreaker {
 	}
 }
 
-func (cb *CircuitBreaker) AllowRequest() bool {
+func (cb *CircuitBreaker) AllowRequest(ctx context.Context) bool {
 	cb.mu.Lock()
 	defer cb.mu.Unlock()
 
@@ -74,7 +75,7 @@ func (cb *CircuitBreaker) AllowRequest() bool {
 	case CircuitOpen:
 		if time.Since(cb.lastFailureTime) >= cb.config.RecoveryTimeout {
 			cb.setState(CircuitHalfOpen)
-			logrus.Infof("[CircuitBreaker][%s] Recovery timeout elapsed, moving to half-open", cb.name)
+			WithRequestEngine(ctx, cb.name).Info("Recovery timeout elapsed, moving to half-open")
 			return true
 		}
 		return false
@@ -85,10 +86,19 @@ func (cb *CircuitBreaker) AllowRequest() bool {
 	}
 }
 
-func (cb *CircuitBreaker) RecordSuccess() {
+func (cb *CircuitBreaker) RecordSuccess(ctx context.Context) {
+	cb.RecordSuccessDuration(ctx, 0)
+}
+
+func (cb *CircuitBreaker) RecordSuccessDuration(ctx context.Context, elapsed time.Duration) {
 	cb.mu.Lock()
 	defer cb.mu.Unlock()
 
+	if elapsed > 0 {
+		cb.successLatency += elapsed
+		cb.successSamples++
+	}
+
 	switch cb.state {
 	case CircuitHalfOpen:
 		cb.successCount++
@@ -96,14 +106,14 @@ func (cb *CircuitBreaker) RecordSuccess() {
 			cb.setState(CircuitClosed)
 			cb.failureCount = 0
 			cb.successCount = 0
-			logrus.Infof("[CircuitBreaker][%s] Recovered, circuit closed", cb.name)
+			WithRequestEngine(ctx, cb.name).Info("Circuit recovered, closed")
 		}
 	case CircuitClosed:
 		cb.failureCount = 0
 	}
 }
 
-func (cb *CircuitBreaker) RecordFailure() {
+func (cb *CircuitBreaker) RecordFailure(ctx context.Context) {
 	cb.mu.Lock()
 	defer cb.mu.Unlock()
 
@@ -114,13 +124,15 @@ func (cb *CircuitBreaker) RecordFailure() {
 		cb.failureCount++
 		if cb.failureCount >= cb.config.FailureThreshold {
 			cb.setState(CircuitOpen)
-			logrus.Warnf("[CircuitBreaker][%s] Circuit OPENED after %d consecutive failures (will retry in %s)",
-				cb.name, cb.failureCount, cb.config.RecoveryTimeout)
+			WithRequestEngine(ctx, cb.name).
+				WithField("failure_count", cb.failureCount).
+				WithField("recovery_timeout", cb.config.RecoveryTimeout.String()).
+				Warn("Circuit opened after consecutive failures")
 		}
 	case CircuitHalfOpen:
 		cb.setState(CircuitOpen)
 		cb.successCount = 0
-		logrus.Warnf("[CircuitBreaker][%s] Failed during half-open, circuit re-opened", cb.name)
+		WithRequestEngine(ctx, cb.name).Warn("Failed during half-open, circuit re-opened")
 	}
 }
 
@@ -154,10 +166,22 @@ func (cb *CircuitBreaker) Stats() map[string]interface{} {
 		}
 		stats["retry_in"] = retryInSeconds
 	}
+	if cb.successSamples > 0 {
+		stats["avg_response_ms"] = int64((cb.successLatency / time.Duration(cb.successSamples)) / time.Millisecond)
+	}
 
 	return stats
 }
 
+func (cb *CircuitBreaker) AvgSuccessLatency() (time.Duration, bool) {
+	cb.mu.RLock()
+	defer cb.mu.RUnlock()
+	if cb.successSamples == 0 {
+		return 0, false
+	}
+	return cb.successLatency / time.Duration(cb.successSamples), true
+}
+
 func (cb *CircuitBreaker) setState(state CircuitState) {
 	cb.state = state
 	cb.lastStateChange = time.Now()

core/circuit_breaker_test.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"context"
 	"testing"
 	"time"
 )
@@ -20,17 +21,17 @@ func TestCircuitBreaker_OpensAfterThreshold(t *testing.T) {
 	}
 	cb := newTestCircuitBreaker(t, cfg)
 
-	cb.RecordFailure()
-	cb.RecordFailure()
+	cb.RecordFailure(context.Background())
+	cb.RecordFailure(context.Background())
 	if cb.State() != CircuitClosed {
 		t.Fatalf("expected closed after 2 failures, got: %s", cb.State())
 	}
 
-	cb.RecordFailure()
+	cb.RecordFailure(context.Background())
 	if cb.State() != CircuitOpen {
 		t.Fatalf("expected open after %d failures, got: %s", cfg.FailureThreshold, cb.State())
 	}
-	if cb.AllowRequest() {
+	if cb.AllowRequest(context.Background()) {
 		t.Error("expected request blocked in open state")
 	}
 }
@@ -45,14 +46,14 @@ func TestCircuitBreaker_RecoveryToHalfOpen(t *testing.T) {
 	}
 	cb := newTestCircuitBreaker(t, cfg)
 
-	cb.RecordFailure()
-	cb.RecordFailure()
+	cb.RecordFailure(context.Background())
+	cb.RecordFailure(context.Background())
 	if cb.State() != CircuitOpen {
 		t.Fatal("expected open")
 	}
 
 	time.Sleep(60 * time.Millisecond)
-	if !cb.AllowRequest() {
+	if !cb.AllowRequest(context.Background()) {
 		t.Error("should allow request after recovery timeout")
 	}
 	if cb.State() != CircuitHalfOpen {
@@ -70,25 +71,25 @@ func TestCircuitBreaker_HalfOpenSuccessClosesCircuit(t *testing.T) {
 	}
 	cb := newTestCircuitBreaker(t, cfg)
 
-	cb.RecordFailure()
+	cb.RecordFailure(context.Background())
 	if cb.State() != CircuitOpen {
 		t.Fatalf("expected open, got: %s", cb.State())
 	}
 
 	time.Sleep(30 * time.Millisecond)
-	if !cb.AllowRequest() {
+	if !cb.AllowRequest(context.Background()) {
 		t.Fatal("expected request to pass in recovery window")
 	}
 	if cb.State() != CircuitHalfOpen {
 		t.Fatalf("expected half-open after recovery timeout, got: %s", cb.State())
 	}
 
-	cb.RecordSuccess()
+	cb.RecordSuccess(context.Background())
 	if cb.State() != CircuitHalfOpen {
 		t.Fatalf("expected to stay half-open until success threshold reached, got: %s", cb.State())
 	}
 
-	cb.RecordSuccess()
+	cb.RecordSuccess(context.Background())
 	if cb.State() != CircuitClosed {
 		t.Fatalf("expected closed after success threshold reached, got: %s", cb.State())
 	}
@@ -104,16 +105,16 @@ func TestCircuitBreaker_HalfOpenFailureReopens(t *testing.T) {
 	}
 	cb := newTestCircuitBreaker(t, cfg)
 
-	cb.RecordFailure()
+	cb.RecordFailure(context.Background())
 	time.Sleep(30 * time.Millisecond)
-	if !cb.AllowRequest() {
+	if !cb.AllowRequest(context.Background()) {
 		t.Fatal("expected probe request in half-open")
 	}
 	if cb.State() != CircuitHalfOpen {
 		t.Fatalf("expected half-open, got: %s", cb.State())
 	}
 
-	cb.RecordFailure()
+	cb.RecordFailure(context.Background())
 	if cb.State() != CircuitOpen {
 		t.Fatalf("expected open after failed half-open probe, got: %s", cb.State())
 	}
@@ -123,7 +124,7 @@ func TestCircuitBreaker_HalfOpenFailureReopens(t *testing.T) {
 // only when breaker is open.
 func TestCircuitBreaker_Stats(t *testing.T) {
 	cb := NewCircuitBreaker("test-engine", DefaultCircuitBreakerConfig())
-	cb.RecordFailure()
+	cb.RecordFailure(context.Background())
 
 	stats := cb.Stats()
 	if stats["engine"] != "test-engine" {
@@ -138,10 +139,17 @@ func TestCircuitBreaker_Stats(t *testing.T) {
 	if _, ok := stats["retry_in"]; ok {
 		t.Fatalf("did not expect retry_in in closed state, got: %v", stats["retry_in"])
 	}
+	latencyCB := NewCircuitBreaker("latency-engine", DefaultCircuitBreakerConfig())
+	latencyCB.RecordSuccessDuration(context.Background(), 25*time.Millisecond)
+	latencyStats := latencyCB.Stats()
+	avg, ok := latencyStats["avg_response_ms"].(int64)
+	if !ok || avg <= 0 {
+		t.Fatalf("expected avg_response_ms int64 > 0, got: %v (%T)", latencyStats["avg_response_ms"], latencyStats["avg_response_ms"])
+	}
 
 	openCfg := CircuitBreakerConfig{FailureThreshold: 1, RecoveryTimeout: time.Second, SuccessThreshold: 1}
 	openCB := NewCircuitBreaker("open-engine", openCfg)
-	openCB.RecordFailure()
+	openCB.RecordFailure(context.Background())
 	openStats := openCB.Stats()
 	retryIn, ok := openStats["retry_in"].(int64)
 	if !ok {

core/clusters.go

@@ -0,0 +1,104 @@
+package core
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"sort"
+)
+
+// BuildClusters groups results by normalized URL and scores them by cross-engine
+// agreement. enginesQueried is the total number of engines that were asked
+// (denominator for the score formula).
+//
+// Score = sum(1/rank for each occurrence) / enginesQueried, capped at 1.0.
+func BuildClusters(results []Result, enginesQueried int) []Cluster {
+	if enginesQueried <= 0 {
+		enginesQueried = 1
+	}
+
+	type clusterAccum struct {
+		occurrences  []ClusterOccurrence
+		scoreSum     float64
+		bestRank     int
+		title        string
+		canonicalURL string
+		domain       string
+	}
+
+	// Group by result ID (which is derived from normalized URL + engine).
+	// For clustering we group by normalized URL regardless of engine, so we
+	// re-key on NormalizeURLForClustering(result.URL).
+	byURL := map[string]*clusterAccum{}
+	urlOrder := []string{}
+
+	for _, r := range results {
+		norm := NormalizeURLForClustering(r.URL)
+		if norm == "" {
+			continue
+		}
+		acc, exists := byURL[norm]
+		if !exists {
+			acc = &clusterAccum{
+				bestRank:     r.Rank,
+				title:        r.Title,
+				canonicalURL: r.URL,
+				domain:       r.Domain,
+			}
+			byURL[norm] = acc
+			urlOrder = append(urlOrder, norm)
+		}
+		rank := r.Rank
+		if rank <= 0 {
+			rank = 1
+		}
+		acc.scoreSum += 1.0 / float64(rank)
+		acc.occurrences = append(acc.occurrences, ClusterOccurrence{
+			Engine:   r.Engine,
+			Rank:     r.Rank,
+			ResultID: r.ID,
+		})
+		if r.Rank > 0 && (acc.bestRank <= 0 || r.Rank < acc.bestRank) {
+			acc.bestRank = r.Rank
+			acc.title = r.Title
+			acc.canonicalURL = r.URL
+			acc.domain = r.Domain
+		}
+	}
+
+	clusters := make([]Cluster, 0, len(byURL))
+	for _, norm := range urlOrder {
+		acc := byURL[norm]
+		score := acc.scoreSum / float64(enginesQueried)
+		if score > 1.0 {
+			score = 1.0
+		}
+		clusters = append(clusters, Cluster{
+			ID:           buildClusterID(norm),
+			CanonicalURL: acc.canonicalURL,
+			Domain:       acc.domain,
+			Title:        acc.title,
+			Occurrences:  acc.occurrences,
+			EnginesCount: len(acc.occurrences),
+			BestRank:     acc.bestRank,
+			Score:        roundScore(score),
+		})
+	}
+
+	// Sort by score descending, then best_rank ascending as tiebreak.
+	sort.Slice(clusters, func(i, j int) bool {
+		if clusters[i].Score != clusters[j].Score {
+			return clusters[i].Score > clusters[j].Score
+		}
+		return clusters[i].BestRank < clusters[j].BestRank
+	})
+	return clusters
+}
+
+func buildClusterID(normalizedURL string) string {
+	h := md5.Sum([]byte(normalizedURL))
+	return "c_" + hex.EncodeToString(h[:responseIDBytes])
+}
+
+func roundScore(s float64) float64 {
+	return float64(int(s*100+0.5)) / 100
+}

core/common.go

@@ -1,25 +1,132 @@
 package core
 
 import (
+	"context"
 	"errors"
+	"fmt"
+	"net"
 	"sort"
 	"strconv"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"golang.org/x/time/rate"
 )
 
+// Extraction depth bounds for the unified extract=N query param. The default
+// is 1 (extract=true == extract=1 == "extract one result"); callers raise it up
+// to maxExtractTop. These mirror the CLI's --extract flag limits.
+const (
+	defaultExtractTop = 1
+	maxExtractTop     = 5
+)
+
+// ErrCaptcha is returned when the engine detects a captcha challenge page.
+// This error is treated as non-retryable by resilient search policies.
 var ErrCaptcha = errors.New("captcha detected")
+
+// ErrSearchTimeout is returned when required SERP elements are not found before
+// selector or page timeouts expire.
 var ErrSearchTimeout = errors.New("timeout. Cannot find element on page")
 
-type SearchResult struct {
-	Rank        int    `json:"rank"`
-	URL         string `json:"url"`
-	Title       string `json:"title"`
-	Description string `json:"description"`
-	Ad          bool   `json:"ad"`
+// ErrParser is returned when SERP parsing selectors drift or expected fields
+// cannot be extracted from an otherwise loaded page.
+var ErrParser = errors.New("parser failure")
+
+// ErrEngineInternal is returned when an engine recovered from an unexpected
+// panic and converted it into a typed error.
+var ErrEngineInternal = errors.New("engine internal error")
+
+// ErrProxyConnect is returned when the proxy cannot establish a network
+// connection. Proxy health is degraded on this error.
+var ErrProxyConnect = errors.New("proxy_connect")
+
+// ErrProxyAuth is returned when proxy credentials are rejected.
+// Proxy health is degraded on this error.
+var ErrProxyAuth = errors.New("proxy_auth")
+
+// ErrTimeout is returned when a network-level timeout occurs on the proxy path.
+// Proxy health is degraded on this error.
+var ErrTimeout = errors.New("timeout")
+
+// ErrEmptyResult signals a successful fetch that returned zero organic results.
+// It is not a failure; the proxy stays healthy and no credit is charged.
+var ErrEmptyResult = errors.New("empty_result")
+
+// ErrBlocked is returned when the search engine blocks the browser request.
+var ErrBlocked = errors.New("blocked")
+
+// ErrRateLimited is returned when the search engine returns HTTP 429.
+var ErrRateLimited = errors.New("rate_limited")
+
+// IsProxyNetworkError reports whether err is a network-level error that
+// indicates a faulty proxy (connect failure, auth rejection, or timeout).
+// Parser drift, captcha pages, and engine errors must NOT degrade proxy health.
+func IsProxyNetworkError(err error) bool {
+	return errors.Is(err, ErrProxyConnect) ||
+		errors.Is(err, ErrProxyAuth) ||
+		errors.Is(err, ErrTimeout)
 }
 
+// classifyProxyNetworkError wraps common transport errors with proxy-health
+// sentinels while preserving the original error for callers.
+func classifyProxyNetworkError(err error) error {
+	if err == nil || IsProxyNetworkError(err) || errors.Is(err, context.Canceled) {
+		return err
+	}
+
+	msg := strings.ToLower(err.Error())
+	if strings.Contains(msg, "407") || strings.Contains(msg, "proxy authentication") {
+		return fmt.Errorf("%w: %w", ErrProxyAuth, err)
+	}
+
+	var netErr net.Error
+	if (errors.As(err, &netErr) && netErr.Timeout()) ||
+		errors.Is(err, context.DeadlineExceeded) ||
+		strings.Contains(msg, "timeout") ||
+		strings.Contains(msg, "deadline exceeded") {
+		return fmt.Errorf("%w: %w", ErrTimeout, err)
+	}
+
+	if strings.Contains(msg, "proxyconnect") ||
+		strings.Contains(msg, "connection refused") ||
+		strings.Contains(msg, "connection reset") ||
+		strings.Contains(msg, "err_tunnel_connection_failed") ||
+		strings.Contains(msg, "no such host") ||
+		strings.Contains(msg, "network is unreachable") ||
+		strings.Contains(msg, "socks connect") {
+		return fmt.Errorf("%w: %w", ErrProxyConnect, err)
+	}
+
+	return err
+}
+
+// SearchResult represents one normalized result item returned by any engine.
+type SearchResult struct {
+	// Rank is the 1-based position within this result type. For SEO callers,
+	// organic rank must not be shifted by ads.
+	Rank int `json:"rank"`
+	// AbsoluteRank is the 1-based position in the mixed SERP stream.
+	AbsoluteRank int `json:"absolute_rank,omitempty"`
+	// Type is the SERP block type when an engine can classify a non-standard
+	// SERP module without changing the public SearchEngine interface.
+	Type ResultType `json:"type,omitempty"`
+	// URL is the canonical result URL.
+	URL string `json:"url"`
+	// Title is the result headline shown on the SERP.
+	Title string `json:"title"`
+	// Description is the snippet text associated with the result.
+	Description string `json:"description"`
+	// Ad reports whether the result is sponsored.
+	Ad bool `json:"ad"`
+	// Features carries extracted SERP modules alongside the legacy result stream.
+	Features []SerpFeature `json:"-"`
+}
+
+// DeduplicateResults removes items with duplicate URLs and returns a result set
+// sorted by rank in ascending order.
 func DeduplicateResults(results []SearchResult) []SearchResult {
 	unique := make(map[string]bool)
 	var deduped []SearchResult
@@ -28,18 +135,21 @@ func DeduplicateResults(results []SearchResult) []SearchResult {
 		if result.URL == "" {
 			continue
 		}
-		if !unique[result.URL] {
-			unique[result.URL] = true
+		key := resultDedupKey(result)
+		if !unique[key] {
+			unique[key] = true
 			deduped = append(deduped, result)
 		}
 	}
 
 	sort.Slice(deduped, func(i, j int) bool {
-		return deduped[i].Rank < deduped[j].Rank
+		return resultLess(deduped[i], deduped[j])
 	})
 	return deduped
 }
 
+// ConvertSearchResultsMap converts a map-based collection to a rank-sorted
+// slice and returns it by pointer.
 func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult {
 	searchResults := []SearchResult{}
 
@@ -48,26 +158,176 @@ func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]Search
 	}
 
 	sort.Slice(searchResults, func(i, j int) bool {
-		return searchResults[i].Rank < searchResults[j].Rank
+		return resultLess(searchResults[i], searchResults[j])
 	})
 	return &searchResults
 }
 
-type Query struct {
-	Text          string
-	LangCode      string // eg. EN, ES, RU...
-	DateInterval  string // format: YYYYMMDD..YYYMMDD - 20181010..20231010
-	Filetype      string // File extension to search.
-	Site          string // Search site
-	Limit         int    // Limit the number of results
-	Start         int    // Search offset for pagination (Google uses 0, 10, 20...)
-	Filter        bool   // Filter duplicates (google) (false: include similar, true: hide similar)
-	Answers       bool   // Include question and answers from SERP page to results with negative indexes
-	ProxyURL      string // Proxy URL for raw requests
-	ProxyOverride string // Request-scoped proxy override: tag or direct
-	Insecure      bool   // Allow insecure TLS connections
+// CountOrganicResults returns the number of non-ad results in a mixed SERP.
+func CountOrganicResults(results []SearchResult) int {
+	count := 0
+	for _, result := range results {
+		if !result.Ad {
+			count++
+		}
+	}
+	return count
 }
 
+// OrganicLimitReached reports whether enough organic results have been
+// collected to satisfy limit. A non-positive limit means "no limit", so it
+// is never reached and pagination continues until the engine runs out.
+func OrganicLimitReached(results []SearchResult, limit int) bool {
+	return limit > 0 && CountOrganicResults(results) >= limit
+}
+
+// ShouldFetchResultPage reports whether a paginated engine should fetch another
+// SERP page. Small/default limits should use the first SERP page as-is instead
+// of chasing a target count across multiple page loads.
+func ShouldFetchResultPage(collected, limit, pagesFetched int) bool {
+	if pagesFetched <= 0 {
+		return true
+	}
+	if limit > 0 && collected >= limit {
+		return false
+	}
+	return limit > defaultQueryLimit
+}
+
+// LimitOrganicResults keeps all ads and at most limit non-ad results.
+func LimitOrganicResults(results []SearchResult, limit int) []SearchResult {
+	if limit <= 0 {
+		return results
+	}
+	out := make([]SearchResult, 0, len(results))
+	organicCount := 0
+	for _, result := range results {
+		if result.Ad {
+			out = append(out, result)
+			continue
+		}
+		if organicCount >= limit {
+			continue
+		}
+		organicCount++
+		out = append(out, result)
+	}
+	return out
+}
+
+func resultDedupKey(result SearchResult) string {
+	resultType := "organic"
+	if result.Ad {
+		resultType = "ad"
+	}
+	return resultType + "\x00" + result.URL
+}
+
+func resultLess(left, right SearchResult) bool {
+	leftPos := resultSortPosition(left)
+	rightPos := resultSortPosition(right)
+	if leftPos != rightPos {
+		return leftPos < rightPos
+	}
+	if left.Ad != right.Ad {
+		return left.Ad
+	}
+	if left.Rank != right.Rank {
+		return left.Rank < right.Rank
+	}
+	return left.URL < right.URL
+}
+
+func resultSortPosition(result SearchResult) int {
+	if result.AbsoluteRank > 0 {
+		return result.AbsoluteRank
+	}
+	if result.Rank < 0 {
+		return -result.Rank
+	}
+	if result.Rank > 0 {
+		return result.Rank
+	}
+	return int(^uint(0) >> 1)
+}
+
+// Query holds request parameters used by HTTP handlers and search engines.
+// Example minimal query: Query{Text: "golang", Limit: 10}.
+type Query struct {
+	// Text is the search phrase, for example "golang fiber tutorial".
+	Text string
+	// LangCode is an engine language hint such as "EN", "DE", or "RU".
+	LangCode string
+	// Region is an engine market/location hint. Yandex accepts numeric lr IDs;
+	// global engines accept country-style hints such as "RU" or "en-RU".
+	Region string
+	// DateInterval filters by date range in YYYYMMDD..YYYYMMDD format.
+	// Example: "20250101..20250331".
+	DateInterval string
+	// Filetype is a file extension filter, for example "pdf" or "docx".
+	Filetype string
+	// Site restricts results to a specific domain, for example "github.com".
+	Site string
+	// Limit is the maximum number of results requested by the client.
+	Limit int
+	// Start is an engine pagination offset. Values are engine-specific:
+	// Google commonly uses 0,10,20 while some engines use page indexes.
+	Start int
+	// Filter controls duplicate filtering when supported by the engine.
+	// For Google, false includes similar results and true hides them.
+	Filter bool
+	// Features enables parsing SERP feature modules (AI summaries, answer boxes,
+	// people-also-ask, related searches) on the browser Search path when
+	// supported by the engine. Such entries may be returned with non-positive
+	// internal rank values.
+	Features bool
+	// Extract fetches and embeds cleaned target-page content for top results.
+	Extract bool
+	// ExtractTop limits how many top results are enriched when Extract is true.
+	ExtractTop int
+	// ExtractMode selects auto, fast, or rendered extraction.
+	ExtractMode string
+	// ExtractMinRunes overrides the auto-mode escalation floor (0 = default).
+	ExtractMinRunes int
+	// ProxyURL is a direct proxy URL used by raw HTTP search paths.
+	ProxyURL string
+	// ProxyCountry identifies the proxy market country for cache/error metadata.
+	ProxyCountry string
+	// ProxyClass identifies the proxy class such as datacenter or residential.
+	ProxyClass string
+	// ProxyProvider identifies the upstream proxy provider.
+	ProxyProvider string
+	// ProxySessionID identifies a sticky balancer session/lane.
+	ProxySessionID string
+	// ProxyOverride is a request-scoped proxy policy override (tag or "direct"),
+	// typically parsed from the X-Use-Proxy header.
+	ProxyOverride string
+	// Insecure enables insecure TLS for request/browser execution.
+	Insecure bool
+	// GuardPrivateNetworks rejects raw HTTP targets that resolve to private,
+	// loopback, link-local, multicast, or otherwise non-public addresses.
+	GuardPrivateNetworks bool
+}
+
+// String renders Query for logs with the proxy URL credentials masked. The
+// default %+v formatter calls this method, so logging Query through %v/%+v
+// never leaks proxy passwords.
+func (q Query) String() string {
+	maskedProxyURL := ""
+	if q.ProxyURL != "" {
+		maskedProxyURL = MaskProxyURL(q.ProxyURL)
+	}
+	return fmt.Sprintf(
+		"{Text:%s LangCode:%s Region:%s DateInterval:%s Filetype:%s Site:%s Limit:%d Start:%d Filter:%t Features:%t Extract:%t ExtractTop:%d ExtractMode:%s ProxyURL:%s ProxyCountry:%s ProxyClass:%s ProxyProvider:%s ProxySessionID:%s ProxyOverride:%s Insecure:%t}",
+		q.Text, q.LangCode, q.Region, q.DateInterval, q.Filetype, q.Site,
+		q.Limit, q.Start, q.Filter, q.Features, q.Extract, q.ExtractTop, q.ExtractMode,
+		maskedProxyURL, q.ProxyCountry, q.ProxyClass, q.ProxyProvider,
+		q.ProxySessionID, q.ProxyOverride, q.Insecure,
+	)
+}
+
+// ComputePagination translates an absolute start offset into page index and
+// in-page offset for a fixed page size.
 func ComputePagination(start int, pageSize int) (int, int, error) {
 	if pageSize <= 0 {
 		return 0, 0, errors.New("pageSize must be > 0")
@@ -78,6 +338,7 @@ func ComputePagination(start int, pageSize int) (int, int, error) {
 	return start / pageSize, start % pageSize, nil
 }
 
+// IsEmpty reports whether query text operators are all absent.
 func (q Query) IsEmpty() bool {
 	if q.Site == "" && q.Filetype == "" && q.Text == "" {
 		return true
@@ -85,57 +346,178 @@ func (q Query) IsEmpty() bool {
 	return false
 }
 
-func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
-	searchQuery.Text = reqCtx.Query("text")
-	searchQuery.LangCode = reqCtx.Query("lang")
-	searchQuery.DateInterval = reqCtx.Query("date")
-	searchQuery.Filetype = reqCtx.Query("file")
-	searchQuery.Site = reqCtx.Query("site")
+// MaxQueryLimit is the maximum allowed value for the limit parameter.
+const MaxQueryLimit = 100
 
-	limit, err := strconv.Atoi(reqCtx.Query("limit", "25"))
+// defaultQueryLimit is the assumed limit when a request omits it (InitFromContext)
+// and the fallback used by pagination math for internally-built queries that
+// leave Limit unset.
+const defaultQueryLimit = 10
+
+// InitFromContext populates Query from HTTP query parameters and request
+// headers. It validates numeric/boolean inputs and returns an *APIError for
+// invalid client input (400) or a plain error for internal failures.
+func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
+	searchQuery.Text = strings.TrimSpace(reqCtx.Query("text"))
+	searchQuery.LangCode = strings.TrimSpace(reqCtx.Query("lang"))
+	searchQuery.Region = strings.TrimSpace(reqCtx.Query("region"))
+	searchQuery.DateInterval = strings.TrimSpace(reqCtx.Query("date"))
+	searchQuery.Filetype = strings.TrimSpace(reqCtx.Query("file"))
+	searchQuery.Site = strings.TrimSpace(reqCtx.Query("site"))
+
+	limitRaw := reqCtx.Query("limit", strconv.Itoa(defaultQueryLimit))
+	limit, err := strconv.Atoi(limitRaw)
 	if err != nil {
-		return err
+		return errInvalidLimit("limit must be an integer")
+	}
+	if limit < 1 || limit > MaxQueryLimit {
+		return errInvalidLimit(fmt.Sprintf("limit must be between 1 and %d", MaxQueryLimit))
 	}
 	searchQuery.Limit = limit
 
-	start, err := strconv.Atoi(reqCtx.Query("start", "0"))
+	startRaw := reqCtx.Query("start", "0")
+	start, err := strconv.Atoi(startRaw)
 	if err != nil {
-		return err
+		return errInvalidStart("start must be a non-negative integer")
 	}
 	if start < 0 {
-		return errors.New("start must be >= 0")
+		return errInvalidStart("start must be >= 0")
 	}
 	searchQuery.Start = start
 
 	searchQuery.Filter, err = strconv.ParseBool(reqCtx.Query("filter", "1"))
 	if err != nil {
-		return err
+		return errInvalidParam(fmt.Sprintf("filter: %v", err))
 	}
 
-	searchQuery.Answers, err = strconv.ParseBool(reqCtx.Query("answers", "0"))
+	searchQuery.Features, err = strconv.ParseBool(reqCtx.Query("features", "1"))
 	if err != nil {
+		return errInvalidParam(fmt.Sprintf("features: %v", err))
+	}
+	// extract is a unified bool-or-int knob: extract=0/false disables, extract=N
+	// (or true/1) extracts the top N results. The tuning params extract_mode and
+	// min_runes also imply extraction (extract=0 still overrides them). The
+	// default depth is 1 — true == 1 == "extract one result".
+	if err := parseExtractParams(reqCtx, searchQuery); err != nil {
 		return err
 	}
 
 	searchQuery.ProxyOverride, err = NormalizeProxyRequestOverride(reqCtx.Get("X-Use-Proxy"))
 	if err != nil {
-		return err
+		return errInvalidParam(fmt.Sprintf("X-Use-Proxy: %v", err))
 	}
+	rawProxyURL := strings.TrimSpace(reqCtx.Get("X-Proxy-URL"))
+	if rawProxyURL != "" {
+		normalized, err := NormalizeProxyURL(rawProxyURL)
+		if err != nil {
+			return errInvalidParam(fmt.Sprintf("X-Proxy-URL: %v", err))
+		}
+		searchQuery.ProxyURL = normalized
+	}
+	searchQuery.ProxyCountry = strings.ToLower(strings.TrimSpace(reqCtx.Get("X-Proxy-Country")))
+	searchQuery.ProxyClass = strings.ToLower(strings.TrimSpace(reqCtx.Get("X-Proxy-Class")))
+	searchQuery.ProxyProvider = strings.ToLower(strings.TrimSpace(reqCtx.Get("X-Proxy-Provider")))
+	searchQuery.ProxySessionID = strings.TrimSpace(reqCtx.Get("X-Proxy-Session-ID"))
 
 	if searchQuery.IsEmpty() {
-		return errors.New("Query cannot be empty")
+		return errEmptyQuery()
 	}
 	return nil
 }
 
-type SearchEngineOptions struct {
-	RateRequests    int   `mapstructure:"rate_requests"`
-	RateTime        int64 `mapstructure:"rate_seconds"`
-	RateBurst       int   `mapstructure:"rate_burst"`
-	SelectorTimeout int64 `mapstructure:"selector_timeout"` // CSS selector timeout in seconds
-	IsSolveCaptcha  bool  `mapstructure:"captcha"`
+// parseExtractParams reads the unified extract knob plus its tuning params onto
+// q. The extract param is bool-or-int:
+//
+//	extract=0 / extract=false  → extraction off
+//	extract=true / extract=1   → on, top 1
+//	extract=N (1..5)           → on, top N (clamped to maxExtractTop)
+//
+// extract_mode and min_runes tune how extraction runs and imply extraction when
+// present, unless extract is explicitly set (extract=0 wins over them). When
+// extraction is on but no depth is given, ExtractTop defaults to 1.
+func parseExtractParams(reqCtx *fiber.Ctx, q *Query) error {
+	q.ExtractTop = defaultExtractTop
+
+	// extract accepts both bool spellings (true/false/1/0) and an integer depth.
+	// Try bool first so legacy true/false keep working, then fall back to int.
+	extractExplicit := false
+	if raw := strings.TrimSpace(reqCtx.Query("extract")); raw != "" {
+		extractExplicit = true
+		if b, err := strconv.ParseBool(raw); err == nil {
+			q.Extract = b
+			if b {
+				q.ExtractTop = 1
+			}
+		} else if n, err := strconv.Atoi(raw); err == nil {
+			q.Extract = n > 0
+			if n > 0 {
+				q.ExtractTop = clampExtractTop(n)
+			}
+		} else {
+			return errInvalidParam("extract must be a boolean or an integer (0 disables, N extracts top N)")
+		}
+	}
+
+	q.ExtractMode = strings.ToLower(strings.TrimSpace(reqCtx.Query("extract_mode", "auto")))
+	switch q.ExtractMode {
+	case "auto", "fast", "rendered":
+	default:
+		return errInvalidParam("extract_mode must be one of auto, fast, rendered")
+	}
+	if !extractExplicit && strings.TrimSpace(reqCtx.Query("extract_mode")) != "" {
+		q.Extract = true
+	}
+
+	minRunes, err := parseNonNegativeIntQuery(reqCtx.Query("min_runes"), 0)
+	if err != nil {
+		return errInvalidParam("min_runes must be a non-negative integer")
+	}
+	q.ExtractMinRunes = minRunes
+	if !extractExplicit && strings.TrimSpace(reqCtx.Query("min_runes")) != "" {
+		q.Extract = true
+	}
+
+	return nil
 }
 
+// clampExtractTop bounds a requested extraction depth to [1, maxExtractTop].
+func clampExtractTop(n int) int {
+	if n < 1 {
+		return 1
+	}
+	if n > maxExtractTop {
+		return maxExtractTop
+	}
+	return n
+}
+
+// SearchEngineOptions controls engine pacing, selector waits, and captcha
+// handling behavior shared by browser and raw implementations.
+type SearchEngineOptions struct {
+	// RateRequests is the allowed number of requests within RateTime seconds.
+	RateRequests int `mapstructure:"rate_requests"`
+	// RateTime defines the rate-limiting window size in seconds.
+	RateTime int64 `mapstructure:"rate_seconds"`
+	// RateBurst is the token bucket burst size for short spikes.
+	RateBurst int `mapstructure:"rate_burst"`
+	// SelectorTimeout is the per-selector wait timeout in seconds.
+	SelectorTimeout int64 `mapstructure:"selector_timeout"`
+	// IsSolveCaptcha enables automatic captcha solving when engine support and
+	// solver credentials are configured.
+	IsSolveCaptcha bool `mapstructure:"captcha"`
+
+	limiterState *rateLimiterState
+}
+
+type rateLimiterState struct {
+	limiter *rate.Limiter
+	every   time.Duration
+	burst   int
+}
+
+var searchEngineOptionsLimiterMu sync.Mutex
+
+// Init sets default option values when fields are zero.
 func (o *SearchEngineOptions) Init() {
 	if o.RateRequests == 0 {
 		o.RateRequests = 6
@@ -151,10 +533,32 @@ func (o *SearchEngineOptions) Init() {
 	}
 }
 
+// GetRatelimit returns the interval between two allowed requests.
+// Call Init() first so RateRequests / RateTime are non-zero.
 func (o *SearchEngineOptions) GetRatelimit() time.Duration {
 	return (time.Duration(o.RateTime) * time.Second) / time.Duration(o.RateRequests)
 }
 
+// GetRateLimiter returns a cached limiter configured from SearchEngineOptions.
+// Call Init() first so RateBurst is non-zero. Do not copy SearchEngineOptions
+// after first use; the limiter state is intentionally shared by each engine.
+func (o *SearchEngineOptions) GetRateLimiter() *rate.Limiter {
+	every := o.GetRatelimit()
+	burst := o.RateBurst
+	searchEngineOptionsLimiterMu.Lock()
+	defer searchEngineOptionsLimiterMu.Unlock()
+	if o.limiterState == nil {
+		o.limiterState = &rateLimiterState{}
+	}
+	if o.limiterState.limiter == nil || o.limiterState.every != every || o.limiterState.burst != burst {
+		o.limiterState.limiter = rate.NewLimiter(rate.Every(every), burst)
+		o.limiterState.every = every
+		o.limiterState.burst = burst
+	}
+	return o.limiterState.limiter
+}
+
+// GetSelectorTimeout returns the selector wait timeout as time.Duration.
 func (o *SearchEngineOptions) GetSelectorTimeout() time.Duration {
 	return time.Duration(o.SelectorTimeout) * time.Second
 }

core/common_test.go

@@ -0,0 +1,58 @@
+package core
+
+import "testing"
+
+func TestShouldFetchResultPage(t *testing.T) {
+	tests := []struct {
+		name         string
+		collected    int
+		limit        int
+		pagesFetched int
+		want         bool
+	}{
+		{
+			name:         "always fetch first page",
+			limit:        10,
+			pagesFetched: 0,
+			want:         true,
+		},
+		{
+			name:         "unset limit stops after first page",
+			collected:    8,
+			limit:        0,
+			pagesFetched: 1,
+			want:         false,
+		},
+		{
+			name:         "default limit stops after first page even if short",
+			collected:    8,
+			limit:        10,
+			pagesFetched: 1,
+			want:         false,
+		},
+		{
+			name:         "larger limit can fetch another short page",
+			collected:    8,
+			limit:        11,
+			pagesFetched: 1,
+			want:         true,
+		},
+		{
+			name:         "larger limit stops when satisfied",
+			collected:    11,
+			limit:        11,
+			pagesFetched: 1,
+			want:         false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ShouldFetchResultPage(tt.collected, tt.limit, tt.pagesFetched)
+			if got != tt.want {
+				t.Fatalf("ShouldFetchResultPage(%d, %d, %d) = %t, want %t",
+					tt.collected, tt.limit, tt.pagesFetched, got, tt.want)
+			}
+		})
+	}
+}

core/context.go

@@ -0,0 +1,61 @@
+package core
+
+import (
+	"context"
+	"errors"
+	"time"
+)
+
+// EnsureContext returns ctx when set; otherwise a non-nil placeholder context.
+func EnsureContext(ctx context.Context) context.Context {
+	if ctx != nil {
+		return ctx
+	}
+	return context.TODO()
+}
+
+// SleepContext blocks for d or until ctx is canceled.
+func SleepContext(ctx context.Context, d time.Duration) error {
+	if d <= 0 {
+		return nil
+	}
+	ctx = EnsureContext(ctx)
+
+	timer := time.NewTimer(d)
+	defer timer.Stop()
+
+	select {
+	case <-timer.C:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// IsContextDone reports whether err is a cancellation/deadline error.
+func IsContextDone(err error) bool {
+	return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
+}
+
+// PrepareEngineContext applies request-scoped metadata expected by all engine
+// search implementations.
+func PrepareEngineContext(ctx context.Context, query Query, engineName string) context.Context {
+	ctx = WithEngine(EnsureContext(ctx), engineName)
+	ctx = WithProfileRegion(ctx, profileRegionHint(query))
+	return WithQueryHash(ctx, QueryHashFromQuery(query))
+}
+
+// profileRegionHint picks the strongest market signal for browser fingerprint
+// matching. An explicit country-code Region wins (e.g. region=DE → de or en-DE),
+// since it's what the user asked the engine to localize to. Engine-native
+// numeric region IDs (Yandex lr) are ignored here and we fall back to LangCode.
+func profileRegionHint(q Query) string {
+	country := CountryFromRegion(q.Region)
+	if country == "" {
+		return q.LangCode
+	}
+	if lang := ParseLocale(q.LangCode).Language; lang != "" {
+		return lang + "-" + country
+	}
+	return country
+}

core/context_test.go

@@ -0,0 +1,26 @@
+package core
+
+import "testing"
+
+func TestProfileRegionHint(t *testing.T) {
+	tests := []struct {
+		name string
+		q    Query
+		want string
+	}{
+		{name: "empty", q: Query{}, want: ""},
+		{name: "lang only", q: Query{LangCode: "en"}, want: "en"},
+		{name: "region country combines with lang", q: Query{LangCode: "en", Region: "DE"}, want: "en-DE"},
+		{name: "region locale combines with lang", q: Query{LangCode: "en", Region: "en-GB"}, want: "en-GB"},
+		{name: "region country without lang", q: Query{Region: "DE"}, want: "DE"},
+		{name: "yandex numeric region falls back to lang", q: Query{LangCode: "ru", Region: "213"}, want: "ru"},
+		{name: "yandex numeric region without lang stays empty", q: Query{Region: "213"}, want: ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := profileRegionHint(tt.q); got != tt.want {
+				t.Fatalf("profileRegionHint = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}

core/enrichment_domain.go

@@ -0,0 +1,232 @@
+package core
+
+import (
+	_ "embed"
+	"os"
+	"strings"
+	"sync"
+
+	"golang.org/x/net/publicsuffix"
+	"gopkg.in/yaml.v3"
+)
+
+//go:embed enrichment_domains.yaml
+var defaultEnrichmentDomainsYAML []byte
+
+type enrichmentDomainsFile struct {
+	DomainSourceHints  map[string]string `yaml:"domain_source_hints"`
+	NewsDomains        []string          `yaml:"news_domains"`
+	ForumDomains       []string          `yaml:"forum_domains"`
+	MarketplaceDomains []string          `yaml:"marketplace_domains"`
+	SocialDomains      []string          `yaml:"social_domains"`
+}
+
+type enrichmentDomainsConfig struct {
+	DomainSourceHints  map[string]string
+	NewsDomains        map[string]bool
+	ForumDomains       map[string]bool
+	MarketplaceDomains map[string]bool
+	SocialDomains      map[string]bool
+}
+
+var (
+	enrichmentDomainsOnce sync.Once
+	enrichmentDomains     enrichmentDomainsConfig
+)
+
+// EnrichDomainInfo derives TLD/category signals from a bare hostname.
+func EnrichDomainInfo(domain string) *DomainInfo {
+	if domain == "" {
+		return nil
+	}
+
+	domain = normalizeDomain(domain)
+	tld, sld := splitDomain(domain)
+	cfg := loadEnrichmentDomains()
+
+	info := &DomainInfo{
+		TLD:      tld,
+		SLD:      sld,
+		Category: domainCategory(domain, tld, cfg),
+	}
+	return info
+}
+
+// ClassifyURL returns a rough content-type and source hint derived from the
+// URL path alone; no network calls.
+func ClassifyURL(rawURL, domain string) *Classification {
+	if rawURL == "" && domain == "" {
+		return nil
+	}
+
+	contentType := classifyContentType(rawURL)
+	sourceHint := classifySourceHint(domain)
+	if contentType == "webpage" && sourceHint == "" {
+		return nil
+	}
+
+	return &Classification{
+		ContentType: contentType,
+		SourceHint:  sourceHint,
+	}
+}
+
+func domainCategory(domain, tld string, cfg enrichmentDomainsConfig) string {
+	switch {
+	case isGovTLD(domain, tld):
+		return "gov"
+	case isEduTLD(domain, tld):
+		return "edu"
+	case isMilTLD(tld):
+		return "mil"
+	case cfg.NewsDomains[domain]:
+		return "news"
+	case cfg.ForumDomains[domain]:
+		return "forum"
+	case cfg.MarketplaceDomains[domain]:
+		return "marketplace"
+	case cfg.SocialDomains[domain]:
+		return "social"
+	default:
+		return ""
+	}
+}
+
+// splitDomain returns (public suffix, registrable domain label).
+func splitDomain(domain string) (tld, sld string) {
+	domain = normalizeDomain(domain)
+	if domain == "" {
+		return "", ""
+	}
+
+	suffix, icann := publicsuffix.PublicSuffix(domain)
+	if suffix == "" || !icann {
+		parts := strings.Split(domain, ".")
+		if len(parts) < 2 {
+			return domain, ""
+		}
+		return parts[len(parts)-1], parts[len(parts)-2]
+	}
+
+	registrable, err := publicsuffix.EffectiveTLDPlusOne(domain)
+	if err != nil {
+		parts := strings.Split(domain, ".")
+		if len(parts) < 2 {
+			return suffix, ""
+		}
+		return suffix, parts[len(parts)-2]
+	}
+
+	sld = strings.TrimSuffix(registrable, "."+suffix)
+	return suffix, sld
+}
+
+func isGovTLD(domain, tld string) bool {
+	return tld == "gov" || strings.HasSuffix(tld, ".gov") || strings.HasSuffix(domain, ".gov")
+}
+
+func isEduTLD(domain, tld string) bool {
+	return tld == "edu" || strings.HasSuffix(tld, ".edu") || tld == "ac.uk" ||
+		strings.HasSuffix(domain, ".edu") || strings.HasSuffix(domain, ".ac.uk")
+}
+
+func isMilTLD(tld string) bool {
+	return tld == "mil"
+}
+
+func classifyContentType(rawURL string) string {
+	lower := strings.ToLower(rawURL)
+	switch {
+	case strings.Contains(lower, "/wiki/"):
+		return "article"
+	case strings.HasSuffix(lower, ".pdf") || strings.Contains(lower, ".pdf?"):
+		return "document"
+	case strings.Contains(lower, "/watch?v=") || strings.Contains(lower, "/video/") ||
+		strings.Contains(lower, "/videos/"):
+		return "video"
+	case strings.Contains(lower, "/forum/") || strings.Contains(lower, "/thread/") ||
+		strings.Contains(lower, "/discussion/") || strings.Contains(lower, "/t/") ||
+		strings.Contains(lower, "/questions/") || strings.Contains(lower, "/q/"):
+		return "forum_thread"
+	case strings.Contains(lower, "/blog/") || strings.Contains(lower, "/post/") ||
+		strings.Contains(lower, "/article/") || strings.Contains(lower, "/news/"):
+		return "article"
+	default:
+		return "webpage"
+	}
+}
+
+func classifySourceHint(domain string) string {
+	cfg := loadEnrichmentDomains()
+	domain = normalizeDomain(domain)
+	if hint, ok := cfg.DomainSourceHints[domain]; ok {
+		return hint
+	}
+	// Fall back to the registrable domain so subdomain hosts (e.g.
+	// megadeth.fandom.com, zh.m.wikipedia.org) match a hint keyed on the
+	// registrable domain (fandom.com, wikipedia.org).
+	if _, sld := splitDomain(domain); sld != "" {
+		if registrable, err := publicsuffix.EffectiveTLDPlusOne(domain); err == nil {
+			if hint, ok := cfg.DomainSourceHints[registrable]; ok {
+				return hint
+			}
+		}
+	}
+	return ""
+}
+
+func loadEnrichmentDomains() enrichmentDomainsConfig {
+	enrichmentDomainsOnce.Do(func() {
+		enrichmentDomains = parseEnrichmentDomains(defaultEnrichmentDomainsYAML)
+		if path := strings.TrimSpace(os.Getenv("OPENSERP_ENRICHMENT_DOMAINS_FILE")); path != "" {
+			if data, err := os.ReadFile(path); err == nil {
+				enrichmentDomains = parseEnrichmentDomains(data)
+			}
+		}
+	})
+	return enrichmentDomains
+}
+
+func parseEnrichmentDomains(data []byte) enrichmentDomainsConfig {
+	cfg := enrichmentDomainsConfig{
+		DomainSourceHints:  map[string]string{},
+		NewsDomains:        map[string]bool{},
+		ForumDomains:       map[string]bool{},
+		MarketplaceDomains: map[string]bool{},
+		SocialDomains:      map[string]bool{},
+	}
+
+	var file enrichmentDomainsFile
+	if err := yaml.Unmarshal(data, &file); err != nil {
+		return cfg
+	}
+
+	for domain, hint := range file.DomainSourceHints {
+		domain = normalizeDomain(domain)
+		hint = strings.TrimSpace(hint)
+		if domain != "" && hint != "" {
+			cfg.DomainSourceHints[domain] = hint
+		}
+	}
+	fillDomainSet(cfg.NewsDomains, file.NewsDomains)
+	fillDomainSet(cfg.ForumDomains, file.ForumDomains)
+	fillDomainSet(cfg.MarketplaceDomains, file.MarketplaceDomains)
+	fillDomainSet(cfg.SocialDomains, file.SocialDomains)
+
+	return cfg
+}
+
+func fillDomainSet(dst map[string]bool, domains []string) {
+	for _, domain := range domains {
+		domain = normalizeDomain(domain)
+		if domain != "" {
+			dst[domain] = true
+		}
+	}
+}
+
+func normalizeDomain(domain string) string {
+	domain = strings.ToLower(strings.TrimSpace(domain))
+	domain = strings.TrimPrefix(domain, "www.")
+	return strings.TrimSuffix(domain, ".")
+}

core/enrichment_domains.yaml

@@ -0,0 +1,108 @@
+domain_source_hints:
+  wikipedia.org: encyclopedia
+  en.wikipedia.org: encyclopedia
+  zh.wikipedia.org: encyclopedia
+  baike.baidu.com: encyclopedia
+  wiki.mbalib.com: encyclopedia
+  britannica.com: encyclopedia
+  fandom.com: encyclopedia
+  github.com: code_repository
+  gitlab.com: code_repository
+  stackoverflow.com: qa_forum
+  stackexchange.com: qa_forum
+  reddit.com: social_forum
+  nytimes.com: news
+  bbc.com: news
+  bbc.co.uk: news
+  reuters.com: news
+  theguardian.com: news
+  washingtonpost.com: news
+  forbes.com: news
+  techcrunch.com: news
+  medium.com: blog_platform
+  scholar.google.com: academic
+  arxiv.org: academic
+  pubmed.ncbi.nlm.nih.gov: academic
+  amazon.com: marketplace
+  ebay.com: marketplace
+  etsy.com: marketplace
+  docs.google.com: document
+  youtube.com: video_platform
+  vimeo.com: video_platform
+  twitter.com: social_media
+  x.com: social_media
+  facebook.com: social_media
+  linkedin.com: professional_network
+  instagram.com: social_media
+
+news_domains:
+  - nytimes.com
+  - bbc.com
+  - bbc.co.uk
+  - reuters.com
+  - apnews.com
+  - theguardian.com
+  - washingtonpost.com
+  - forbes.com
+  - techcrunch.com
+  - wired.com
+  - bloomberg.com
+  - cnn.com
+  - nbcnews.com
+  - cbsnews.com
+  - abcnews.go.com
+  - foxnews.com
+  - theverge.com
+  - engadget.com
+  - arstechnica.com
+  - zdnet.com
+  - venturebeat.com
+  - axios.com
+  - politico.com
+  - theatlantic.com
+  - economist.com
+  - ft.com
+  - wsj.com
+  - usatoday.com
+  - latimes.com
+  - nypost.com
+
+forum_domains:
+  - reddit.com
+  - news.ycombinator.com
+  - stackoverflow.com
+  - stackexchange.com
+  - superuser.com
+  - serverfault.com
+  - quora.com
+  - discourse.org
+  - boards.4chan.org
+  - hackernews.com
+
+marketplace_domains:
+  - amazon.com
+  - amazon.co.uk
+  - amazon.de
+  - ebay.com
+  - etsy.com
+  - walmart.com
+  - target.com
+  - bestbuy.com
+  - newegg.com
+  - aliexpress.com
+  - alibaba.com
+  - shopify.com
+
+social_domains:
+  - twitter.com
+  - x.com
+  - facebook.com
+  - instagram.com
+  - tiktok.com
+  - snapchat.com
+  - pinterest.com
+  - tumblr.com
+  - linkedin.com
+  - youtube.com
+  - twitch.tv
+  - discord.com

core/errors.go

@@ -0,0 +1,47 @@
+package core
+
+import "fmt"
+
+// APIError represents a client-facing error with a stable machine-readable reason code.
+type APIError struct {
+	HTTPStatus int
+	ErrorCode  string
+	Reason     string
+	Message    string
+	Meta       map[string]interface{}
+}
+
+func (e *APIError) Error() string {
+	if e.Reason == "" {
+		return e.Message
+	}
+	return fmt.Sprintf("%s: %s", e.Reason, e.Message)
+}
+
+// Common validation reason codes.
+const (
+	ReasonInvalidLimit            = "INVALID_LIMIT"
+	ReasonInvalidStart            = "INVALID_START"
+	ReasonInvalidParam            = "INVALID_PARAM"
+	ReasonEmptyQuery              = "EMPTY_QUERY"
+	ReasonNoEngines               = "NO_ENGINES"
+	ReasonUnknownFormat           = "UNKNOWN_FORMAT"
+	ReasonRequestProxyURLDisabled = "REQUEST_PROXY_URL_DISABLED"
+	ReasonUnsupportedProxyScheme  = "UNSUPPORTED_PROXY_SCHEME"
+)
+
+func errInvalidLimit(msg string) *APIError {
+	return &APIError{HTTPStatus: 400, Reason: ReasonInvalidLimit, Message: msg}
+}
+
+func errInvalidStart(msg string) *APIError {
+	return &APIError{HTTPStatus: 400, Reason: ReasonInvalidStart, Message: msg}
+}
+
+func errInvalidParam(msg string) *APIError {
+	return &APIError{HTTPStatus: 400, Reason: ReasonInvalidParam, Message: msg}
+}
+
+func errEmptyQuery() *APIError {
+	return &APIError{HTTPStatus: 400, Reason: ReasonEmptyQuery, Message: "query cannot be empty: provide text, site, or file parameter"}
+}

core/feature_selectors.go

@@ -0,0 +1,294 @@
+package core
+
+import (
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// blockLevelTags are HTML elements whose boundaries should become line breaks
+// when flattening a feature container to text, so block content (headings,
+// paragraphs, list items, code blocks) does not fuse into the neighbouring text.
+// div/section are deliberately excluded: some engines (e.g. Google's streaming
+// AI Overview) wrap every word in its own <div>, which would otherwise put each
+// word on its own line. Structure there comes from p/h*/li/br instead.
+var blockLevelTags = map[atom.Atom]bool{
+	atom.P: true, atom.Br: true, atom.Li: true,
+	atom.Tr: true, atom.Pre: true,
+	atom.H1: true, atom.H2: true, atom.H3: true, atom.H4: true, atom.H5: true, atom.H6: true,
+	atom.Blockquote: true,
+}
+
+// blockAwareText flattens a selection to text while inserting line breaks at
+// block-element boundaries, then collapses horizontal whitespace per line and
+// drops blank lines. The result keeps logical structure (one line per heading/
+// paragraph/list item) instead of fusing words across element edges, which is
+// what goquery's raw .Text() does.
+func blockAwareText(sel *goquery.Selection) string {
+	var sb strings.Builder
+	for _, node := range sel.Nodes {
+		writeNodeText(&sb, node)
+	}
+	lines := strings.Split(sb.String(), "\n")
+	cleaned := make([]string, 0, len(lines))
+	for _, line := range lines {
+		if line = cleanFeatureText(line); line != "" {
+			cleaned = append(cleaned, line)
+		}
+	}
+	return strings.Join(cleaned, "\n")
+}
+
+func isASCIISpace(b byte) bool {
+	return b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\f' || b == '\v'
+}
+
+func writeNodeText(sb *strings.Builder, node *html.Node) {
+	switch node.Type {
+	case html.TextNode:
+		// Collapse whitespace inside the text node (including source-formatting
+		// newlines) to single spaces, so only the block-boundary breaks inserted
+		// below survive. Preserve a single leading/trailing space so adjacent
+		// inline fragments ("global " + "fetch()") keep their word gap.
+		text := node.Data
+		collapsed := strings.Join(strings.Fields(text), " ")
+		if collapsed == "" {
+			return
+		}
+		if len(text) > 0 && isASCIISpace(text[0]) {
+			sb.WriteByte(' ')
+		}
+		sb.WriteString(collapsed)
+		if len(text) > 0 && isASCIISpace(text[len(text)-1]) {
+			sb.WriteByte(' ')
+		}
+		return
+	case html.ElementNode:
+		if node.DataAtom == atom.Script || node.DataAtom == atom.Style {
+			return
+		}
+		block := blockLevelTags[node.DataAtom]
+		if block {
+			sb.WriteByte('\n')
+		}
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			writeNodeText(sb, child)
+		}
+		if block {
+			sb.WriteByte('\n')
+		}
+	default:
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			writeNodeText(sb, child)
+		}
+	}
+}
+
+// SerpFeatureSelector describes one engine-native SERP module shape.
+type SerpFeatureSelector struct {
+	Type          ResultType
+	Title         string
+	Container     []string
+	TitleSelector []string
+	TextSelector  []string
+	ItemSelector  []string
+	LinkSelector  []string
+	Position      int
+	Confidence    float64
+	// SingleMatch emits at most one feature for this spec: the first container
+	// node (across Container selectors, in order) that yields content. Use it
+	// for modules whose container selector also matches nested sub-panels, which
+	// would otherwise fragment one logical module into many features.
+	SingleMatch bool
+}
+
+// ExtractSerpFeaturesBySelectors converts engine-native SERP module markup into
+// normalized features. It is intentionally conservative: a matched container is
+// emitted only when it yields text, grouped items, or source links.
+func ExtractSerpFeaturesBySelectors(doc *goquery.Document, specs []SerpFeatureSelector) []SerpFeature {
+	var features []SerpFeature
+	for _, spec := range specs {
+		matched := false
+		for _, selector := range spec.Container {
+			if spec.SingleMatch && matched {
+				break
+			}
+			doc.Find(selector).EachWithBreak(func(_ int, container *goquery.Selection) bool {
+				feature := SerpFeature{
+					Type:       spec.Type,
+					Title:      firstNonEmpty(spec.Title, firstSelectedText(container, spec.TitleSelector)),
+					Text:       firstSelectedText(container, spec.TextSelector),
+					Items:      selectedFeatureItems(container, spec.ItemSelector),
+					Links:      selectedFeatureLinks(container, spec.LinkSelector),
+					Confidence: spec.Confidence,
+				}
+				if spec.Position > 0 {
+					feature.Position = &Position{Absolute: spec.Position}
+				}
+				if feature.Text == "" && len(feature.Items) == 0 && len(feature.Links) == 0 {
+					return true
+				}
+				features = append(features, feature)
+				matched = true
+				// Stop after the first content-bearing container when SingleMatch.
+				return !spec.SingleMatch
+			})
+		}
+	}
+	return DeduplicateSerpFeatures(features)
+}
+
+// AttachFeaturesToFirstResult keeps ParseHTML signatures unchanged while
+// letting server response building split features onto the new top-level field.
+func AttachFeaturesToFirstResult(results []SearchResult, features []SerpFeature) []SearchResult {
+	if len(features) == 0 {
+		return results
+	}
+	if len(results) == 0 {
+		return []SearchResult{{Features: features}}
+	}
+	results[0].Features = append(results[0].Features, features...)
+	return results
+}
+
+// StripResultFeatures drops parser-attached features unless keep is true.
+func StripResultFeatures(results []SearchResult, keep bool) []SearchResult {
+	if keep {
+		return results
+	}
+	for i := range results {
+		results[i].Features = nil
+	}
+	return results
+}
+
+// DeduplicateSerpFeatures removes duplicate modules emitted by overlapping
+// selectors while preserving original order.
+func DeduplicateSerpFeatures(features []SerpFeature) []SerpFeature {
+	seen := map[string]struct{}{}
+	unique := make([]SerpFeature, 0, len(features))
+	for _, feature := range features {
+		key := serpFeatureKey(feature)
+		if _, ok := seen[key]; ok {
+			continue
+		}
+		seen[key] = struct{}{}
+		unique = append(unique, feature)
+	}
+	return unique
+}
+
+func firstSelectedText(container *goquery.Selection, selectors []string) string {
+	for _, selector := range selectors {
+		var text string
+		container.Find(selector).EachWithBreak(func(_ int, item *goquery.Selection) bool {
+			text = blockAwareText(item)
+			return text == ""
+		})
+		if text != "" {
+			return text
+		}
+	}
+	return ""
+}
+
+func selectedFeatureItems(container *goquery.Selection, selectors []string) []FeatureItem {
+	var items []FeatureItem
+	for _, selector := range selectors {
+		container.Find(selector).Each(func(_ int, item *goquery.Selection) {
+			text := cleanFeatureText(item.Text())
+			title := cleanFeatureText(firstAttr(item, "data-q", "data-title", "aria-label", "title"))
+			// Some modules keep the question in an attr and fill text lazily.
+			if text == "" {
+				text = title
+			}
+			if text == "" {
+				return
+			}
+			if title == "" {
+				title = text
+			}
+			link := firstAttr(item, "href", "data-url", "data-link")
+			items = append(items, FeatureItem{
+				Title: title,
+				Text:  text,
+				Link:  strings.TrimSpace(link),
+			})
+		})
+		if len(items) > 0 {
+			break
+		}
+	}
+	return items
+}
+
+func selectedFeatureLinks(container *goquery.Selection, selectors []string) []FeatureLink {
+	var links []FeatureLink
+	for _, selector := range selectors {
+		container.Find(selector).Each(func(_ int, item *goquery.Selection) {
+			href := strings.TrimSpace(firstAttr(item, "href", "data-url", "data-link"))
+			if href == "" {
+				return
+			}
+			title := cleanFeatureText(firstAttr(item, "data-title", "aria-label", "title"))
+			if title == "" {
+				title = cleanFeatureText(item.Text())
+			}
+			links = append(links, FeatureLink{Title: title, URL: href})
+		})
+		if len(links) > 0 {
+			break
+		}
+	}
+	return links
+}
+
+func firstAttr(item *goquery.Selection, names ...string) string {
+	for _, name := range names {
+		value, ok := item.Attr(name)
+		if ok && strings.TrimSpace(value) != "" {
+			return value
+		}
+	}
+	return ""
+}
+
+func firstNonEmpty(values ...string) string {
+	for _, value := range values {
+		if strings.TrimSpace(value) != "" {
+			return strings.TrimSpace(value)
+		}
+	}
+	return ""
+}
+
+var invisibleFormatChars = strings.NewReplacer(
+	"\u00AD", "",
+	"\u200B", "",
+	"\u2060", "",
+	"\uFEFF", "",
+)
+
+func cleanFeatureText(value string) string {
+	return strings.Join(strings.Fields(invisibleFormatChars.Replace(value)), " ")
+}
+
+func serpFeatureKey(feature SerpFeature) string {
+	firstLink := ""
+	if len(feature.Links) > 0 {
+		firstLink = feature.Links[0].URL
+	}
+	firstItem := ""
+	if len(feature.Items) > 0 {
+		firstItem = feature.Items[0].Text + "|" + feature.Items[0].Link
+	}
+	return strings.Join([]string{
+		string(feature.Type),
+		strings.ToLower(cleanFeatureText(feature.Title)),
+		strings.ToLower(cleanFeatureText(feature.Text)),
+		strings.ToLower(firstItem),
+		strings.ToLower(firstLink),
+	}, "|")
+}

core/format_markdown.go

@@ -0,0 +1,148 @@
+package core
+
+import (
+	"fmt"
+	"strings"
+)
+
+// RenderMarkdown formats an Envelope as a Markdown document suitable for
+// Slack/Discord/email nodes in n8n workflows.
+func RenderMarkdown(env *Envelope) []byte {
+	var b strings.Builder
+
+	enginesStr := strings.Join(env.Query.EnginesRequested, ", ")
+	fmt.Fprintf(&b, "# Search results for %q\n\n", env.Query.Text)
+	fmt.Fprintf(&b, "**Query:** %s - **Engines:** %s - **Took:** %dms\n\n",
+		env.Query.Text, enginesStr, env.Meta.TookMs)
+
+	if len(env.Meta.EnginesFailed) > 0 {
+		fmt.Fprintf(&b, "> Engines that failed: %s\n\n", strings.Join(env.Meta.EnginesFailed, ", "))
+	}
+
+	renderMarkdownFeatures(&b, env.SerpFeatures, featureRenderOrderBeforeResults())
+
+	if len(env.Results) > 0 {
+		b.WriteString("## Results\n\n")
+	}
+	for i, r := range env.Results {
+		fmt.Fprintf(&b, "### %d. %s\n\n", i+1, escapeMarkdown(r.Title))
+		typeLabel := string(r.Type)
+		fmt.Fprintf(&b, "**%s** - %s\n\n", r.DisplayURL, typeLabel)
+		if r.Snippet != "" {
+			fmt.Fprintf(&b, "%s\n\n", r.Snippet)
+		}
+		fmt.Fprintf(&b, "-> %s\n\n", r.URL)
+		if r.Extracted != nil && r.Extracted.Content != "" {
+			b.WriteString("#### Extracted content\n\n")
+			b.WriteString(shiftMarkdownHeadings(r.Extracted.Content, 4))
+			b.WriteString("\n\n")
+		}
+	}
+
+	renderMarkdownFeatures(&b, env.SerpFeatures, featureRenderOrderAfterResults(env.SerpFeatures))
+
+	return []byte(b.String())
+}
+
+func shiftMarkdownHeadings(markdown string, minLevel int) string {
+	lines := strings.Split(markdown, "\n")
+	for i, line := range lines {
+		trimmed := strings.TrimLeft(line, " ")
+		indent := line[:len(line)-len(trimmed)]
+		if !strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+		count := 0
+		for count < len(trimmed) && trimmed[count] == '#' {
+			count++
+		}
+		if count == 0 || count >= len(trimmed) || trimmed[count] != ' ' {
+			continue
+		}
+		target := count + minLevel
+		if target > 6 {
+			target = 6
+		}
+		lines[i] = indent + strings.Repeat("#", target) + trimmed[count:]
+	}
+	return strings.TrimSpace(strings.Join(lines, "\n"))
+}
+
+// RenderMarkdownImage formats an ImageEnvelope as Markdown.
+func RenderMarkdownImage(env *ImageEnvelope) []byte {
+	var b strings.Builder
+
+	enginesStr := strings.Join(env.Query.EnginesRequested, ", ")
+	fmt.Fprintf(&b, "# Image results for %q\n\n", env.Query.Text)
+	fmt.Fprintf(&b, "**Query:** %s - **Engines:** %s - **Took:** %dms\n\n",
+		env.Query.Text, enginesStr, env.Meta.TookMs)
+
+	for i, r := range env.Results {
+		fmt.Fprintf(&b, "## %d. %s\n\n", i+1, escapeMarkdown(r.Title))
+		fmt.Fprintf(&b, "**Source:** %s\n\n", r.Source.Domain)
+		fmt.Fprintf(&b, "-> Image: %s\n", r.Image.URL)
+		fmt.Fprintf(&b, "-> Page: %s\n\n", r.Source.PageURL)
+	}
+
+	return []byte(b.String())
+}
+
+func renderMarkdownFeatures(b *strings.Builder, features []SerpFeature, order []ResultType) {
+	forEachFeatureInOrder(features, order, func(feature SerpFeature) {
+		renderMarkdownFeature(b, feature)
+	})
+}
+
+func renderMarkdownFeature(b *strings.Builder, feature SerpFeature) {
+	heading := featureHeading(feature)
+	if feature.Type == ResultTypeKnowledgePanel && feature.Title != "" {
+		heading += " - " + feature.Title
+	}
+	fmt.Fprintf(b, "## %s\n\n", heading)
+	if feature.Type == ResultTypeFeaturedSnippet && feature.Text != "" {
+		fmt.Fprintf(b, "> %s\n", feature.Text)
+		if len(feature.Links) > 0 {
+			fmt.Fprintf(b, "> - [%s](%s)\n", escapeMarkdown(feature.Links[0].Title), feature.Links[0].URL)
+		}
+		b.WriteString("\n")
+		return
+	}
+	if feature.Text != "" {
+		fmt.Fprintf(b, "%s\n\n", feature.Text)
+	}
+	if len(feature.Items) > 0 {
+		for _, item := range feature.Items {
+			switch {
+			case item.Title != "" && item.Text != "":
+				fmt.Fprintf(b, "- **%s** - %s\n", escapeMarkdown(item.Title), item.Text)
+			case item.Text != "":
+				fmt.Fprintf(b, "- %s\n", item.Text)
+			case item.Title != "":
+				fmt.Fprintf(b, "- %s\n", escapeMarkdown(item.Title))
+			}
+		}
+		b.WriteString("\n")
+	}
+	if len(feature.Links) > 0 {
+		b.WriteString("Sources:\n")
+		for _, link := range feature.Links {
+			title := link.Title
+			if title == "" {
+				title = link.URL
+			}
+			fmt.Fprintf(b, "- [%s](%s)\n", escapeMarkdown(title), link.URL)
+		}
+		b.WriteString("\n")
+	}
+}
+
+func escapeMarkdown(s string) string {
+	replacer := strings.NewReplacer(
+		"*", `\*`,
+		"_", `\_`,
+		"`", "\\`",
+		"[", `\[`,
+		"]", `\]`,
+	)
+	return replacer.Replace(s)
+}

core/format_text.go

@@ -0,0 +1,238 @@
+package core
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+// RenderText formats an Envelope as a minimal plain-text block optimised for
+// LLM context windows (~25-30% fewer tokens than JSON for the same data).
+func RenderText(env *Envelope) []byte {
+	var b strings.Builder
+
+	fmt.Fprintf(&b, "Search: %s\n", env.Query.Text)
+	enginesStr := strings.Join(env.Query.EnginesRequested, ", ")
+	if enginesStr != "" {
+		fmt.Fprintf(&b, "Engines: %s\n", enginesStr)
+	}
+	if len(env.Meta.EnginesFailed) > 0 {
+		fmt.Fprintf(&b, "Failed: %s\n", strings.Join(env.Meta.EnginesFailed, ", "))
+	}
+	b.WriteString("\n")
+
+	renderTextFeatures(&b, env.SerpFeatures, featureRenderOrderBeforeResults())
+
+	if len(env.Results) > 0 {
+		b.WriteString("Results\n\n")
+	}
+	for i, r := range env.Results {
+		fmt.Fprintf(&b, "[%d] %s (%s)\n", i+1, r.Title, r.Domain)
+		if r.Snippet != "" {
+			fmt.Fprintf(&b, "%s\n", r.Snippet)
+		}
+		fmt.Fprintf(&b, "URL: %s\n\n", r.URL)
+		if r.Extracted != nil && r.Extracted.Content != "" {
+			b.WriteString("Extracted content:\n")
+			b.WriteString(r.Extracted.Content)
+			b.WriteString("\n\n")
+		}
+	}
+
+	renderTextFeatures(&b, env.SerpFeatures, featureRenderOrderAfterResults(env.SerpFeatures))
+
+	return []byte(b.String())
+}
+
+// RenderTextImage formats an ImageEnvelope as plain text.
+func RenderTextImage(env *ImageEnvelope) []byte {
+	var b strings.Builder
+
+	fmt.Fprintf(&b, "Image search: %s\n\n", env.Query.Text)
+
+	for i, r := range env.Results {
+		fmt.Fprintf(&b, "[%d] %s (%s)\n", i+1, r.Title, r.Source.Domain)
+		fmt.Fprintf(&b, "Image: %s\n", r.Image.URL)
+		fmt.Fprintf(&b, "Page: %s\n\n", r.Source.PageURL)
+	}
+
+	return []byte(b.String())
+}
+
+// RenderNDJSON formats an Envelope as newline-delimited JSON.
+func RenderNDJSON(env *Envelope) []byte {
+	var b strings.Builder
+	for _, r := range env.Results {
+		writeNDJSONLine(&b, "result", r)
+	}
+	for _, feature := range env.SerpFeatures {
+		writeNDJSONLine(&b, "feature", feature)
+	}
+	return []byte(b.String())
+}
+
+// RenderNDJSONImage formats an ImageEnvelope as newline-delimited JSON.
+func RenderNDJSONImage(env *ImageEnvelope) []byte {
+	var b strings.Builder
+	for _, r := range env.Results {
+		writeNDJSONLine(&b, "result", r)
+	}
+	return []byte(b.String())
+}
+
+func renderTextFeatures(b *strings.Builder, features []SerpFeature, order []ResultType) {
+	forEachFeatureInOrder(features, order, func(feature SerpFeature) {
+		renderTextFeature(b, feature)
+	})
+}
+
+// forEachFeatureInOrder invokes render for every feature whose Type appears in
+// order, type by type. It is the single iteration shared by the text and
+// markdown renderers.
+func forEachFeatureInOrder(features []SerpFeature, order []ResultType, render func(SerpFeature)) {
+	for _, featureType := range order {
+		for _, feature := range features {
+			if feature.Type == featureType {
+				render(feature)
+			}
+		}
+	}
+}
+
+func renderTextFeature(b *strings.Builder, feature SerpFeature) {
+	heading := featureHeading(feature)
+	if feature.Type == ResultTypeKnowledgePanel && feature.Title != "" {
+		heading += " - " + feature.Title
+	}
+	fmt.Fprintf(b, "%s\n", heading)
+	if feature.Text != "" {
+		fmt.Fprintf(b, "%s", feature.Text)
+		if len(feature.Links) == 1 {
+			fmt.Fprintf(b, " (source: %s)", feature.Links[0].URL)
+		}
+		b.WriteString("\n")
+	}
+	for _, item := range feature.Items {
+		switch {
+		case item.Title != "" && item.Text != "":
+			fmt.Fprintf(b, "- %s - %s\n", item.Title, item.Text)
+		case item.Text != "":
+			fmt.Fprintf(b, "- %s\n", item.Text)
+		case item.Title != "":
+			fmt.Fprintf(b, "- %s\n", item.Title)
+		}
+	}
+	if len(feature.Links) > 1 {
+		b.WriteString("Sources:\n")
+		for _, link := range feature.Links {
+			fmt.Fprintf(b, "- %s\n", link.URL)
+		}
+	}
+	b.WriteString("\n")
+}
+
+func writeNDJSONLine(b *strings.Builder, kind string, value any) {
+	data, err := json.Marshal(value)
+	if err != nil {
+		return
+	}
+	var object map[string]any
+	if err := json.Unmarshal(data, &object); err != nil {
+		return
+	}
+	object["kind"] = kind
+	data, err = json.Marshal(object)
+	if err != nil {
+		return
+	}
+	b.Write(data)
+	b.WriteByte('\n')
+}
+
+// featureRenderOrderBeforeResults lists the feature sections rendered above the
+// results list, in fixed order (spec: AI summary -> answer box -> featured
+// snippet -> PAA -> related questions -> knowledge panel -> results -> ...).
+func featureRenderOrderBeforeResults() []ResultType {
+	return []ResultType{
+		ResultTypeAISummary,
+		ResultTypeAnswerBox,
+		ResultTypeFeaturedSnippet,
+		ResultTypePeopleAlsoAsk,
+		ResultTypeRelatedQuestions,
+		ResultTypeKnowledgePanel,
+	}
+}
+
+// featureRenderOrderAfterResults lists the feature sections rendered below the
+// results list (related searches and the module gallery), in fixed order. Any
+// feature type present in features but absent from both fixed orders is appended
+// at the end, so a newly added feature enum is never silently dropped from text
+// or markdown output.
+func featureRenderOrderAfterResults(features []SerpFeature) []ResultType {
+	order := []ResultType{
+		ResultTypeRelatedSearches,
+		ResultTypeNews,
+		ResultTypeVideo,
+		ResultTypeVideos,
+		ResultTypeShopping,
+		ResultTypeImagesInline,
+		ResultTypeLocal,
+		ResultTypeSitelinks,
+		ResultTypeCalculator,
+		ResultTypeWeather,
+		ResultTypeDictionary,
+	}
+	placed := make(map[ResultType]bool, len(order)+len(featureRenderOrderBeforeResults()))
+	for _, t := range featureRenderOrderBeforeResults() {
+		placed[t] = true
+	}
+	for _, t := range order {
+		placed[t] = true
+	}
+	for _, feature := range features {
+		if !placed[feature.Type] {
+			order = append(order, feature.Type)
+			placed[feature.Type] = true
+		}
+	}
+	return order
+}
+
+func featureHeading(feature SerpFeature) string {
+	switch feature.Type {
+	case ResultTypeAISummary:
+		return "AI summary"
+	case ResultTypeAnswerBox:
+		return "Answer box"
+	case ResultTypeFeaturedSnippet:
+		return "Featured snippet"
+	case ResultTypePeopleAlsoAsk:
+		return "People also ask"
+	case ResultTypeRelatedQuestions:
+		return "Related questions"
+	case ResultTypeKnowledgePanel:
+		return "Knowledge panel"
+	case ResultTypeRelatedSearches:
+		return "Related searches"
+	case ResultTypeNews:
+		return "News"
+	case ResultTypeVideo, ResultTypeVideos:
+		return "Videos"
+	case ResultTypeShopping:
+		return "Shopping"
+	case ResultTypeImagesInline:
+		return "Images"
+	case ResultTypeLocal:
+		return "Local pack"
+	case ResultTypeSitelinks:
+		return "Sitelinks"
+	case ResultTypeCalculator:
+		return "Calculator"
+	case ResultTypeWeather:
+		return "Weather"
+	case ResultTypeDictionary:
+		return "Dictionary"
+	default:
+		return strings.ReplaceAll(string(feature.Type), "_", " ")
+	}
+}

core/fpcheck/detector.go

@@ -0,0 +1,45 @@
+package fpcheck
+
+import (
+	"context"
+
+	"github.com/go-rod/rod"
+)
+
+// Detection represents a single anti-bot signal verdict from a detector page.
+type Detection struct {
+	Detected    bool     `json:"detected"`
+	Description string   `json:"description"`
+	Severity    string   `json:"severity,omitempty"`
+	Numeric     *float64 `json:"numeric,omitempty"`
+}
+
+// Summary contains aggregate counters for a detector report.
+type Summary struct {
+	Passed   int      `json:"passed"`
+	Failed   int      `json:"failed"`
+	Critical []string `json:"critical,omitempty"`
+}
+
+// Report is the normalized output for one detector run.
+type Report struct {
+	DetectorName  string               `json:"detector_name"`
+	URL           string               `json:"url"`
+	CapturedAtUTC string               `json:"captured_at_utc"`
+	Screenshot    string               `json:"screenshot_path"`
+	Detections    map[string]Detection `json:"detections"`
+	Summary       Summary              `json:"summary"`
+	RawNotes      string               `json:"raw_notes,omitempty"`
+}
+
+// Detector knows how to extract normalized anti-bot verdicts from one site.
+type Detector interface {
+	Name() string
+	URL() string
+	Extract(ctx context.Context, page *rod.Page) (map[string]Detection, string, error)
+}
+
+// BrowserNavigator is the minimal browser contract used by fpcheck runner.
+type BrowserNavigator interface {
+	Navigate(ctx context.Context, URL string) (*rod.Page, error)
+}

core/fpcheck/detectors/browserscan.go

@@ -0,0 +1,62 @@
+package detectors
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/go-rod/rod"
+	"github.com/karust/openserp/core/fpcheck"
+)
+
+const browserscanURL = "https://www.browserscan.net/bot-detection"
+
+type BrowserScan struct{}
+
+func NewBrowserScan() fpcheck.Detector {
+	return BrowserScan{}
+}
+
+func (BrowserScan) Name() string {
+	return "browserscan"
+}
+
+func (BrowserScan) URL() string {
+	return browserscanURL
+}
+
+func (BrowserScan) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
+	err := waitFor(ctx, 25*time.Second, 250*time.Millisecond, func() (bool, error) {
+		res, err := page.Eval(`() => {
+			const text = (document.body && document.body.innerText ? document.body.innerText : "").toLowerCase();
+			return text.includes("bot") && (text.includes("detected") || text.includes("pass") || text.includes("fail"));
+		}`)
+		if err != nil {
+			return false, nil
+		}
+
+		var ready bool
+		if err := res.Value.Unmarshal(&ready); err != nil {
+			return false, nil
+		}
+		return ready, nil
+	})
+	if err != nil {
+		return nil, "", fmt.Errorf("browserscan readiness: %w", err)
+	}
+
+	rows, err := parseRows(page)
+	if err != nil {
+		return nil, "", err
+	}
+	if len(rows) == 0 {
+		return nil, "", fmt.Errorf("browserscan detector rows not found")
+	}
+
+	detections := rowsToDetections(rows, []string{"bot", "webdriver", "automation", "headless", "fingerprint"})
+	if len(detections) == 0 {
+		return nil, "", fmt.Errorf("browserscan detections are empty")
+	}
+
+	return detections, "", nil
+}

core/fpcheck/detectors/custom.go

@@ -0,0 +1,154 @@
+package detectors
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/go-rod/rod"
+	"github.com/karust/openserp/core/fpcheck"
+)
+
+const customDetectorName = "custom"
+const defaultCustomSelector = "body"
+
+type Custom struct {
+	targetURL string
+	selector  string
+}
+
+type customPayload struct {
+	Found        bool   `json:"found"`
+	Error        string `json:"error"`
+	Title        string `json:"title"`
+	URL          string `json:"url"`
+	ReadyState   string `json:"readyState"`
+	Selector     string `json:"selector"`
+	SelectedText string `json:"selectedText"`
+	SelectedHTML string `json:"selectedHTML"`
+}
+
+func NewCustom(rawURL string) (fpcheck.Detector, error) {
+	return NewCustomWithSelector(rawURL, "")
+}
+
+func NewCustomWithSelector(rawURL string, selector string) (fpcheck.Detector, error) {
+	normalized, err := normalizeCustomURL(rawURL)
+	if err != nil {
+		return nil, err
+	}
+	return Custom{
+		targetURL: normalized,
+		selector:  normalizeCustomSelector(selector),
+	}, nil
+}
+
+func (c Custom) Name() string {
+	return customDetectorName
+}
+
+func (c Custom) URL() string {
+	return c.targetURL
+}
+
+func (c Custom) Selector() string {
+	return normalizeCustomSelector(c.selector)
+}
+
+func (c Custom) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
+	selector := c.Selector()
+	var payload customPayload
+	err := waitFor(ctx, 15*time.Second, 200*time.Millisecond, func() (bool, error) {
+		current, err := extractCustomPayload(page, selector)
+		if err != nil {
+			return false, err
+		}
+		if !current.Found {
+			return false, nil
+		}
+		payload = current
+		return true, nil
+	})
+	if err != nil {
+		return nil, "", fmt.Errorf("custom page readiness for selector %q: %w", selector, err)
+	}
+
+	rawOut := strings.TrimSpace(payload.SelectedText)
+	if rawOut == "" {
+		rawOut = strings.TrimSpace(payload.SelectedHTML)
+	}
+
+	return map[string]fpcheck.Detection{
+		"selected_page_output": {
+			Detected:    false,
+			Description: fmt.Sprintf("captured selector %q", payload.Selector),
+		},
+	}, rawOut, nil
+}
+
+func extractCustomPayload(page *rod.Page, selector string) (customPayload, error) {
+	res, err := page.Timeout(2*time.Second).Eval(`(selector) => {
+		const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
+		let selected = null;
+		try {
+			selected = document.querySelector(selector);
+		 } catch (err) {
+			return {
+				found: false,
+				error: err && err.message ? err.message : String(err),
+				selector,
+			};
+		}
+		return {
+			found: !!selected,
+			title: document.title || "",
+			url: location.href || "",
+			readyState: document.readyState || "",
+			selector,
+			selectedText: normalize(selected ? selected.innerText || selected.textContent || "" : ""),
+			selectedHTML: selected ? selected.innerHTML || "" : "",
+		};
+	}`, selector)
+	if err != nil {
+		return customPayload{}, err
+	}
+
+	var payload customPayload
+	if err := res.Value.Unmarshal(&payload); err != nil {
+		return customPayload{}, fmt.Errorf("decode custom detector payload: %w", err)
+	}
+	if strings.TrimSpace(payload.Error) != "" {
+		return customPayload{}, fmt.Errorf("query selector %q: %s", selector, payload.Error)
+	}
+	return payload, nil
+}
+
+func normalizeCustomURL(rawURL string) (string, error) {
+	trimmed := strings.TrimSpace(rawURL)
+	if trimmed == "" {
+		return "", fmt.Errorf("custom detector requires non-empty url query parameter")
+	}
+
+	parsed, err := url.Parse(trimmed)
+	if err != nil {
+		return "", fmt.Errorf("invalid custom detector URL: %w", err)
+	}
+	if parsed.Scheme != "http" && parsed.Scheme != "https" {
+		return "", fmt.Errorf("invalid custom detector URL scheme %q: use http or https", parsed.Scheme)
+	}
+	if strings.TrimSpace(parsed.Host) == "" {
+		return "", fmt.Errorf("invalid custom detector URL: host is required")
+	}
+
+	return parsed.String(), nil
+}
+
+func normalizeCustomSelector(selector string) string {
+	trimmed := strings.TrimSpace(selector)
+	if trimmed == "" {
+		return defaultCustomSelector
+	}
+	return trimmed
+}

core/fpcheck/detectors/deviceandbrowser.go

@@ -0,0 +1,171 @@
+package detectors
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/go-rod/rod"
+	"github.com/karust/openserp/core/fpcheck"
+)
+
+const deviceAndBrowserURL = "https://deviceandbrowserinfo.com/are_you_a_bot"
+
+type DeviceAndBrowser struct{}
+
+func NewDeviceAndBrowser() fpcheck.Detector {
+	return DeviceAndBrowser{}
+}
+
+func (DeviceAndBrowser) Name() string {
+	return "deviceandbrowser"
+}
+
+func (DeviceAndBrowser) URL() string {
+	return deviceAndBrowserURL
+}
+
+func (DeviceAndBrowser) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
+	err := waitFor(ctx, 25*time.Second, 250*time.Millisecond, func() (bool, error) {
+		res, err := page.Eval(`() => {
+			const hasJson = !!document.querySelector("#jsonResult");
+			const hasCard = !!document.querySelector("#resultsBotTest");
+			const text = (document.body && document.body.innerText ? document.body.innerText : "").toLowerCase();
+			return hasJson || hasCard || text.includes("are you a bot");
+		}`)
+		if err != nil {
+			return false, nil
+		}
+
+		var ready bool
+		if err := res.Value.Unmarshal(&ready); err != nil {
+			return false, nil
+		}
+		return ready, nil
+	})
+	if err != nil {
+		return nil, "", fmt.Errorf("deviceandbrowser readiness: %w", err)
+	}
+
+	res, err := page.Eval(`() => {
+		const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
+		const decodeHTML = (value) => {
+			const textarea = document.createElement("textarea");
+			textarea.innerHTML = value;
+			return textarea.value;
+		};
+
+		const out = { isBot: null, details: {}, rawJson: "", cardText: "", body: "" };
+
+		const card = document.querySelector("#resultsBotTest");
+		if (card) {
+			out.cardText = normalize(card.innerText || card.textContent || "");
+			const low = out.cardText.toLowerCase();
+			if (low.includes("you are a bot")) out.isBot = true;
+			if (low.includes("not a bot")) out.isBot = false;
+		}
+
+		const jsonNode = document.querySelector("#jsonResult");
+		if (jsonNode) {
+			let raw = (jsonNode.textContent || jsonNode.innerText || "").replace(/\u00a0/g, " ").trim();
+			if (!raw && jsonNode.innerHTML) {
+				raw = decodeHTML(jsonNode.innerHTML.replace(/<br\s*\/?>/gi, "\n")).replace(/\u00a0/g, " ").trim();
+			}
+			out.rawJson = raw;
+			try {
+				const parsed = JSON.parse(raw);
+				if (typeof parsed.isBot === "boolean") out.isBot = parsed.isBot;
+				if (parsed.details && typeof parsed.details === "object") {
+					for (const [k, v] of Object.entries(parsed.details)) {
+						if (typeof v === "boolean") out.details[k] = v;
+					}
+				}
+			} catch (_) {}
+		}
+
+		out.body = normalize(document.body && document.body.innerText ? document.body.innerText : "").slice(0, 6000);
+		return out;
+	}`)
+	if err != nil {
+		return nil, "", err
+	}
+
+	var payload struct {
+		IsBot    *bool           `json:"isBot"`
+		Details  map[string]bool `json:"details"`
+		RawJSON  string          `json:"rawJson"`
+		CardText string          `json:"cardText"`
+		Body     string          `json:"body"`
+	}
+	if err := res.Value.Unmarshal(&payload); err != nil {
+		return nil, "", fmt.Errorf("decode deviceandbrowser payload: %w", err)
+	}
+
+	detections := make(map[string]fpcheck.Detection)
+	if payload.IsBot != nil {
+		severity := ""
+		if *payload.IsBot {
+			severity = "critical"
+		}
+		detections["overall_is_bot"] = fpcheck.Detection{
+			Detected:    *payload.IsBot,
+			Description: strings.TrimSpace(payload.CardText),
+			Severity:    severity,
+		}
+	}
+
+	for key, value := range payload.Details {
+		norm := normalizeKey(key)
+		if norm == "unknown" {
+			continue
+		}
+
+		severity := ""
+		if value && hasKeyword(norm, []string{"webdriver", "cdp", "headless", "bot", "playwright", "selenium"}) {
+			severity = "critical"
+		}
+		detections[norm] = fpcheck.Detection{
+			Detected:    value,
+			Description: fmt.Sprintf("%t", value),
+			Severity:    severity,
+		}
+	}
+
+	if len(detections) == 0 && strings.TrimSpace(payload.RawJSON) != "" {
+		// Fallback parse when JSON was extracted but JS-side parser missed fields.
+		var decoded struct {
+			IsBot   *bool           `json:"isBot"`
+			Details map[string]bool `json:"details"`
+		}
+		if err := json.Unmarshal([]byte(payload.RawJSON), &decoded); err == nil {
+			if decoded.IsBot != nil {
+				detections["overall_is_bot"] = fpcheck.Detection{
+					Detected:    *decoded.IsBot,
+					Description: strings.TrimSpace(payload.CardText),
+				}
+			}
+			for key, value := range decoded.Details {
+				norm := normalizeKey(key)
+				if norm == "unknown" {
+					continue
+				}
+				detections[norm] = fpcheck.Detection{
+					Detected:    value,
+					Description: fmt.Sprintf("%t", value),
+				}
+			}
+		}
+	}
+
+	if len(detections) == 0 {
+		return nil, payload.Body, fmt.Errorf("deviceandbrowser detections are empty")
+	}
+
+	rawNotes := strings.TrimSpace(payload.RawJSON)
+	if rawNotes == "" {
+		rawNotes = strings.TrimSpace(payload.Body)
+	}
+	return detections, rawNotes, nil
+}

core/fpcheck/detectors/helpers.go

@@ -0,0 +1,248 @@
+package detectors
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/go-rod/rod"
+	"github.com/karust/openserp/core/fpcheck"
+)
+
+var (
+	reNonAlphanumUnderscore = regexp.MustCompile(`[^a-z0-9_]+`)
+	reMultiUnderscore       = regexp.MustCompile(`_+`)
+	reExtractScore          = regexp.MustCompile(`(?i)(score|overall|risk)[^\d]{0,20}(\d+(?:\.\d+)?)`)
+)
+
+type detectorRow struct {
+	Name   string `json:"name"`
+	Status string `json:"status"`
+	Detail string `json:"detail"`
+}
+
+func waitFor(ctx context.Context, timeout time.Duration, poll time.Duration, probe func() (bool, error)) error {
+	if timeout <= 0 {
+		timeout = 20 * time.Second
+	}
+	if poll <= 0 {
+		poll = 250 * time.Millisecond
+	}
+
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+
+		ok, err := probe()
+		if err != nil {
+			return err
+		}
+		if ok {
+			return nil
+		}
+
+		timer := time.NewTimer(poll)
+		select {
+		case <-timer.C:
+		case <-ctx.Done():
+			timer.Stop()
+			return ctx.Err()
+		}
+	}
+
+	return fmt.Errorf("ready condition not met after %s", timeout)
+}
+
+func normalizeKey(name string) string {
+	name = strings.ToLower(strings.TrimSpace(name))
+	if name == "" {
+		return "unknown"
+	}
+
+	replacer := strings.NewReplacer(
+		" ", "_",
+		"-", "_",
+		"/", "_",
+		"\\", "_",
+		":", "_",
+		".", "_",
+	)
+	name = replacer.Replace(name)
+	name = reNonAlphanumUnderscore.ReplaceAllString(name, "")
+	name = reMultiUnderscore.ReplaceAllString(name, "_")
+	name = strings.Trim(name, "_")
+	if name == "" {
+		return "unknown"
+	}
+	return name
+}
+
+func classifyStatus(status string) bool {
+	value := strings.ToLower(strings.TrimSpace(status))
+	if value == "" {
+		return false
+	}
+
+	if strings.Contains(value, "🔴") {
+		return true
+	}
+	if strings.Contains(value, "🟢") || strings.Contains(value, "⚪") {
+		return false
+	}
+
+	notDetected := []string{"not detected", "not found", "clean", "clear", "pass", "passed", "ok", "safe", "green", "false", "no"}
+	for _, marker := range notDetected {
+		if strings.Contains(value, marker) {
+			return false
+		}
+	}
+
+	detected := []string{"detected", "fail", "failed", "bot", "leak", "warning", "critical", "red", "true", "yes"}
+	for _, marker := range detected {
+		if strings.Contains(value, marker) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func parseRows(page *rod.Page) ([]detectorRow, error) {
+	res, err := page.Eval(`() => {
+		const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
+		const rows = [];
+		const seen = new Set();
+
+		const tableRows = Array.from(document.querySelectorAll("table tr"));
+		for (const row of tableRows) {
+			const cells = Array.from(row.querySelectorAll("th, td"));
+			if (cells.length < 2) continue;
+			const name = normalize(cells[0].innerText || cells[0].textContent || "");
+			if (!name) continue;
+			if (/^(test(\s+name)?|property|status|result|check)$/i.test(name)) continue;
+
+			const valueCell = cells[cells.length - 1];
+			const status = normalize(valueCell.innerText || valueCell.textContent || "");
+			if (!status) continue;
+
+			const key = name.toLowerCase();
+			if (seen.has(key)) continue;
+			seen.add(key);
+
+			rows.push({
+				name,
+				status,
+				detail: normalize(row.innerText || row.textContent || ""),
+			});
+		}
+
+		const candidates = Array.from(document.querySelectorAll("[data-test], [data-testid], [data-check], [data-name], .check, .result, li"));
+		for (const node of candidates) {
+			const text = normalize(node.innerText || node.textContent || "");
+			if (!text || text.length > 260) continue;
+			if (!/(pass|fail|detected|not detected|warning|critical|true|false|yes|no|leak|bot)/i.test(text)) continue;
+
+			let name = normalize(node.getAttribute("data-check") || node.getAttribute("data-name") || node.getAttribute("data-testid") || "");
+			let status = "";
+
+			if (!name) {
+				const parts = text.split(/[:\-|]/).map(normalize).filter(Boolean);
+				if (parts.length >= 2) {
+					name = parts[0];
+					status = parts.slice(1).join(" ");
+				}
+			}
+
+			if (!name) {
+				const lines = text.split(/\n+/).map(normalize).filter(Boolean);
+				if (lines.length >= 2) {
+					name = lines[0];
+					status = lines.slice(1).join(" ");
+				}
+			}
+
+			if (!name) continue;
+			if (!status) {
+				status = text;
+			}
+
+			const key = name.toLowerCase();
+			if (seen.has(key)) continue;
+			seen.add(key);
+
+			rows.push({ name, status, detail: text });
+		}
+
+		return rows;
+	}`)
+	if err != nil {
+		return nil, err
+	}
+
+	var rows []detectorRow
+	if err := res.Value.Unmarshal(&rows); err != nil {
+		return nil, fmt.Errorf("decode detector rows: %w", err)
+	}
+	return rows, nil
+}
+
+func rowsToDetections(rows []detectorRow, criticalKeywords []string) map[string]fpcheck.Detection {
+	out := make(map[string]fpcheck.Detection, len(rows))
+	for _, row := range rows {
+		key := normalizeKey(row.Name)
+		if key == "unknown" {
+			continue
+		}
+
+		detected := classifyStatus(row.Status)
+		severity := ""
+		if detected && hasKeyword(key+" "+strings.ToLower(row.Detail), criticalKeywords) {
+			severity = "critical"
+		}
+
+		description := strings.TrimSpace(row.Status)
+		if description == "" {
+			description = strings.TrimSpace(row.Detail)
+		}
+
+		out[key] = fpcheck.Detection{
+			Detected:    detected,
+			Description: description,
+			Severity:    severity,
+		}
+	}
+	return out
+}
+
+func hasKeyword(value string, keywords []string) bool {
+	if len(keywords) == 0 {
+		return false
+	}
+	for _, keyword := range keywords {
+		k := strings.ToLower(strings.TrimSpace(keyword))
+		if k == "" {
+			continue
+		}
+		if strings.Contains(value, k) {
+			return true
+		}
+	}
+	return false
+}
+
+func extractScore(text string) (float64, bool) {
+	matches := reExtractScore.FindStringSubmatch(text)
+	if len(matches) < 3 {
+		return 0, false
+	}
+	score, err := strconv.ParseFloat(matches[2], 64)
+	if err != nil {
+		return 0, false
+	}
+	return score, true
+}

core/fpcheck/detectors/pixelscan.go

@@ -0,0 +1,204 @@
+package detectors
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/go-rod/rod"
+	"github.com/karust/openserp/core/fpcheck"
+)
+
+const (
+	pixelscanURL = "https://pixelscan.net/bot-check"
+)
+
+type PixelScan struct{}
+
+func NewPixelScan() fpcheck.Detector {
+	return PixelScan{}
+}
+
+func (PixelScan) Name() string {
+	return "pixelscan"
+}
+
+func (PixelScan) URL() string {
+	return pixelscanURL
+}
+
+func (PixelScan) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
+	err := waitFor(ctx, 25*time.Second, 250*time.Millisecond, func() (bool, error) {
+		res, err := page.Eval(`() => {
+			const hasSummary = !!document.querySelector(".bot-check-summary, .bot-check__summary, .bot-check-accordion__row");
+			const hasState = !!document.querySelector(".state-success, .state-error");
+			const body = (document.body && document.body.innerText ? document.body.innerText : "").toLowerCase();
+			return hasSummary || hasState || body.includes("running bot detection") || body.includes("definitely a human") || body.includes("bot behavior detected");
+		}`)
+		if err != nil {
+			return false, nil
+		}
+
+		var ready bool
+		if err := res.Value.Unmarshal(&ready); err != nil {
+			return false, nil
+		}
+		return ready, nil
+	})
+	if err != nil {
+		return nil, "", fmt.Errorf("pixelscan readiness: %w", err)
+	}
+
+	res, err := page.Eval(`() => {
+		const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
+		const isVisible = (node) => !!node && !!(node.offsetParent || node.getClientRects().length) &&
+			window.getComputedStyle(node).display !== "none" &&
+			window.getComputedStyle(node).visibility !== "hidden" &&
+			window.getComputedStyle(node).opacity !== "0";
+
+		const successNode = document.querySelector(".state-success");
+		const errorNode = document.querySelector(".state-error");
+		let state = "unknown";
+		if (isVisible(successNode)) state = "human";
+		if (isVisible(errorNode)) state = "bot";
+
+		if (state === "unknown") {
+			const text = normalize(document.body && document.body.innerText ? document.body.innerText : "").toLowerCase();
+			if (text.includes("you're definitely a human")) state = "human";
+			if (text.includes("bot behavior detected")) state = "bot";
+		}
+
+		const summary = [];
+		for (const section of Array.from(document.querySelectorAll(".summary-section"))) {
+			const full = normalize(section.innerText || section.textContent || "");
+			if (!full) continue;
+
+			const statusNode = section.querySelector(".summary-section__status");
+			const status = normalize(statusNode ? (statusNode.innerText || statusNode.textContent || "") : "");
+			let name = full;
+			if (status) {
+				name = normalize(full.replace(new RegExp("\\\\s*" + status + "\\\\s*\\\\d*\\\\s*parameters?$", "i"), ""));
+			}
+			if (!name || !status) continue;
+			summary.push({name, status});
+		}
+
+		const rows = [];
+		for (const row of Array.from(document.querySelectorAll(".bot-check-accordion__row"))) {
+			const statusNode = row.querySelector(".bot-check-accordion__status");
+			const labelNode = row.querySelector(".bot-check-accordion__label");
+			const status = normalize(statusNode ? (statusNode.innerText || statusNode.textContent || "") : "");
+			if (!status) continue;
+
+			let name = normalize(labelNode ? (labelNode.innerText || labelNode.textContent || "") : "");
+			if (!name) {
+				const full = normalize(row.innerText || row.textContent || "");
+				name = normalize(full.replace(new RegExp("\\\\s*" + status + "\\\\s*$", "i"), ""));
+			}
+			if (!name) continue;
+
+			rows.push({name, status});
+		}
+
+		const body = normalize(document.body && document.body.innerText ? document.body.innerText : "");
+		return {
+			state,
+			summary,
+			rows,
+			body: body.slice(0, 6000),
+		};
+	}`)
+	if err != nil {
+		return nil, "", err
+	}
+
+	var payload struct {
+		State   string `json:"state"`
+		Summary []struct {
+			Name   string `json:"name"`
+			Status string `json:"status"`
+		} `json:"summary"`
+		Rows []struct {
+			Name   string `json:"name"`
+			Status string `json:"status"`
+		} `json:"rows"`
+		Body string `json:"body"`
+	}
+	if err := res.Value.Unmarshal(&payload); err != nil {
+		return nil, "", fmt.Errorf("decode pixelscan payload: %w", err)
+	}
+	payload.Body = strings.TrimSpace(payload.Body)
+	if payload.Body == "" {
+		return nil, "", fmt.Errorf("pixelscan body is empty")
+	}
+
+	detections := make(map[string]fpcheck.Detection)
+	if payload.State != "" {
+		overallDetected := strings.EqualFold(strings.TrimSpace(payload.State), "bot")
+		overallStatus := strings.TrimSpace(payload.State)
+		if overallStatus == "" {
+			overallStatus = "unknown"
+		}
+		overallSeverity := ""
+		if overallDetected {
+			overallSeverity = "critical"
+		}
+		detections["overall_verdict"] = fpcheck.Detection{
+			Detected:    overallDetected,
+			Description: overallStatus,
+			Severity:    overallSeverity,
+		}
+	}
+
+	for _, item := range payload.Summary {
+		key := "summary_" + normalizeKey(item.Name)
+		if key == "summary_unknown" {
+			continue
+		}
+		detected := classifyStatus(item.Status)
+		severity := ""
+		if detected && hasKeyword(strings.ToLower(item.Name), []string{"webdriver", "cdp", "bot"}) {
+			severity = "critical"
+		}
+		detections[key] = fpcheck.Detection{
+			Detected:    detected,
+			Description: strings.TrimSpace(item.Status),
+			Severity:    severity,
+		}
+	}
+
+	for _, row := range payload.Rows {
+		key := normalizeKey(row.Name)
+		if key == "unknown" {
+			continue
+		}
+		detected := classifyStatus(row.Status)
+		severity := ""
+		if detected && hasKeyword(strings.ToLower(row.Name), []string{"webdriver", "cdp", "headless", "automation"}) {
+			severity = "critical"
+		}
+		detections[key] = fpcheck.Detection{
+			Detected:    detected,
+			Description: strings.TrimSpace(row.Status),
+			Severity:    severity,
+		}
+	}
+
+	if len(detections) == 0 {
+		if score, ok := extractScore(payload.Body); ok {
+			scoreValue := score
+			detections["overall_score"] = fpcheck.Detection{
+				Detected:    false,
+				Description: fmt.Sprintf("score %.2f", score),
+				Numeric:     &scoreValue,
+			}
+		}
+	}
+
+	if len(detections) == 0 {
+		return nil, payload.Body, fmt.Errorf("pixelscan detections not found")
+	}
+
+	return detections, payload.Body, nil
+}

core/fpcheck/detectors/rebrowser.go

@@ -0,0 +1,192 @@
+package detectors
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/go-rod/rod"
+	"github.com/karust/openserp/core/fpcheck"
+)
+
+const rebrowserURL = "https://bot-detector.rebrowser.net/"
+
+type Rebrowser struct{}
+
+func NewRebrowser() fpcheck.Detector {
+	return Rebrowser{}
+}
+
+func (Rebrowser) Name() string {
+	return "rebrowser"
+}
+
+func (Rebrowser) URL() string {
+	return rebrowserURL
+}
+
+type rebrowserCheck struct {
+	Type   string  `json:"type"`
+	Icon   string  `json:"icon"`
+	Rating float64 `json:"rating"`
+	Note   string  `json:"note"`
+	Debug  string  `json:"debug"`
+}
+
+func (Rebrowser) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
+	err := waitFor(ctx, 20*time.Second, 250*time.Millisecond, func() (bool, error) {
+		hasBody, _, err := page.Has("body")
+		if err != nil {
+			return false, err
+		}
+		if !hasBody {
+			return false, nil
+		}
+
+		res, err := page.Eval(`() => {
+			const output = document.querySelector('#detections-json');
+			if (!output || !output.value) return false;
+			try {
+				const parsed = JSON.parse(output.value);
+				return Array.isArray(parsed) && parsed.length > 0;
+			} catch (_) {
+				return false;
+			}
+		}`)
+		if err != nil {
+			return false, nil
+		}
+
+		var ready bool
+		if err := res.Value.Unmarshal(&ready); err != nil {
+			return false, nil
+		}
+		return ready, nil
+	})
+	if err != nil {
+		return nil, "", fmt.Errorf("rebrowser readiness: %w", err)
+	}
+
+	res, err := page.Eval(`() => {
+		const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
+		const stripHTML = (value) => {
+			const div = document.createElement('div');
+			div.innerHTML = value || '';
+			return normalize(div.textContent || div.innerText || '');
+		};
+		const checks = new Map();
+
+		const put = (item) => {
+			const type = normalize(item.type);
+			if (!type) return;
+
+			const current = checks.get(type) || { type, icon: "", rating: 0, note: "", debug: "" };
+			const next = {
+				type,
+				icon: normalize(item.icon || current.icon),
+				rating: Number.isFinite(item.rating) ? item.rating : current.rating,
+				note: normalize(item.note || current.note),
+				debug: normalize(item.debug || current.debug),
+			};
+			checks.set(type, next);
+		};
+
+		try {
+			const raw = document.querySelector('#detections-json')?.value || '[]';
+			const parsed = JSON.parse(raw);
+			if (Array.isArray(parsed)) {
+				for (const item of parsed) {
+					put({
+						type: item.type || '',
+						rating: Number(item.rating),
+						note: stripHTML(item.note || ''),
+						debug: typeof item.debug === 'string' ? normalize(item.debug) : normalize(JSON.stringify(item.debug || {})),
+					});
+				}
+			}
+		} catch (_) {}
+
+		for (const row of Array.from(document.querySelectorAll('#detections-table tbody tr'))) {
+			const cells = Array.from(row.querySelectorAll('td'));
+			if (cells.length < 1) continue;
+			const rawName = normalize(cells[0].innerText || cells[0].textContent || "");
+			if (!rawName) continue;
+			const chars = Array.from(rawName);
+			const icon = chars.length > 0 ? chars[0] : "";
+			const type = normalize(rawName.replace(icon, ""));
+			const note = cells.length > 2 ? normalize(cells[2].innerText || cells[2].textContent || "") : "";
+			put({
+				type,
+				icon,
+				note,
+			});
+		}
+
+		return Array.from(checks.values());
+	}`)
+	if err != nil {
+		return nil, "", fmt.Errorf("rebrowser extraction failed: %w", err)
+	}
+
+	var checks []rebrowserCheck
+	if err := res.Value.Unmarshal(&checks); err != nil {
+		return nil, "", fmt.Errorf("decode rebrowser checks: %w", err)
+	}
+	if len(checks) == 0 {
+		return nil, "", fmt.Errorf("rebrowser detector checks not found")
+	}
+
+	detections := rebrowserChecksToDetections(checks)
+	if len(detections) == 0 {
+		return nil, "", fmt.Errorf("rebrowser detections are empty")
+	}
+
+	raw, _ := json.MarshalIndent(checks, "", "  ")
+	return detections, string(raw), nil
+}
+
+func rebrowserChecksToDetections(checks []rebrowserCheck) map[string]fpcheck.Detection {
+	detections := make(map[string]fpcheck.Detection, len(checks))
+	for _, check := range checks {
+		key := normalizeKey(check.Type)
+		if key == "unknown" {
+			continue
+		}
+
+		detected := false
+		switch check.Icon {
+		case "🔴":
+			detected = true
+		case "🟢", "🟡", "⚪️", "⚪":
+			detected = false
+		default:
+			detected = check.Rating >= 1
+		}
+
+		description := strings.TrimSpace(check.Note)
+		if strings.TrimSpace(check.Debug) != "" {
+			if description != "" {
+				description = description + " | " + strings.TrimSpace(check.Debug)
+			} else {
+				description = strings.TrimSpace(check.Debug)
+			}
+		}
+		if description == "" {
+			description = fmt.Sprintf("rating=%.2f", check.Rating)
+		}
+
+		severity := ""
+		if detected {
+			severity = "critical"
+		}
+
+		detections[key] = fpcheck.Detection{
+			Detected:    detected,
+			Description: description,
+			Severity:    severity,
+		}
+	}
+	return detections
+}

core/fpcheck/detectors/rebrowser_test.go

@@ -0,0 +1,38 @@
+package detectors
+
+import "testing"
+
+func TestRebrowserChecksToDetections_UsesIconAndRating(t *testing.T) {
+	checks := []rebrowserCheck{
+		{
+			Type:   "navigatorWebdriver",
+			Icon:   "🔴",
+			Rating: -1,
+			Note:   "Own properties detected",
+		},
+		{
+			Type:   "runtimeEnableLeak",
+			Icon:   "",
+			Rating: 1,
+			Note:   "runtime leak",
+		},
+		{
+			Type:   "viewport",
+			Icon:   "🟢",
+			Rating: 1,
+			Note:   "looks fine",
+		},
+	}
+
+	detections := rebrowserChecksToDetections(checks)
+
+	if !detections["navigatorwebdriver"].Detected {
+		t.Fatal("expected red icon check to be detected")
+	}
+	if !detections["runtimeenableleak"].Detected {
+		t.Fatal("expected rating>=1 check to be detected")
+	}
+	if detections["viewport"].Detected {
+		t.Fatal("expected green icon check to be not detected")
+	}
+}

core/fpcheck/detectors/registry.go

@@ -0,0 +1,69 @@
+package detectors
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/karust/openserp/core/fpcheck"
+)
+
+var standardDetectorFactories = []struct {
+	name string
+	new  func() fpcheck.Detector
+}{
+	{name: "sannysoft", new: NewSannysoft},
+	{name: "rebrowser", new: NewRebrowser},
+	{name: "browserscan", new: NewBrowserScan},
+	{name: "pixelscan", new: NewPixelScan},
+	{name: "deviceandbrowser", new: NewDeviceAndBrowser},
+}
+
+func All() []fpcheck.Detector {
+	detectors := make([]fpcheck.Detector, 0, len(standardDetectorFactories))
+	for _, item := range standardDetectorFactories {
+		detectors = append(detectors, item.new())
+	}
+	return detectors
+}
+
+func Select(name string, customURL string) ([]fpcheck.Detector, error) {
+	return SelectWithCustomSelector(name, customURL, "")
+}
+
+func SelectWithCustomSelector(name string, customURL string, customSelector string) ([]fpcheck.Detector, error) {
+	trimmed := strings.ToLower(strings.TrimSpace(name))
+	if trimmed == "" || trimmed == "all" {
+		return All(), nil
+	}
+
+	if IsCustom(trimmed) {
+		customDetector, err := NewCustomWithSelector(customURL, customSelector)
+		if err != nil {
+			return nil, err
+		}
+		return []fpcheck.Detector{customDetector}, nil
+	}
+
+	for _, item := range standardDetectorFactories {
+		if strings.EqualFold(item.name, trimmed) {
+			return []fpcheck.Detector{item.new()}, nil
+		}
+	}
+
+	return nil, fmt.Errorf("unknown detector %q (allowed: %s)", name, strings.Join(Names(), ","))
+}
+
+func Names() []string {
+	names := make([]string, 0, len(standardDetectorFactories)+1)
+	for _, item := range standardDetectorFactories {
+		names = append(names, item.name)
+	}
+	names = append(names, customDetectorName)
+	sort.Strings(names)
+	return names
+}
+
+func IsCustom(name string) bool {
+	return strings.EqualFold(strings.TrimSpace(name), customDetectorName)
+}

core/fpcheck/detectors/registry_test.go

@@ -0,0 +1,62 @@
+package detectors
+
+import "testing"
+
+func TestSelectCustomRequiresURL(t *testing.T) {
+	_, err := Select("custom", "")
+	if err == nil {
+		t.Fatal("expected custom detector selection without URL to fail")
+	}
+}
+
+func TestSelectCustomAcceptsHTTPSURL(t *testing.T) {
+	detectorList, err := Select("custom", "https://localhost:9000")
+	if err != nil {
+		t.Fatalf("expected custom detector URL to be accepted, got %v", err)
+	}
+	if len(detectorList) != 1 {
+		t.Fatalf("expected one detector, got %d", len(detectorList))
+	}
+	if detectorList[0].Name() != "custom" {
+		t.Fatalf("expected custom detector, got %q", detectorList[0].Name())
+	}
+	if detectorList[0].URL() != "https://localhost:9000" {
+		t.Fatalf("expected normalized custom URL to be preserved, got %q", detectorList[0].URL())
+	}
+	custom, ok := detectorList[0].(Custom)
+	if !ok {
+		t.Fatalf("expected custom detector type, got %T", detectorList[0])
+	}
+	if custom.Selector() != "body" {
+		t.Fatalf("expected default custom selector body, got %q", custom.Selector())
+	}
+}
+
+func TestSelectCustomAcceptsSelector(t *testing.T) {
+	detectorList, err := SelectWithCustomSelector("custom", "https://localhost:9000", "pre")
+	if err != nil {
+		t.Fatalf("expected custom detector URL to be accepted, got %v", err)
+	}
+
+	custom, ok := detectorList[0].(Custom)
+	if !ok {
+		t.Fatalf("expected custom detector type, got %T", detectorList[0])
+	}
+	if custom.Selector() != "pre" {
+		t.Fatalf("expected custom selector pre, got %q", custom.Selector())
+	}
+}
+
+func TestNamesIncludesCustom(t *testing.T) {
+	names := Names()
+	found := false
+	for _, name := range names {
+		if name == "custom" {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatalf("expected names to include custom detector, got %#v", names)
+	}
+}

core/fpcheck/detectors/sannysoft.go

@@ -0,0 +1,162 @@
+package detectors
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/go-rod/rod"
+	"github.com/karust/openserp/core/fpcheck"
+)
+
+const sannysoftURL = "https://bot.sannysoft.com"
+
+type Sannysoft struct{}
+
+func NewSannysoft() fpcheck.Detector {
+	return Sannysoft{}
+}
+
+func (Sannysoft) Name() string {
+	return "sannysoft"
+}
+
+func (Sannysoft) URL() string {
+	return sannysoftURL
+}
+
+func (Sannysoft) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
+	var checks []sannysoftRow
+	err := waitFor(ctx, 20*time.Second, 250*time.Millisecond, func() (bool, error) {
+		hasRows, _, err := page.Has("table tr")
+		if err != nil {
+			return false, fmt.Errorf("table probe failed: %w", err)
+		}
+		if !hasRows {
+			return false, nil
+		}
+
+		rows, err := extractSannysoftRows(page)
+		if err != nil {
+			return false, nil
+		}
+		checks = rows
+		return len(checks) >= 5, nil
+	})
+	if err != nil {
+		return nil, "", err
+	}
+	if len(checks) == 0 {
+		return nil, "", fmt.Errorf("sannysoft did not return any fingerprint check rows")
+	}
+
+	detections := make(map[string]fpcheck.Detection, len(checks))
+	for _, check := range checks {
+		key := normalizeKey(check.Name)
+		if key == "unknown" {
+			continue
+		}
+
+		detected := check.Status == "fail"
+		severity := ""
+		if detected && strings.Contains(key, "webdriver") {
+			severity = "critical"
+		}
+
+		description := check.Status
+		if description == "" {
+			description = "unknown"
+		}
+
+		detections[key] = fpcheck.Detection{
+			Detected:    detected,
+			Description: description,
+			Severity:    severity,
+		}
+	}
+
+	return detections, "", nil
+}
+
+type sannysoftRow struct {
+	Name   string `json:"name"`
+	Status string `json:"status"`
+}
+
+func extractSannysoftRows(page *rod.Page) ([]sannysoftRow, error) {
+	res, err := page.Eval(`() => {
+		const parseRGB = (value) => {
+			const match = (value || "").match(/rgba?\((\d+),\s*(\d+),\s*(\d+)/i);
+			if (!match) return null;
+			return [parseInt(match[1], 10), parseInt(match[2], 10), parseInt(match[3], 10)];
+		};
+
+		const classify = (text, className, bgColor) => {
+			const normalizedText = (text || "").toLowerCase();
+			const normalizedClass = (className || "").toLowerCase();
+			const rgb = parseRGB(bgColor);
+
+			if (/\b(fail(?:ed)?|detected|bot)\b/.test(normalizedText)) return "fail";
+			if (/\b(pass(?:ed)?|ok|success)\b/.test(normalizedText)) return "pass";
+			if (/\b(fail(?:ed)?|error|danger|bad|red)\b/.test(normalizedClass)) return "fail";
+			if (/\b(pass(?:ed)?|success|ok|good|green)\b/.test(normalizedClass)) return "pass";
+
+			if (rgb) {
+				const [r, g, b] = rgb;
+				if (r > g + 35 && r > b + 35) return "fail";
+				if (g > r + 20 && g > b + 20) return "pass";
+			}
+
+			return "unknown";
+		};
+
+		const rows = Array.from(document.querySelectorAll("table tr"));
+		const seen = new Set();
+		const checks = [];
+
+		for (const row of rows) {
+			const cells = Array.from(row.querySelectorAll("th, td"));
+			if (cells.length < 2) continue;
+
+			const nameCell = cells[0];
+			const name = (nameCell.innerText || nameCell.textContent || "").replace(/\s+/g, " ").trim();
+			if (!name) continue;
+			if (/^(test(\s+name)?|property|status|result)$/i.test(name)) continue;
+
+			const resultCells = cells.slice(1);
+			const statusCell =
+				resultCells.find((cell) => /\b(result|pass|fail|success|ok)\b/i.test(cell.className || "")) ||
+				resultCells.find((cell) => {
+					const bg = window.getComputedStyle(cell).backgroundColor || "";
+					return bg !== "" && bg !== "transparent" && bg !== "rgba(0, 0, 0, 0)";
+				}) ||
+				resultCells[0];
+			if (!statusCell) continue;
+
+			const statusText = (statusCell.innerText || statusCell.textContent || "").replace(/\s+/g, " ").trim();
+			const className = (row.className || "") + " " + (statusCell.className || "");
+			const bgColor = window.getComputedStyle(statusCell).backgroundColor || "";
+			const dedupeKey = name.toLowerCase();
+			if (seen.has(dedupeKey)) continue;
+			seen.add(dedupeKey);
+
+			checks.push({
+				name,
+				status: classify(statusText, className, bgColor),
+			});
+		}
+
+		return checks;
+	}`)
+	if err != nil {
+		return nil, err
+	}
+
+	var checks []sannysoftRow
+	if err := res.Value.Unmarshal(&checks); err != nil {
+		return nil, fmt.Errorf("decode sannysoft results: %w", err)
+	}
+
+	return checks, nil
+}

core/fpcheck/runner.go

@@ -0,0 +1,174 @@
+package fpcheck
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/go-rod/rod"
+	"github.com/go-rod/rod/lib/proto"
+)
+
+// RunOptions controls detector run behavior.
+type RunOptions struct {
+	ArtifactDir       string
+	WaitBeforeExtract time.Duration
+	WaitBeforeClose   time.Duration
+}
+
+// Run navigates the given browser to detector URL, extracts verdicts,
+// captures a screenshot artifact, and returns a normalized report.
+func Run(ctx context.Context, browser BrowserNavigator, detector Detector, artifactDir string) (Report, error) {
+	return RunWithOptions(ctx, browser, detector, RunOptions{
+		ArtifactDir: artifactDir,
+	})
+}
+
+// RunWithOptions navigates the given browser to detector URL, extracts
+// verdicts, captures a screenshot artifact, optionally waits, and returns a
+// normalized report.
+func RunWithOptions(ctx context.Context, browser BrowserNavigator, detector Detector, options RunOptions) (Report, error) {
+	report := Report{
+		DetectorName: detector.Name(),
+		URL:          detector.URL(),
+		Detections:   map[string]Detection{},
+	}
+
+	artifactDir := strings.TrimSpace(options.ArtifactDir)
+	if artifactDir == "" {
+		artifactDir = "testdata"
+	}
+
+	screenshotPath := filepath.Join(artifactDir, fmt.Sprintf("fpcheck_%s.png", sanitizeFilePart(detector.Name())))
+	report.Screenshot = filepath.ToSlash(screenshotPath)
+
+	page, err := browser.Navigate(ctx, detector.URL())
+	if err != nil {
+		return report, fmt.Errorf("navigate %s: %w", detector.Name(), err)
+	}
+	defer func() {
+		if options.WaitBeforeClose > 0 {
+			_ = sleepWithContext(ctx, options.WaitBeforeClose)
+		}
+		closePageWithTimeout(context.Background(), browser, page, time.Second)
+	}()
+
+	if options.WaitBeforeExtract > 0 {
+		_ = sleepWithContext(ctx, options.WaitBeforeExtract)
+	}
+
+	detections, rawNotes, err := detector.Extract(ctx, page)
+	if err != nil {
+		_ = saveScreenshot(page, screenshotPath)
+		return report, fmt.Errorf("extract %s: %w", detector.Name(), err)
+	}
+
+	if err := saveScreenshot(page, screenshotPath); err != nil {
+		return report, fmt.Errorf("capture screenshot %s: %w", detector.Name(), err)
+	}
+
+	report.CapturedAtUTC = time.Now().UTC().Format(time.RFC3339)
+	report.Detections = detections
+	report.Summary = summarize(detections)
+	report.RawNotes = strings.TrimSpace(rawNotes)
+
+	return report, nil
+}
+
+func sleepWithContext(ctx context.Context, d time.Duration) error {
+	if d <= 0 {
+		return nil
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	timer := time.NewTimer(d)
+	defer timer.Stop()
+
+	select {
+	case <-timer.C:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func summarize(detections map[string]Detection) Summary {
+	summary := Summary{}
+	for key, detection := range detections {
+		if detection.Detected {
+			summary.Failed++
+			if strings.EqualFold(strings.TrimSpace(detection.Severity), "critical") {
+				summary.Critical = append(summary.Critical, key)
+			}
+			continue
+		}
+		summary.Passed++
+	}
+	sort.Strings(summary.Critical)
+	return summary
+}
+
+func saveScreenshot(page *rod.Page, path string) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("create screenshot directory: %w", err)
+	}
+
+	bytes, err := page.Screenshot(true, nil)
+	if err != nil {
+		return fmt.Errorf("capture screenshot: %w", err)
+	}
+	if err := os.WriteFile(path, bytes, 0o644); err != nil {
+		return fmt.Errorf("write screenshot file %s: %w", path, err)
+	}
+	return nil
+}
+
+type pageCloser interface {
+	ClosePage(context.Context, *rod.Page, time.Duration) error
+}
+
+func closePageWithTimeout(ctx context.Context, browser BrowserNavigator, page *rod.Page, timeout time.Duration) {
+	if page == nil {
+		return
+	}
+	if closer, ok := browser.(pageCloser); ok {
+		_ = closer.ClosePage(ctx, page, timeout)
+		return
+	}
+	if timeout <= 0 {
+		timeout = time.Second
+	}
+	closeCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	pageWithTimeout := page.Context(closeCtx)
+	info, _ := pageWithTimeout.Info()
+
+	_ = pageWithTimeout.Close()
+
+	if info != nil && info.BrowserContextID != "" {
+		_ = (proto.TargetDisposeBrowserContext{BrowserContextID: info.BrowserContextID}).Call(page.Browser().Context(closeCtx))
+	}
+}
+
+func sanitizeFilePart(value string) string {
+	value = strings.TrimSpace(strings.ToLower(value))
+	if value == "" {
+		return "detector"
+	}
+
+	parts := strings.FieldsFunc(value, func(r rune) bool {
+		return !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9')
+	})
+	if len(parts) == 0 {
+		return "detector"
+	}
+
+	return strings.Join(parts, "_")
+}

core/html_parser.go

@@ -0,0 +1,10 @@
+package core
+
+import "io"
+
+// HTMLParser is implemented by engines that can parse a SERP HTML document
+// without a live browser. Used to expose POST /parse/{engine} endpoints.
+type HTMLParser interface {
+	Name() string
+	ParseHTML(io.Reader) ([]SearchResult, error)
+}

core/http_client.go

@@ -1,74 +1,538 @@
 package core
 
 import (
+	"bytes"
 	"context"
-	"crypto/tls"
-	"net"
+	"fmt"
+	"hash/fnv"
+	"io"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 
-	utls "github.com/refraction-networking/utls"
+	fhttp "github.com/bogdanfinn/fhttp"
+	tlsclient "github.com/bogdanfinn/tls-client"
+	"github.com/bogdanfinn/tls-client/profiles"
+	browserprofile "github.com/karust/openserp/core/browser"
 )
 
-const rawHTTPTimeout = 10 * time.Second
+const rawHTTPTimeout = 30 * time.Second
+const rawHTTPClientCacheMaxEntries = 64
 
+// fallbackRawUserAgent guards against tls-client's "Go-http-client" UA leaking.
+const fallbackRawUserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"
+
+// rawChromeProfiles pairs each TLS fingerprint with its Chrome major so the UA
+// and Sec-CH-UA stay coherent. We round-robin the major here. Add presets as
+// tls-client ships them.
+var rawChromeProfiles = []struct {
+	major int
+	tls   profiles.ClientProfile
+}{
+	{133, profiles.Chrome_133},
+	{144, profiles.Chrome_144},
+	{146, profiles.Chrome_146},
+}
+
+// pickRawChromeProfile hashes salt to a stable but varied fingerprint.
+func pickRawChromeProfile(salt string) (int, profiles.ClientProfile) {
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(salt))
+	p := rawChromeProfiles[int(h.Sum32())%len(rawChromeProfiles)]
+	return p.major, p.tls
+}
+
+// rawHeaderOrder controls request header order; tls-client profiles do not.
+var rawHeaderOrder = []string{
+	"host",
+	"user-agent",
+	"accept",
+	"accept-language",
+	"accept-encoding",
+	"upgrade-insecure-requests",
+	"sec-ch-ua",
+	"sec-ch-ua-mobile",
+	"sec-ch-ua-platform",
+	"sec-fetch-site",
+	"sec-fetch-mode",
+	"sec-fetch-user",
+	"sec-fetch-dest",
+}
+
+var rawHTTPClientCache = struct {
+	sync.Mutex
+	clients map[rawHTTPClientKey]*rawHTTPClientEntry
+}{
+	clients: map[rawHTTPClientKey]*rawHTTPClientEntry{},
+}
+
+type rawHTTPClientKey struct {
+	proxyURL             string
+	profile              string
+	insecure             bool
+	guardPrivateNetworks bool
+}
+
+type rawHTTPClientEntry struct {
+	client   tlsclient.HttpClient
+	lastUsed time.Time
+}
+
+type rawRequestProfile struct {
+	id             string
+	userAgent      string
+	acceptLanguage string
+	secCHUA        string
+	platform       string
+	mobile         bool
+	tlsProfile     profiles.ClientProfile
+}
+
+// DrainAndCloseResponse drains then closes the body so the connection can be reused.
+func DrainAndCloseResponse(resp *http.Response) {
+	if resp == nil || resp.Body == nil {
+		return
+	}
+
+	_, _ = io.Copy(io.Discard, resp.Body)
+	_ = resp.Body.Close()
+}
+
+// RawSearchRequest executes a raw-mode GET and returns a stdlib response.
+func RawSearchRequest(ctx context.Context, searchURL string, query Query) (*http.Response, error) {
+	profile := rawRequestProfileFor(ctx, query)
+	client, err := cachedRawHTTPClient(query, profile.cacheKey(), profile.tlsProfile)
+	if err != nil {
+		return nil, err
+	}
+	SetBrowserProfileID(ctx, profile.id)
+
+	// Guarded path validates every hop, including the first.
+	if query.GuardPrivateNetworks {
+		return doGuardedRawRequest(ctx, client, searchURL, profile, query)
+	}
+	return doRawRequest(ctx, client, searchURL, profile, query)
+}
+
+// doRawRequest issues one GET and converts the response at the boundary.
+func doRawRequest(ctx context.Context, client tlsclient.HttpClient, searchURL string, profile rawRequestProfile, query Query) (*http.Response, error) {
+	req, err := fhttp.NewRequestWithContext(ctx, fhttp.MethodGet, searchURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	applyRawRequestHeaders(req, profile)
+	return execRawRequest(ctx, client, req, rawRequestUsesProxy(query))
+}
+
+// execRawRequest runs the request and converts proxy errors and the response.
+func execRawRequest(ctx context.Context, client tlsclient.HttpClient, req *fhttp.Request, proxied bool) (*http.Response, error) {
+	resp, err := client.Do(req)
+	if err != nil {
+		if proxied {
+			return nil, classifyProxyNetworkError(err)
+		}
+		return nil, err
+	}
+	return convertRawResponse(ctx, resp), nil
+}
+
+const maxGuardedRedirects = 10
+
+// doGuardedRawRequest validates every redirect hop before fetching it.
+func doGuardedRawRequest(ctx context.Context, client tlsclient.HttpClient, searchURL string, profile rawRequestProfile, query Query) (*http.Response, error) {
+	current := searchURL
+	for hop := 0; ; hop++ {
+		if err := ValidatePublicHTTPURL(ctx, current); err != nil {
+			return nil, err
+		}
+		resp, err := doRawRequest(ctx, client, current, profile, query)
+		if err != nil {
+			return nil, err
+		}
+		location, ok := redirectLocation(resp)
+		if !ok {
+			return resp, nil
+		}
+		if hop >= maxGuardedRedirects {
+			DrainAndCloseResponse(resp)
+			return nil, fmt.Errorf("%w: stopped after %d redirects", ErrEngineInternal, maxGuardedRedirects)
+		}
+		next, err := resolveRedirectURL(current, location)
+		if err != nil {
+			DrainAndCloseResponse(resp)
+			return nil, err
+		}
+		DrainAndCloseResponse(resp)
+		current = next
+	}
+}
+
+func redirectLocation(resp *http.Response) (string, bool) {
+	if resp == nil {
+		return "", false
+	}
+	switch resp.StatusCode {
+	case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,
+		http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
+		location := strings.TrimSpace(resp.Header.Get("Location"))
+		return location, location != ""
+	default:
+		return "", false
+	}
+}
+
+func resolveRedirectURL(base, location string) (string, error) {
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return "", err
+	}
+	locURL, err := url.Parse(location)
+	if err != nil {
+		return "", err
+	}
+	return baseURL.ResolveReference(locURL).String(), nil
+}
+
+func ReadRawSearchBody(resp *http.Response) ([]byte, error) {
+	if resp == nil {
+		return nil, fmt.Errorf("%w: nil raw search response", ErrEngineInternal)
+	}
+	if err := ClassifySearchHTTPStatus(resp.StatusCode); err != nil {
+		return nil, err
+	}
+	return io.ReadAll(resp.Body)
+}
+
+func ClassifySearchHTTPStatus(status int) error {
+	switch status {
+	case 0:
+		return nil
+	case http.StatusForbidden, http.StatusUnauthorized:
+		return ErrBlocked
+	case http.StatusTooManyRequests:
+		return ErrRateLimited
+	}
+	if status >= 500 {
+		return fmt.Errorf("%w: search engine returned HTTP %d", ErrBlocked, status)
+	}
+	if status < 200 || status >= 300 {
+		return fmt.Errorf("%w: search engine returned HTTP %d", ErrParser, status)
+	}
+	return nil
+}
+
+// NewRawHTTPClient returns a stdlib client backed by tls-client.
 func NewRawHTTPClient(query Query) (*http.Client, error) {
-	transport, err := newRawTransport(query)
+	profile := rawRequestProfileFor(context.Background(), query)
+	client, err := cachedRawHTTPClient(query, profile.cacheKey(), profile.tlsProfile)
 	if err != nil {
 		return nil, err
 	}
 
-	return &http.Client{
-		Transport: transport,
-		Timeout:   rawHTTPTimeout,
-	}, nil
+	stdClient := &http.Client{
+		Transport: rawTLSRoundTripper{
+			client:               client,
+			proxied:              rawRequestUsesProxy(query),
+			guardPrivateNetworks: query.GuardPrivateNetworks,
+			profile:              profile,
+		},
+		Timeout: rawHTTPTimeout,
+	}
+	if query.GuardPrivateNetworks {
+		stdClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+			return ValidatePublicHTTPURL(req.Context(), req.URL.String())
+		}
+	}
+	return stdClient, nil
 }
 
-func newRawTransport(query Query) (*http.Transport, error) {
-	transport := &http.Transport{}
-	if query.Insecure {
-		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-	}
-
+func cachedRawHTTPClient(query Query, profileKey string, tlsProfile profiles.ClientProfile) (tlsclient.HttpClient, error) {
 	proxyURL, err := NormalizeProxyURL(query.ProxyURL)
 	if err != nil {
 		return nil, err
 	}
-	if proxyURL != "" {
-		parsed, err := url.Parse(proxyURL)
-		if err != nil {
-			return nil, err
-		}
 
-		// Keep proxied requests on the standard transport path so SOCKS5/SOCKS5H
-		// resolution and routing are handled by the configured proxy correctly.
-		transport.Proxy = http.ProxyURL(parsed)
-		return transport, nil
+	key := rawHTTPClientKey{
+		proxyURL:             proxyURL,
+		profile:              profileKey,
+		insecure:             query.Insecure,
+		guardPrivateNetworks: query.GuardPrivateNetworks,
 	}
 
-	transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
-		dialer := &net.Dialer{}
-		rawConn, err := dialer.DialContext(ctx, network, addr)
-		if err != nil {
-			return nil, err
-		}
+	rawHTTPClientCache.Lock()
+	defer rawHTTPClientCache.Unlock()
 
-		hostname := strings.Split(addr, ":")[0]
-		config := &utls.Config{
-			ServerName:         hostname,
-			InsecureSkipVerify: query.Insecure,
-		}
-
-		uconn := utls.UClient(rawConn, config, utls.HelloChrome_Auto)
-		if err := uconn.Handshake(); err != nil {
-			rawConn.Close()
-			return nil, err
-		}
-
-		return uconn, nil
+	now := time.Now()
+	if entry := rawHTTPClientCache.clients[key]; entry != nil {
+		entry.lastUsed = now
+		return entry.client, nil
 	}
 
-	return transport, nil
+	client, err := newRawTLSClient(query, proxyURL, tlsProfile)
+	if err != nil {
+		return nil, err
+	}
+	rawHTTPClientCache.clients[key] = &rawHTTPClientEntry{
+		client:   client,
+		lastUsed: now,
+	}
+	evictRawHTTPClientCacheLocked()
+	return client, nil
+}
+
+func rawRequestUsesProxy(query Query) bool {
+	return strings.TrimSpace(query.ProxyURL) != ""
+}
+
+func evictRawHTTPClientCacheLocked() {
+	for len(rawHTTPClientCache.clients) > rawHTTPClientCacheMaxEntries {
+		var (
+			oldestKey   rawHTTPClientKey
+			oldestEntry *rawHTTPClientEntry
+		)
+		for key, entry := range rawHTTPClientCache.clients {
+			if oldestEntry == nil || entry.lastUsed.Before(oldestEntry.lastUsed) {
+				oldestKey = key
+				oldestEntry = entry
+			}
+		}
+		if oldestEntry == nil {
+			return
+		}
+		delete(rawHTTPClientCache.clients, oldestKey)
+		oldestEntry.client.CloseIdleConnections()
+	}
+}
+
+// newRawTLSClient builds a pooled Chrome-profile transport; proxyURL must be normalized.
+func newRawTLSClient(query Query, proxyURL string, tlsProfile profiles.ClientProfile) (tlsclient.HttpClient, error) {
+	options := []tlsclient.HttpClientOption{
+		tlsclient.WithClientProfile(tlsProfile),
+		tlsclient.WithTimeout(int(rawHTTPTimeout / time.Second)),
+		tlsclient.WithNotFollowRedirects(),
+	}
+	if query.Insecure {
+		options = append(options, tlsclient.WithInsecureSkipVerify())
+	}
+
+	if proxyURL != "" {
+		options = append(options, tlsclient.WithProxyUrl(proxyURL))
+	} else if query.GuardPrivateNetworks {
+		options = append(options, tlsclient.WithDialContext(GuardedDialContext))
+	}
+
+	return tlsclient.NewHttpClient(tlsclient.NewNoopLogger(), options...)
+}
+
+// convertRawResponse keeps fhttp from leaking past this file.
+func convertRawResponse(ctx context.Context, resp *fhttp.Response) *http.Response {
+	if resp == nil {
+		return nil
+	}
+	std := &http.Response{
+		Status:        resp.Status,
+		StatusCode:    resp.StatusCode,
+		Proto:         resp.Proto,
+		ProtoMajor:    resp.ProtoMajor,
+		ProtoMinor:    resp.ProtoMinor,
+		Header:        http.Header(resp.Header),
+		ContentLength: resp.ContentLength,
+		Body:          resp.Body,
+	}
+	if std.Body == nil {
+		std.Body = io.NopCloser(bytes.NewReader(nil))
+	}
+	std.Body = networkUsageReadCloser{ReadCloser: std.Body, ctx: ctx}
+	return std
+}
+
+type rawTLSRoundTripper struct {
+	client               tlsclient.HttpClient
+	proxied              bool
+	guardPrivateNetworks bool
+	profile              rawRequestProfile
+}
+
+func (rt rawTLSRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	if rt.guardPrivateNetworks {
+		if err := ValidatePublicHTTPURL(req.Context(), req.URL.String()); err != nil {
+			return nil, err
+		}
+	}
+
+	freq, err := fhttp.NewRequestWithContext(req.Context(), req.Method, req.URL.String(), req.Body)
+	if err != nil {
+		return nil, err
+	}
+	for key, values := range req.Header {
+		freq.Header[key] = values
+	}
+	applyRawRequestHeaders(freq, rt.profile)
+	SetBrowserProfileID(req.Context(), rt.profile.id)
+	return execRawRequest(req.Context(), rt.client, freq, rt.proxied)
+}
+
+func rawRequestProfileFor(ctx context.Context, query Query) rawRequestProfile {
+	engine := engineFromContext(ctx)
+	region := rawProfileRegion(ctx, query)
+	salt := rawProfileSalt(ctx, engine, region)
+
+	profile := browserprofile.Profile{}
+	if forcedID := forcedProfileIDFromContext(ctx); forcedID != "" {
+		if forced, ok := browserprofile.ProfileByID(forcedID); ok {
+			profile = forced
+		}
+	}
+	if strings.TrimSpace(profile.ID) == "" {
+		profile = browserprofile.SelectProfileForSession(engine, region, salt)
+	}
+	profile = applyProfileLanguageHint(profile, region)
+
+	major, tlsProfile := pickRawChromeProfile(salt + "\x00" + strings.TrimSpace(profile.ID))
+	profile = applyRawChromeMajor(profile, major)
+
+	userAgent := strings.TrimSpace(profile.UserAgent)
+	if userAgent == "" {
+		userAgent = fallbackRawUserAgent
+	}
+	acceptLanguage := strings.TrimSpace(profile.AcceptLanguage)
+	if acceptLanguage == "" {
+		acceptLanguage = BuildAcceptLanguageHeader(region)
+	}
+	if acceptLanguage == "" {
+		acceptLanguage = BuildAcceptLanguageHeader(query.LangCode)
+	}
+
+	return rawRequestProfile{
+		id:             strings.TrimSpace(profile.ID),
+		userAgent:      userAgent,
+		acceptLanguage: acceptLanguage,
+		secCHUA:        formatSecCHUA(profile.UACHBrands),
+		platform:       strings.TrimSpace(profile.Platform),
+		mobile:         profile.Mobile,
+		tlsProfile:     tlsProfile,
+	}
+}
+
+func applyRawChromeMajor(profile browserprofile.Profile, major int) browserprofile.Profile {
+	version := strconv.Itoa(major)
+	if template := strings.TrimSpace(profile.UserAgentTemplate); template != "" {
+		profile.UserAgent = strings.ReplaceAll(template, "{chrome_major}", version)
+	} else if extractChromeVersion(profile.UserAgent) == "" {
+		profile.UserAgent = fallbackRawUserAgent
+	} else {
+		profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, version+".0.0.0")
+	}
+	if len(profile.UACHBrands) == 0 {
+		profile.UACHBrands = rawUACHBrands(version, false)
+	} else {
+		profile.UACHBrands = patchBrandVersions(profile.UACHBrands, version, false)
+	}
+	if len(profile.UACHFullVerList) == 0 {
+		profile.UACHFullVerList = rawUACHBrands(version+".0.0.0", true)
+	} else {
+		profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, version+".0.0.0", true)
+	}
+	return profile
+}
+
+func rawUACHBrands(version string, full bool) []browserprofile.BrandVersion {
+	notABrandVersion := "24"
+	if full {
+		notABrandVersion = "24.0.0.0"
+	}
+	return []browserprofile.BrandVersion{
+		{Brand: "Not_A Brand", Version: notABrandVersion},
+		{Brand: "Chromium", Version: version},
+		{Brand: "Google Chrome", Version: version},
+	}
+}
+
+func rawProfileRegion(ctx context.Context, query Query) string {
+	if region := profileRegionFromContext(ctx); region != "" {
+		return region
+	}
+	if query.ProxyCountry != "" {
+		return query.ProxyCountry
+	}
+	return profileRegionHint(query)
+}
+
+func rawProfileSalt(ctx context.Context, engine, region string) string {
+	if laneKey := proxyLaneKeyFromContext(ctx); !laneKey.Empty() {
+		return laneKey.SessionID
+	}
+	return browserprofile.LaneKey(engine, region)
+}
+
+// cacheKey includes all headers that affect the pooled fingerprint.
+func (p rawRequestProfile) cacheKey() string {
+	return strings.Join([]string{p.id, p.userAgent, p.acceptLanguage, p.secCHUA, p.platform, fmt.Sprint(p.mobile)}, "\x00")
+}
+
+// applyRawRequestHeaders sets the Chrome identity headers and order; tls-client
+// owns Host and Accept-Encoding.
+func applyRawRequestHeaders(req *fhttp.Request, profile rawRequestProfile) {
+	if req == nil {
+		return
+	}
+	secCHUAMobile := "?0"
+	if profile.mobile {
+		secCHUAMobile = "?1"
+	}
+	platform := ""
+	if profile.platform != "" {
+		platform = quoteSecCHValue(profile.platform)
+	}
+	for _, h := range [][2]string{
+		{"User-Agent", profile.userAgent},
+		{"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"},
+		{"Accept-Language", profile.acceptLanguage},
+		{"Upgrade-Insecure-Requests", "1"},
+		{"Sec-CH-UA", profile.secCHUA},
+		{"Sec-CH-UA-Mobile", secCHUAMobile},
+		{"Sec-CH-UA-Platform", platform},
+		{"Sec-Fetch-Site", "none"},
+		{"Sec-Fetch-Mode", "navigate"},
+		{"Sec-Fetch-User", "?1"},
+		{"Sec-Fetch-Dest", "document"},
+	} {
+		if h[1] != "" {
+			req.Header.Set(h[0], h[1])
+		}
+	}
+	req.Header[fhttp.HeaderOrderKey] = rawHeaderOrder
+}
+
+func formatSecCHUA(brands []browserprofile.BrandVersion) string {
+	parts := make([]string, 0, len(brands))
+	for _, brand := range brands {
+		name := strings.TrimSpace(brand.Brand)
+		version := strings.TrimSpace(brand.Version)
+		if name == "" || version == "" {
+			continue
+		}
+		parts = append(parts, quoteSecCHValue(name)+`;v=`+quoteSecCHValue(version))
+	}
+	return strings.Join(parts, ", ")
+}
+
+func quoteSecCHValue(value string) string {
+	value = strings.ReplaceAll(value, `\`, `\\`)
+	value = strings.ReplaceAll(value, `"`, `\"`)
+	return `"` + value + `"`
+}
+
+type networkUsageReadCloser struct {
+	io.ReadCloser
+	ctx context.Context
+}
+
+func (r networkUsageReadCloser) Read(p []byte) (int, error) {
+	n, err := r.ReadCloser.Read(p)
+	AddNetworkBytes(r.ctx, int64(n))
+	return n, err
 }

core/http_client_test.go

@@ -0,0 +1,306 @@
+package core
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+)
+
+type trackingBody struct {
+	io.Reader
+	closed bool
+}
+
+func (b *trackingBody) Close() error {
+	b.closed = true
+	return nil
+}
+
+func TestDrainAndCloseResponseDrainsAndCloses(t *testing.T) {
+	body := &trackingBody{Reader: strings.NewReader("unread payload")}
+	resp := &http.Response{Body: body}
+
+	DrainAndCloseResponse(resp)
+
+	if !body.closed {
+		t.Fatal("expected body to be closed")
+	}
+
+	n, err := body.Read(make([]byte, 1))
+	if err != io.EOF || n != 0 {
+		t.Fatalf("expected body drained to EOF, got n=%d err=%v", n, err)
+	}
+}
+
+func TestClassifySearchHTTPStatus(t *testing.T) {
+	tests := []struct {
+		name   string
+		status int
+		want   error
+	}{
+		{name: "unknown browser status", status: 0, want: nil},
+		{name: "ok", status: http.StatusOK, want: nil},
+		{name: "blocked", status: http.StatusForbidden, want: ErrBlocked},
+		{name: "rate limited", status: http.StatusTooManyRequests, want: ErrRateLimited},
+		{name: "server error", status: http.StatusBadGateway, want: ErrBlocked},
+		{name: "unexpected status", status: http.StatusNotFound, want: ErrParser},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ClassifySearchHTTPStatus(tt.status)
+			if tt.want == nil {
+				if err != nil {
+					t.Fatalf("expected nil error, got %v", err)
+				}
+				return
+			}
+			if !errors.Is(err, tt.want) {
+				t.Fatalf("expected %v, got %v", tt.want, err)
+			}
+		})
+	}
+}
+
+func TestDrainAndCloseResponseNilSafe(t *testing.T) {
+	DrainAndCloseResponse(nil)
+	DrainAndCloseResponse(&http.Response{})
+}
+
+func TestRawHTTPClientTracksNetworkBytes(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		_, _ = w.Write([]byte("payload"))
+	}))
+	defer server.Close()
+
+	client, err := NewRawHTTPClient(Query{})
+	if err != nil {
+		t.Fatalf("new raw client: %v", err)
+	}
+
+	ctx := WithNetworkUsage(context.Background())
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("do request: %v", err)
+	}
+	defer DrainAndCloseResponse(resp)
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("read body: %v", err)
+	}
+	if string(body) != "payload" {
+		t.Fatalf("unexpected body: %q", string(body))
+	}
+	if got := NetworkBytesFromContext(ctx); got < int64(len(body)) {
+		t.Fatalf("expected tracked bytes >= body length, got %d", got)
+	}
+}
+
+func TestRawHTTPClientAppliesBrowserHeaderDefaults(t *testing.T) {
+	resetRawHTTPClientCache(t)
+
+	var userAgent, acceptLanguage, secCHUA string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		userAgent = r.Header.Get("User-Agent")
+		acceptLanguage = r.Header.Get("Accept-Language")
+		secCHUA = r.Header.Get("Sec-CH-UA")
+		_, _ = w.Write([]byte("ok"))
+	}))
+	defer server.Close()
+
+	client, err := NewRawHTTPClient(Query{LangCode: "fr", Region: "FR"})
+	if err != nil {
+		t.Fatalf("new raw client: %v", err)
+	}
+
+	ctx := WithBrowserProfileUsage(context.Background())
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("do request: %v", err)
+	}
+	DrainAndCloseResponse(resp)
+
+	if userAgent == "" || strings.Contains(userAgent, "Go-http-client") {
+		t.Fatalf("unexpected User-Agent %q", userAgent)
+	}
+	if want := BuildAcceptLanguageHeader("fr-FR"); acceptLanguage != want {
+		t.Fatalf("Accept-Language = %q, want %q", acceptLanguage, want)
+	}
+	if secCHUA == "" {
+		t.Fatal("expected Sec-CH-UA to be set")
+	}
+	if ids := BrowserProfileIDsFromContext(ctx); len(ids) != 1 || ids[0] == "" {
+		t.Fatalf("expected one recorded profile id, got %v", ids)
+	}
+}
+
+func TestRawHTTPClientGuardRejectsInitialPrivateURLWithProxy(t *testing.T) {
+	client, err := NewRawHTTPClient(Query{
+		ProxyURL:             "http://127.0.0.1:1",
+		GuardPrivateNetworks: true,
+	})
+	if err != nil {
+		t.Fatalf("new raw client: %v", err)
+	}
+
+	resp, err := client.Get("http://127.0.0.1/")
+	DrainAndCloseResponse(resp)
+	if !errors.Is(err, ErrTargetNotAllowed) {
+		t.Fatalf("expected target guard error, got %v", err)
+	}
+}
+
+func TestRawHTTPClientProxyAuthError(t *testing.T) {
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusProxyAuthRequired)
+		_, _ = w.Write([]byte("proxy auth required"))
+	}))
+	defer proxy.Close()
+
+	client, err := NewRawHTTPClient(Query{ProxyURL: proxy.URL})
+	if err != nil {
+		t.Fatalf("new raw client: %v", err)
+	}
+
+	ctx := WithNetworkUsage(context.Background())
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://example.com/", nil)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+	resp, err := client.Do(req)
+	DrainAndCloseResponse(resp)
+	if !errors.Is(err, ErrProxyAuth) {
+		t.Fatalf("expected proxy auth error, got %v", err)
+	}
+}
+
+// TestRawSearchRequestReusesPooledClient checks that same-profile calls share
+// one connection and a stable Chrome UA. Bytes/headers are covered elsewhere.
+func TestRawSearchRequestReusesPooledClient(t *testing.T) {
+	resetRawHTTPClientCache(t)
+
+	var connCount atomic.Int32
+	var mu sync.Mutex
+	var userAgents []string
+
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		userAgents = append(userAgents, r.Header.Get("User-Agent"))
+		mu.Unlock()
+		_, _ = w.Write([]byte("ok"))
+	}))
+	server.Config.ConnState = func(_ net.Conn, state http.ConnState) {
+		if state == http.StateNew {
+			connCount.Add(1)
+		}
+	}
+	server.Start()
+	defer server.Close()
+
+	query := Query{LangCode: "de", Region: "DE"}
+	for i := 0; i < 2; i++ {
+		ctx := WithEngine(WithBrowserProfileUsage(context.Background()), "google")
+		readRawSearchBodyForTest(t, ctx, server.URL, query)
+		if ids := BrowserProfileIDsFromContext(ctx); len(ids) != 1 || ids[0] == "" {
+			t.Fatalf("request %d recorded profile ids %v, want exactly one", i, ids)
+		}
+	}
+
+	if got := connCount.Load(); got != 1 {
+		t.Fatalf("expected one reused TCP connection, got %d", got)
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	if len(userAgents) != 2 {
+		t.Fatalf("expected two captured User-Agents, got %d", len(userAgents))
+	}
+	if userAgents[0] == "" || strings.Contains(userAgents[0], "Go-http-client") || userAgents[0] != userAgents[1] {
+		t.Fatalf("expected stable Chrome User-Agent, got %q then %q", userAgents[0], userAgents[1])
+	}
+}
+
+// TestRawRequestProfilesRoundRobinCoherently checks rotation hits every Chrome
+// major and keeps UA major == Sec-CH-UA major == TLS fingerprint.
+func TestRawRequestProfilesRoundRobinCoherently(t *testing.T) {
+	tlsByMajor := map[int]string{}
+	for _, p := range rawChromeProfiles {
+		tlsByMajor[p.major] = p.tls.GetClientHelloStr()
+	}
+
+	seen := map[int]bool{}
+	for i := 0; i < 200; i++ {
+		ctx := WithProxyLaneKey(WithEngine(context.Background(), "google"),
+			ProxyLaneKey{Engine: "google", SessionID: "sid-" + strconv.Itoa(i)})
+
+		profile := rawRequestProfileFor(ctx, Query{Region: "US"})
+		major, err := strconv.Atoi(chromeMajorVersion(extractChromeVersion(profile.userAgent)))
+		if err != nil {
+			t.Fatalf("raw User-Agent has no Chrome major: %q", profile.userAgent)
+		}
+		wantTLS, ok := tlsByMajor[major]
+		if !ok {
+			t.Fatalf("UA major %d has no configured tls-client profile", major)
+		}
+		if profile.tlsProfile.GetClientHelloStr() != wantTLS {
+			t.Fatalf("major %d: TLS fingerprint does not match UA", major)
+		}
+		if !strings.Contains(profile.secCHUA, `;v="`+strconv.Itoa(major)+`"`) {
+			t.Fatalf("Sec-CH-UA %q does not match UA major %d", profile.secCHUA, major)
+		}
+		seen[major] = true
+	}
+
+	for _, p := range rawChromeProfiles {
+		if !seen[p.major] {
+			t.Fatalf("configured Chrome major %d never selected; seen=%v", p.major, seen)
+		}
+	}
+}
+
+func readRawSearchBodyForTest(t *testing.T, ctx context.Context, searchURL string, query Query) string {
+	t.Helper()
+
+	resp, err := RawSearchRequest(ctx, searchURL, query)
+	if err != nil {
+		t.Fatalf("raw search request: %v", err)
+	}
+	defer DrainAndCloseResponse(resp)
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("read raw search body: %v", err)
+	}
+	return string(body)
+}
+
+func resetRawHTTPClientCache(t *testing.T) {
+	t.Helper()
+
+	rawHTTPClientCache.Lock()
+	entries := rawHTTPClientCache.clients
+	rawHTTPClientCache.clients = map[rawHTTPClientKey]*rawHTTPClientEntry{}
+	rawHTTPClientCache.Unlock()
+
+	for _, entry := range entries {
+		entry.client.CloseIdleConnections()
+	}
+}

core/locale.go

@@ -0,0 +1,100 @@
+package core
+
+import "github.com/karust/openserp/core/region"
+
+// Locale is a parsed language/region pair derived from a BCP47-style code.
+// The canonical definition lives in the dependency-free core/region subpackage;
+// this alias preserves the historical core.Locale name for existing callers.
+type Locale = region.Locale
+
+var defaultLocaleCountryByLanguage = map[string]string{
+	"en": "US",
+	"de": "DE",
+	"ru": "RU",
+	"fr": "FR",
+	"es": "ES",
+	"it": "IT",
+	"pt": "BR",
+	"zh": "CN",
+	"ja": "JP",
+	"ko": "KR",
+	"nl": "NL",
+	"pl": "PL",
+	"tr": "TR",
+	"ar": "SA",
+}
+
+// ParseLocale parses a language code such as "en", "EN-us", or "de_AT" into a
+// Locale. See region.ParseLocale for details.
+func ParseLocale(code string) Locale {
+	return region.ParseLocale(code)
+}
+
+// CountryFromRegion extracts a two-letter country/market code from a region
+// hint. See region.CountryFromRegion for details.
+func CountryFromRegion(hint string) string {
+	return region.CountryFromRegion(hint)
+}
+
+// PrimaryLanguageTag returns the BCP47 primary tag for a lang code, filling in
+// a default country for bare languages (e.g. "de" -> "de-DE"). Returns "" when
+// the input has no language subtag.
+func PrimaryLanguageTag(langCode string) string {
+	locale := ParseLocale(langCode)
+	if locale.Language == "" {
+		return ""
+	}
+	country := locale.Country
+	if country == "" {
+		country = defaultLocaleCountryByLanguage[locale.Language]
+	}
+	if country == "" {
+		return locale.Language
+	}
+	return locale.Language + "-" + country
+}
+
+var defaultTimezoneByCountry = map[string]string{
+	"US": "America/New_York", "GB": "Europe/London", "DE": "Europe/Berlin",
+	"FR": "Europe/Paris", "ES": "Europe/Madrid", "IT": "Europe/Rome",
+	"RU": "Europe/Moscow", "BR": "America/Sao_Paulo", "JP": "Asia/Tokyo",
+	"CN": "Asia/Shanghai", "KR": "Asia/Seoul", "IN": "Asia/Kolkata",
+	"AU": "Australia/Sydney", "CA": "America/Toronto",
+	"MX": "America/Mexico_City", "PL": "Europe/Warsaw",
+	"NL": "Europe/Amsterdam", "TR": "Europe/Istanbul",
+	"AR": "America/Argentina/Buenos_Aires", "SA": "Asia/Riyadh",
+	"BE": "Europe/Brussels", "KZ": "Asia/Almaty", "UA": "Europe/Kyiv",
+}
+
+// TimezoneForLocale returns an IANA timezone for a locale's country, or for
+// the default country of the language when country is empty. Returns "" for
+// unknown locales — caller should retain the existing profile timezone.
+func TimezoneForLocale(loc Locale) string {
+	if loc.Country != "" {
+		if tz, ok := defaultTimezoneByCountry[loc.Country]; ok {
+			return tz
+		}
+	}
+	if loc.Language != "" {
+		if country, ok := defaultLocaleCountryByLanguage[loc.Language]; ok {
+			if tz, ok2 := defaultTimezoneByCountry[country]; ok2 {
+				return tz
+			}
+		}
+	}
+	return ""
+}
+
+// BuildAcceptLanguageHeader formats an Accept-Language value from a lang code.
+// Example: "de" -> "de-DE,de;q=0.9", "en-GB" -> "en-GB,en;q=0.9", "sw" -> "sw".
+func BuildAcceptLanguageHeader(langCode string) string {
+	primary := PrimaryLanguageTag(langCode)
+	if primary == "" {
+		return ""
+	}
+	language := ParseLocale(langCode).Language
+	if primary == language {
+		return language
+	}
+	return primary + "," + language + ";q=0.9"
+}

core/locale_test.go

@@ -0,0 +1,105 @@
+package core
+
+import (
+	"testing"
+)
+
+func TestParseLocale(t *testing.T) {
+	tests := []struct {
+		name     string
+		in       string
+		wantLang string
+		wantCC   string
+	}{
+		{"empty", "", "", ""},
+		{"whitespace only", "   ", "", ""},
+		{"language only", "EN", "en", ""},
+		{"language with region dash", "en-US", "en", "US"},
+		{"language with region underscore", "de_AT", "de", "AT"},
+		{"mixed casing", "Pt-bR", "pt", "BR"},
+		{"trailing whitespace", "  fr-CA  ", "fr", "CA"},
+		{"empty region after dash", "ru-", "ru", ""},
+		{"language only after split", "-US", "", ""},
+		{"extra subtags ignored", "en-US-x-private", "en", "US"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ParseLocale(tt.in)
+			if got.Language != tt.wantLang || got.Country != tt.wantCC {
+				t.Fatalf("ParseLocale(%q) = {%q, %q}, want {%q, %q}",
+					tt.in, got.Language, got.Country, tt.wantLang, tt.wantCC)
+			}
+		})
+	}
+}
+
+func TestCountryFromRegion(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{name: "empty", in: "", want: ""},
+		{name: "country lowercase", in: "ru", want: "RU"},
+		{name: "country uppercase", in: "US", want: "US"},
+		{name: "locale dash", in: "en-GB", want: "GB"},
+		{name: "locale underscore", in: "de_AT", want: "AT"},
+		{name: "numeric engine region is not country", in: "213", want: ""},
+		{name: "unknown shape", in: "moscow", want: ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := CountryFromRegion(tt.in); got != tt.want {
+				t.Fatalf("CountryFromRegion(%q) = %q, want %q", tt.in, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestBuildAcceptLanguageHeader(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{name: "empty", in: "", want: ""},
+		{name: "language only with default country", in: "de", want: "de-DE,de;q=0.9"},
+		{name: "language only with mapped country", in: "pt", want: "pt-BR,pt;q=0.9"},
+		{name: "explicit region", in: "en-GB", want: "en-GB,en;q=0.9"},
+		{name: "unknown language emits bare tag", in: "sw", want: "sw"},
+		{name: "invalid locale", in: "-US", want: ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := BuildAcceptLanguageHeader(tt.in)
+			if got != tt.want {
+				t.Fatalf("BuildAcceptLanguageHeader(%q) = %q, want %q", tt.in, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestTimezoneForLocale(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{name: "de-DE explicit country", in: "de-DE", want: "Europe/Berlin"},
+		{name: "pt-BR explicit country", in: "pt-BR", want: "America/Sao_Paulo"},
+		{name: "bare de via default country", in: "de", want: "Europe/Berlin"},
+		{name: "unknown locale", in: "xx-YY", want: ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := TimezoneForLocale(ParseLocale(tt.in))
+			if got != tt.want {
+				t.Fatalf("TimezoneForLocale(%q) = %q, want %q", tt.in, got, tt.want)
+			}
+		})
+	}
+}

core/logger.go

@@ -1,132 +1,296 @@
 package core
 
 import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"os"
+	"sort"
 	"strings"
+	"time"
 
 	"github.com/sirupsen/logrus"
 )
 
-type customFormatter struct {
-	logrus.TextFormatter
+type loggerContextKey string
+
+const (
+	requestIDContextKey loggerContextKey = "request_id"
+	tenantContextKey    loggerContextKey = "tenant"
+	engineContextKey    loggerContextKey = "engine"
+	queryHashContextKey loggerContextKey = "query_hash"
+
+	LogFormatJSON = "json"
+	LogFormatText = "text"
+)
+
+func NormalizeLogFormat(raw string) (string, error) {
+	format := strings.ToLower(strings.TrimSpace(raw))
+	if format == "" {
+		return LogFormatText, nil
+	}
+	switch format {
+	case LogFormatJSON, LogFormatText:
+		return format, nil
+	default:
+		return "", fmt.Errorf("invalid logging.format %q: expected json or text", raw)
+	}
 }
 
-func (f *customFormatter) Format(entry *logrus.Entry) ([]byte, error) {
-	message := entry.Message
-
-	// Check if engine name is provided as a field
-	engineName := ""
-	if engine, exists := entry.Data["engine"]; exists {
-		if engineStr, ok := engine.(string); ok {
-			engineName = engineStr
-		}
+func WithRequestID(ctx context.Context, requestID string) context.Context {
+	requestID = strings.TrimSpace(requestID)
+	if requestID == "" {
+		return EnsureContext(ctx)
 	}
-
-	// Format: [timestamp][level][engine] message
-	if engineName != "" {
-		return []byte(fmt.Sprintf("[%s][%s][%s] %s\n",
-			entry.Time.Format(f.TimestampFormat),
-			strings.ToUpper(entry.Level.String()),
-			engineName,
-			message)), nil
-	}
-
-	// Format: [timestamp][level] message (no engine)
-	return []byte(fmt.Sprintf("[%s][%s] %s\n",
-		entry.Time.Format(f.TimestampFormat),
-		strings.ToUpper(entry.Level.String()),
-		message)), nil
+	return context.WithValue(EnsureContext(ctx), requestIDContextKey, requestID)
 }
 
-// EngineLogger provides simplified logging for search engines
+func WithTenant(ctx context.Context, tenant string) context.Context {
+	tenant = strings.TrimSpace(tenant)
+	if tenant == "" {
+		return EnsureContext(ctx)
+	}
+	return context.WithValue(EnsureContext(ctx), tenantContextKey, tenant)
+}
+
+func WithEngine(ctx context.Context, engine string) context.Context {
+	engine = strings.TrimSpace(engine)
+	if engine == "" {
+		return EnsureContext(ctx)
+	}
+	return context.WithValue(EnsureContext(ctx), engineContextKey, engine)
+}
+
+func WithQueryHash(ctx context.Context, queryHash string) context.Context {
+	queryHash = strings.TrimSpace(queryHash)
+	if queryHash == "" {
+		return EnsureContext(ctx)
+	}
+	return context.WithValue(EnsureContext(ctx), queryHashContextKey, queryHash)
+}
+
+func RequestIDFromContext(ctx context.Context) string {
+	value, _ := EnsureContext(ctx).Value(requestIDContextKey).(string)
+	return strings.TrimSpace(value)
+}
+
+func TenantFromContext(ctx context.Context) string {
+	value, _ := EnsureContext(ctx).Value(tenantContextKey).(string)
+	return strings.TrimSpace(value)
+}
+
+func WithRequest(ctx context.Context) *logrus.Entry {
+	ctx = EnsureContext(ctx)
+	fields := logrus.Fields{}
+
+	if requestID, ok := ctx.Value(requestIDContextKey).(string); ok && strings.TrimSpace(requestID) != "" {
+		fields["request_id"] = strings.TrimSpace(requestID)
+	}
+	if tenant, ok := ctx.Value(tenantContextKey).(string); ok && strings.TrimSpace(tenant) != "" {
+		fields["tenant"] = strings.TrimSpace(tenant)
+	}
+	if engine, ok := ctx.Value(engineContextKey).(string); ok && strings.TrimSpace(engine) != "" {
+		fields["engine"] = strings.TrimSpace(engine)
+	}
+	if queryHash, ok := ctx.Value(queryHashContextKey).(string); ok && strings.TrimSpace(queryHash) != "" {
+		fields["query_hash"] = strings.TrimSpace(queryHash)
+	}
+	profileIDs := BrowserProfileIDsFromContext(ctx)
+	if len(profileIDs) == 1 {
+		fields["browser_profile_id"] = profileIDs[0]
+	} else if len(profileIDs) > 1 {
+		fields["browser_profile_ids"] = strings.Join(profileIDs, ",")
+	}
+
+	return logrus.WithFields(fields)
+}
+
+func WithRequestEngine(ctx context.Context, engine string) *logrus.Entry {
+	return WithRequest(WithEngine(ctx, engine))
+}
+
+func QueryHash(raw string) string {
+	normalized := strings.TrimSpace(strings.ToLower(raw))
+	if normalized == "" {
+		return ""
+	}
+	hash := md5.Sum([]byte(normalized))
+	return hex.EncodeToString(hash[:])
+}
+
+func QueryHashFromQuery(q Query) string {
+	raw := strings.Join([]string{
+		q.Text,
+		q.Site,
+		q.Filetype,
+		q.LangCode,
+		q.Region,
+		q.DateInterval,
+	}, "|")
+	return QueryHash(raw)
+}
+
+func formatMessage(message string, args ...any) string {
+	if len(args) == 0 {
+		return message
+	}
+	return fmt.Sprintf(message, args...)
+}
+
+// EngineLogger provides structured logging for search engines with a fixed engine field.
 type EngineLogger struct {
 	engine string
-	logger *logrus.Entry
+	entry  *logrus.Entry
 }
 
-// NewEngineLogger creates a new logger for a specific search engine
 func NewEngineLogger(engine string) *EngineLogger {
-	return &EngineLogger{
-		engine: engine,
-		logger: logrus.WithField("engine", engine),
-	}
+	engine = strings.ToLower(strings.TrimSpace(engine))
+	return &EngineLogger{engine: engine, entry: logrus.WithField("engine", engine)}
 }
 
-// Debug logs a debug message
-func (el *EngineLogger) Debug(message string, args ...interface{}) {
-	el.logger.Debugf(message, args...)
+func (el *EngineLogger) WithRequest(ctx context.Context) *EngineLogger {
+	return &EngineLogger{engine: el.engine, entry: WithRequestEngine(ctx, el.engine)}
 }
 
-// Info logs an info message
-func (el *EngineLogger) Info(message string, args ...interface{}) {
-	el.logger.Infof(message, args...)
+// Fields returns a new EngineLogger with additional structured fields merged in.
+func (el *EngineLogger) Fields(fields logrus.Fields) *EngineLogger {
+	return &EngineLogger{engine: el.engine, entry: el.entry.WithFields(fields)}
 }
 
-// Warn logs a warning message
-func (el *EngineLogger) Warn(message string, args ...interface{}) {
-	el.logger.Warnf(message, args...)
+func (el *EngineLogger) Debug(message string, args ...any) {
+	el.entry.Debug(formatMessage(message, args...))
 }
 
-// Error logs an error message
-func (el *EngineLogger) Error(message string, args ...interface{}) {
-	el.logger.Errorf(message, args...)
+func (el *EngineLogger) Info(message string, args ...any) {
+	el.entry.Info(formatMessage(message, args...))
 }
 
-// Fatal logs a fatal message
-func (el *EngineLogger) Fatal(message string, args ...interface{}) {
-	el.logger.Fatalf(message, args...)
+func (el *EngineLogger) Warn(message string, args ...any) {
+	el.entry.Warn(formatMessage(message, args...))
 }
 
-// Panic logs a panic message
-func (el *EngineLogger) Panic(message string, args ...interface{}) {
-	el.logger.Panicf(message, args...)
+func (el *EngineLogger) Error(message string, args ...any) {
+	el.entry.Error(formatMessage(message, args...))
 }
 
-// LogWithEngine logs a message with engine information (deprecated - use EngineLogger instead)
-func LogWithEngine(level logrus.Level, engine, message string, args ...interface{}) {
-	entry := logrus.WithField("engine", engine)
-	switch level {
-	case logrus.DebugLevel:
-		entry.Debugf(message, args...)
-	case logrus.InfoLevel:
-		entry.Infof(message, args...)
-	case logrus.WarnLevel:
-		entry.Warnf(message, args...)
-	case logrus.ErrorLevel:
-		entry.Errorf(message, args...)
-	case logrus.FatalLevel:
-		entry.Fatalf(message, args...)
-	case logrus.PanicLevel:
-		entry.Panicf(message, args...)
-	}
+func (el *EngineLogger) Fatal(message string, args ...any) {
+	el.entry.Fatal(formatMessage(message, args...))
 }
 
-func InitLogger(isVerbose, isDebug bool) {
-	logrus.SetFormatter(&customFormatter{logrus.TextFormatter{
-		FullTimestamp:          true,
-		TimestampFormat:        "2006-01-02 15:04:05",
-		ForceColors:            true,
-		DisableLevelTruncation: true,
-	}})
+func (el *EngineLogger) Panic(message string, args ...any) {
+	el.entry.Panic(formatMessage(message, args...))
+}
 
-	if isVerbose {
-		logrus.SetLevel(logrus.DebugLevel)
+// bracketFormatter emits bracket-delimited fields:
+//
+//	[time][level][engine=..][request_id=..][query_hash=..][extra fields sorted][msg]
+//
+// request_id is truncated to last 8 chars; query_hash to first 12.
+type bracketFormatter struct {
+	TimestampFormat string
+}
+
+func (f *bracketFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+	ts := entry.Time.Format(f.TimestampFormat)
+
+	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "[%s][%s]", ts, entry.Level.String())
+
+	// Context identity fields in fixed order, then remaining fields sorted, then msg last.
+	priority := []string{"engine", "tenant", "request_id", "query_hash"}
+	written := make(map[string]bool, len(entry.Data))
+
+	for _, key := range priority {
+		val, ok := entry.Data[key]
+		if !ok {
+			continue
+		}
+		s := fmt.Sprintf("%v", val)
+		switch key {
+		case "request_id":
+			if len(s) > 8 {
+				s = s[len(s)-8:]
+			}
+		case "query_hash":
+			if len(s) > 12 {
+				s = s[:12]
+			}
+		}
+		fmt.Fprintf(&buf, "[%s=%s]", key, quoteIfNeeded(s))
+		written[key] = true
 	}
 
-	if isDebug {
-		logrus.SetOutput(io.MultiWriter(os.Stdout))
-		logrus.SetLevel(logrus.TraceLevel)
+	rest := make([]string, 0, len(entry.Data))
+	for k := range entry.Data {
+		if !written[k] {
+			rest = append(rest, k)
+		}
+	}
+	sort.Strings(rest)
+	for _, k := range rest {
+		fmt.Fprintf(&buf, "[%s=%s]", k, quoteIfNeeded(fmt.Sprintf("%v", entry.Data[k])))
+	}
+
+	// Message last so context fields are scannable without scrolling past a long msg.
+	if entry.Message != "" {
+		fmt.Fprintf(&buf, "[%s]", quoteIfNeeded(entry.Message))
+	}
+
+	buf.WriteByte('\n')
+	return buf.Bytes(), nil
+}
+
+// quoteIfNeeded wraps s in double-quotes if it contains spaces.
+func quoteIfNeeded(s string) string {
+	if strings.ContainsAny(s, " \t") {
+		return `"` + strings.ReplaceAll(s, `"`, `\"`) + `"`
+	}
+	return s
+}
+
+func InitLogger(isVerbose, isDebug, isQuiet bool, format string) {
+	switch format {
+	case LogFormatText:
+		logrus.SetFormatter(&bracketFormatter{TimestampFormat: "2006-01-02 15:04:05"})
+	case LogFormatJSON:
+		logrus.SetFormatter(&logrus.JSONFormatter{
+			TimestampFormat: time.RFC3339Nano,
+		})
+	}
+
+	// Logs go to stderr (+ optional file) so stdout carries only the payload.
+	switch {
+	case isDebug:
+		logrus.SetOutput(io.MultiWriter(os.Stderr))
 		logrus.SetReportCaller(true)
-	} else {
+	case isQuiet:
+		// One-shot CLI default: stderr only, no ./logs.txt in the user's CWD.
+		logrus.SetOutput(os.Stderr)
+		logrus.SetReportCaller(false)
+	default:
 		f, err := os.OpenFile("./logs.txt", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
 		if err != nil {
-			fmt.Println("Failed to create logsfile: ./logs.txt")
-			panic(err)
+			fmt.Fprintf(os.Stderr, "Failed to open logs file ./logs.txt: %v\n", err)
+			logrus.SetOutput(io.MultiWriter(os.Stderr))
+		} else {
+			logrus.SetOutput(io.MultiWriter(f, os.Stderr))
 		}
-
-		logrus.SetOutput(io.MultiWriter(f, os.Stdout))
-		logrus.SetLevel(logrus.DebugLevel)
+		logrus.SetReportCaller(false)
 	}
+
+	level := logrus.InfoLevel
+	if isVerbose {
+		level = logrus.DebugLevel
+	}
+	if isDebug {
+		level = logrus.TraceLevel
+	}
+	if isQuiet && !isVerbose && !isDebug {
+		// Quiet keeps only warnings/errors on stderr.
+		level = logrus.WarnLevel
+	}
+	logrus.SetLevel(level)
 }

core/middleware.go

@@ -1,18 +1,23 @@
 package core
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"time"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
 )
 
 type JSONErrorResponse struct {
-	Error   string `json:"error"`
-	Code    int    `json:"code"`
-	Message string `json:"message,omitempty"`
+	Error     string                 `json:"error"`
+	Code      int                    `json:"code"`
+	RequestID string                 `json:"request_id,omitempty"`
+	Message   string                 `json:"message,omitempty"`
+	Reason    string                 `json:"reason,omitempty"`
+	Meta      map[string]interface{} `json:"meta,omitempty"`
 }
 
 type CORSConfig struct {
@@ -22,15 +27,57 @@ type CORSConfig struct {
 	MaxAge       int
 }
 
+const browserProfileIDHeader = "X-Browser-Profile-Id"
+const useProfileHeader = "X-Use-Profile"
+
+const exposedResponseHeaders = "X-Request-ID, X-Cache, X-Fallback-Engine, X-Proxy-Mode, X-Proxy-Tag, X-Proxy-Used, X-Proxy-Attempts, X-Network-Bytes, " + browserProfileIDHeader
+
 func DefaultCORSConfig() CORSConfig {
 	return CORSConfig{
 		AllowOrigins: "*",
 		AllowMethods: "GET, POST, OPTIONS",
-		AllowHeaders: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy",
+		AllowHeaders: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Proxy-URL, X-Proxy-Country, X-Proxy-Class, X-Proxy-Provider, X-Proxy-Session-ID, X-Request-ID, X-Tenant, X-Use-Profile",
 		MaxAge:       86400,
 	}
 }
 
+func RequestContextMiddleware() fiber.Handler {
+	return func(c *fiber.Ctx) error {
+		requestID := strings.TrimSpace(c.Get("X-Request-ID"))
+		if requestID == "" {
+			id, err := uuid.NewV7()
+			if err != nil {
+				requestID = uuid.NewString()
+			} else {
+				requestID = id.String()
+			}
+		}
+
+		requestCtx := WithRequestID(c.UserContext(), requestID)
+		requestCtx = WithTenant(requestCtx, strings.TrimSpace(c.Get("X-Tenant")))
+		requestCtx = WithQueryHash(requestCtx, QueryHash(c.Query("text")))
+		c.SetUserContext(requestCtx)
+
+		c.Set("X-Request-ID", requestID)
+		return c.Next()
+	}
+}
+
+// RequestTimeoutMiddleware bounds wall-clock time per request by attaching a
+// deadline to the user context, which fasthttp never cancels on client
+// disconnect. /mega/* (MegaTimeout) and /extract* (batch budget) are exempt.
+func RequestTimeoutMiddleware(timeout time.Duration) fiber.Handler {
+	return func(c *fiber.Ctx) error {
+		if strings.HasPrefix(c.Path(), "/mega/") || c.Path() == "/extract" || c.Path() == "/extract/batch" {
+			return c.Next()
+		}
+		ctx, cancel := context.WithTimeout(c.UserContext(), timeout)
+		defer cancel()
+		c.SetUserContext(ctx)
+		return c.Next()
+	}
+}
+
 func CORSMiddleware(cfg CORSConfig) fiber.Handler {
 	cfg = normalizeCORSConfig(cfg)
 
@@ -39,6 +86,7 @@ func CORSMiddleware(cfg CORSConfig) fiber.Handler {
 		c.Set("Access-Control-Allow-Methods", cfg.AllowMethods)
 		c.Set("Access-Control-Allow-Headers", cfg.AllowHeaders)
 		c.Set("Access-Control-Max-Age", fmt.Sprintf("%d", cfg.MaxAge))
+		c.Set("Access-Control-Expose-Headers", exposedResponseHeaders)
 
 		if c.Method() == "OPTIONS" {
 			return c.SendStatus(fiber.StatusNoContent)
@@ -77,46 +125,93 @@ func RequestLoggerMiddleware() fiber.Handler {
 		if err != nil {
 			if e, ok := err.(*fiber.Error); ok {
 				status = e.Code
+			} else if apiErr, ok := err.(*APIError); ok {
+				status = apiErr.HTTPStatus
 			} else {
 				status = fiber.StatusInternalServerError
 			}
 		}
 
 		logFields := logrus.Fields{
-			"method":  c.Method(),
-			"path":    c.Path(),
-			"status":  status,
-			"latency": latency.String(),
-			"ip":      c.IP(),
+			"method": c.Method(),
+			"path":   c.Path(),
+			"status": status,
+			"ip":     c.IP(),
 		}
+		logFields["latency_ms"] = latency.Milliseconds()
 		if query := c.Query("text"); query != "" {
-			logFields["query"] = query
+			logFields["query_hash"] = QueryHash(query)
 		}
+		addProxyLogFields(c, logFields)
 
-		entry := logrus.WithFields(logFields)
+		entry := WithRequest(c.UserContext()).WithFields(logFields)
 		if status >= 500 {
-			entry.Errorf("%s - request failed", c.Path())
+			entry.Error("request failed")
 		} else if status >= 400 {
-			entry.Warnf("%s - request error", c.Path())
+			entry.Warn("request error")
 		} else {
-			entry.Infof("%s - request completed", c.Path())
+			entry.Info("request completed")
 		}
 
 		return err
 	}
 }
 
+func addProxyLogFields(c *fiber.Ctx, fields logrus.Fields) {
+	if country := strings.ToLower(strings.TrimSpace(c.Get("X-Proxy-Country"))); country != "" {
+		fields["proxy_country"] = country
+	}
+	if class := strings.ToLower(strings.TrimSpace(c.Get("X-Proxy-Class"))); class != "" {
+		fields["proxy_class"] = class
+	}
+	if provider := strings.ToLower(strings.TrimSpace(c.Get("X-Proxy-Provider"))); provider != "" {
+		fields["proxy_provider"] = provider
+	}
+
+	if sessionID := strings.TrimSpace(c.Get("X-Proxy-Session-ID")); sessionID != "" {
+		fields["proxy_session_id"] = sessionID
+	}
+
+	if proxyURL := strings.TrimSpace(c.Get("X-Proxy-URL")); proxyURL != "" {
+		fields["proxy_used"] = MaskProxyURL(proxyURL)
+	}
+
+	if laneKey := proxyLaneKeyFromContext(c.UserContext()); !laneKey.Empty() {
+		fields["lane_id"] = laneKey.ID()
+	}
+}
+
 func JSONErrorMiddleware() fiber.ErrorHandler {
 	return func(c *fiber.Ctx, err error) error {
 		code := fiber.StatusInternalServerError
+		errorCode := ""
+		reason := ""
+		var meta map[string]interface{}
+
 		if e, ok := err.(*fiber.Error); ok {
 			code = e.Code
 		}
+		if apiErr, ok := err.(*APIError); ok {
+			code = apiErr.HTTPStatus
+			errorCode = apiErr.ErrorCode
+			reason = apiErr.Reason
+			meta = apiErr.Meta
+		}
+		if errorCode == "" {
+			errorCode = statusText(code)
+		}
+		requestID := RequestIDFromContext(c.UserContext())
+		if requestID == "" {
+			requestID = strings.TrimSpace(c.Get("X-Request-ID"))
+		}
 
 		resp := JSONErrorResponse{
-			Error:   statusText(code),
-			Code:    code,
-			Message: err.Error(),
+			Error:     errorCode,
+			Code:      code,
+			RequestID: requestID,
+			Message:   err.Error(),
+			Reason:    reason,
+			Meta:      meta,
 		}
 
 		c.Set("Content-Type", "application/json")

core/middleware_test.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -62,6 +63,9 @@ func TestCORSMiddleware_UsesConfiguredHeaders(t *testing.T) {
 	if got := resp.Header.Get("Access-Control-Max-Age"); got != "1200" {
 		t.Fatalf("unexpected max-age: %q", got)
 	}
+	if got := resp.Header.Get("Access-Control-Expose-Headers"); !strings.Contains(got, "X-Network-Bytes") || !strings.Contains(got, "X-Browser-Profile-Id") || !strings.Contains(got, "X-Proxy-Attempts") {
+		t.Fatalf("expected exposed headers to include network, browser profile, and proxy-attempts headers, got %q", got)
+	}
 }
 
 func TestCORSMiddleware_OPTIONSReturnsNoContent(t *testing.T) {
@@ -107,3 +111,57 @@ func TestDefaultCORSConfig_IncludesProxyOverrideHeader(t *testing.T) {
 		t.Fatalf("expected allow_headers to include X-Use-Proxy, got %q", got)
 	}
 }
+
+func TestRequestContextMiddleware_EchoesProvidedRequestID(t *testing.T) {
+	app := fiber.New()
+	app.Use(RequestContextMiddleware())
+	app.Get("/id", func(c *fiber.Ctx) error {
+		return c.SendString(RequestIDFromContext(c.UserContext()))
+	})
+
+	req := httptest.NewRequest(http.MethodGet, "/id", nil)
+	req.Header.Set("X-Request-ID", "foo")
+	resp, err := app.Test(req, -1)
+	if err != nil {
+		t.Fatalf("request failed: %v", err)
+	}
+
+	if got := resp.Header.Get("X-Request-ID"); got != "foo" {
+		t.Fatalf("expected response X-Request-ID=foo, got %q", got)
+	}
+	if got := readBody(t, resp); got != "foo" {
+		t.Fatalf("expected context request id to be echoed, got %q", got)
+	}
+}
+
+func TestRequestContextMiddleware_GeneratesRequestID(t *testing.T) {
+	app := fiber.New()
+	app.Use(RequestContextMiddleware())
+	app.Get("/id", func(c *fiber.Ctx) error {
+		return c.SendString(RequestIDFromContext(c.UserContext()))
+	})
+
+	req := httptest.NewRequest(http.MethodGet, "/id", nil)
+	resp, err := app.Test(req, -1)
+	if err != nil {
+		t.Fatalf("request failed: %v", err)
+	}
+
+	requestID := resp.Header.Get("X-Request-ID")
+	if requestID == "" {
+		t.Fatal("expected generated X-Request-ID")
+	}
+	if got := readBody(t, resp); got != requestID {
+		t.Fatalf("expected request id in context to match header: body=%q header=%q", got, requestID)
+	}
+}
+
+func readBody(t *testing.T, resp *http.Response) string {
+	t.Helper()
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("read body failed: %v", err)
+	}
+	return string(body)
+}

core/network_guard.go

@@ -0,0 +1,159 @@
+package core
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"net/url"
+	"strings"
+)
+
+// ErrTargetNotAllowed marks URL-guard policy rejections (bad scheme, missing
+// host, or a target resolving to a non-public IP). Handlers match it with
+// errors.Is to report a client error instead of an upstream failure.
+var ErrTargetNotAllowed = errors.New("target not allowed")
+
+var carrierGradeNATPrefix = netip.MustParsePrefix("100.64.0.0/10")
+
+// GuardedDialContext dials only public IP targets. Hostnames are resolved first
+// and the returned connection is made to the vetted IP, so DNS rebinding cannot
+// swap in a private address between validation and dial.
+func GuardedDialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+	ips, err := resolvePublicDialTargets(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+
+	dialer := &net.Dialer{}
+	var lastErr error
+	for _, ip := range ips {
+		if !ipMatchesNetwork(ip, network) {
+			continue
+		}
+		conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
+		if err == nil {
+			return conn, nil
+		}
+		lastErr = err
+	}
+	if lastErr != nil {
+		return nil, lastErr
+	}
+	return nil, fmt.Errorf("no public IPs available for %s", host)
+}
+
+func ValidatePublicHTTPURL(ctx context.Context, rawURL string) error {
+	parsed, err := validateHTTPURL(rawURL)
+	if err != nil {
+		return err
+	}
+	return validatePublicHost(ctx, parsed.Hostname())
+}
+
+func validateHTTPURL(rawURL string) (*url.URL, error) {
+	parsed, err := url.Parse(strings.TrimSpace(rawURL))
+	if err != nil {
+		return nil, fmt.Errorf("%w: invalid URL: %v", ErrTargetNotAllowed, err)
+	}
+	if parsed.Scheme != "http" && parsed.Scheme != "https" {
+		return nil, fmt.Errorf("%w: unsupported URL scheme %q: only http and https are allowed", ErrTargetNotAllowed, parsed.Scheme)
+	}
+	if parsed.Hostname() == "" {
+		return nil, fmt.Errorf("%w: URL host is required", ErrTargetNotAllowed)
+	}
+	return parsed, nil
+}
+
+// resolveHostIPs resolves host (or parses a literal IP) and partitions the
+// addresses by the public-IP policy.
+func resolveHostIPs(ctx context.Context, host string) (public, blocked []netip.Addr, err error) {
+	if ip, parseErr := netip.ParseAddr(host); parseErr == nil {
+		ip = ip.Unmap()
+		if isPublicIP(ip) {
+			return []netip.Addr{ip}, nil, nil
+		}
+		return nil, []netip.Addr{ip}, nil
+	}
+	ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host)
+	if err != nil {
+		return nil, nil, err
+	}
+	for _, ip := range ips {
+		ip = ip.Unmap()
+		if isPublicIP(ip) {
+			public = append(public, ip)
+		} else {
+			blocked = append(blocked, ip)
+		}
+	}
+	return public, blocked, nil
+}
+
+func resolvePublicDialTargets(ctx context.Context, host string) ([]netip.Addr, error) {
+	public, blocked, err := resolveHostIPs(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+	if len(public) == 0 {
+		if len(blocked) == 0 {
+			return nil, fmt.Errorf("%w: target host %q resolved to no IP addresses", ErrTargetNotAllowed, host)
+		}
+		return nil, fmt.Errorf("%w: target host %q resolves only to non-public IPs: %s", ErrTargetNotAllowed, host, joinAddrs(blocked))
+	}
+	return public, nil
+}
+
+// validatePublicHost rejects a host when any of its addresses is non-public —
+// stricter than the dial guard on purpose. The rendered path hands the URL to
+// Chrome, which resolves DNS on its own, so a mixed public/private record set
+// must fail closed here rather than rely on dial-time pinning.
+func validatePublicHost(ctx context.Context, host string) error {
+	public, blocked, err := resolveHostIPs(ctx, host)
+	if err != nil {
+		return err
+	}
+	if len(blocked) > 0 {
+		return fmt.Errorf("%w: target host %q resolves to non-public IPs: %s", ErrTargetNotAllowed, host, joinAddrs(blocked))
+	}
+	if len(public) == 0 {
+		return fmt.Errorf("%w: target host %q resolved to no IP addresses", ErrTargetNotAllowed, host)
+	}
+	return nil
+}
+
+func joinAddrs(addrs []netip.Addr) string {
+	parts := make([]string, len(addrs))
+	for i, addr := range addrs {
+		parts[i] = addr.String()
+	}
+	return strings.Join(parts, ", ")
+}
+
+func isPublicIP(ip netip.Addr) bool {
+	ip = ip.Unmap()
+	return ip.IsValid() &&
+		!ip.IsUnspecified() &&
+		!ip.IsLoopback() &&
+		!ip.IsPrivate() &&
+		!ip.IsLinkLocalUnicast() &&
+		!ip.IsLinkLocalMulticast() &&
+		!ip.IsMulticast() &&
+		!carrierGradeNATPrefix.Contains(ip)
+}
+
+func ipMatchesNetwork(ip netip.Addr, network string) bool {
+	switch network {
+	case "tcp4":
+		return ip.Is4()
+	case "tcp6":
+		return ip.Is6()
+	default:
+		return true
+	}
+}

core/network_guard_test.go

@@ -0,0 +1,42 @@
+package core
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+)
+
+func TestValidatePublicHTTPURLRejectsPrivateTargets(t *testing.T) {
+	tests := []string{
+		"http://127.0.0.1/",
+		"http://[::1]/",
+		"http://10.0.0.1/",
+		"http://172.16.0.1/",
+		"http://192.168.1.1/",
+		"http://169.254.169.254/",
+		"http://100.64.0.1/",
+	}
+
+	for _, rawURL := range tests {
+		t.Run(rawURL, func(t *testing.T) {
+			err := ValidatePublicHTTPURL(context.Background(), rawURL)
+			if !errors.Is(err, ErrTargetNotAllowed) {
+				t.Fatalf("expected ErrTargetNotAllowed, got %v", err)
+			}
+			if !strings.Contains(strings.ToLower(err.Error()), "non-public") {
+				t.Fatalf("expected non-public error, got %v", err)
+			}
+		})
+	}
+}
+
+func TestValidatePublicHTTPURLRejectsUnsupportedScheme(t *testing.T) {
+	err := ValidatePublicHTTPURL(context.Background(), "file:///etc/passwd")
+	if !errors.Is(err, ErrTargetNotAllowed) {
+		t.Fatalf("expected ErrTargetNotAllowed, got %v", err)
+	}
+	if !strings.Contains(err.Error(), "only http and https") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}