mirror of
https://github.com/karust/openserp.git
synced 2026-08-15 21:15:56 +08:00
fix: derive a per-request deadline from the retry budget.
Add `RequestTimeout` config that bounds wall-clock time of any request that does not manage its own deadline budget. It is derived from the engine timeout and retry budget via RequestTimeoutForRetries.
This commit is contained in:
Rustem Kamalov
1
.gitignore
vendored
1
.gitignore
vendored
@@ -38,3 +38,4 @@ sanitize_html_fixtures.py
|
||||
core/testdata/*
|
||||
.tmp-gocache
|
||||
.tmpcache/
|
||||
google/data/geotargets-2026-05-28.csv
|
||||
|
||||
16
cmd/serve.go
16
cmd/serve.go
@@ -151,6 +151,14 @@ func buildFingerprintBrowserOptions() core.BrowserOpts {
|
||||
}
|
||||
|
||||
func buildServerOptions(corsCfg core.CORSConfig, proxyCfg core.ProxyConfig, fingerprintBrowserOpts core.BrowserOpts) core.ServerOptions {
|
||||
retryCfg := core.RetryConfig{
|
||||
MaxRetries: config.Resilience.MaxRetries,
|
||||
InitialBackoff: 1 * time.Second,
|
||||
MaxBackoff: 30 * time.Second,
|
||||
BackoffFactor: 2.0,
|
||||
}
|
||||
engineTimeout := time.Duration(config.App.Timeout) * time.Second
|
||||
|
||||
return core.ServerOptions{
|
||||
CacheTTL: time.Duration(config.Cache.TTLSeconds) * time.Second,
|
||||
CacheMaxSize: config.Cache.MaxSize,
|
||||
@@ -161,14 +169,10 @@ func buildServerOptions(corsCfg core.CORSConfig, proxyCfg core.ProxyConfig, fing
|
||||
FingerprintArtifactDir: core.DefaultFingerprintArtifactDir,
|
||||
FingerprintBrowserOpts: fingerprintBrowserOpts,
|
||||
MegaTimeout: config.App.MegaTimeout,
|
||||
RequestTimeout: core.RequestTimeoutForRetries(engineTimeout, retryCfg),
|
||||
Extract: config.Extract,
|
||||
Resilience: core.ResilientConfig{
|
||||
Retry: core.RetryConfig{
|
||||
MaxRetries: config.Resilience.MaxRetries,
|
||||
InitialBackoff: 1 * time.Second,
|
||||
MaxBackoff: 30 * time.Second,
|
||||
BackoffFactor: 2.0,
|
||||
},
|
||||
Retry: retryCfg,
|
||||
CircuitBreaker: core.CircuitBreakerConfig{
|
||||
FailureThreshold: config.CircuitBreaker.Failures,
|
||||
RecoveryTimeout: time.Duration(config.CircuitBreaker.RecoverySeconds) * time.Second,
|
||||
|
||||
@@ -8,7 +8,7 @@ server:
|
||||
|
||||
app:
|
||||
log_format: "text" # json|text
|
||||
timeout: 15 # Browser/search timeout in seconds
|
||||
timeout: 15 # Browser/search timeout in seconds, per attempt; the request deadline is derived from this x retries
|
||||
browser_path: "" # Custom browser binary path (chrome/chromium/edge..)
|
||||
profiles: "" # Overriding built-in browser profiles
|
||||
head: false # Headful mode
|
||||
@@ -18,6 +18,8 @@ app:
|
||||
idle_ttl: 5m # close a Chrome that has not served traffic for this long
|
||||
|
||||
mega_timeout: 90s # max total wait for /mega/* requests; slow engines return partial results
|
||||
block_trackers: true
|
||||
block_resources: "image,font,css,media"
|
||||
|
||||
extract:
|
||||
enabled: true
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -62,6 +63,21 @@ func RequestContextMiddleware() fiber.Handler {
|
||||
}
|
||||
}
|
||||
|
||||
// RequestTimeoutMiddleware bounds wall-clock time per request by attaching a
|
||||
// deadline to the user context, which fasthttp never cancels on client
|
||||
// disconnect. /mega/* (MegaTimeout) and /extract (batch budget) are exempt.
|
||||
func RequestTimeoutMiddleware(timeout time.Duration) fiber.Handler {
|
||||
return func(c *fiber.Ctx) error {
|
||||
if strings.HasPrefix(c.Path(), "/mega/") || c.Path() == "/extract" {
|
||||
return c.Next()
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(c.UserContext(), timeout)
|
||||
defer cancel()
|
||||
c.SetUserContext(ctx)
|
||||
return c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
func CORSMiddleware(cfg CORSConfig) fiber.Handler {
|
||||
cfg = normalizeCORSConfig(cfg)
|
||||
|
||||
|
||||
@@ -98,6 +98,33 @@ func RetryableSearch(ctx context.Context, cfg RetryConfig, engineName string, se
|
||||
}
|
||||
}
|
||||
|
||||
// requestTimeoutSlack covers per-request pipeline overhead that happens
|
||||
// outside engine attempts: rate-limiter waits, proxy selection, parsing.
|
||||
const requestTimeoutSlack = 5 * time.Second
|
||||
|
||||
// RequestTimeoutForRetries derives a per-request deadline that a healthy
|
||||
// request exhausting its full retry budget cannot hit: worst-case attempts
|
||||
// (each bounded by attemptTimeout) plus worst-case jittered backoffs plus
|
||||
// slack. Used as the server-wide request timeout so raising MaxRetries or
|
||||
// the engine timeout never silently truncates retries.
|
||||
func RequestTimeoutForRetries(attemptTimeout time.Duration, cfg RetryConfig) time.Duration {
|
||||
if cfg.BackoffFactor <= 0 {
|
||||
cfg.BackoffFactor = 2.0
|
||||
}
|
||||
|
||||
budget := time.Duration(cfg.MaxRetries+1) * attemptTimeout
|
||||
for attempt := 1; attempt <= cfg.MaxRetries; attempt++ {
|
||||
// calculateBackoff jitters by (0.5 + rand[0,1)), so 1.5x is the worst
|
||||
// case before the MaxBackoff cap.
|
||||
worst := time.Duration(1.5 * float64(cfg.InitialBackoff) * math.Pow(cfg.BackoffFactor, float64(attempt-1)))
|
||||
if worst > cfg.MaxBackoff || worst < 0 {
|
||||
worst = cfg.MaxBackoff
|
||||
}
|
||||
budget += worst
|
||||
}
|
||||
return budget + requestTimeoutSlack
|
||||
}
|
||||
|
||||
var nonRetryableSentinels = []struct {
|
||||
err error
|
||||
reason string
|
||||
|
||||
@@ -90,6 +90,13 @@ type ServerOptions struct {
|
||||
// as failed with a context-deadline error. Zero disables the bound
|
||||
// (legacy behavior — wait until the slowest engine finishes).
|
||||
MegaTimeout time.Duration
|
||||
// RequestTimeout bounds wall-clock time of any request that does not
|
||||
// manage its own deadline budget (/mega/* uses MegaTimeout, /extract has
|
||||
// a per-batch budget). Exceeding it returns 504 request_timeout. Zero
|
||||
// disables the bound. The serve command derives it from the engine
|
||||
// timeout and retry budget via RequestTimeoutForRetries instead of
|
||||
// exposing a separate config knob.
|
||||
RequestTimeout time.Duration
|
||||
// BrowserResolver returns a pooled browser for rendered extraction.
|
||||
BrowserResolver BrowserResolver
|
||||
// Extract configures the URL extraction endpoint and search enrichment.
|
||||
@@ -115,7 +122,10 @@ func DefaultServerOptions() ServerOptions {
|
||||
},
|
||||
Resilience: DefaultResilientConfig(),
|
||||
MegaTimeout: 90 * time.Second,
|
||||
Extract: extractpkg.DefaultConfig(),
|
||||
// 30s matches the default engine timeout (app.timeout); the serve
|
||||
// command re-derives from the configured values.
|
||||
RequestTimeout: RequestTimeoutForRetries(30*time.Second, DefaultRetryConfig()),
|
||||
Extract: extractpkg.DefaultConfig(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,6 +171,9 @@ func NewServerWithOptions(host string, port int, opts ServerOptions, searchEngin
|
||||
// extract, stats) so the process survives.
|
||||
app.Use(fiberrecover.New(fiberrecover.Config{EnableStackTrace: true}))
|
||||
app.Use(RequestContextMiddleware())
|
||||
if opts.RequestTimeout > 0 {
|
||||
app.Use(RequestTimeoutMiddleware(opts.RequestTimeout))
|
||||
}
|
||||
if opts.EnableCORS {
|
||||
app.Use(CORSMiddleware(opts.CORS))
|
||||
}
|
||||
|
||||
146
core/server_timeout_test.go
Normal file
146
core/server_timeout_test.go
Normal file
@@ -0,0 +1,146 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
)
|
||||
|
||||
// FP-2: every endpoint that doesn't manage its own deadline budget must get
|
||||
// one from RequestTimeoutMiddleware; /mega/* (MegaTimeout) and /extract
|
||||
// (batch budget) keep theirs.
|
||||
func TestRequestTimeoutMiddlewareSetsDeadlineExceptBudgetedPaths(t *testing.T) {
|
||||
app := fiber.New()
|
||||
app.Use(RequestTimeoutMiddleware(time.Minute))
|
||||
|
||||
deadlines := map[string]bool{}
|
||||
// c.Path() aliases fasthttp's reusable buffer, so capture the path
|
||||
// explicitly instead of using it as a map key after the request ends.
|
||||
record := func(path string) fiber.Handler {
|
||||
return func(c *fiber.Ctx) error {
|
||||
_, ok := c.UserContext().Deadline()
|
||||
deadlines[path] = ok
|
||||
return c.SendStatus(http.StatusOK)
|
||||
}
|
||||
}
|
||||
app.Get("/google/search", record("/google/search"))
|
||||
app.Post("/google/parse", record("/google/parse"))
|
||||
app.Get("/mega/search", record("/mega/search"))
|
||||
app.Get("/extract", record("/extract"))
|
||||
|
||||
for path, method := range map[string]string{
|
||||
"/google/search": http.MethodGet,
|
||||
"/google/parse": http.MethodPost,
|
||||
"/mega/search": http.MethodGet,
|
||||
"/extract": http.MethodGet,
|
||||
} {
|
||||
req := httptest.NewRequest(method, path, nil)
|
||||
resp, err := app.Test(req, -1)
|
||||
if err != nil {
|
||||
t.Fatalf("request failed for %s: %v", path, err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("%s: handler did not run, status %d", path, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
for path, want := range map[string]bool{
|
||||
"/google/search": true,
|
||||
"/google/parse": true,
|
||||
"/mega/search": false,
|
||||
"/extract": false,
|
||||
} {
|
||||
if deadlines[path] != want {
|
||||
t.Errorf("%s: deadline attached = %v, want %v", path, deadlines[path], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The derived deadline must never truncate a healthy request that exhausts
|
||||
// its full retry budget: attempts + worst-case jittered backoffs + slack.
|
||||
func TestRequestTimeoutForRetriesCoversFullRetryBudget(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
attemptTimeout time.Duration
|
||||
cfg RetryConfig
|
||||
want time.Duration
|
||||
}{
|
||||
{
|
||||
name: "defaults: 4x15s attempts + (1.5+3+6)s backoff + 5s slack",
|
||||
attemptTimeout: 15 * time.Second,
|
||||
cfg: DefaultRetryConfig(),
|
||||
want: 75500 * time.Millisecond,
|
||||
},
|
||||
{
|
||||
name: "no retries: one attempt + slack",
|
||||
attemptTimeout: 15 * time.Second,
|
||||
cfg: RetryConfig{MaxRetries: 0},
|
||||
want: 20 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "backoffs capped at MaxBackoff",
|
||||
attemptTimeout: 10 * time.Second,
|
||||
cfg: RetryConfig{
|
||||
MaxRetries: 2,
|
||||
InitialBackoff: time.Minute,
|
||||
MaxBackoff: 2 * time.Second,
|
||||
BackoffFactor: 2.0,
|
||||
},
|
||||
want: 30*time.Second + 4*time.Second + 5*time.Second,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
if got := RequestTimeoutForRetries(tc.attemptTimeout, tc.cfg); got != tc.want {
|
||||
t.Errorf("%s: got %s, want %s", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHungEngineReturns504RequestTimeoutAndServerSurvives(t *testing.T) {
|
||||
hung := func(ctx context.Context, _ Query) ([]SearchResult, error) {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
engine := &engineMock{
|
||||
name: "google",
|
||||
initialized: true,
|
||||
searchFn: hung,
|
||||
imageFn: hung,
|
||||
}
|
||||
|
||||
opts := DefaultServerOptions()
|
||||
opts.RequestTimeout = 100 * time.Millisecond
|
||||
opts.Resilience.Retry.MaxRetries = 0
|
||||
srv := NewServerWithOptions("127.0.0.1", 7140, opts, engine)
|
||||
|
||||
for _, path := range []string{"/google/search?text=golang", "/google/image?text=golang"} {
|
||||
started := time.Now()
|
||||
resp := request(t, srv, path)
|
||||
if resp.StatusCode != http.StatusGatewayTimeout {
|
||||
t.Fatalf("%s: expected 504 for hung engine, got %d", path, resp.StatusCode)
|
||||
}
|
||||
if elapsed := time.Since(started); elapsed > 5*time.Second {
|
||||
t.Fatalf("%s: request took %s, deadline did not bound it", path, elapsed)
|
||||
}
|
||||
var payload JSONErrorResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
|
||||
t.Fatalf("%s: decode error response: %v", path, err)
|
||||
}
|
||||
if payload.Error != "request_timeout" {
|
||||
t.Fatalf("%s: expected error=request_timeout, got %q", path, payload.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// Server keeps serving after timed-out requests.
|
||||
engine.searchFn = nil
|
||||
second := request(t, srv, "/google/search?text=golang")
|
||||
if second.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected server to keep serving after timeouts, got %d", second.StatusCode)
|
||||
}
|
||||
}
|
||||
322
docs/PROXY_SETUP.md
Normal file
322
docs/PROXY_SETUP.md
Normal file
@@ -0,0 +1,322 @@
|
||||
# Proxy Setup
|
||||
|
||||
This document is for the **backend engineer building the balancer** in front of OpenSERP. It explains the request/response contract for the SaaS `X-Proxy-URL` path, what each header does, what errors mean, and how to size and operate workers.
|
||||
|
||||
If you only need a self-contained OpenSERP with locally configured proxies, scroll to [Configured Proxies (no balancer)](#configured-proxies-no-balancer).
|
||||
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
client ─▶ your balancer ─▶ OpenSERP worker ─▶ search engine
|
||||
owns: applies:
|
||||
- proxy provider - the proxy you supply
|
||||
- country/class - sticky cookies/profile per session
|
||||
- session minting - cache keyed by market metadata
|
||||
- rotation policy - stable typed errors
|
||||
- usage accounting
|
||||
```
|
||||
|
||||
The balancer is the source of truth for _which_ proxy to use and _when_ to rotate. OpenSERP is stateless w.r.t. provider choice — it just executes the search through whatever proxy URL you hand it and reports back what happened.
|
||||
|
||||
## Worker Configuration
|
||||
|
||||
Enable the request-proxy-URL path on every worker fronted by your balancer:
|
||||
|
||||
```yaml
|
||||
proxies:
|
||||
allow_request_proxy_url: true # required for X-Proxy-URL to be honored
|
||||
lanes:
|
||||
enabled: true
|
||||
max_lanes: 100 # LRU cap on sticky lanes per worker
|
||||
drop_cookies_on_challenge: true
|
||||
|
||||
app:
|
||||
max_processes: 4 # LRU cap on Chrome processes per worker
|
||||
idle_ttl: 10m # close a Chrome that has not served traffic for this long
|
||||
```
|
||||
|
||||
Worker rejects `X-Proxy-URL` with `400 bad_request` (`reason=REQUEST_PROXY_URL_DISABLED`) when the flag is off. Keep this off on any worker reachable by untrusted clients.
|
||||
|
||||
CORS already lists every header the balancer sends; if you customise `cors.allow_headers`, keep `X-Proxy-URL, X-Proxy-Country, X-Proxy-Class, X-Proxy-Provider, X-Proxy-Session-ID, X-Tenant, X-Use-Proxy, X-Request-ID`.
|
||||
|
||||
## Request Contract
|
||||
|
||||
### Headers your balancer sends
|
||||
|
||||
| Header | Required | Purpose |
|
||||
| --------------------- | :------: | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `X-Proxy-URL` | ✅ | The actual proxy URL to route this request through. `http://`, `https://`, or unauthenticated `socks5://`/`socks5h://`. Authenticated SOCKS is rejected. |
|
||||
| `X-Proxy-Country` | ⚠️ | Two-letter market code (`us`, `de`). **Cache key**. Without it, proxied responses bypass cache. |
|
||||
| `X-Proxy-Class` | ⚠️ | `datacenter`, `residential`, `mobile`, etc. **Cache key**. |
|
||||
| `X-Proxy-Provider` | ⚠️ | Provider id (`webshare`, `brightdata`, `internal`). **Cache key**. |
|
||||
| `X-Proxy-Session-ID` | 🟢 | Sticky session id. **Lane key** — same id reuses cookies and profile. Rotate to get a clean lane. |
|
||||
| `X-Tenant` | 🟢 | Multi-tenant scope. Lanes become `tenant + engine + session_id`. |
|
||||
| `X-Use-Proxy: direct` | 🟢 | Force-disable proxy for this request, even if `X-Proxy-URL` is set. |
|
||||
| `X-Request-ID` | 🟢 | Pass-through correlation id (UUID v7 generated if absent). |
|
||||
|
||||
✅ required, ⚠️ recommended (cache won't engage without it), 🟢 optional.
|
||||
|
||||
### Precedence
|
||||
|
||||
1. `X-Use-Proxy: direct`
|
||||
2. `X-Proxy-URL` (when allowed)
|
||||
3. `X-Use-Proxy: <tag>`
|
||||
4. Per-engine configured tag (worker config)
|
||||
5. `proxies.global` (worker config)
|
||||
6. Direct (no proxy)
|
||||
|
||||
### Endpoints
|
||||
|
||||
| Path | What |
|
||||
| ---------------------- | ---------------------------------------------------------- |
|
||||
| `GET /{engine}/search` | Single engine: `google`, `yandex`, `baidu`, `bing`, `duck` |
|
||||
| `GET /{engine}/image` | Single engine image search |
|
||||
| `GET /mega/search` | Parallel across all (or `?engines=...`) engines |
|
||||
| `GET /mega/image` | Parallel image search |
|
||||
| `GET /stats/proxy` | Pool, lane, and Chrome-process stats |
|
||||
| `GET /health` | Engine + circuit-breaker health |
|
||||
| `GET /ready` | `503 draining` during graceful shutdown |
|
||||
|
||||
Full schema at `/openapi.yaml` and Swagger UI at `/docs`.
|
||||
|
||||
### Example call
|
||||
|
||||
```bash
|
||||
curl -G \
|
||||
-H "X-Proxy-URL: http://USER:PASS@proxy.example:8080" \
|
||||
-H "X-Proxy-Country: us" \
|
||||
-H "X-Proxy-Class: residential" \
|
||||
-H "X-Proxy-Provider: webshare" \
|
||||
-H "X-Proxy-Session-ID: sid-abc" \
|
||||
--data-urlencode "text=golang" \
|
||||
--data-urlencode "lang=EN" \
|
||||
--data-urlencode "limit=10" \
|
||||
http://worker.internal:7000/google/search
|
||||
```
|
||||
|
||||
## Response Contract
|
||||
|
||||
Every response carries:
|
||||
|
||||
```text
|
||||
X-Request-ID: 01HXYZ... # mirrors meta.request_id
|
||||
X-Proxy-Mode: off | tag_pool | request_url # what actually ran
|
||||
X-Proxy-Used: direct | http://proxy.example:8080 | multiple | mixed | pooled
|
||||
X-Proxy-Tag: us # only when X-Proxy-Mode=tag_pool
|
||||
X-Cache: HIT | MISS | BYPASS # only when cache is enabled
|
||||
```
|
||||
|
||||
A balancer-driven request always sees `X-Proxy-Mode: request_url` (unless overridden via `X-Use-Proxy: direct`). `X-Proxy-Used` is the masked `scheme://host:port` of the proxy you sent — credentials are stripped. Use these headers to confirm OpenSERP actually applied your proxy and didn't quietly fall back.
|
||||
|
||||
Successful body is the v1 envelope (`/openapi.yaml#/components/schemas/SearchEnvelope`).
|
||||
|
||||
## Error Playbook
|
||||
|
||||
All errors are JSON of shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"error": "<stable code>",
|
||||
"code": 503,
|
||||
"message": "<human readable>",
|
||||
"reason": "<sub-code, only on 400>",
|
||||
"meta": {
|
||||
"engine": "google",
|
||||
"proxy_used": "http://proxy.example:8080",
|
||||
"proxy_country": "us",
|
||||
"proxy_class": "residential",
|
||||
"proxy_provider": "webshare",
|
||||
"proxy_session_id": "sid-abc"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Credentials are **never** present in `meta.proxy_used`, response headers, logs, or stats.
|
||||
|
||||
| HTTP | `error` | What it means | Balancer action |
|
||||
| ---: | -------------------------------------------- | ---------------------------------------------------------------------------------------------- | ---------------------------------------------------------- |
|
||||
| 400 | `bad_request` | Bad client input. See `reason`. | Don't retry. Surface to caller. |
|
||||
| 400 | `bad_request` (`REQUEST_PROXY_URL_DISABLED`) | Worker has the feature off. | Misconfigured worker. Page oncall. |
|
||||
| 400 | `bad_request` (`UNSUPPORTED_PROXY_SCHEME`) | Authenticated SOCKS in browser mode. | Don't send authenticated SOCKS. Use HTTP/HTTPS. |
|
||||
| 403 | `blocked` | The search engine returned 403 to the proxy. | Rotate session id. Maybe rotate proxy. |
|
||||
| 429 | `captcha_detected` | Captcha challenge page. Worker has dropped this lane's cookies. | Rotate session id. Cool the proxy. |
|
||||
| 429 | `rate_limited` | Engine returned 429. | Slow the proxy. Rotate session id. |
|
||||
| 502 | `parser_failure` | SERP parser drift on a successfully fetched page. | Don't blame the proxy. Page oncall — engine update needed. |
|
||||
| 502 | `engine_internal` | Engine panic recovered, or all engines failed (mega). | Retry once on a different worker. |
|
||||
| 503 | `proxy_connect` | TCP/TLS to the proxy failed. | Mark proxy bad. Retry on another proxy. |
|
||||
| 503 | `proxy_auth` | 407 Proxy Auth Required. | Credentials wrong/expired. Don't retry the same one. |
|
||||
| 503 | `proxy_timeout` | Network timeout on the proxy path. | Retry on another proxy. |
|
||||
| 503 | `proxy_unavailable` | No healthy proxy left in the configured tag pool (worker-side, not your concern in SaaS mode). | Worker config issue. |
|
||||
| 504 | `search_timeout` | Required SERP elements never appeared before timeout. | Retry once. If repeated, page oncall. |
|
||||
|
||||
### Decision rules for the balancer
|
||||
|
||||
- **Rotate session id** (mint a new `X-Proxy-Session-ID`) on `captcha_detected`, `blocked`, `rate_limited`. The lane keeps its profile but cookies are dropped on captcha; a new id starts fresh anyway.
|
||||
- **Mark proxy bad** on `proxy_connect`, `proxy_auth`, `proxy_timeout`. Don't degrade the proxy on captcha/parser/engine errors — those aren't the proxy's fault.
|
||||
- **Don't retry** on 400. Those are client bugs.
|
||||
|
||||
## Sticky Lanes
|
||||
|
||||
A **lane** is a (tenant, engine, session) tuple owned by one worker.
|
||||
|
||||
```yaml
|
||||
proxies:
|
||||
lanes:
|
||||
enabled: true
|
||||
max_lanes: 100
|
||||
drop_cookies_on_challenge: true
|
||||
```
|
||||
|
||||
What a lane holds:
|
||||
|
||||
- The browser **profile** picked at first use (UA, viewport, languages, UA-CH brand list).
|
||||
- The **cookies** harvested during navigation.
|
||||
|
||||
Lane key:
|
||||
|
||||
- `tenant + engine + session_id` when `X-Tenant` is present.
|
||||
- `engine + session_id` otherwise.
|
||||
- If `X-Proxy-Session-ID` is missing, the lane id is derived from `sha256(host:port|username)[:16]`. The password is never part of the key, so rotating credentials on the same proxy keeps the lane.
|
||||
|
||||
What invalidates a lane:
|
||||
|
||||
- `X-Proxy-Session-ID` change → fresh lane (the old one stays warm until LRU evicts).
|
||||
- Captcha response → cookies dropped, profile retained (when `drop_cookies_on_challenge: true`).
|
||||
- `max_lanes` LRU eviction.
|
||||
|
||||
What does **not** invalidate a lane:
|
||||
|
||||
- Block, rate-limit, parser, engine, or proxy-network errors. The balancer decides whether to rotate.
|
||||
|
||||
If your balancer issues a sticky-session credential pattern (e.g. Bright Data `session-SID`), reuse the same `X-Proxy-Session-ID` for the same upstream sticky window. When you rotate, change both the proxy URL session token AND the `X-Proxy-Session-ID` in the same step.
|
||||
|
||||
## Cache
|
||||
|
||||
Cache key includes query fields + `country + class + provider`. It does **not** include the proxy URL, username, password, or session id.
|
||||
|
||||
Behavior with proxied requests:
|
||||
|
||||
- All three of country/class/provider missing → request bypasses cache (`X-Cache: BYPASS`).
|
||||
- Country missing but `lang` present → falls back to `lang` as a weak market hint.
|
||||
- Cross-engine fallback responses (`X-Fallback-Engine` set) are not cached.
|
||||
|
||||
Translation: **send country/class/provider on every proxied request** if you want cache hits.
|
||||
|
||||
## Browser Process Pool
|
||||
|
||||
OpenSERP keeps one Chrome process per _authenticated proxy identity_ (`scheme + host + port + username`):
|
||||
|
||||
| Upstream proxy | Chrome used |
|
||||
| ----------------------------------------------- | -------------------------------------------- |
|
||||
| `http://userA:passA@proxy:8080` | Dedicated Chrome `[http\|proxy:8080\|userA]` |
|
||||
| `http://userA:passB@proxy:8080` (rotated pass) | Same Chrome (password ignored in key) |
|
||||
| `http://userB:pass@proxy:8080` (different user) | New dedicated Chrome |
|
||||
| `socks5://proxy:1080` (unauthenticated) | Shared "no-auth" Chrome, per-context proxy |
|
||||
| no proxy / direct | Shared "no-auth" Chrome |
|
||||
|
||||
Why dedicated processes for authenticated HTTP proxies: Chrome's per-`BrowserContext` auth callback is process-global and only answers the _next_ pending challenge — so concurrent requests with different credentials would race and subresources hang. Launching Chrome with `--proxy-server=...` per identity lets the OS-level Chrome auth path handle 407s natively for the main document AND every subresource.
|
||||
|
||||
The pool grows lazily and is bounded by `app.max_processes` with LRU eviction, plus an idle sweeper that closes Chromes idle for `app.idle_ttl`. **One Chrome serves many concurrent requests** via per-page `BrowserContext` isolation — `max_processes` does NOT bound concurrent search requests, only the number of distinct authenticated proxy identities a worker can keep warm.
|
||||
|
||||
### Sizing the pool
|
||||
|
||||
If you expect `N` distinct authenticated identities to hit one worker concurrently, set `max_processes ≥ N`. Below that, the pool LRU-closes Chromes mid-burst, which adds Chrome-startup latency to the next request that needs the evicted identity.
|
||||
|
||||
Rough memory budget: each Chrome ≈ 150-300 MiB resident. `max_processes: 4` → plan for ~1 GiB of Chrome RAM per worker, plus the rest of the Go process.
|
||||
|
||||
`/debug/fingerprint-check` (when `app.debug_endpoints: true`) is intentionally **not** pooled — it spawns a fresh Chrome per call so you can verify each profile in isolation.
|
||||
|
||||
## Stats Endpoint
|
||||
|
||||
`GET /stats/proxy` returns:
|
||||
|
||||
```json
|
||||
{
|
||||
"configured_count": 0,
|
||||
"healthy_count": 0,
|
||||
"unhealthy_count": 0,
|
||||
"request_proxy_url_enabled": true,
|
||||
"lanes": {
|
||||
"active": 12,
|
||||
"evicted_lru": 7,
|
||||
"cookies_dropped": 20
|
||||
},
|
||||
"browser_processes": {
|
||||
"active": 3,
|
||||
"max": 4,
|
||||
"evicted_lru": 12,
|
||||
"evicted_idle": 5
|
||||
},
|
||||
"tags": {},
|
||||
"entries": []
|
||||
}
|
||||
```
|
||||
|
||||
Watch for:
|
||||
|
||||
- `browser_processes.evicted_lru` rising → bump `max_processes`.
|
||||
- `browser_processes.evicted_idle` rising while `active` stays low → traffic is bursty; that's healthy.
|
||||
- `lanes.cookies_dropped` rising → the proxy is hitting captchas; consider rotating the session more aggressively.
|
||||
|
||||
## Provider Examples
|
||||
|
||||
Bright Data (residential, sticky session):
|
||||
|
||||
```text
|
||||
X-Proxy-URL: http://brd-customer-CUSTOMER-zone-res-country-us-session-SID:PASS@brd.superproxy.io:22225
|
||||
X-Proxy-Provider: brightdata
|
||||
X-Proxy-Class: residential
|
||||
X-Proxy-Country: us
|
||||
X-Proxy-Session-ID: SID
|
||||
```
|
||||
|
||||
Webshare:
|
||||
|
||||
```text
|
||||
X-Proxy-URL: http://USER:PASS@p.webshare.io:80
|
||||
X-Proxy-Provider: webshare
|
||||
X-Proxy-Class: datacenter
|
||||
X-Proxy-Country: us
|
||||
```
|
||||
|
||||
Internal HTTP datacenter:
|
||||
|
||||
```text
|
||||
X-Proxy-URL: http://user:pass@dc-proxy.example:8080
|
||||
X-Proxy-Provider: internal
|
||||
X-Proxy-Class: datacenter
|
||||
X-Proxy-Country: de
|
||||
```
|
||||
|
||||
Authenticated SOCKS proxies are rejected by browser mode because Chrome cannot safely answer SOCKS auth challenges via the DevTools auth callback. Unauthenticated `socks5://` and `socks5h://` are accepted on both `X-Proxy-URL` and configured pools.
|
||||
|
||||
## Configured Proxies (no balancer)
|
||||
|
||||
For OSS and local deployments without a balancer, OpenSERP can manage proxies directly:
|
||||
|
||||
```yaml
|
||||
proxies:
|
||||
global: http://user:pass@127.0.0.1:8080 # one proxy for everything
|
||||
|
||||
# OR a tagged pool:
|
||||
proxies:
|
||||
entries:
|
||||
- url: http://user:pass@proxy-us.example:8080
|
||||
tags: [default, us]
|
||||
- url: socks5h://127.0.0.1:1080
|
||||
tags: [eu]
|
||||
health:
|
||||
failure_threshold: 3 # disable a proxy after N consecutive network errors
|
||||
|
||||
google:
|
||||
proxy: default # opt this engine into the "default" tag pool
|
||||
```
|
||||
|
||||
Per-request override (no balancer needed):
|
||||
|
||||
```bash
|
||||
curl -H "X-Use-Proxy: us" "http://127.0.0.1:7000/google/search?text=golang"
|
||||
curl -H "X-Use-Proxy: direct" "http://127.0.0.1:7000/google/search?text=golang"
|
||||
```
|
||||
|
||||
Pool health: a tag pool that exhausts (every member disabled) goes into a 5-minute quarantine. After quarantine, one proxy is re-enabled as a recovery probe. Network errors degrade health; captcha/parser/engine errors do not.
|
||||
Reference in New Issue
Block a user