diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9bf976..52f6aa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,14 +23,22 @@ jobs: go-version: "1.24" cache: true + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + - name: Run unit tests run: go test -race -count=1 ./... - name: Run go vet run: go vet ./... + - name: Lint OpenAPI spec + run: npx --yes @redocly/cli lint docs/openapi.yaml + - name: Build run: go build . - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6 \ No newline at end of file + uses: golangci/golangci-lint-action@v6 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..6e99915 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,120 @@ +# Contributing to OpenSERP + +## Development Setup + +### Prerequisites + +- Go 1.24+ +- Chromium/Chrome (only required for browser-mode work and integration tests) +- Optional: Docker + +### Clone, build, run + +```bash +git clone https://github.com/karust/openserp.git +cd openserp +go build -o openserp . +./openserp serve +``` + +### Test commands + +Unit tests (default, no browser/network assumptions): + +```bash +go test -race ./... +``` + +Integration tests (explicitly enabled): + +```bash +OPENSERP_INTEGRATION_TESTS=1 go test -race -timeout=120s ./... +``` + +Notes: + +- Integration tests are gated by `testutil.RequireIntegration(t)`. +- Do not create browser instances in `init()` or package-level variables. + +## Adding a New Search Engine + +### 1) Create engine package + +Create a new folder (example: `myengine/`) with: + +- `myengine/url.go` (`BuildURL`, and `BuildImageURL` when image support exists) +- `myengine/search.go` (browser mode implementation) +- `myengine/search_raw.go` (optional raw mode implementation) + +### 2) Implement `core.SearchEngine` + +Your engine type must implement: + +- `Search(core.Query) ([]core.SearchResult, error)` +- `SearchImage(core.Query) ([]core.SearchResult, error)` +- `IsInitialized() bool` +- `Name() string` +- `GetRateLimiter() *rate.Limiter` + +Use the existing engines (for example `google/`) as the reference pattern. + +### 3) Register the engine in server wiring + +Update [`cmd/serve.go`](cmd/serve.go): + +- Add engine spec in `browserEngineSpecs()` +- Add raw-mode handling if raw support exists + +### 4) Add config block + +Update [`config.yaml`](config.yaml) with your engine section: + +- `rate_requests` +- `rate_burst` +- optional `proxy` tag +- optional engine-specific fields + +### 5) Add tests + +- URL builder tests (table-driven) +- Parser tests (prefer deterministic fixtures in `testdata/`) +- Integration tests guarded by `testutil.RequireIntegration(t)` + +## Code Style and Quality Checks + +Run these before opening a PR: + +```bash +gofmt -w . +go vet ./... +golangci-lint run +go test -race ./... +``` + +Guidelines: + +- Return `error` values instead of panicking in library code. +- Reuse existing patterns in `core/` and existing engines. +- Add comments only for non-obvious decisions (why, not what). + +## Test Categories + +- Unit tests: deterministic tests that run with `go test ./...` and do not require browser/network. +- Integration tests: live/browser/network dependent tests gated by `OPENSERP_INTEGRATION_TESTS=1`. + +When adding tests, keep unit and integration behavior clearly separated. + +## Pull Request Process + +For each PR: + +1. Describe what changed and why. +2. Link the related issue (if available). +3. Include or update tests for behavior changes. +4. Include updated docs when API/config/contracts change. + +If you change API behavior, update: + +- [`docs/openapi.yaml`](docs/openapi.yaml) +- [`README.md`](README.md) +- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) when flow/design changes diff --git a/core/server.go b/core/server.go index 54172f9..73dc354 100644 --- a/core/server.go +++ b/core/server.go @@ -4,12 +4,14 @@ import ( "encoding/json" "errors" "fmt" + "html" "runtime" "sort" "strings" "time" "github.com/gofiber/fiber/v2" + apidocs "github.com/karust/openserp/docs" "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -84,6 +86,9 @@ func NewServerWithOptions(host string, port int, opts ServerOptions, searchEngin } app.Use(RequestLoggerMiddleware()) + app.Get("/openapi.yaml", serv.handleOpenAPISpec) + app.Get("/docs", serv.handleSwaggerUI) + app.Get("/docs/", serv.handleSwaggerUI) app.Get("/health", serv.handleHealthCheck) app.Get("/stats", serv.handleStats) app.Get("/stats/cache", serv.handleCacheStats) @@ -563,6 +568,45 @@ func (s *Server) applyProxyHeaders(c *fiber.Ctx, meta ProxyExecutionMeta) { c.Set("X-Proxy-Used", used) } +func (s *Server) handleOpenAPISpec(c *fiber.Ctx) error { + c.Set("Content-Type", "application/yaml; charset=utf-8") + return c.Send(apidocs.OpenAPIYAML) +} + +func (s *Server) handleSwaggerUI(c *fiber.Ctx) error { + const specPath = "/openapi.yaml" + page := fmt.Sprintf(` + + + + + OpenSERP API Docs + + + + +
+ + + +`, html.EscapeString(specPath)) + c.Set("Content-Type", "text/html; charset=utf-8") + return c.SendString(page) +} + func (s *Server) Listen() error { return s.app.Listen(s.addr) } diff --git a/core/server_test.go b/core/server_test.go index 346e004..6aa51b2 100644 --- a/core/server_test.go +++ b/core/server_test.go @@ -3,6 +3,7 @@ package core import ( "encoding/json" "errors" + "io" "net/http" "net/http/httptest" "strings" @@ -72,6 +73,52 @@ func requestWithHeader(t *testing.T, s *Server, path string, header string, valu return resp } +func TestOpenAPISpecEndpoint(t *testing.T) { + engine := &engineMock{name: "google", initialized: true} + srv := NewServerWithOptions("127.0.0.1", 7107, DefaultServerOptions(), engine) + + resp := request(t, srv, "/openapi.yaml") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected /openapi.yaml to return 200, got %d", resp.StatusCode) + } + if got := resp.Header.Get("Content-Type"); !strings.Contains(got, "application/yaml") { + t.Fatalf("expected YAML content-type, got %q", got) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read /openapi.yaml body: %v", err) + } + if !strings.Contains(string(body), "openapi: 3.0.3") { + t.Fatalf("expected OpenAPI version marker in body") + } +} + +func TestDocsEndpointServesSwaggerUI(t *testing.T) { + engine := &engineMock{name: "google", initialized: true} + srv := NewServerWithOptions("127.0.0.1", 7108, DefaultServerOptions(), engine) + + resp := request(t, srv, "/docs") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected /docs to return 200, got %d", resp.StatusCode) + } + if got := resp.Header.Get("Content-Type"); !strings.Contains(got, "text/html") { + t.Fatalf("expected HTML content-type, got %q", got) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read /docs body: %v", err) + } + content := string(body) + if !strings.Contains(content, "SwaggerUIBundle") { + t.Fatalf("expected SwaggerUI bundle script on docs page") + } + if !strings.Contains(content, "/openapi.yaml") { + t.Fatalf("expected docs page to reference /openapi.yaml") + } +} + func TestInvalidQueryParametersReturnJSONError(t *testing.T) { engine := &engineMock{name: "google", initialized: true} srv := NewServerWithOptions("127.0.0.1", 7104, DefaultServerOptions(), engine) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..37807f1 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,265 @@ +# OpenSERP Architecture + +## 1. Overview + +OpenSERP is a Go API + CLI for search results extraction from Google, Yandex, Baidu, Bing, and DuckDuckGo. + +It supports two execution modes: + +- Browser mode (default): headless Chromium via `go-rod`, with engine-specific DOM parsing. +- Raw HTTP mode: direct requests + HTML parsing (`goquery`) for engines that implement raw parsing. + +Browser mode is the primary path and supports all engines. Raw mode currently supports Google, Yandex, and Baidu only. + +## 2. Directory Structure + +```text +openserp/ +├── main.go # Entry point, executes cmd.RootCmd +├── AGENTS.md # Contributor + agent project guidance +├── README.md # User-facing quickstart and API overview +├── config.yaml # Runtime configuration (loaded by Viper) +├── docs/ +│ ├── ARCHITECTURE.md # This architecture reference +│ ├── openapi.yaml # OpenAPI 3.0 specification +│ └── embed.go # Embeds openapi.yaml for /openapi.yaml endpoint +├── cmd/ +│ ├── root.go # Cobra root command + Viper config binding/defaults +│ ├── serve.go # HTTP server bootstrap, engine wiring, browser pooling +│ ├── search.go # CLI one-shot search command +│ └── proxy_policy.go # Proxy policy mapping from config to runtime +├── core/ +│ ├── common.go # Shared domain types: Query, SearchResult, SearchEngine +│ ├── server.go # Fiber routes, request handlers, cache/proxy headers +│ ├── middleware.go # CORS, request logging, JSON error envelope +│ ├── browser.go # Chromium navigation lifecycle and page orchestration +│ ├── http_client.go # Raw HTTP client (uTLS fingerprinting) +│ ├── resilient.go # Retry + CB + rate limiting + proxy orchestration +│ ├── retry.go # Backoff retry runner and retry conditions +│ ├── circuit_breaker.go # Per-engine circuit breaker state machine +│ ├── cache.go # In-memory TTL cache for API responses +│ ├── proxy.go # Proxy normalization, pools, health/rotation, stats +│ ├── logger.go # Logging setup helpers +│ └── captcha.go # Captcha-related helpers/errors +├── google/ # Google engine implementation +│ ├── url.go # URL builders +│ ├── search.go # Browser mode parser +│ └── search_raw.go # Raw HTTP parser +├── yandex/ # Yandex engine implementation +│ ├── url.go +│ ├── search.go +│ └── search_raw.go +├── baidu/ # Baidu engine implementation +│ ├── url.go +│ ├── search.go +│ └── search_raw.go +├── bing/ # Bing engine implementation (browser-only) +│ ├── url.go +│ └── search.go +├── duckduckgo/ # DuckDuckGo engine implementation (browser-only) +│ ├── url.go +│ └── search.go +├── testutil/ # Integration gating and shared test fixtures/helpers +└── .github/workflows/ci.yml # CI checks (test/vet/build/lint/openapi lint) +``` + +## 3. Key Interfaces and Types + +### `core.SearchEngine` + +Contract for all engines: + +- `Search(Query) ([]SearchResult, error)` for web results +- `SearchImage(Query) ([]SearchResult, error)` for image results +- `IsInitialized() bool` for health readiness +- `Name() string` for endpoint and stats identity +- `GetRateLimiter() *rate.Limiter` for per-engine throttling + +### `core.Query` + +Parsed from query parameters and request headers: + +- `Text` (`text`) +- `LangCode` (`lang`) +- `DateInterval` (`date`, format `YYYYMMDD..YYYYMMDD`) +- `Filetype` (`file`) +- `Site` (`site`) +- `Limit` (`limit`, default `25`) +- `Start` (`start`, default `0`) +- `Filter` (`filter`, default `true`) +- `Answers` (`answers`, default `false`) +- `ProxyOverride` (`X-Use-Proxy` header: `` or `direct`) +- Internal runtime fields: `ProxyURL`, `Insecure` + +Validation summary: + +- `start` must be `>= 0` +- At least one of `text`, `site`, or `file` must be non-empty +- Invalid query parsing is returned as JSON error response + +### `core.SearchResult` + +Single SERP item shape: + +- `rank` (int) +- `url` (string) +- `title` (string) +- `description` (string) +- `ad` (bool) + +Mega endpoints return `core.MegaSearchResult`, which extends `SearchResult` with: + +- `engine` (string) + +## 4. Request Flow + +```text +HTTP request + -> Fiber router + -> handleDedicatedEndpoint / handleMegaEndpoint + -> Query.InitFromContext + -> ResilientSearcher.SearchPrimary/SearchWithFallback (or mega parallel search) + -> CircuitBreaker.AllowRequest + -> RateLimiter.Wait + -> Proxy policy resolution and proxy selection + -> RetryableSearch (backoff/retry loop) + -> Engine.Search / Engine.SearchImage + Browser path: Browser.Navigate(url) -> DOM parse -> []SearchResult + Raw path: raw HTTP request -> goquery parse -> []SearchResult + -> De-duplication (mega endpoints) + -> Cache.Set (if enabled and cacheable) + -> JSON response + X-Cache/X-Proxy-*/X-Fallback-Engine headers +``` + +## 5. Browser vs Raw Mode + +### Browser Mode (default) + +- Enabled when `server.raw_requests: false` +- Uses Chromium + `go-rod` navigation and page parsing +- Supported engines: Google, Yandex, Baidu, Bing, DuckDuckGo +- Best compatibility, but heavier resource usage + +### Raw HTTP Mode + +- Enabled when `server.raw_requests: true` +- Uses direct HTTP + HTML parsing without launching a browser +- Supported engines: Google, Yandex, Baidu +- Faster/lighter, but less reliable for anti-bot protected pages and missing image support + +Mode switch options: + +- Config: `server.raw_requests` +- CLI flag: `--raw` + +## 6. Resilience Stack + +The effective request protection sequence is: + +1. Rate limiter (`engine.GetRateLimiter().Wait`) +2. Retry with exponential backoff (`core/retry.go`) +3. Circuit breaker per engine (`core/circuit_breaker.go`) +4. Proxy selection/rotation + health tracking (`core/proxy.go`) +5. Response cache (API-level TTL cache in `core/cache.go`) + +Important behaviors: + +- `ErrCaptcha` is non-retryable. +- `ErrProxyUnavailable` does not record circuit-breaker failure. +- Dedicated endpoints are engine-pure by default (`allow_endpoint_fallback: false`). +- Fallback responses are not cached on dedicated endpoints. + +## 7. Config Reference + +Defaults below are the shipped defaults in `config.yaml` (if present). If the config file is missing, fallback defaults from `cmd/root.go` are applied. + +### `server` + +| Key | Default | Description | +| --- | --- | --- | +| `server.host` | `0.0.0.0` | API bind host | +| `server.port` | `7000` | API bind port | +| `server.debug` | `false` | Debug mode, forces headful browser | +| `server.verbose` | `true` | Info-level request logs | +| `server.raw_requests` | `false` | `true` = raw HTTP mode | +| `server.insecure` | `true` | Allow insecure TLS connections | + +### `app` + +| Key | Default | Description | +| --- | --- | --- | +| `app.timeout` | `15` | Request timeout in seconds | +| `app.browser_path` | `""` | Custom browser binary path | +| `app.head` | `false` | Headful browser UI | +| `app.leakless` | `false` | Force browser process cleanup | +| `app.leave_head` | `false` | Keep browser tabs open | +| `app.stealth` | `false` | Enable stealth plugin | + +### `proxies` + +| Key | Default | Description | +| --- | --- | --- | +| `proxies.global` | unset | Force single proxy for all engines | +| `proxies.entries[]` | empty | Tagged proxy pool entries (`url`, `tags`) | +| `proxies.health.failure_threshold` | `3` | Disable proxy after N failures | + +Per-engine optional proxy tag: + +- `google.proxy` +- `yandex.proxy` +- `baidu.proxy` +- `bing.proxy` +- `duckduckgo.proxy` + +### `cache` + +| Key | Default | Description | +| --- | --- | --- | +| `cache.ttl_seconds` | `60` | Response cache TTL (0 disables cache) | +| `cache.max_size` | `1000` | Max cached entries | + +### `resilience` + +| Key | Default | Description | +| --- | --- | --- | +| `resilience.max_retries` | `2` | Retry attempts per request | +| `resilience.allow_endpoint_fallback` | `false` | Allow dedicated endpoints to fallback to other engines | + +### `circuit_breaker` + +| Key | Default | Description | +| --- | --- | --- | +| `circuit_breaker.failures` | `5` | Failures before opening circuit | +| `circuit_breaker.recovery_seconds` | `60` | Open -> half-open wait time | +| `circuit_breaker.successes` | `2` | Half-open successes to close circuit | + +### `cors` + +| Key | Default | Description | +| --- | --- | --- | +| `cors.enabled` | `true` | Enable CORS middleware | +| `cors.allow_origins` | `"*"` | Allowed origins | +| `cors.allow_methods` | `"GET, POST, OPTIONS"` | Allowed methods | +| `cors.allow_headers` | `"Origin, Content-Type, Accept, Authorization, X-Use-Proxy"` | Allowed headers | +| `cors.max_age` | `86400` | Preflight cache max age (seconds) | + +### `2captcha` + +| Key | Default | Description | +| --- | --- | --- | +| `2captcha.apikey` | unset | Optional captcha solver key | + +### Engine rate-limit defaults + +For each engine (`google`, `yandex`, `baidu`, `bing`, `duckduckgo`): + +| Key | Default | Description | +| --- | --- | --- | +| `.rate_requests` | `4` | Average requests per minute | +| `.rate_burst` | `2` | Burst capacity | +| `.rate_seconds` | `60` (implicit) | Rate window seconds | +| `.selector_timeout` | `5` (implicit) | Selector wait timeout seconds | + +Google-only additional toggle: + +- `google.captcha` (default: `true`) diff --git a/docs/embed.go b/docs/embed.go new file mode 100644 index 0000000..b21fa32 --- /dev/null +++ b/docs/embed.go @@ -0,0 +1,8 @@ +package docs + +import _ "embed" + +// OpenAPIYAML is the bundled OpenAPI specification served by /openapi.yaml. +// +//go:embed openapi.yaml +var OpenAPIYAML []byte diff --git a/docs/openapi.yaml b/docs/openapi.yaml new file mode 100644 index 0000000..1287d90 --- /dev/null +++ b/docs/openapi.yaml @@ -0,0 +1,880 @@ +openapi: 3.0.3 +info: + title: OpenSERP API + version: 0.6.3 + description: > + OpenSERP provides dedicated and multi-engine search endpoints for Google, Yandex, + Baidu, Bing, and DuckDuckGo. Responses are normalized into a shared schema and + include runtime metadata via response headers. + license: + name: MIT + url: https://opensource.org/licenses/MIT +servers: + - url: http://127.0.0.1:7000 + description: Local default server +security: [] +tags: + - name: Search + description: Dedicated per-engine search endpoints + - name: Mega + description: Cross-engine aggregated search endpoints + - name: Health + description: Health and readiness endpoints + - name: Stats + description: Runtime statistics endpoints + - name: Docs + description: OpenAPI and Swagger UI endpoints +paths: + /{engine}/search: + get: + tags: [Search] + operationId: searchWeb + summary: Search web results from a specific engine + description: > + Engine path values are `google`, `yandex`, `baidu`, `bing`, and `duck` (`duck` maps + to DuckDuckGo internally). + parameters: + - $ref: "#/components/parameters/EnginePath" + - $ref: "#/components/parameters/TextQuery" + - $ref: "#/components/parameters/LangQuery" + - $ref: "#/components/parameters/DateQuery" + - $ref: "#/components/parameters/FileQuery" + - $ref: "#/components/parameters/SiteQuery" + - $ref: "#/components/parameters/LimitQuery" + - $ref: "#/components/parameters/StartQuery" + - $ref: "#/components/parameters/FilterQuery" + - $ref: "#/components/parameters/AnswersQuery" + - $ref: "#/components/parameters/UseProxyHeader" + responses: + "200": + description: Search results + headers: + X-Cache: + $ref: "#/components/headers/XCache" + X-Fallback-Engine: + $ref: "#/components/headers/XFallbackEngine" + X-Proxy-Mode: + $ref: "#/components/headers/XProxyMode" + X-Proxy-Tag: + $ref: "#/components/headers/XProxyTag" + X-Proxy-Used: + $ref: "#/components/headers/XProxyUsed" + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/SearchResult" + examples: + default: + value: + - rank: 1 + url: https://go.dev/doc/ + title: The Go Programming Language + description: Official Go language documentation. + ad: false + "503": + $ref: "#/components/responses/ServiceUnavailableError" + "404": + $ref: "#/components/responses/NotFoundError" + "500": + $ref: "#/components/responses/InternalServerError" + /{engine}/image: + get: + tags: [Search] + operationId: searchImages + summary: Search image results from a specific engine + parameters: + - $ref: "#/components/parameters/EnginePath" + - $ref: "#/components/parameters/TextQuery" + - $ref: "#/components/parameters/LangQuery" + - $ref: "#/components/parameters/DateQuery" + - $ref: "#/components/parameters/FileQuery" + - $ref: "#/components/parameters/SiteQuery" + - $ref: "#/components/parameters/LimitQuery" + - $ref: "#/components/parameters/StartQuery" + - $ref: "#/components/parameters/FilterQuery" + - $ref: "#/components/parameters/AnswersQuery" + - $ref: "#/components/parameters/UseProxyHeader" + responses: + "200": + description: Image search results + headers: + X-Cache: + $ref: "#/components/headers/XCache" + X-Fallback-Engine: + $ref: "#/components/headers/XFallbackEngine" + X-Proxy-Mode: + $ref: "#/components/headers/XProxyMode" + X-Proxy-Tag: + $ref: "#/components/headers/XProxyTag" + X-Proxy-Used: + $ref: "#/components/headers/XProxyUsed" + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/SearchResult" + examples: + default: + value: + - rank: 1 + url: https://upload.wikimedia.org/example/golang.png + title: Golang logo image + description: Example image result + ad: false + "503": + $ref: "#/components/responses/ServiceUnavailableError" + "404": + $ref: "#/components/responses/NotFoundError" + "500": + $ref: "#/components/responses/InternalServerError" + /mega/search: + get: + tags: [Mega] + operationId: megaSearch + summary: Search across multiple engines in parallel + parameters: + - $ref: "#/components/parameters/TextQuery" + - $ref: "#/components/parameters/LangQuery" + - $ref: "#/components/parameters/DateQuery" + - $ref: "#/components/parameters/FileQuery" + - $ref: "#/components/parameters/SiteQuery" + - $ref: "#/components/parameters/LimitQuery" + - $ref: "#/components/parameters/StartQuery" + - $ref: "#/components/parameters/FilterQuery" + - $ref: "#/components/parameters/AnswersQuery" + - $ref: "#/components/parameters/EnginesQuery" + - $ref: "#/components/parameters/UseProxyHeader" + responses: + "200": + description: Aggregated and de-duplicated results + headers: + X-Cache: + $ref: "#/components/headers/XCache" + X-Proxy-Mode: + $ref: "#/components/headers/XProxyMode" + X-Proxy-Tag: + $ref: "#/components/headers/XProxyTag" + X-Proxy-Used: + $ref: "#/components/headers/XProxyUsed" + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/MegaSearchResult" + examples: + default: + value: + - rank: 1 + url: https://go.dev/doc/ + title: The Go Programming Language + description: Official documentation. + ad: false + engine: google + - rank: 2 + url: https://pkg.go.dev/ + title: Go Packages + description: Go package documentation. + ad: false + engine: bing + "400": + $ref: "#/components/responses/BadRequestError" + "500": + $ref: "#/components/responses/InternalServerError" + /mega/image: + get: + tags: [Mega] + operationId: megaImageSearch + summary: Image search across multiple engines in parallel + parameters: + - $ref: "#/components/parameters/TextQuery" + - $ref: "#/components/parameters/LangQuery" + - $ref: "#/components/parameters/DateQuery" + - $ref: "#/components/parameters/FileQuery" + - $ref: "#/components/parameters/SiteQuery" + - $ref: "#/components/parameters/LimitQuery" + - $ref: "#/components/parameters/StartQuery" + - $ref: "#/components/parameters/FilterQuery" + - $ref: "#/components/parameters/AnswersQuery" + - $ref: "#/components/parameters/EnginesQuery" + - $ref: "#/components/parameters/UseProxyHeader" + responses: + "200": + description: Aggregated and de-duplicated image results + headers: + X-Cache: + $ref: "#/components/headers/XCache" + X-Proxy-Mode: + $ref: "#/components/headers/XProxyMode" + X-Proxy-Tag: + $ref: "#/components/headers/XProxyTag" + X-Proxy-Used: + $ref: "#/components/headers/XProxyUsed" + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/MegaSearchResult" + examples: + default: + value: + - rank: 1 + url: https://upload.wikimedia.org/example/gopher.png + title: Go Gopher + description: Example image + ad: false + engine: duckduckgo + "400": + $ref: "#/components/responses/BadRequestError" + "500": + $ref: "#/components/responses/InternalServerError" + /mega/engines: + get: + tags: [Mega] + operationId: listMegaEngines + summary: List available engines and runtime state + responses: + "200": + description: Engine list + content: + application/json: + schema: + $ref: "#/components/schemas/MegaEnginesResponse" + examples: + default: + value: + total: 5 + engines: + - name: google + initialized: true + circuit_state: closed + - name: bing + initialized: true + circuit_state: closed + "404": + $ref: "#/components/responses/NotFoundError" + /health: + get: + tags: [Health] + operationId: healthCheck + summary: Service health status + responses: + "200": + description: Healthy or degraded service + content: + application/json: + schema: + $ref: "#/components/schemas/HealthStatus" + examples: + healthy: + value: + status: healthy + uptime: 1h2m3s + engines: + - name: google + initialized: true + status: ready + system: + goroutines: 32 + memory_mb: 128 + go_version: go1.24.6 + "503": + description: Unhealthy service + content: + application/json: + schema: + $ref: "#/components/schemas/HealthStatus" + examples: + unhealthy: + value: + status: unhealthy + uptime: 12m10s + engines: + - name: google + initialized: false + status: not_initialized + system: + goroutines: 14 + memory_mb: 96 + go_version: go1.24.6 + "404": + $ref: "#/components/responses/NotFoundError" + /stats: + get: + tags: [Stats] + operationId: getStats + summary: Combined cache, proxy, and circuit-breaker stats + responses: + "200": + description: Runtime statistics + content: + application/json: + schema: + $ref: "#/components/schemas/StatsResponse" + examples: + default: + value: + cache: + status: true + entries: 10 + hits: 200 + misses: 15 + bypasses: 4 + evictions: 0 + ttl_seconds: 60 + max_size: 1000 + proxy: + configured_count: 2 + healthy_count: 2 + unhealthy_count: 0 + tags: + default: + configured: 2 + healthy: 2 + entries: + - proxy: http://proxy1:8080 + tags: [default] + healthy: true + failures: 0 + disabled: false + engines: + google: + tag: default + selected_proxy: pooled + circuit_breakers: + - engine: google + state: closed + failure_count: 0 + last_changed: 2026-04-15T08:00:00Z + "404": + $ref: "#/components/responses/NotFoundError" + /stats/cache: + get: + tags: [Stats] + operationId: getCacheStats + summary: Cache statistics only + responses: + "200": + description: Cache status + content: + application/json: + schema: + $ref: "#/components/schemas/CacheStats" + examples: + enabled: + value: + status: true + entries: 1 + hits: 1 + misses: 2 + bypasses: 1 + evictions: 0 + ttl_seconds: 60 + max_size: 1000 + disabled: + value: + status: false + "404": + $ref: "#/components/responses/NotFoundError" + /stats/proxy: + get: + tags: [Stats] + operationId: getProxyStats + summary: Proxy pool and per-engine proxy policy statistics + responses: + "200": + description: Proxy stats payload + content: + application/json: + schema: + $ref: "#/components/schemas/ProxyStats" + examples: + default: + value: + configured_count: 1 + healthy_count: 1 + unhealthy_count: 0 + tags: + us: + configured: 1 + healthy: 1 + entries: + - proxy: http://proxy-us:8080 + tags: [us] + healthy: true + failures: 0 + disabled: false + engines: + google: + tag: us + selected_proxy: pooled + yandex: + selected_proxy: direct + "404": + $ref: "#/components/responses/NotFoundError" + /stats/cb: + get: + tags: [Stats] + operationId: getCircuitBreakerStats + summary: Circuit breaker state per engine + responses: + "200": + description: Circuit breaker stats payload + content: + application/json: + schema: + $ref: "#/components/schemas/CircuitBreakerStatsResponse" + examples: + default: + value: + circuit_breakers: + - engine: google + state: open + failure_count: 5 + last_changed: 2026-04-15T08:00:00Z + retry_in: 43 + "404": + $ref: "#/components/responses/NotFoundError" + /openapi.yaml: + get: + tags: [Docs] + operationId: getOpenAPISpec + summary: Get raw OpenAPI YAML + responses: + "200": + description: OpenAPI YAML + content: + application/yaml: + schema: + type: string + "404": + $ref: "#/components/responses/NotFoundError" + /docs: + get: + tags: [Docs] + operationId: getSwaggerUI + summary: Swagger UI for interactive API docs + responses: + "200": + description: HTML page loading Swagger UI from CDN + content: + text/html: + schema: + type: string + "404": + $ref: "#/components/responses/NotFoundError" +components: + parameters: + EnginePath: + name: engine + in: path + required: true + description: Search engine endpoint alias (`duck` is DuckDuckGo). + schema: + type: string + enum: [google, yandex, baidu, bing, duck] + TextQuery: + name: text + in: query + required: false + description: > + Search query text. At least one of `text`, `site`, or `file` must be non-empty. + schema: + type: string + example: golang + LangQuery: + name: lang + in: query + required: false + description: Language code (engine-specific behavior). + schema: + type: string + example: EN + DateQuery: + name: date + in: query + required: false + description: Date interval in `YYYYMMDD..YYYYMMDD` format. + schema: + type: string + pattern: "^[0-9]{8}\\.\\.[0-9]{8}$" + example: 20250101..20250131 + FileQuery: + name: file + in: query + required: false + description: File extension filter (for engines that support it). + schema: + type: string + example: PDF + SiteQuery: + name: site + in: query + required: false + description: Site/domain filter. + schema: + type: string + example: github.com + LimitQuery: + name: limit + in: query + required: false + description: Maximum results to return. + schema: + type: integer + default: 25 + example: 10 + StartQuery: + name: start + in: query + required: false + description: Pagination offset (must be >= 0). + schema: + type: integer + minimum: 0 + default: 0 + example: 20 + FilterQuery: + name: filter + in: query + required: false + description: Duplicate filtering flag (primarily used by Google parser behavior). + schema: + type: boolean + default: true + example: true + AnswersQuery: + name: answers + in: query + required: false + description: Include answer box style results when supported. + schema: + type: boolean + default: false + example: false + EnginesQuery: + name: engines + in: query + required: false + description: > + Comma-separated engine list for mega endpoints. If omitted, all available engines are used. + schema: + type: string + example: google,bing,duckduckgo + UseProxyHeader: + name: X-Use-Proxy + in: header + required: false + description: > + Request-scoped proxy override. Use `direct` to disable proxy for the request or + pass a proxy tag name (for example `us`) to force a tagged proxy pool. + schema: + type: string + examples: + direct: + value: direct + tag: + value: us + headers: + XCache: + description: Cache status when cache is enabled (`HIT`, `MISS`, `BYPASS`). + schema: + type: string + XFallbackEngine: + description: Engine name used when dedicated endpoint fallback served the response. + schema: + type: string + XProxyMode: + description: Effective proxy mode for the request (`off` or `tag_pool`). + schema: + type: string + enum: [off, tag_pool] + XProxyTag: + description: Effective proxy tag when `X-Proxy-Mode=tag_pool`. + schema: + type: string + XProxyUsed: + description: Effective proxy target used (`direct`, masked proxy, `pooled`, `multiple`, `mixed`). + schema: + type: string + responses: + BadRequestError: + description: Invalid request parameters + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + examples: + invalidEngines: + value: + error: bad_request + code: 400 + message: No valid search engines specified + ServiceUnavailableError: + description: Search failed and no result could be produced + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + examples: + primaryFailed: + value: + error: service_unavailable + code: 503 + message: all search engines failed + NotFoundError: + description: Endpoint not found + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + examples: + notFound: + value: + error: not_found + code: 404 + message: Cannot GET /unknown + InternalServerError: + description: Internal error while parsing/handling request + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + examples: + parseError: + value: + error: server_error + code: 500 + message: invalid syntax + schemas: + SearchResult: + type: object + required: [rank, url, title, description, ad] + properties: + rank: + type: integer + example: 1 + url: + type: string + example: https://go.dev/doc/ + title: + type: string + example: The Go Programming Language + description: + type: string + example: Official Go language documentation. + ad: + type: boolean + example: false + MegaSearchResult: + allOf: + - $ref: "#/components/schemas/SearchResult" + - type: object + required: [engine] + properties: + engine: + type: string + example: google + ErrorResponse: + type: object + required: [error, code] + properties: + error: + type: string + example: service_unavailable + code: + type: integer + example: 503 + message: + type: string + example: all search engines failed + EngineHealth: + type: object + required: [name, initialized, status] + properties: + name: + type: string + example: google + initialized: + type: boolean + example: true + status: + type: string + enum: [ready, not_initialized, circuit_open] + example: ready + HealthStatus: + type: object + required: [status, uptime, engines, system] + properties: + status: + type: string + enum: [healthy, degraded, unhealthy] + uptime: + type: string + example: 1h12m3s + engines: + type: array + items: + $ref: "#/components/schemas/EngineHealth" + system: + type: object + additionalProperties: true + properties: + goroutines: + type: integer + memory_mb: + type: integer + go_version: + type: string + CacheStatsEnabled: + type: object + required: [status, entries, hits, misses, bypasses, evictions, ttl_seconds, max_size] + properties: + status: + type: boolean + enum: [true] + entries: + type: integer + hits: + type: integer + misses: + type: integer + bypasses: + type: integer + evictions: + type: integer + ttl_seconds: + type: integer + max_size: + type: integer + CacheStatsDisabled: + type: object + required: [status] + properties: + status: + type: boolean + enum: [false] + CacheStats: + oneOf: + - $ref: "#/components/schemas/CacheStatsEnabled" + - $ref: "#/components/schemas/CacheStatsDisabled" + ProxyTagSummary: + type: object + required: [configured, healthy] + properties: + configured: + type: integer + healthy: + type: integer + ProxyStatsEntry: + type: object + required: [proxy, tags, healthy, failures, disabled] + properties: + proxy: + type: string + example: http://proxy-us:8080 + tags: + type: array + items: + type: string + healthy: + type: boolean + failures: + type: integer + disabled: + type: boolean + ProxyEngineStats: + type: object + required: [selected_proxy] + properties: + tag: + type: string + selected_proxy: + type: string + example: direct + ProxyStats: + type: object + required: [configured_count, healthy_count, unhealthy_count, tags, entries] + properties: + configured_count: + type: integer + healthy_count: + type: integer + unhealthy_count: + type: integer + tags: + type: object + additionalProperties: + $ref: "#/components/schemas/ProxyTagSummary" + entries: + type: array + items: + $ref: "#/components/schemas/ProxyStatsEntry" + engines: + type: object + additionalProperties: + $ref: "#/components/schemas/ProxyEngineStats" + CircuitBreakerStat: + type: object + required: [engine, state, failure_count, last_changed] + properties: + engine: + type: string + example: google + state: + type: string + enum: [closed, open, half-open] + failure_count: + type: integer + last_changed: + type: string + format: date-time + retry_in: + type: integer + description: Seconds until next half-open attempt (present when state is open). + CircuitBreakerStatsResponse: + type: object + required: [circuit_breakers] + properties: + circuit_breakers: + type: array + items: + $ref: "#/components/schemas/CircuitBreakerStat" + StatsResponse: + type: object + required: [cache, proxy, circuit_breakers] + properties: + cache: + $ref: "#/components/schemas/CacheStats" + proxy: + $ref: "#/components/schemas/ProxyStats" + circuit_breakers: + type: array + items: + $ref: "#/components/schemas/CircuitBreakerStat" + MegaEngineInfo: + type: object + required: [name, initialized] + properties: + name: + type: string + example: google + initialized: + type: boolean + circuit_state: + type: string + enum: [closed, open, half-open] + MegaEnginesResponse: + type: object + required: [engines, total] + properties: + engines: + type: array + items: + $ref: "#/components/schemas/MegaEngineInfo" + total: + type: integer