mirror of
https://github.com/karust/openserp.git
synced 2026-08-15 21:15:56 +08:00
Merge pull request #22 from karust/dev
Add runtime proxy override, update readme
This commit is contained in:
Rustem Kamalov
161
README.md
161
README.md
@@ -6,18 +6,18 @@
|
||||
[](https://pkg.go.dev/github.com/karust/openserp)
|
||||
[](https://github.com/karust/openserp/releases)
|
||||
|
||||
<!--[](https://hub.docker.com/repository/docker/karust/openserp)-->
|
||||
<!-- [](https://hub.docker.com/repository/docker/karust/openserp) -->
|
||||
|
||||
**OpenSERP** provides free API and CLI access to multiple search engines including **[Google, Yandex, Baidu, Bing, DuckDuckGo]**. Get comprehensive search results without expensive API subscriptions!
|
||||
**OpenSERP** provides free API and CLI access to multiple search engines including **Google, Yandex, Baidu, Bing, and DuckDuckGo**. Get comprehensive search results without expensive API subscriptions!
|
||||
|
||||
## Features
|
||||
|
||||
- 🔍 **Multi-Engine Support**: Google, Yandex, Baidu, Bing, DuckDuckGo...
|
||||
- 🌐 **Megasearch**: Aggregate results from multiple engines simultaneously
|
||||
- 🖼 **Images**: Image search is also available!
|
||||
- 🎯 **Advanced Filtering**: Language, date range, file type, site-specific searches
|
||||
- 🌍 **Proxy Support**: HTTP/SOCKS5 proxy support
|
||||
- 🐳 **Docker Ready**: Easy deployment with Docker
|
||||
- 🔍 **Multi-engine** - search with dedicated endpoints for each engine
|
||||
- 🌐 **Megasearch** - cross-engine aggregation with deduplication
|
||||
- 🖼 **Images** - image search is also available
|
||||
- 🎯 **Advanced filters** - language, date range, file type, and site queries
|
||||
- 🌍 **Configurable** - proxy, cache, and resilient mode support
|
||||
- 🐳 **Docker-ready** - local and container deployment
|
||||
|
||||
## Quick Start⚡️
|
||||
|
||||
@@ -31,36 +31,36 @@ docker run -p 127.0.0.1:7000:7000 -it karust/openserp serve -a 0.0.0.0 -p 7000
|
||||
docker compose up --build
|
||||
```
|
||||
|
||||
### From Source
|
||||
### From source
|
||||
|
||||
```bash
|
||||
# Clone and build
|
||||
git clone https://github.com/karust/openserp.git
|
||||
cd openserp
|
||||
go build -o openserp .
|
||||
|
||||
# Run the server
|
||||
./openserp serve
|
||||
```
|
||||
|
||||
## 🌐 Megasearch & Megaimage - Search Everything at Once!
|
||||
## 🌐 Megasearch & Megaimage
|
||||
|
||||
**Megasearch** aggregates results from multiple engines simultaneously with automatic deduplication. **Megaimage** does the same for image searches!
|
||||
|
||||
### Megasearch (Web Results)
|
||||
Search all engines at once:
|
||||
|
||||
```bash
|
||||
# Search ALL engines at once
|
||||
curl "http://localhost:7000/mega/search?text=golang&limit=10"
|
||||
|
||||
# Pick specific engines
|
||||
curl "http://localhost:7000/mega/search?text=golang&engines=duckduckgo,bing&limit=15"
|
||||
|
||||
# Advanced filtering
|
||||
curl "http://localhost:7000/mega/search?text=Donald+Trump&engines=duckduckgo,bing&limit=20&date=20251005..20251005&lang=EN"
|
||||
curl "http://127.0.0.1:7000/mega/search?text=golang&limit=10"
|
||||
```
|
||||
|
||||
- API response example:
|
||||
Search only selected engines:
|
||||
|
||||
```bash
|
||||
curl "http://127.0.0.1:7000/mega/search?text=golang&engines=duckduckgo,bing&limit=15"
|
||||
```
|
||||
|
||||
Advanced filtering:
|
||||
|
||||
```bash
|
||||
curl "http://127.0.0.1:7000/mega/search?text=Donald+Trump&engines=duckduckgo,bing&limit=20&date=20251005..20251005&lang=EN"
|
||||
```
|
||||
|
||||
API response example:
|
||||
|
||||
```json
|
||||
[
|
||||
@@ -68,48 +68,46 @@ curl "http://localhost:7000/mega/search?text=Donald+Trump&engines=duckduckgo,bin
|
||||
"rank": 1,
|
||||
"url": "https://en.wikipedia.org/wiki/Golden_Retriever",
|
||||
"title": "Golden Retriever - Wikipedia",
|
||||
"description": "The Golden Retriever is a Scottish breed of retriever dog of medium size. It is characterised by a gentle and affectionate nature and a striking golden coat. It is a working dog, and registration is subject to successful completion of a working trial. [2] It is commonly kept as a companion dog and is among the most frequently registered breeds in several Western countries; some may compete in ...",
|
||||
"description": "The Golden Retriever is a Scottish breed of retriever dog of medium size. It is characterised by a gentle and affectionate nature and a striking golden coat.",
|
||||
"ad": false,
|
||||
"engine": "duckduckgo"
|
||||
},
|
||||
{
|
||||
"rank": 2,
|
||||
"url": "https://www.bing.com/ck/a?!&&p=6f15ac4589858d0a104cd6f55cc8e91e8d8d6da91f905b626921f67f2323a467JmltdHM9MTc1OTE5MDQwMA&ptn=3&ver=2&hsh=4&fclid=2357c2f4-6131-68de-359f-d48c607c691d&u=a1aHR0cHM6Ly93d3cuZ29sZGVucmV0cmlldmVyZm9ydW0uY29tL3RocmVhZHMvdW5kZXJzdGFuZGluZy13aHktZ29sZGVuLXJldHJpZXZlciVFMiU4MCU5OXMtbGlmZXNwYW4taGFsdmVkLWluLXRoZS1sYXN0LTM1LXllYXJzLjM1NzMyMi8&ntb=1",
|
||||
"title": "Golden Retriever Dog Forums\nhttps://www.goldenretrieverforum.com › threads › understanding-why-g...",
|
||||
"description": "Oct 20, 2024 · Back in the 1970s, Golden Retrievers routinely lived until 16 and 17 years old, they are now living until 9 or 10 years old. Golden Retrievers seem to be dying mostly of bone ...",
|
||||
"url": "https://www.bing.com/ck/a?!&&p=6f15ac4589858d0a104cd6f55cc8",
|
||||
"title": "Golden Retriever Dog Forums",
|
||||
"description": "Oct 20, 2024 · Back in the 1970s, Golden Retrievers routinely lived until 16 and 17 years old, they are now...",
|
||||
"ad": false,
|
||||
"engine": "bing"
|
||||
},
|
||||
{
|
||||
"rank": 3,
|
||||
"url": "http://www.baidu.com/link?url=2544q3ugc68j0scVxdpWCSX-gl2AmuCy1l7uRR3loIfS1hmJWMiJKW4MDGWoZrLE7X-ybu1L7T8PspoL7iy_dK",
|
||||
"title": "golden retrievers是什么意思_golden retrievers怎么读_解释_用法...",
|
||||
"description": "\n\n2025年9月21日golden retrievers 读音:美英 golden retrievers基本解释 金毛猎犬 分词解释 golden金(黄)色的 retrievers寻猎物犬( retriever的名词复数 ) 词组短语 golden retrieversfor sale出售金毛寻回犬 golden retrieversnear me我附近的金毛寻回犬 golden retrieverspuppies金毛寻回犬幼犬...\ndanci.gei6.com/golden...retrievers...",
|
||||
"url": "http://www.baidu.com/link?url==2544q3ugc68j0scVxdpWCSX-gl2AmuCy1l7uRR3loIfS1",
|
||||
"title": "golden retrievers是什么意思",
|
||||
"description": "2025年9月21日golden retrievers 读音:美英 golden retrievers基本解释 金毛猎犬 分词解释 golden金(黄)色的...",
|
||||
"ad": false,
|
||||
"engine": "baidu"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### Megaimage (Image Results)
|
||||
Image search:
|
||||
|
||||
```bash
|
||||
# Search images across ALL engines
|
||||
curl "http://localhost:7000/mega/image?text=golang logo&limit=20"
|
||||
curl "http://127.0.0.1:7000/mega/image?text=golang logo&limit=20"
|
||||
```
|
||||
|
||||
### Available Engines
|
||||
List available engines:
|
||||
|
||||
```bash
|
||||
# Check which engines are available
|
||||
curl "http://localhost:7000/mega/engines"
|
||||
curl "http://127.0.0.1:7000/mega/engines"
|
||||
```
|
||||
|
||||
**Available engines:** `google`, `yandex`, `baidu`, `bing`, `duckduckgo`
|
||||
|
||||
## 🔍 Individual Engine APIs
|
||||
|
||||
### Search Parameters
|
||||
Common query parameters:
|
||||
|
||||
| Parameter | Description | Example |
|
||||
| --------- | -------------------- | --------------------------------- |
|
||||
@@ -120,85 +118,62 @@ curl "http://localhost:7000/mega/engines"
|
||||
| `site` | Site-specific search | `github.com`, `stackoverflow.com` |
|
||||
| `limit` | Number of results | `10`, `25`, `50` |
|
||||
|
||||
### Engine-Specific Parameters
|
||||
Engine-specific parameters:
|
||||
|
||||
| Parameter | Supported engines | Notes |
|
||||
| --------- | ----------------------------------- | ---------------------------------------------------------------------- |
|
||||
| `start` | `google`, `bing`, `yandex`, `baidu` | Web search pagination offset. |
|
||||
| `filter` | `google` | Duplicate filter (`true` => hide similar, `false` => include similar). |
|
||||
| `answers` | `google` | Include Google answer boxes in output (negative ranks). |
|
||||
| Parameter | Supported engines | Notes |
|
||||
| --------- | ----------------------------------- | ------------------------------------------------------------------ |
|
||||
| `start` | `google`, `bing`, `yandex`, `baidu` | Web search pagination offset. |
|
||||
| `filter` | `google` | Duplicate filter (`true` hides similar, `false` includes similar). |
|
||||
| `answers` | `google` | Include Google answer boxes in output with negative ranks. |
|
||||
|
||||
### Individual Engine Examples
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
# DuckDuckGo search
|
||||
curl "http://localhost:7000/duck/search?text=golang&limit=7"
|
||||
|
||||
# Google search
|
||||
curl "http://localhost:7000/google/search?text=golang&lang=EN&limit=10"
|
||||
|
||||
# Google search (pagination + include similar/hidden results)
|
||||
curl "http://localhost:7000/google/search?text=golang&lang=EN&limit=10&start=10&filter=false"
|
||||
|
||||
# Bing search (offset 20 => around page 3 for 10 results/page)
|
||||
curl "http://localhost:7000/bing/search?text=golang&limit=10&start=20"
|
||||
|
||||
# Yandex search (offset 10 => second page)
|
||||
curl "http://localhost:7000/yandex/search?text=golang&limit=10&start=10"
|
||||
curl "http://127.0.0.1:7000/duck/search?text=golang&limit=7"
|
||||
curl "http://127.0.0.1:7000/google/search?text=golang&lang=EN&limit=10"
|
||||
curl "http://127.0.0.1:7000/bing/search?text=golang&limit=10&start=20"
|
||||
curl "http://127.0.0.1:7000/yandex/search?text=golang&limit=10&start=10"
|
||||
curl "http://127.0.0.1:7000/bing/image?text=golang&limit=20"
|
||||
```
|
||||
|
||||
### Image Search
|
||||
## 🌍 Proxy Support
|
||||
|
||||
OpenSERP supports HTTP and SOCKS5 proxies.
|
||||
|
||||
Simple global proxy:
|
||||
|
||||
```bash
|
||||
# Bing Images
|
||||
curl "http://localhost:7000/bing/image?text=golang&limit=20"
|
||||
|
||||
# Baidu Images
|
||||
curl "http://localhost:7000/baidu/image?text=golang&limit=15"
|
||||
```
|
||||
|
||||
## 🌐 Proxy Support
|
||||
|
||||
OpenSERP supports HTTP and SOCKS5 proxies with authentication:
|
||||
|
||||
```bash
|
||||
# SOCKS5 proxy
|
||||
./openserp serve --proxy socks5://127.0.0.1:1080
|
||||
|
||||
# HTTP proxy with authentication
|
||||
./openserp search bing "query" --proxy http://user:pass@127.0.0.1:8080
|
||||
```
|
||||
|
||||
## Health Check
|
||||
Advanced proxy configuration is available in [config.yaml](./config.yaml).
|
||||
You can enable tagged proxy pools and per-request override via `X-Use-Proxy: <tag>` or `X-Use-Proxy: direct`.
|
||||
|
||||
## Health & Stats
|
||||
|
||||
```bash
|
||||
curl -i "http://127.0.0.1:7000/health"
|
||||
curl "http://127.0.0.1:7000/stats"
|
||||
curl "http://127.0.0.1:7000/stats/cache"
|
||||
curl "http://127.0.0.1:7000/stats/proxy"
|
||||
curl "http://127.0.0.1:7000/stats/cb"
|
||||
```
|
||||
|
||||
Response includes:
|
||||
|
||||
- `status`: `healthy`, `degraded`, or `unhealthy`
|
||||
- `uptime`
|
||||
- per-engine readiness in `engines`
|
||||
- basic runtime stats in `system`
|
||||
|
||||
HTTP status behavior:
|
||||
|
||||
- `200` for `healthy` and `degraded`
|
||||
- `503` for `unhealthy`
|
||||
Useful response headers in server mode: `X-Cache`, `X-Fallback-Engine`,`X-Proxy-Mode`, `X-Proxy-Tag`, `X-Proxy-Used`
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
||||
This project is licensed under the MIT License. See [LICENSE](LICENSE).
|
||||
|
||||
## 🤝 Contributing
|
||||
|
||||
Contributions are welcome! Please feel free to submit a Pull Request.
|
||||
Contributions are welcome. Please feel free to submit a pull request.
|
||||
|
||||
## 👾 Issues & Support
|
||||
|
||||
If you encounter any issues or have questions:
|
||||
If you encounter issues or have questions:
|
||||
|
||||
- Open an issue on GitHub
|
||||
- Check existing issues for solutions
|
||||
- Review the documentation above
|
||||
- Check existing issues for similar reports
|
||||
- Review the documentation and example config
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
version = "0.5.9"
|
||||
version = "0.5.10"
|
||||
defaultConfigFilename = "config"
|
||||
envPrefix = "OPENSERP"
|
||||
)
|
||||
@@ -307,7 +307,7 @@ func setConfigDefaults(v *viper.Viper) {
|
||||
v.SetDefault("cors.enabled", true)
|
||||
v.SetDefault("cors.allow_origins", "*")
|
||||
v.SetDefault("cors.allow_methods", "GET, POST, OPTIONS")
|
||||
v.SetDefault("cors.allow_headers", "Origin, Content-Type, Accept, Authorization")
|
||||
v.SetDefault("cors.allow_headers", "Origin, Content-Type, Accept, Authorization, X-Use-Proxy")
|
||||
v.SetDefault("cors.max_age", 86400)
|
||||
}
|
||||
|
||||
|
||||
34
config.yaml
34
config.yaml
@@ -14,19 +14,19 @@ app:
|
||||
leave_head: false # Keep tabs open after request for debugging
|
||||
stealth: false # Enable stealth browser plugin
|
||||
|
||||
#proxies:
|
||||
# # Force a single proxy for all engines.
|
||||
# # Same behavior as passing --proxy on the CLI.
|
||||
# global: http://test:test@127.0.0.1:18888
|
||||
#
|
||||
# # Advanced mode: define tagged proxy pools and opt engines in with `proxy: <tag>`.
|
||||
# entries:
|
||||
# - url: http://test:test@127.0.0.1:18888
|
||||
# tags: [default, us]
|
||||
# - url: socks5://127.0.0.1:19080 # Chrome doesn't support authenticated SOCKS proxies and hostname resolution
|
||||
# tags: [default, eu]
|
||||
# health:
|
||||
# failure_threshold: 3 # Disable proxy after this many consecutive failures
|
||||
proxies:
|
||||
# Force a single proxy for all engines.
|
||||
# Same behavior as passing --proxy on the CLI.
|
||||
#global: socks5h://127.0.0.1:19080
|
||||
|
||||
# Advanced mode: define tagged proxy pools and opt engines in with `proxy: <tag>`.
|
||||
entries:
|
||||
- url: http://test:test@127.0.0.1:18888
|
||||
tags: [default, us]
|
||||
- url: socks5://127.0.0.1:19080 # Browser mode fails fast on authenticated SOCKS proxies
|
||||
tags: [default, eu]
|
||||
health:
|
||||
failure_threshold: 3 # Disable proxy after this many consecutive failures
|
||||
|
||||
cache:
|
||||
ttl_seconds: 60 # Dedicated endpoint cache TTL in seconds (0 disables cache)
|
||||
@@ -45,7 +45,7 @@ cors:
|
||||
enabled: true
|
||||
allow_origins: "*"
|
||||
allow_methods: "GET, POST, OPTIONS"
|
||||
allow_headers: "Origin, Content-Type, Accept, Authorization"
|
||||
allow_headers: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy"
|
||||
max_age: 86400
|
||||
|
||||
# 2captcha:
|
||||
@@ -55,12 +55,12 @@ google:
|
||||
rate_requests: 4 # Allowed average requests per minute
|
||||
rate_burst: 2 # Burst requests before limiter applies
|
||||
captcha: true # Enable captcha solver path
|
||||
#proxy: us
|
||||
proxy: us
|
||||
|
||||
yandex:
|
||||
rate_requests: 4
|
||||
rate_burst: 2
|
||||
#proxy: eu
|
||||
proxy: eu
|
||||
|
||||
baidu:
|
||||
rate_requests: 4
|
||||
@@ -75,4 +75,4 @@ bing:
|
||||
duckduckgo:
|
||||
rate_requests: 4
|
||||
rate_burst: 2
|
||||
#proxy: default
|
||||
proxy: default
|
||||
|
||||
@@ -36,7 +36,7 @@ func NewResponseCache(ttl time.Duration, maxSize int) *ResponseCache {
|
||||
|
||||
func BuildCacheKey(engine string, action string, q Query) string {
|
||||
raw := fmt.Sprintf(
|
||||
"%s|%s|%s|%s|%s|%s|%s|%d|%d|%t|%t",
|
||||
"%s|%s|%s|%s|%s|%s|%s|%d|%d|%t|%t|%s",
|
||||
engine,
|
||||
action,
|
||||
q.Text,
|
||||
@@ -48,6 +48,7 @@ func BuildCacheKey(engine string, action string, q Query) string {
|
||||
q.Start,
|
||||
q.Filter,
|
||||
q.Answers,
|
||||
q.ProxyOverride,
|
||||
)
|
||||
hash := sha256.Sum256([]byte(raw))
|
||||
return hex.EncodeToString(hash[:])
|
||||
|
||||
@@ -54,17 +54,18 @@ func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]Search
|
||||
}
|
||||
|
||||
type Query struct {
|
||||
Text string
|
||||
LangCode string // eg. EN, ES, RU...
|
||||
DateInterval string // format: YYYYMMDD..YYYMMDD - 20181010..20231010
|
||||
Filetype string // File extension to search.
|
||||
Site string // Search site
|
||||
Limit int // Limit the number of results
|
||||
Start int // Search offset for pagination (Google uses 0, 10, 20...)
|
||||
Filter bool // Filter duplicates (google) (false: include similar, true: hide similar)
|
||||
Answers bool // Include question and answers from SERP page to results with negative indexes
|
||||
ProxyURL string // Proxy URL for raw requests
|
||||
Insecure bool // Allow insecure TLS connections
|
||||
Text string
|
||||
LangCode string // eg. EN, ES, RU...
|
||||
DateInterval string // format: YYYYMMDD..YYYMMDD - 20181010..20231010
|
||||
Filetype string // File extension to search.
|
||||
Site string // Search site
|
||||
Limit int // Limit the number of results
|
||||
Start int // Search offset for pagination (Google uses 0, 10, 20...)
|
||||
Filter bool // Filter duplicates (google) (false: include similar, true: hide similar)
|
||||
Answers bool // Include question and answers from SERP page to results with negative indexes
|
||||
ProxyURL string // Proxy URL for raw requests
|
||||
ProxyOverride string // Request-scoped proxy override: tag or direct
|
||||
Insecure bool // Allow insecure TLS connections
|
||||
}
|
||||
|
||||
func ComputePagination(start int, pageSize int) (int, int, error) {
|
||||
@@ -116,6 +117,11 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
|
||||
return err
|
||||
}
|
||||
|
||||
searchQuery.ProxyOverride, err = NormalizeProxyRequestOverride(reqCtx.Get("X-Use-Proxy"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if searchQuery.IsEmpty() {
|
||||
return errors.New("Query cannot be empty")
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ func DefaultCORSConfig() CORSConfig {
|
||||
return CORSConfig{
|
||||
AllowOrigins: "*",
|
||||
AllowMethods: "GET, POST, OPTIONS",
|
||||
AllowHeaders: "Origin, Content-Type, Accept, Authorization",
|
||||
AllowHeaders: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy",
|
||||
MaxAge: 86400,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package core
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
@@ -99,3 +100,10 @@ func TestNormalizeCORSConfig_FillsMissingValues(t *testing.T) {
|
||||
t.Fatalf("expected default max_age, got %d", cfg.MaxAge)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultCORSConfig_IncludesProxyOverrideHeader(t *testing.T) {
|
||||
cfg := DefaultCORSConfig()
|
||||
if got := cfg.AllowHeaders; !strings.Contains(got, "X-Use-Proxy") {
|
||||
t.Fatalf("expected allow_headers to include X-Use-Proxy, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ const (
|
||||
ProxyModeOff = "off"
|
||||
ProxyModeTagPool = "tag_pool"
|
||||
DefaultProxyFailureThreshold = 3
|
||||
ProxyOverrideDirect = "direct"
|
||||
)
|
||||
|
||||
var supportedProxySchemes = map[string]struct{}{
|
||||
@@ -281,6 +282,17 @@ func NormalizeProxyTag(raw string) (string, error) {
|
||||
return tag, nil
|
||||
}
|
||||
|
||||
func NormalizeProxyRequestOverride(raw string) (string, error) {
|
||||
override := normalizeTag(raw)
|
||||
if override == "" {
|
||||
return "", nil
|
||||
}
|
||||
if override == ProxyOverrideDirect {
|
||||
return ProxyOverrideDirect, nil
|
||||
}
|
||||
return NormalizeProxyTag(override)
|
||||
}
|
||||
|
||||
func IsAuthenticatedSocksProxyURL(raw string) bool {
|
||||
normalized, err := NormalizeProxyURL(raw)
|
||||
if err != nil || normalized == "" {
|
||||
|
||||
@@ -227,6 +227,32 @@ func TestNormalizeProxyTag(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeProxyRequestOverride(t *testing.T) {
|
||||
override, err := NormalizeProxyRequestOverride(" direct ")
|
||||
if err != nil {
|
||||
t.Fatalf("normalize direct proxy override: %v", err)
|
||||
}
|
||||
if override != ProxyOverrideDirect {
|
||||
t.Fatalf("expected direct override, got %q", override)
|
||||
}
|
||||
|
||||
override, err = NormalizeProxyRequestOverride(" US ")
|
||||
if err != nil {
|
||||
t.Fatalf("normalize tagged proxy override: %v", err)
|
||||
}
|
||||
if override != "us" {
|
||||
t.Fatalf("expected normalized tag override us, got %q", override)
|
||||
}
|
||||
|
||||
override, err = NormalizeProxyRequestOverride("")
|
||||
if err != nil {
|
||||
t.Fatalf("normalize empty override: %v", err)
|
||||
}
|
||||
if override != "" {
|
||||
t.Fatalf("expected empty override, got %q", override)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsAuthenticatedSocksProxyURL(t *testing.T) {
|
||||
if !IsAuthenticatedSocksProxyURL("socks5h://user:pass@127.0.0.1:1080") {
|
||||
t.Fatal("expected authenticated socks proxy to be detected")
|
||||
|
||||
@@ -139,7 +139,7 @@ func (rs *ResilientSearcher) searchWithProtection(engine SearchEngine, q Query,
|
||||
return nil, ProxyExecutionMeta{}, ErrCircuitOpen
|
||||
}
|
||||
|
||||
policy := rs.effectivePolicyForEngine(engine.Name())
|
||||
policy := rs.effectivePolicyForQuery(engine.Name(), q)
|
||||
attemptMeta := rs.baseProxyMeta(policy)
|
||||
|
||||
result := RetryableSearch(rs.retryCfg, engine.Name(), func() ([]SearchResult, error) {
|
||||
@@ -160,7 +160,7 @@ func (rs *ResilientSearcher) searchWithProtection(engine SearchEngine, q Query,
|
||||
attemptQuery.ProxyURL = ""
|
||||
attemptMeta.Used = "direct"
|
||||
case ProxyModeTagPool:
|
||||
proxyURL = rs.selectProxyForPolicy(policy)
|
||||
proxyURL = rs.selectProxyForQuery(policy, q)
|
||||
if proxyURL == "" {
|
||||
return nil, fmt.Errorf("%w: no healthy proxy available for tag %q", ErrProxyUnavailable, policy.Tag)
|
||||
}
|
||||
@@ -318,7 +318,7 @@ func (rs *ResilientSearcher) GetProxyStats() ProxyStats {
|
||||
return stats
|
||||
}
|
||||
|
||||
func (rs *ResilientSearcher) ResolveMegaProxyMeta(engines []SearchEngine) ProxyExecutionMeta {
|
||||
func (rs *ResilientSearcher) ResolveMegaProxyMeta(q Query, engines []SearchEngine) ProxyExecutionMeta {
|
||||
if len(engines) == 0 {
|
||||
return ProxyExecutionMeta{Mode: ProxyModeOff, Used: "direct"}
|
||||
}
|
||||
@@ -328,7 +328,7 @@ func (rs *ResilientSearcher) ResolveMegaProxyMeta(engines []SearchEngine) ProxyE
|
||||
hasOff := false
|
||||
|
||||
for _, engine := range engines {
|
||||
policy := rs.effectivePolicyForEngine(engine.Name())
|
||||
policy := rs.effectivePolicyForQuery(engine.Name(), q)
|
||||
if policy.Mode == ProxyModeOff {
|
||||
hasOff = true
|
||||
continue
|
||||
@@ -351,9 +351,11 @@ func (rs *ResilientSearcher) ResolveMegaProxyMeta(engines []SearchEngine) ProxyE
|
||||
}
|
||||
}
|
||||
|
||||
if global := strings.TrimSpace(rs.proxyCfg.Proxies.Global); global != "" && !hasOff {
|
||||
meta.Used = MaskProxyURL(global)
|
||||
return meta
|
||||
if q.ProxyOverride == "" {
|
||||
if global := strings.TrimSpace(rs.proxyCfg.Proxies.Global); global != "" && !hasOff {
|
||||
meta.Used = MaskProxyURL(global)
|
||||
return meta
|
||||
}
|
||||
}
|
||||
|
||||
if rs.proxyRuntime == ProxyRuntimeRaw {
|
||||
@@ -388,6 +390,17 @@ func (rs *ResilientSearcher) effectivePolicyForEngine(engineName string) ProxyPo
|
||||
return rs.proxyDefaults
|
||||
}
|
||||
|
||||
func (rs *ResilientSearcher) effectivePolicyForQuery(engineName string, q Query) ProxyPolicy {
|
||||
switch q.ProxyOverride {
|
||||
case "":
|
||||
return rs.effectivePolicyForEngine(engineName)
|
||||
case ProxyOverrideDirect:
|
||||
return ProxyPolicy{Mode: ProxyModeOff}
|
||||
default:
|
||||
return ProxyPolicy{Mode: ProxyModeTagPool, Tag: q.ProxyOverride}
|
||||
}
|
||||
}
|
||||
|
||||
func (rs *ResilientSearcher) selectProxyForTag(tag string) string {
|
||||
if rs.proxyRegistry == nil {
|
||||
return ""
|
||||
@@ -408,12 +421,14 @@ func (rs *ResilientSearcher) reportProxyAttempt(proxyURL string, err error) {
|
||||
rs.proxyRegistry.ReportSuccess(proxyURL)
|
||||
}
|
||||
|
||||
func (rs *ResilientSearcher) selectProxyForPolicy(policy ProxyPolicy) string {
|
||||
func (rs *ResilientSearcher) selectProxyForQuery(policy ProxyPolicy, q Query) string {
|
||||
if policy.Mode != ProxyModeTagPool {
|
||||
return ""
|
||||
}
|
||||
if global := strings.TrimSpace(rs.proxyCfg.Proxies.Global); global != "" {
|
||||
return global
|
||||
if q.ProxyOverride == "" {
|
||||
if global := strings.TrimSpace(rs.proxyCfg.Proxies.Global); global != "" {
|
||||
return global
|
||||
}
|
||||
}
|
||||
return rs.selectProxyForTag(policy.Tag)
|
||||
}
|
||||
|
||||
@@ -334,7 +334,7 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string, run func(Query,
|
||||
engineNames[i] = engine.Name()
|
||||
}
|
||||
engineNamesJoined := strings.Join(engineNames, ",")
|
||||
s.applyProxyHeaders(c, s.resilient.ResolveMegaProxyMeta(enginesToUse))
|
||||
s.applyProxyHeaders(c, s.resilient.ResolveMegaProxyMeta(q, enginesToUse))
|
||||
logrus.Infof("Starting SERP mega %s request using engines: %s for query: %s", action, engineNamesJoined, q.Text)
|
||||
|
||||
cacheHitCandidates := []cacheHitCandidate{
|
||||
|
||||
@@ -60,6 +60,17 @@ func request(t *testing.T, s *Server, path string) *http.Response {
|
||||
return resp
|
||||
}
|
||||
|
||||
func requestWithHeader(t *testing.T, s *Server, path string, header string, value string) *http.Response {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
req.Header.Set(header, value)
|
||||
resp, err := s.app.Test(req, -1)
|
||||
if err != nil {
|
||||
t.Fatalf("request failed for %s: %v", path, err)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func TestHealthEndpointStatusSemantics(t *testing.T) {
|
||||
ready := &engineMock{name: "google", initialized: true, limiter: rate.NewLimiter(rate.Every(time.Second), 1)}
|
||||
notReady := &engineMock{name: "yandex", initialized: false, limiter: rate.NewLimiter(rate.Every(time.Second), 1)}
|
||||
@@ -663,6 +674,79 @@ func TestGlobalProxyForcesAllEnginesRaw(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestProxyOverrideDirectBeatsGlobal(t *testing.T) {
|
||||
var googleProxy string
|
||||
engine := &engineMock{
|
||||
name: "google",
|
||||
initialized: true,
|
||||
searchFn: func(q Query) ([]SearchResult, error) {
|
||||
googleProxy = q.ProxyURL
|
||||
return []SearchResult{{Rank: 1, URL: "https://example.com/google", Title: "google"}}, nil
|
||||
},
|
||||
}
|
||||
|
||||
opts := DefaultServerOptions()
|
||||
opts.Resilience.Proxy = ProxyConfig{
|
||||
Runtime: ProxyRuntimeRaw,
|
||||
Proxies: ProxiesConfig{
|
||||
Global: "http://global-proxy:8080",
|
||||
},
|
||||
}
|
||||
|
||||
srv := NewServerWithOptions("127.0.0.1", 7099, opts, engine)
|
||||
resp := requestWithHeader(t, srv, "/google/search?text=golang", "X-Use-Proxy", "direct")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected direct override request to succeed, got %d", resp.StatusCode)
|
||||
}
|
||||
if googleProxy != "" {
|
||||
t.Fatalf("expected direct override to disable proxy, got %q", googleProxy)
|
||||
}
|
||||
if got := resp.Header.Get("X-Proxy-Mode"); got != ProxyModeOff {
|
||||
t.Fatalf("expected X-Proxy-Mode=%s, got %q", ProxyModeOff, got)
|
||||
}
|
||||
if got := resp.Header.Get("X-Proxy-Used"); got != "direct" {
|
||||
t.Fatalf("expected X-Proxy-Used=direct, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestProxyOverrideTagBeatsGlobal(t *testing.T) {
|
||||
var googleProxy string
|
||||
engine := &engineMock{
|
||||
name: "google",
|
||||
initialized: true,
|
||||
searchFn: func(q Query) ([]SearchResult, error) {
|
||||
googleProxy = q.ProxyURL
|
||||
return []SearchResult{{Rank: 1, URL: "https://example.com/google", Title: "google"}}, nil
|
||||
},
|
||||
}
|
||||
|
||||
opts := DefaultServerOptions()
|
||||
opts.Resilience.Proxy = ProxyConfig{
|
||||
Runtime: ProxyRuntimeRaw,
|
||||
Proxies: ProxiesConfig{
|
||||
Global: "http://global-proxy:8080",
|
||||
Entries: []ProxyEntryConfig{
|
||||
{URL: "http://proxy-us:8080", Tags: []string{"us"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
srv := NewServerWithOptions("127.0.0.1", 7100, opts, engine)
|
||||
resp := requestWithHeader(t, srv, "/google/search?text=golang", "X-Use-Proxy", "us")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected tagged override request to succeed, got %d", resp.StatusCode)
|
||||
}
|
||||
if googleProxy != "http://proxy-us:8080" {
|
||||
t.Fatalf("expected tagged override to use pool proxy, got %q", googleProxy)
|
||||
}
|
||||
if got := resp.Header.Get("X-Proxy-Tag"); got != "us" {
|
||||
t.Fatalf("expected X-Proxy-Tag=us, got %q", got)
|
||||
}
|
||||
if got := resp.Header.Get("X-Proxy-Used"); got != "http://proxy-us:8080" {
|
||||
t.Fatalf("expected X-Proxy-Used to reflect override proxy, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBrowserProxyPoolRotatesPerRequest(t *testing.T) {
|
||||
var attemptedProxies []string
|
||||
engine := &engineMock{
|
||||
@@ -702,6 +786,64 @@ func TestBrowserProxyPoolRotatesPerRequest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestProxyOverrideMissingTagFailsClosed(t *testing.T) {
|
||||
engine := &engineMock{name: "google", initialized: true}
|
||||
opts := DefaultServerOptions()
|
||||
opts.Resilience.Proxy = ProxyConfig{
|
||||
Runtime: ProxyRuntimeRaw,
|
||||
Proxies: ProxiesConfig{
|
||||
Global: "http://global-proxy:8080",
|
||||
},
|
||||
}
|
||||
|
||||
srv := NewServerWithOptions("127.0.0.1", 7101, opts, engine)
|
||||
resp := requestWithHeader(t, srv, "/google/search?text=golang", "X-Use-Proxy", "missing")
|
||||
if resp.StatusCode != http.StatusServiceUnavailable {
|
||||
t.Fatalf("expected missing tag override to fail closed, got %d", resp.StatusCode)
|
||||
}
|
||||
if got := resp.Header.Get("X-Proxy-Mode"); got != ProxyModeTagPool {
|
||||
t.Fatalf("expected X-Proxy-Mode=%s on missing tag response, got %q", ProxyModeTagPool, got)
|
||||
}
|
||||
if got := resp.Header.Get("X-Proxy-Tag"); got != "missing" {
|
||||
t.Fatalf("expected X-Proxy-Tag=missing, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMegaProxyOverrideHeaderBeatsGlobal(t *testing.T) {
|
||||
var googleProxy string
|
||||
engine := &engineMock{
|
||||
name: "google",
|
||||
initialized: true,
|
||||
searchFn: func(q Query) ([]SearchResult, error) {
|
||||
googleProxy = q.ProxyURL
|
||||
return []SearchResult{{Rank: 1, URL: "https://example.com/google", Title: "google"}}, nil
|
||||
},
|
||||
}
|
||||
|
||||
opts := DefaultServerOptions()
|
||||
opts.Resilience.Proxy = ProxyConfig{
|
||||
Runtime: ProxyRuntimeRaw,
|
||||
Proxies: ProxiesConfig{
|
||||
Global: "http://global-proxy:8080",
|
||||
Entries: []ProxyEntryConfig{
|
||||
{URL: "http://proxy-us:8080", Tags: []string{"us"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
srv := NewServerWithOptions("127.0.0.1", 7102, opts, engine)
|
||||
resp := requestWithHeader(t, srv, "/mega/search?text=golang&engines=google", "X-Use-Proxy", "us")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected mega override request to succeed, got %d", resp.StatusCode)
|
||||
}
|
||||
if googleProxy != "http://proxy-us:8080" {
|
||||
t.Fatalf("expected mega override to use pool proxy, got %q", googleProxy)
|
||||
}
|
||||
if got := resp.Header.Get("X-Proxy-Tag"); got != "us" {
|
||||
t.Fatalf("expected mega X-Proxy-Tag=us, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyFailClosedWhenNoHealthyProxy(t *testing.T) {
|
||||
engine := &engineMock{name: "google", initialized: true}
|
||||
opts := DefaultServerOptions()
|
||||
|
||||
Reference in New Issue
Block a user