From 205f722ddc2399c036d23ef6507200793c4b51a9 Mon Sep 17 00:00:00 2001 From: Pachakutiq Date: Sun, 1 Dec 2024 23:35:11 +0800 Subject: [PATCH 01/12] fix: Fix search crash (slice bounds out of range [4:1]) --- google/search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/search.go b/google/search.go index 118280a..e0a29e4 100644 --- a/google/search.go +++ b/google/search.go @@ -293,7 +293,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { // Get description text := resEl.MustText() textSliced := strings.Split(text, "\n") - srchRes.Description = strings.Join(textSliced[4:], "\n") + srchRes.Description = strings.Join(textSliced[:], "\n") } else { //fmt.Println(i, attrs) From 84e2d46c4ba181b05f54de3e5aed5213b35d450a Mon Sep 17 00:00:00 2001 From: Pachakutiq <101460915+PACHAKUTlQ@users.noreply.github.com> Date: Tue, 8 Apr 2025 16:06:41 +0800 Subject: [PATCH 02/12] feat: Support http and socks5 proxy with auth for raw search --- baidu/search_raw.go | 30 ++++++++++++++++++++++++++---- cmd/root.go | 4 ++++ cmd/search.go | 6 ++++-- core/common.go | 2 ++ google/search_raw.go | 29 +++++++++++++++++++++++++---- yandex/search_raw.go | 30 ++++++++++++++++++++++++++---- 6 files changed, 87 insertions(+), 14 deletions(-) diff --git a/baidu/search_raw.go b/baidu/search_raw.go index 695586d..188fd7b 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -1,9 +1,12 @@ package baidu import ( + "crypto/tls" "fmt" "net/http" + "net/url" "strings" + "time" "github.com/PuerkitoBio/goquery" "github.com/corpix/uarand" @@ -11,8 +14,27 @@ import ( "github.com/sirupsen/logrus" ) -func baiduRequest(searchURL string) (*http.Response, error) { - baseClient := &http.Client{} +func baiduRequest(searchURL string, query core.Query) (*http.Response, error) { + // Create HTTP transport with proxy + transport := &http.Transport{} + if query.ProxyURL != "" { + proxyUrl, err := url.Parse(query.ProxyURL) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyUrl) + } + + // Set insecure TLS + if query.Insecure { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + baseClient := &http.Client{ + Transport: transport, + Timeout: time.Second * 10, + } + req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, err @@ -27,7 +49,7 @@ func baiduRequest(searchURL string) (*http.Response, error) { } func baiduResultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromResponse(response) + doc, err := goquery.NewDocumentFromReader(response.Body) if err != nil { return nil, err } @@ -79,7 +101,7 @@ func Search(query core.Query) ([]core.SearchResult, error) { } logrus.Debugf("Baidu URL built: %s", googleURL) - res, err := baiduRequest(googleURL) + res, err := baiduRequest(googleURL, query) if err != nil { return nil, err } diff --git a/cmd/root.go b/cmd/root.go index a799355..ed72e3c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -40,6 +40,8 @@ type AppConfig struct { IsDebug bool `mapstructure:"debug"` IsVerbose bool `mapstructure:"verbose"` IsRawRequests bool `mapstructure:"raw_requests"` + ProxyURL string `mapstructure:"proxy"` + Insecure bool `mapstructure:"insecure"` } var config = Config{} @@ -133,4 +135,6 @@ func init() { RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests") RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made") RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key") + RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "", "", "HTTP proxy URL (e.g. http://user:pass@proxy:8080)") + RootCmd.PersistentFlags().BoolVarP(&config.App.Insecure, "insecure", "k", false, "Allow insecure TLS connections") } diff --git a/cmd/search.go b/cmd/search.go index c137735..823d694 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -26,8 +26,10 @@ func search(cmd *cobra.Command, args []string) { var err error engineType := args[0] query := core.Query{ - Text: args[1], - Limit: 10, + Text: args[1], + Limit: 10, + ProxyURL: config.App.ProxyURL, + Insecure: config.App.Insecure, } results := []core.SearchResult{} diff --git a/core/common.go b/core/common.go index 7a52a66..bf0cfad 100644 --- a/core/common.go +++ b/core/common.go @@ -41,6 +41,8 @@ type Query struct { Site string // Search site Limit int // Limit the number of results Answers bool // Include question and answers from SERP page to results with negative indexes + ProxyURL string // Proxy URL for raw requests + Insecure bool // Allow insecure TLS connections } func (q Query) IsEmpty() bool { diff --git a/google/search_raw.go b/google/search_raw.go index 3e482ff..287a355 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -1,8 +1,11 @@ package google import ( + "crypto/tls" "net/http" + "net/url" "strings" + "time" "github.com/PuerkitoBio/goquery" "github.com/corpix/uarand" @@ -10,8 +13,26 @@ import ( "github.com/sirupsen/logrus" ) -func googleRequest(searchURL string) (*http.Response, error) { - baseClient := &http.Client{} +func googleRequest(searchURL string, query core.Query) (*http.Response, error) { + // Create HTTP transport with proxy + transport := &http.Transport{} + if query.ProxyURL != "" { + proxyUrl, err := url.Parse(query.ProxyURL) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyUrl) + } + + // Set insecure TLS + if query.Insecure { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + baseClient := &http.Client{ + Transport: transport, + Timeout: time.Second * 10, + } req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, err @@ -26,7 +47,7 @@ func googleRequest(searchURL string) (*http.Response, error) { } func googleResultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromResponse(response) + doc, err := goquery.NewDocumentFromReader(response.Body) if err != nil { return nil, err } @@ -77,7 +98,7 @@ func Search(query core.Query) ([]core.SearchResult, error) { } logrus.Debugf("Google URL built: %s", googleURL) - res, err := googleRequest(googleURL) + res, err := googleRequest(googleURL, query) if err != nil { return nil, err } diff --git a/yandex/search_raw.go b/yandex/search_raw.go index 8779050..102a721 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -1,8 +1,11 @@ package yandex import ( + "crypto/tls" "net/http" + "net/url" "strings" + "time" "github.com/PuerkitoBio/goquery" "github.com/corpix/uarand" @@ -10,8 +13,27 @@ import ( "github.com/sirupsen/logrus" ) -func yandexRequest(searchURL string) (*http.Response, error) { - baseClient := &http.Client{} +func yandexRequest(searchURL string, query core.Query) (*http.Response, error) { + // Create HTTP transport with proxy + transport := &http.Transport{} + if query.ProxyURL != "" { + proxyUrl, err := url.Parse(query.ProxyURL) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyUrl) + } + + // Set insecure TLS + if query.Insecure { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + baseClient := &http.Client{ + Transport: transport, + Timeout: time.Second * 10, + } + req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, err @@ -26,7 +48,7 @@ func yandexRequest(searchURL string) (*http.Response, error) { } func yandexResultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromResponse(response) + doc, err := goquery.NewDocumentFromReader(response.Body) if err != nil { return nil, err } @@ -77,7 +99,7 @@ func Search(query core.Query) ([]core.SearchResult, error) { } logrus.Debugf("Yandex URL built: %s", googleURL) - res, err := yandexRequest(googleURL) + res, err := yandexRequest(googleURL, query) if err != nil { return nil, err } From 21f12c5996757e1848259f6cfd2fa1077d954754 Mon Sep 17 00:00:00 2001 From: Pachakutiq <101460915+PACHAKUTlQ@users.noreply.github.com> Date: Tue, 8 Apr 2025 18:48:07 +0800 Subject: [PATCH 03/12] fix: Google updated DOM of search result page --- google/search.go | 70 +++++++++++++++++++++++++++----------------- google/search_raw.go | 43 ++++++++++++++++++++++----- 2 files changed, 78 insertions(+), 35 deletions(-) diff --git a/google/search.go b/google/search.go index e0a29e4..78e27e3 100644 --- a/google/search.go +++ b/google/search.go @@ -157,8 +157,8 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { gogl.acceptCookies(page) } - // Find all results - results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid]") + // Find all results using stable attributes + results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved]") if err != nil { logrus.Errorf("Cannot parse search results: %s", err) return nil, core.ErrSearchTimeout @@ -265,35 +265,51 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { searchResults = append(searchResults, srchRes) } continue - } else if strings.Contains(attrs, "data-ved") && strings.Contains(attrs, "lang") { - // 3. Parse regular search results - // Get URL - link, err := resEl.Element("a") + } else if strings.Contains(attrs, "data-ved") { + // Parse regular search results + // Get title from h3 + titleTag, err := resEl.Element("h3") if err != nil { continue } - href, err := link.Property("href") - if err != nil { - logrus.Debug("No `href` tag found") + srchRes.Title, _ = titleTag.Text() + + // Get URL from parent link of h3 + link, err := titleTag.Parent() + if err == nil && link.MustMatches("a") { + href, _ := link.Property("href") + srchRes.URL = href.String() } - srchRes.URL = href.String() + + // Get description using multiple fallback strategies + desc := "" + if descTag, err := resEl.Element("div[data-sncf='1'] div"); err == nil { + desc = descTag.MustText() + } else if descTag, err := resEl.Element("div.VwiC3b"); err == nil { + desc = descTag.MustText() + } else { + // Structural fallback + parent, err := titleTag.Parent() + if err == nil { + parent, err = parent.Parent() + if err == nil { + parent, err = parent.Parent() + if err == nil { + if descTag, err := parent.Next(); err == nil { + if descDiv, err := descTag.Element("div"); err == nil { + desc = descDiv.MustText() + } + } + } + } + } + } + srchRes.Description = desc + rank += 1 - - // Get title - titleTag, err := link.Element("h3") - if err != nil { - continue - } - - srchRes.Title, err = titleTag.Text() - if err != nil { - logrus.Debug("Cannot extract text from title") - } - - // Get description - text := resEl.MustText() - textSliced := strings.Split(text, "\n") - srchRes.Description = strings.Join(textSliced[:], "\n") + srchRes.Rank = rank + searchResults = append(searchResults, srchRes) + continue } else { //fmt.Println(i, attrs) @@ -324,7 +340,7 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) { page.Mouse.Scroll(0, 1000000, 1) page.WaitLoad() - results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction][jsdata]") + results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]") if err != nil { logrus.Errorf("Cannot parse search results: %s", err) return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout diff --git a/google/search_raw.go b/google/search_raw.go index 287a355..59c15ff 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -33,6 +33,7 @@ func googleRequest(searchURL string, query core.Query) (*http.Response, error) { Transport: transport, Timeout: time.Second * 10, } + req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, err @@ -55,23 +56,49 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) { results := []core.SearchResult{} rank := 1 - // Get individual results - sel := doc.Find("div.g") + // Use data attributes instead of class names to find results + // Both old and new DOM have data-hveid and data-ved attributes + sel := doc.Find("div[data-hveid][data-ved]") for i := range sel.Nodes { item := sel.Eq(i) - // Find URL - linkTag := item.Find("a") - link, _ := linkTag.Attr("href") + // Skip items without an h3 element (which indicates a search result) + if item.Find("h3").Length() == 0 { + continue + } + + // Find URL - look for the anchor that contains the h3 title + linkTag := item.Find("h3").Parent() + if linkTag.Is("a") == false { + linkTag = item.Find("h3").Closest("a") + } + + link, exists := linkTag.Attr("href") + if !exists || link == "" || link == "#" { + continue + } link = strings.Trim(link, " ") - // Find title + // Find title - this is inside the h3 element titleTag := item.Find("h3") title := titleTag.Text() - // Find description - descTag := item.Find(`div[data-sncf~="1"]`) + // Find description - find div with text content after the heading + // Using attribute selectors that match the description container + descTag := item.Find("div[data-sncf='1']").Find("div").First() + if descTag.Length() == 0 { + // Try another selector approach if the first one fails + descTag = item.Find("div.VwiC3b") + if descTag.Length() == 0 { + // As a last resort, look for any div after the title that might contain description + titleParent := titleTag.Parent() + if titleParent.Is("a") { + titleParent = titleParent.Parent().Parent() + } + descTag = titleParent.NextAll().First().Find("div").First() + } + } desc := descTag.Text() if link != "" && link != "#" { From b2ffcf8477f7a25cb58a19996b273e70e6a804fd Mon Sep 17 00:00:00 2001 From: Pachakutiq <101460915+PACHAKUTlQ@users.noreply.github.com> Date: Tue, 8 Apr 2025 23:51:44 +0800 Subject: [PATCH 04/12] feat: Support http and socks5 proxy (support authentication) for chrome driver mode --- cmd/search.go | 2 ++ core/browser.go | 54 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/cmd/search.go b/cmd/search.go index 823d694..2c506d4 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -61,6 +61,8 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er Timeout: time.Second * time.Duration(config.App.Timeout), LeavePageOpen: config.App.IsLeaveHead, CaptchaSolverApiKey: config.Config2Capcha.ApiKey, + ProxyURL: config.App.ProxyURL, + Insecure: config.App.Insecure, } if config.App.IsDebug { diff --git a/core/browser.go b/core/browser.go index b739cd4..293a3d3 100644 --- a/core/browser.go +++ b/core/browser.go @@ -1,6 +1,8 @@ package core import ( + "fmt" + "net/url" "time" "github.com/go-rod/rod" @@ -19,6 +21,8 @@ type BrowserOpts struct { LeavePageOpen bool // Leave pages and browser open WaitLoadTime time.Duration // Time to wait till page loads CaptchaSolverApiKey string // 2Captcha api key + ProxyURL string // Proxy URL + Insecure bool // Allow insecure TLS connections } @@ -47,9 +51,33 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) { path, has := launcher.LookPath() logrus.Debug("Browser found: ", has) + // Create launcher + l := launcher.New().Bin(path).Leakless(opts.IsLeakless).Headless(opts.IsHeadless) + + // Configure proxy if specified + if opts.ProxyURL != "" { + proxyUrl, err := url.Parse(opts.ProxyURL) + if err != nil { + return nil, fmt.Errorf("invalid proxy URL: %v", err) + } + + // Make sure the proxy URL includes the scheme when passed to launcher + // This ensures proper handling of SOCKS5 proxies + proxyStr := proxyUrl.String() + logrus.Debugf("Setting up proxy: %s", proxyStr) + l = l.Proxy(proxyStr) + + // Check if proxy has auth credentials + if proxyUrl.User != nil { + username := proxyUrl.User.Username() + logrus.Debugf("Using proxy authentication: %s:****", username) + // We'll handle auth in the Navigate method + } + } + var err error b := Browser{BrowserOpts: opts} - b.browserAddr, err = launcher.New().Bin(path).Leakless(opts.IsLeakless).Headless(opts.IsHeadless).Launch() + b.browserAddr, err = l.Launch() if opts.CaptchaSolverApiKey != "" { b.CaptchaSolver = NewSolver(opts.CaptchaSolverApiKey) @@ -76,10 +104,27 @@ func (b *Browser) Navigate(URL string) *rod.Page { b.browser.MustConnect() b.browser.SetCookies(nil) - //page := b.browser.MustPage(URL) + // Handle proxy authentication before any navigations + if b.ProxyURL != "" { + proxyUrl, _ := url.Parse(b.ProxyURL) + + // Always ignore certificate errors when using proxies + // This fixes the ERR_CERT_AUTHORITY_INVALID error for SOCKS5 proxies + b.browser.MustIgnoreCertErrors(true) + + if proxyUrl.User != nil { + username := proxyUrl.User.Username() + password, _ := proxyUrl.User.Password() + // Launch auth handler before any navigation occurs + go b.browser.MustHandleAuth(username, password)() + } + } else if b.Insecure { + // Still respect the insecure flag if no proxy is used + b.browser.MustIgnoreCertErrors(true) + } + page := stealth.MustPage(b.browser) page.MustEmulate(devices.Device{ - //UserAgent: uarand.GetRandom(), AcceptLanguage: b.LanguageCode, }) page.MustNavigate(URL) @@ -90,9 +135,6 @@ func (b *Browser) Navigate(URL string) *rod.Page { wait() } - // Wait till page loads - //time.Sleep(b.WaitLoadTime) - return page } From 5ad89b2ea8401428d70272b4c7d53725c10591b5 Mon Sep 17 00:00:00 2001 From: Pachakutiq <101460915+PACHAKUTlQ@users.noreply.github.com> Date: Wed, 9 Apr 2025 02:08:01 +0800 Subject: [PATCH 05/12] feat: Support proxy for serve --- cmd/serve.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/serve.go b/cmd/serve.go index 3a14e3f..0b4e632 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -26,6 +26,8 @@ func serve(cmd *cobra.Command, args []string) { Timeout: time.Second * time.Duration(config.App.Timeout), LeavePageOpen: config.App.IsLeaveHead, CaptchaSolverApiKey: config.Config2Capcha.ApiKey, + ProxyURL: config.App.ProxyURL, + Insecure: config.App.Insecure, } if config.App.IsDebug { From ec03439b2f7824f606c257456278558bbae45f91 Mon Sep 17 00:00:00 2001 From: Pachakutiq <101460915+PACHAKUTlQ@users.noreply.github.com> Date: Wed, 9 Apr 2025 02:11:22 +0800 Subject: [PATCH 06/12] style: Format README.md using prettier --- README.md | 85 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 0565f75..9f8962a 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,23 @@ # OpenSERP (Search Engine Results Page) + ![OpenSERP](/logo.png) [![Go Report Card](https://goreportcard.com/badge/github.com/karust/openserp)](https://goreportcard.com/report/github.com/karust/openserp) [![Go Reference](https://pkg.go.dev/badge/github.com/karust/openserp.svg)](https://pkg.go.dev/github.com/karust/openserp) [![release](https://img.shields.io/github/release-pre/karust/openserp.svg)](https://github.com/karust/openserp/releases) + + API access for search engines results if available isn't free. Using OpenSERP, you can get search results from **Google**, **Yandex**, **Baidu** via API or CLI! See [Docker](#docker) and [CLI](#cli) usage examples below ([search](#search), [images](#images)). -## Docker usage 馃惓 -* Run API server: +## Docker usage 馃惓 + +- Run API server: + ```bash # Use prebuilt image docker run -p 127.0.0.1:7000:7000 -it karust/openserp serve -a 0.0.0.0 -p 7000 @@ -22,26 +27,32 @@ docker-compose up --build ``` ### Request parameters -| Param | Description | -|-------|--------------------------------------------------------------| -| text | Text to search | -| lang | Search pages in selected language (`EN`, `DE`, `RU`...) | -| date | Date in `YYYYMMDD..YYYYMMDD` format (e.g. 20181010..20231010) | -| file | File extension to search (e.g. `PDF`, `DOC`) | -| site | Search within a specific website | -| limit | Limit the number of results -| answers | Include google answers as negative rank indexes (e.g. `true`, `false`) + +| Param | Description | +| ------- | ---------------------------------------------------------------------- | +| text | Text to search | +| lang | Search pages in selected language (`EN`, `DE`, `RU`...) | +| date | Date in `YYYYMMDD..YYYYMMDD` format (e.g. 20181010..20231010) | +| file | File extension to search (e.g. `PDF`, `DOC`) | +| site | Search within a specific website | +| limit | Limit the number of results | +| answers | Include google answers as negative rank indexes (e.g. `true`, `false`) | ### **Search** -### *Example request* + +### _Example request_ + Get 20 **Google** results for `hello world`, only in English: + ``` GET http:/127.0.0.1:7000/google/search?lang=EN&limit=20&text=hello world ``` -You can replace `google` to `yandex` or `baidu` in query to change search engine. - | -### *Example response* +You can replace `google` to `yandex` or `baidu` in query to change search engine. +| + +### _Example response_ + ```JSON [ { @@ -53,39 +64,51 @@ You can replace `google` to `yandex` or `baidu` in query to change search engine }, ] ``` + ### **Images** **[WIP]** -### *Example request* + +### _Example request_ + Get 100 **Google** results for `golden puppy`: + ``` GET http://127.0.0.1:7000/google/image?text=golden puppy&limit=100 ``` - ## CLI 鈱笍 -* Use `-h` flag to see commands. -* You can use `serve` command to serve API: + +- Use `-h` flag to see commands. +- You can use `serve` command to serve API: + ```bash -openserp serve +openserp serve ``` -* Or print results in CLI using `search` command: + +- Or print results in CLI using `search` command: + ```bash openserp search google "how to get banned from google fast" # Change `google` to `yandex` or `baidu` ``` + As a result you should get JSON output containting search results: + ```json [ - { - "rank": 1, - "url": "https://www.cyberoptik.net/blog/6-sure-fire-ways-to-get-banned-from-google/", - "title": "11 Sure-Fire Ways to Get Banned From Google | CyberOptik", - "description": "How To Get Banned From Google 路 1. Cloaking: The Art of Deception 路 2. Plagiarism: Because Originality is Overrated 路 3. Keyword Stuffing: More is Always Better 路 4 ...", - "ad": false - }, + { + "rank": 1, + "url": "https://www.cyberoptik.net/blog/6-sure-fire-ways-to-get-banned-from-google/", + "title": "11 Sure-Fire Ways to Get Banned From Google | CyberOptik", + "description": "How To Get Banned From Google 路 1. Cloaking: The Art of Deception 路 2. Plagiarism: Because Originality is Overrated 路 3. Keyword Stuffing: More is Always Better 路 4 ...", + "ad": false + } ] - ``` +``` + +## License - ## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details ## Bugs + Questions 馃懢 -If you have some issues/bugs/questions, feel free to open an issue. \ No newline at end of file + +If you have some issues/bugs/questions, feel free to open an issue. + From 90f27c2974bce7e238ca76c4def00f1f3240eb96 Mon Sep 17 00:00:00 2001 From: Pachakutiq <101460915+PACHAKUTlQ@users.noreply.github.com> Date: Wed, 9 Apr 2025 02:54:42 +0800 Subject: [PATCH 07/12] feat: Support short flag `-x` for proxy --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index ed72e3c..043024d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -135,6 +135,6 @@ func init() { RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests") RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made") RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key") - RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "", "", "HTTP proxy URL (e.g. http://user:pass@proxy:8080)") + RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "x", "", "HTTP or Socks5 proxy URL (e.g. http://user:pass@127.0.0.1:8080)") RootCmd.PersistentFlags().BoolVarP(&config.App.Insecure, "insecure", "k", false, "Allow insecure TLS connections") } From 2f77dd6fe52f3efeafb48d1de18b37d909221a38 Mon Sep 17 00:00:00 2001 From: Pachakutiq <101460915+PACHAKUTlQ@users.noreply.github.com> Date: Wed, 9 Apr 2025 02:55:59 +0800 Subject: [PATCH 08/12] doc: Add proxy guide in README.md --- README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9f8962a..37f4b0a 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ GET http://127.0.0.1:7000/google/image?text=golden puppy&limit=100 ## CLI 鈱笍 -- Use `-h` flag to see commands. +- Use `-h` flag to see all commands. - You can use `serve` command to serve API: ```bash @@ -104,6 +104,22 @@ As a result you should get JSON output containting search results: ] ``` +### Proxy Support + +Both browser mode and raw mode support HTTP and SOCKS5 proxies with authentication: + +```bash +# HTTP proxy with auth +openserp search google "query" --proxy http://user:pass@127.0.0.1:8080 + +# Serve with SOCKS5 proxy +openserp serve --proxy socks5://127.0.0.1:1080 + +# For HTTPS sites through HTTP proxy, use --insecure to ignore certificate errors +openserp search google "query" --proxy http://127.0.0.1:8080 --insecure +openserp search google "query" -x http://127.0.0.1:8080 -k +``` + ## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details @@ -111,4 +127,3 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file ## Bugs + Questions 馃懢 If you have some issues/bugs/questions, feel free to open an issue. - From 8099dae3ce17035caeb04f7f9f8b11004bb806d9 Mon Sep 17 00:00:00 2001 From: Pachakutiq <101460915+PACHAKUTlQ@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:35:54 +0800 Subject: [PATCH 09/12] fix: Remove duplicated results in google/search.go --- google/search.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/google/search.go b/google/search.go index 78e27e3..491bdd6 100644 --- a/google/search.go +++ b/google/search.go @@ -281,6 +281,23 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { srchRes.URL = href.String() } + // Skip if URL is empty or we've already seen this URL + if srchRes.URL == "" { + continue + } + + // Check for duplicates + isDuplicate := false + for _, existing := range searchResults { + if existing.URL == srchRes.URL { + isDuplicate = true + break + } + } + if isDuplicate { + continue + } + // Get description using multiple fallback strategies desc := "" if descTag, err := resEl.Element("div[data-sncf='1'] div"); err == nil { From 6691ebaadd00171491ff441a3f50c727f9aed42f Mon Sep 17 00:00:00 2001 From: Pachakutiq <101460915+PACHAKUTlQ@users.noreply.github.com> Date: Thu, 10 Apr 2025 16:59:31 +0800 Subject: [PATCH 10/12] fix: Remove duplicated results for all search engines and both modes --- baidu/search.go | 4 ++-- baidu/search_raw.go | 2 +- core/common.go | 20 ++++++++++++++++++++ google/search.go | 17 +++-------------- google/search_raw.go | 2 +- yandex/search.go | 4 ++-- yandex/search_raw.go | 2 +- 7 files changed, 30 insertions(+), 21 deletions(-) diff --git a/baidu/search.go b/baidu/search.go index e4edd1e..ef87845 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -138,7 +138,7 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { } } - return searchResults, nil + return core.DeduplicateResults(searchResults), nil } func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { @@ -231,5 +231,5 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { } } - return searchResults, nil + return core.DeduplicateResults(searchResults), nil } diff --git a/baidu/search_raw.go b/baidu/search_raw.go index 188fd7b..998e3b0 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -113,5 +113,5 @@ func Search(query core.Query) ([]core.SearchResult, error) { } logrus.Debugf("Baidu Raw results : %v", results) - return results, nil + return core.DeduplicateResults(results), nil } diff --git a/core/common.go b/core/common.go index bf0cfad..6670c2c 100644 --- a/core/common.go +++ b/core/common.go @@ -20,6 +20,26 @@ type SearchResult struct { Ad bool `json:"ad"` } +func DeduplicateResults(results []SearchResult) []SearchResult { + unique := make(map[string]bool) + var deduped []SearchResult + + for _, result := range results { + if result.URL == "" { + continue + } + if !unique[result.URL] { + unique[result.URL] = true + deduped = append(deduped, result) + } + } + + sort.Slice(deduped, func(i, j int) bool { + return deduped[i].Rank < deduped[j].Rank + }) + return deduped +} + func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult { searchResults := []SearchResult{} diff --git a/google/search.go b/google/search.go index 491bdd6..9d79e84 100644 --- a/google/search.go +++ b/google/search.go @@ -132,6 +132,7 @@ func (gogl *Google) acceptCookies(page *rod.Page) { btnElms[3].Click(proto.InputMouseButtonLeft, 1) } + func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { logrus.Tracef("Start Google search, query: %+v", query) @@ -281,23 +282,11 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { srchRes.URL = href.String() } - // Skip if URL is empty or we've already seen this URL + // Skip if URL is empty if srchRes.URL == "" { continue } - // Check for duplicates - isDuplicate := false - for _, existing := range searchResults { - if existing.URL == srchRes.URL { - isDuplicate = true - break - } - } - if isDuplicate { - continue - } - // Get description using multiple fallback strategies desc := "" if descTag, err := resEl.Element("div[data-sncf='1'] div"); err == nil { @@ -337,7 +326,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { searchResults = append(searchResults, srchRes) } - return searchResults, nil + return core.DeduplicateResults(searchResults), nil } func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) { diff --git a/google/search_raw.go b/google/search_raw.go index 59c15ff..72543bc 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -115,7 +115,7 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) { } logrus.Tracef("Google search document size: %d", len(doc.Text())) - return results, err + return core.DeduplicateResults(results), err } func Search(query core.Query) ([]core.SearchResult, error) { diff --git a/yandex/search.go b/yandex/search.go index d7ffb5e..853df64 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -169,7 +169,7 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { time.Sleep(yand.pageSleep) } - return allResults, nil + return core.DeduplicateResults(allResults), nil } func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) { @@ -242,5 +242,5 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) { return searchResults[i].Rank < searchResults[j].Rank }) - return searchResults, nil + return core.DeduplicateResults(searchResults), nil } diff --git a/yandex/search_raw.go b/yandex/search_raw.go index 102a721..69d8884 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -89,7 +89,7 @@ func yandexResultParser(response *http.Response) ([]core.SearchResult, error) { } logrus.Tracef("Yandex search document size: %d", len(doc.Text())) - return results, err + return core.DeduplicateResults(results), err } func Search(query core.Query) ([]core.SearchResult, error) { From 2f8ee2164d3e2d4551983c25d1b4db3d7554faa4 Mon Sep 17 00:00:00 2001 From: Pachakutiq <101460915+PACHAKUTlQ@users.noreply.github.com> Date: Thu, 10 Apr 2025 18:07:57 +0800 Subject: [PATCH 11/12] fix: Fix serve mode not supporting raw mode --- cmd/serve.go | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/cmd/serve.go b/cmd/serve.go index 0b4e632..8e8cb96 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -1,6 +1,7 @@ package cmd import ( + "fmt" "time" "github.com/karust/openserp/baidu" @@ -9,8 +10,48 @@ import ( "github.com/karust/openserp/yandex" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "golang.org/x/time/rate" ) +// rawEngine implements SearchEngine interface for raw HTTP requests +type rawEngine struct { + name string +} + +func (r *rawEngine) Search(q core.Query) ([]core.SearchResult, error) { + // Inject proxy settings from config + q.ProxyURL = config.App.ProxyURL + q.Insecure = config.App.Insecure + + switch r.name { + case "google": + return google.Search(q) + case "yandex": + return yandex.Search(q) + case "baidu": + return baidu.Search(q) + default: + return nil, fmt.Errorf("unsupported engine: %s", r.name) + } +} + +func (r *rawEngine) SearchImage(q core.Query) ([]core.SearchResult, error) { + return nil, fmt.Errorf("image search is not supported in raw mode for %s", r.name) +} + +func (r *rawEngine) Name() string { + return r.name +} + +func (r *rawEngine) IsInitialized() bool { + return true +} + +func (r *rawEngine) GetRateLimiter() *rate.Limiter { + // Use default rate limiter for raw requests + return rate.NewLimiter(rate.Every(time.Second), 5) +} + var serveCMD = &cobra.Command{ Use: "serve", Aliases: []string{"listen"}, @@ -20,6 +61,17 @@ var serveCMD = &cobra.Command{ } func serve(cmd *cobra.Command, args []string) { + if config.App.IsRawRequests { + logrus.Warn("Browserless results are very inconsistent or may not even work!") + serv := core.NewServer(config.App.Host, config.App.Port, + &rawEngine{name: "google"}, + &rawEngine{name: "yandex"}, + &rawEngine{name: "baidu"}, + ) + serv.Listen() + return + } + opts := core.BrowserOpts{ IsHeadless: !config.App.IsBrowserHead, // Disable headless if browser head mode is set IsLeakless: config.App.IsLeakless, @@ -37,6 +89,7 @@ func serve(cmd *cobra.Command, args []string) { browser, err := core.NewBrowser(opts) if err != nil { logrus.Error(err) + return } yand := yandex.New(*browser, config.YandexConfig) From c387268d2068bdcf1dd410bbd6c46aa36419e748 Mon Sep 17 00:00:00 2001 From: Pachakutiq <101460915+PACHAKUTlQ@users.noreply.github.com> Date: Tue, 27 May 2025 15:05:53 +0800 Subject: [PATCH 12/12] chore: Exclude .aider* in .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6c8a434..e96c274 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ logrus.txt logs.txt .release core/test/ +.aider*