diff --git a/.gitignore b/.gitignore index 6c8a434..e96c274 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ logrus.txt logs.txt .release core/test/ +.aider* diff --git a/README.md b/README.md index 0565f75..37f4b0a 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,23 @@ # OpenSERP (Search Engine Results Page) + ![OpenSERP](/logo.png) [![Go Report Card](https://goreportcard.com/badge/github.com/karust/openserp)](https://goreportcard.com/report/github.com/karust/openserp) [![Go Reference](https://pkg.go.dev/badge/github.com/karust/openserp.svg)](https://pkg.go.dev/github.com/karust/openserp) [![release](https://img.shields.io/github/release-pre/karust/openserp.svg)](https://github.com/karust/openserp/releases) + + API access for search engines results if available isn't free. Using OpenSERP, you can get search results from **Google**, **Yandex**, **Baidu** via API or CLI! See [Docker](#docker) and [CLI](#cli) usage examples below ([search](#search), [images](#images)). -## Docker usage 馃惓 -* Run API server: +## Docker usage 馃惓 + +- Run API server: + ```bash # Use prebuilt image docker run -p 127.0.0.1:7000:7000 -it karust/openserp serve -a 0.0.0.0 -p 7000 @@ -22,26 +27,32 @@ docker-compose up --build ``` ### Request parameters -| Param | Description | -|-------|--------------------------------------------------------------| -| text | Text to search | -| lang | Search pages in selected language (`EN`, `DE`, `RU`...) | -| date | Date in `YYYYMMDD..YYYYMMDD` format (e.g. 20181010..20231010) | -| file | File extension to search (e.g. `PDF`, `DOC`) | -| site | Search within a specific website | -| limit | Limit the number of results -| answers | Include google answers as negative rank indexes (e.g. `true`, `false`) + +| Param | Description | +| ------- | ---------------------------------------------------------------------- | +| text | Text to search | +| lang | Search pages in selected language (`EN`, `DE`, `RU`...) | +| date | Date in `YYYYMMDD..YYYYMMDD` format (e.g. 20181010..20231010) | +| file | File extension to search (e.g. `PDF`, `DOC`) | +| site | Search within a specific website | +| limit | Limit the number of results | +| answers | Include google answers as negative rank indexes (e.g. `true`, `false`) | ### **Search** -### *Example request* + +### _Example request_ + Get 20 **Google** results for `hello world`, only in English: + ``` GET http:/127.0.0.1:7000/google/search?lang=EN&limit=20&text=hello world ``` -You can replace `google` to `yandex` or `baidu` in query to change search engine. - | -### *Example response* +You can replace `google` to `yandex` or `baidu` in query to change search engine. +| + +### _Example response_ + ```JSON [ { @@ -53,39 +64,66 @@ You can replace `google` to `yandex` or `baidu` in query to change search engine }, ] ``` + ### **Images** **[WIP]** -### *Example request* + +### _Example request_ + Get 100 **Google** results for `golden puppy`: + ``` GET http://127.0.0.1:7000/google/image?text=golden puppy&limit=100 ``` - ## CLI 鈱笍 -* Use `-h` flag to see commands. -* You can use `serve` command to serve API: + +- Use `-h` flag to see all commands. +- You can use `serve` command to serve API: + ```bash -openserp serve +openserp serve ``` -* Or print results in CLI using `search` command: + +- Or print results in CLI using `search` command: + ```bash openserp search google "how to get banned from google fast" # Change `google` to `yandex` or `baidu` ``` + As a result you should get JSON output containting search results: + ```json [ - { - "rank": 1, - "url": "https://www.cyberoptik.net/blog/6-sure-fire-ways-to-get-banned-from-google/", - "title": "11 Sure-Fire Ways to Get Banned From Google | CyberOptik", - "description": "How To Get Banned From Google 路 1. Cloaking: The Art of Deception 路 2. Plagiarism: Because Originality is Overrated 路 3. Keyword Stuffing: More is Always Better 路 4 ...", - "ad": false - }, + { + "rank": 1, + "url": "https://www.cyberoptik.net/blog/6-sure-fire-ways-to-get-banned-from-google/", + "title": "11 Sure-Fire Ways to Get Banned From Google | CyberOptik", + "description": "How To Get Banned From Google 路 1. Cloaking: The Art of Deception 路 2. Plagiarism: Because Originality is Overrated 路 3. Keyword Stuffing: More is Always Better 路 4 ...", + "ad": false + } ] - ``` +``` + +### Proxy Support + +Both browser mode and raw mode support HTTP and SOCKS5 proxies with authentication: + +```bash +# HTTP proxy with auth +openserp search google "query" --proxy http://user:pass@127.0.0.1:8080 + +# Serve with SOCKS5 proxy +openserp serve --proxy socks5://127.0.0.1:1080 + +# For HTTPS sites through HTTP proxy, use --insecure to ignore certificate errors +openserp search google "query" --proxy http://127.0.0.1:8080 --insecure +openserp search google "query" -x http://127.0.0.1:8080 -k +``` + +## License - ## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details ## Bugs + Questions 馃懢 -If you have some issues/bugs/questions, feel free to open an issue. \ No newline at end of file + +If you have some issues/bugs/questions, feel free to open an issue. diff --git a/baidu/search.go b/baidu/search.go index e4edd1e..ef87845 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -138,7 +138,7 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { } } - return searchResults, nil + return core.DeduplicateResults(searchResults), nil } func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { @@ -231,5 +231,5 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { } } - return searchResults, nil + return core.DeduplicateResults(searchResults), nil } diff --git a/baidu/search_raw.go b/baidu/search_raw.go index 695586d..998e3b0 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -1,9 +1,12 @@ package baidu import ( + "crypto/tls" "fmt" "net/http" + "net/url" "strings" + "time" "github.com/PuerkitoBio/goquery" "github.com/corpix/uarand" @@ -11,8 +14,27 @@ import ( "github.com/sirupsen/logrus" ) -func baiduRequest(searchURL string) (*http.Response, error) { - baseClient := &http.Client{} +func baiduRequest(searchURL string, query core.Query) (*http.Response, error) { + // Create HTTP transport with proxy + transport := &http.Transport{} + if query.ProxyURL != "" { + proxyUrl, err := url.Parse(query.ProxyURL) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyUrl) + } + + // Set insecure TLS + if query.Insecure { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + baseClient := &http.Client{ + Transport: transport, + Timeout: time.Second * 10, + } + req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, err @@ -27,7 +49,7 @@ func baiduRequest(searchURL string) (*http.Response, error) { } func baiduResultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromResponse(response) + doc, err := goquery.NewDocumentFromReader(response.Body) if err != nil { return nil, err } @@ -79,7 +101,7 @@ func Search(query core.Query) ([]core.SearchResult, error) { } logrus.Debugf("Baidu URL built: %s", googleURL) - res, err := baiduRequest(googleURL) + res, err := baiduRequest(googleURL, query) if err != nil { return nil, err } @@ -91,5 +113,5 @@ func Search(query core.Query) ([]core.SearchResult, error) { } logrus.Debugf("Baidu Raw results : %v", results) - return results, nil + return core.DeduplicateResults(results), nil } diff --git a/cmd/root.go b/cmd/root.go index a799355..043024d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -40,6 +40,8 @@ type AppConfig struct { IsDebug bool `mapstructure:"debug"` IsVerbose bool `mapstructure:"verbose"` IsRawRequests bool `mapstructure:"raw_requests"` + ProxyURL string `mapstructure:"proxy"` + Insecure bool `mapstructure:"insecure"` } var config = Config{} @@ -133,4 +135,6 @@ func init() { RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests") RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made") RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key") + RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "x", "", "HTTP or Socks5 proxy URL (e.g. http://user:pass@127.0.0.1:8080)") + RootCmd.PersistentFlags().BoolVarP(&config.App.Insecure, "insecure", "k", false, "Allow insecure TLS connections") } diff --git a/cmd/search.go b/cmd/search.go index c137735..2c506d4 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -26,8 +26,10 @@ func search(cmd *cobra.Command, args []string) { var err error engineType := args[0] query := core.Query{ - Text: args[1], - Limit: 10, + Text: args[1], + Limit: 10, + ProxyURL: config.App.ProxyURL, + Insecure: config.App.Insecure, } results := []core.SearchResult{} @@ -59,6 +61,8 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er Timeout: time.Second * time.Duration(config.App.Timeout), LeavePageOpen: config.App.IsLeaveHead, CaptchaSolverApiKey: config.Config2Capcha.ApiKey, + ProxyURL: config.App.ProxyURL, + Insecure: config.App.Insecure, } if config.App.IsDebug { diff --git a/cmd/serve.go b/cmd/serve.go index 3a14e3f..8e8cb96 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -1,6 +1,7 @@ package cmd import ( + "fmt" "time" "github.com/karust/openserp/baidu" @@ -9,8 +10,48 @@ import ( "github.com/karust/openserp/yandex" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "golang.org/x/time/rate" ) +// rawEngine implements SearchEngine interface for raw HTTP requests +type rawEngine struct { + name string +} + +func (r *rawEngine) Search(q core.Query) ([]core.SearchResult, error) { + // Inject proxy settings from config + q.ProxyURL = config.App.ProxyURL + q.Insecure = config.App.Insecure + + switch r.name { + case "google": + return google.Search(q) + case "yandex": + return yandex.Search(q) + case "baidu": + return baidu.Search(q) + default: + return nil, fmt.Errorf("unsupported engine: %s", r.name) + } +} + +func (r *rawEngine) SearchImage(q core.Query) ([]core.SearchResult, error) { + return nil, fmt.Errorf("image search is not supported in raw mode for %s", r.name) +} + +func (r *rawEngine) Name() string { + return r.name +} + +func (r *rawEngine) IsInitialized() bool { + return true +} + +func (r *rawEngine) GetRateLimiter() *rate.Limiter { + // Use default rate limiter for raw requests + return rate.NewLimiter(rate.Every(time.Second), 5) +} + var serveCMD = &cobra.Command{ Use: "serve", Aliases: []string{"listen"}, @@ -20,12 +61,25 @@ var serveCMD = &cobra.Command{ } func serve(cmd *cobra.Command, args []string) { + if config.App.IsRawRequests { + logrus.Warn("Browserless results are very inconsistent or may not even work!") + serv := core.NewServer(config.App.Host, config.App.Port, + &rawEngine{name: "google"}, + &rawEngine{name: "yandex"}, + &rawEngine{name: "baidu"}, + ) + serv.Listen() + return + } + opts := core.BrowserOpts{ IsHeadless: !config.App.IsBrowserHead, // Disable headless if browser head mode is set IsLeakless: config.App.IsLeakless, Timeout: time.Second * time.Duration(config.App.Timeout), LeavePageOpen: config.App.IsLeaveHead, CaptchaSolverApiKey: config.Config2Capcha.ApiKey, + ProxyURL: config.App.ProxyURL, + Insecure: config.App.Insecure, } if config.App.IsDebug { @@ -35,6 +89,7 @@ func serve(cmd *cobra.Command, args []string) { browser, err := core.NewBrowser(opts) if err != nil { logrus.Error(err) + return } yand := yandex.New(*browser, config.YandexConfig) diff --git a/core/browser.go b/core/browser.go index b739cd4..293a3d3 100644 --- a/core/browser.go +++ b/core/browser.go @@ -1,6 +1,8 @@ package core import ( + "fmt" + "net/url" "time" "github.com/go-rod/rod" @@ -19,6 +21,8 @@ type BrowserOpts struct { LeavePageOpen bool // Leave pages and browser open WaitLoadTime time.Duration // Time to wait till page loads CaptchaSolverApiKey string // 2Captcha api key + ProxyURL string // Proxy URL + Insecure bool // Allow insecure TLS connections } @@ -47,9 +51,33 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) { path, has := launcher.LookPath() logrus.Debug("Browser found: ", has) + // Create launcher + l := launcher.New().Bin(path).Leakless(opts.IsLeakless).Headless(opts.IsHeadless) + + // Configure proxy if specified + if opts.ProxyURL != "" { + proxyUrl, err := url.Parse(opts.ProxyURL) + if err != nil { + return nil, fmt.Errorf("invalid proxy URL: %v", err) + } + + // Make sure the proxy URL includes the scheme when passed to launcher + // This ensures proper handling of SOCKS5 proxies + proxyStr := proxyUrl.String() + logrus.Debugf("Setting up proxy: %s", proxyStr) + l = l.Proxy(proxyStr) + + // Check if proxy has auth credentials + if proxyUrl.User != nil { + username := proxyUrl.User.Username() + logrus.Debugf("Using proxy authentication: %s:****", username) + // We'll handle auth in the Navigate method + } + } + var err error b := Browser{BrowserOpts: opts} - b.browserAddr, err = launcher.New().Bin(path).Leakless(opts.IsLeakless).Headless(opts.IsHeadless).Launch() + b.browserAddr, err = l.Launch() if opts.CaptchaSolverApiKey != "" { b.CaptchaSolver = NewSolver(opts.CaptchaSolverApiKey) @@ -76,10 +104,27 @@ func (b *Browser) Navigate(URL string) *rod.Page { b.browser.MustConnect() b.browser.SetCookies(nil) - //page := b.browser.MustPage(URL) + // Handle proxy authentication before any navigations + if b.ProxyURL != "" { + proxyUrl, _ := url.Parse(b.ProxyURL) + + // Always ignore certificate errors when using proxies + // This fixes the ERR_CERT_AUTHORITY_INVALID error for SOCKS5 proxies + b.browser.MustIgnoreCertErrors(true) + + if proxyUrl.User != nil { + username := proxyUrl.User.Username() + password, _ := proxyUrl.User.Password() + // Launch auth handler before any navigation occurs + go b.browser.MustHandleAuth(username, password)() + } + } else if b.Insecure { + // Still respect the insecure flag if no proxy is used + b.browser.MustIgnoreCertErrors(true) + } + page := stealth.MustPage(b.browser) page.MustEmulate(devices.Device{ - //UserAgent: uarand.GetRandom(), AcceptLanguage: b.LanguageCode, }) page.MustNavigate(URL) @@ -90,9 +135,6 @@ func (b *Browser) Navigate(URL string) *rod.Page { wait() } - // Wait till page loads - //time.Sleep(b.WaitLoadTime) - return page } diff --git a/core/common.go b/core/common.go index 7a52a66..6670c2c 100644 --- a/core/common.go +++ b/core/common.go @@ -20,6 +20,26 @@ type SearchResult struct { Ad bool `json:"ad"` } +func DeduplicateResults(results []SearchResult) []SearchResult { + unique := make(map[string]bool) + var deduped []SearchResult + + for _, result := range results { + if result.URL == "" { + continue + } + if !unique[result.URL] { + unique[result.URL] = true + deduped = append(deduped, result) + } + } + + sort.Slice(deduped, func(i, j int) bool { + return deduped[i].Rank < deduped[j].Rank + }) + return deduped +} + func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult { searchResults := []SearchResult{} @@ -41,6 +61,8 @@ type Query struct { Site string // Search site Limit int // Limit the number of results Answers bool // Include question and answers from SERP page to results with negative indexes + ProxyURL string // Proxy URL for raw requests + Insecure bool // Allow insecure TLS connections } func (q Query) IsEmpty() bool { diff --git a/google/search.go b/google/search.go index 118280a..9d79e84 100644 --- a/google/search.go +++ b/google/search.go @@ -132,6 +132,7 @@ func (gogl *Google) acceptCookies(page *rod.Page) { btnElms[3].Click(proto.InputMouseButtonLeft, 1) } + func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { logrus.Tracef("Start Google search, query: %+v", query) @@ -157,8 +158,8 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { gogl.acceptCookies(page) } - // Find all results - results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid]") + // Find all results using stable attributes + results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved]") if err != nil { logrus.Errorf("Cannot parse search results: %s", err) return nil, core.ErrSearchTimeout @@ -265,35 +266,56 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { searchResults = append(searchResults, srchRes) } continue - } else if strings.Contains(attrs, "data-ved") && strings.Contains(attrs, "lang") { - // 3. Parse regular search results - // Get URL - link, err := resEl.Element("a") + } else if strings.Contains(attrs, "data-ved") { + // Parse regular search results + // Get title from h3 + titleTag, err := resEl.Element("h3") if err != nil { continue } - href, err := link.Property("href") - if err != nil { - logrus.Debug("No `href` tag found") + srchRes.Title, _ = titleTag.Text() + + // Get URL from parent link of h3 + link, err := titleTag.Parent() + if err == nil && link.MustMatches("a") { + href, _ := link.Property("href") + srchRes.URL = href.String() } - srchRes.URL = href.String() + + // Skip if URL is empty + if srchRes.URL == "" { + continue + } + + // Get description using multiple fallback strategies + desc := "" + if descTag, err := resEl.Element("div[data-sncf='1'] div"); err == nil { + desc = descTag.MustText() + } else if descTag, err := resEl.Element("div.VwiC3b"); err == nil { + desc = descTag.MustText() + } else { + // Structural fallback + parent, err := titleTag.Parent() + if err == nil { + parent, err = parent.Parent() + if err == nil { + parent, err = parent.Parent() + if err == nil { + if descTag, err := parent.Next(); err == nil { + if descDiv, err := descTag.Element("div"); err == nil { + desc = descDiv.MustText() + } + } + } + } + } + } + srchRes.Description = desc + rank += 1 - - // Get title - titleTag, err := link.Element("h3") - if err != nil { - continue - } - - srchRes.Title, err = titleTag.Text() - if err != nil { - logrus.Debug("Cannot extract text from title") - } - - // Get description - text := resEl.MustText() - textSliced := strings.Split(text, "\n") - srchRes.Description = strings.Join(textSliced[4:], "\n") + srchRes.Rank = rank + searchResults = append(searchResults, srchRes) + continue } else { //fmt.Println(i, attrs) @@ -304,7 +326,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { searchResults = append(searchResults, srchRes) } - return searchResults, nil + return core.DeduplicateResults(searchResults), nil } func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) { @@ -324,7 +346,7 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) { page.Mouse.Scroll(0, 1000000, 1) page.WaitLoad() - results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction][jsdata]") + results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]") if err != nil { logrus.Errorf("Cannot parse search results: %s", err) return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout diff --git a/google/search_raw.go b/google/search_raw.go index 3e482ff..72543bc 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -1,8 +1,11 @@ package google import ( + "crypto/tls" "net/http" + "net/url" "strings" + "time" "github.com/PuerkitoBio/goquery" "github.com/corpix/uarand" @@ -10,8 +13,27 @@ import ( "github.com/sirupsen/logrus" ) -func googleRequest(searchURL string) (*http.Response, error) { - baseClient := &http.Client{} +func googleRequest(searchURL string, query core.Query) (*http.Response, error) { + // Create HTTP transport with proxy + transport := &http.Transport{} + if query.ProxyURL != "" { + proxyUrl, err := url.Parse(query.ProxyURL) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyUrl) + } + + // Set insecure TLS + if query.Insecure { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + baseClient := &http.Client{ + Transport: transport, + Timeout: time.Second * 10, + } + req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, err @@ -26,7 +48,7 @@ func googleRequest(searchURL string) (*http.Response, error) { } func googleResultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromResponse(response) + doc, err := goquery.NewDocumentFromReader(response.Body) if err != nil { return nil, err } @@ -34,23 +56,49 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) { results := []core.SearchResult{} rank := 1 - // Get individual results - sel := doc.Find("div.g") + // Use data attributes instead of class names to find results + // Both old and new DOM have data-hveid and data-ved attributes + sel := doc.Find("div[data-hveid][data-ved]") for i := range sel.Nodes { item := sel.Eq(i) - // Find URL - linkTag := item.Find("a") - link, _ := linkTag.Attr("href") + // Skip items without an h3 element (which indicates a search result) + if item.Find("h3").Length() == 0 { + continue + } + + // Find URL - look for the anchor that contains the h3 title + linkTag := item.Find("h3").Parent() + if linkTag.Is("a") == false { + linkTag = item.Find("h3").Closest("a") + } + + link, exists := linkTag.Attr("href") + if !exists || link == "" || link == "#" { + continue + } link = strings.Trim(link, " ") - // Find title + // Find title - this is inside the h3 element titleTag := item.Find("h3") title := titleTag.Text() - // Find description - descTag := item.Find(`div[data-sncf~="1"]`) + // Find description - find div with text content after the heading + // Using attribute selectors that match the description container + descTag := item.Find("div[data-sncf='1']").Find("div").First() + if descTag.Length() == 0 { + // Try another selector approach if the first one fails + descTag = item.Find("div.VwiC3b") + if descTag.Length() == 0 { + // As a last resort, look for any div after the title that might contain description + titleParent := titleTag.Parent() + if titleParent.Is("a") { + titleParent = titleParent.Parent().Parent() + } + descTag = titleParent.NextAll().First().Find("div").First() + } + } desc := descTag.Text() if link != "" && link != "#" { @@ -67,7 +115,7 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) { } logrus.Tracef("Google search document size: %d", len(doc.Text())) - return results, err + return core.DeduplicateResults(results), err } func Search(query core.Query) ([]core.SearchResult, error) { @@ -77,7 +125,7 @@ func Search(query core.Query) ([]core.SearchResult, error) { } logrus.Debugf("Google URL built: %s", googleURL) - res, err := googleRequest(googleURL) + res, err := googleRequest(googleURL, query) if err != nil { return nil, err } diff --git a/yandex/search.go b/yandex/search.go index d7ffb5e..853df64 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -169,7 +169,7 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) { time.Sleep(yand.pageSleep) } - return allResults, nil + return core.DeduplicateResults(allResults), nil } func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) { @@ -242,5 +242,5 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) { return searchResults[i].Rank < searchResults[j].Rank }) - return searchResults, nil + return core.DeduplicateResults(searchResults), nil } diff --git a/yandex/search_raw.go b/yandex/search_raw.go index 8779050..69d8884 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -1,8 +1,11 @@ package yandex import ( + "crypto/tls" "net/http" + "net/url" "strings" + "time" "github.com/PuerkitoBio/goquery" "github.com/corpix/uarand" @@ -10,8 +13,27 @@ import ( "github.com/sirupsen/logrus" ) -func yandexRequest(searchURL string) (*http.Response, error) { - baseClient := &http.Client{} +func yandexRequest(searchURL string, query core.Query) (*http.Response, error) { + // Create HTTP transport with proxy + transport := &http.Transport{} + if query.ProxyURL != "" { + proxyUrl, err := url.Parse(query.ProxyURL) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyUrl) + } + + // Set insecure TLS + if query.Insecure { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + baseClient := &http.Client{ + Transport: transport, + Timeout: time.Second * 10, + } + req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, err @@ -26,7 +48,7 @@ func yandexRequest(searchURL string) (*http.Response, error) { } func yandexResultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromResponse(response) + doc, err := goquery.NewDocumentFromReader(response.Body) if err != nil { return nil, err } @@ -67,7 +89,7 @@ func yandexResultParser(response *http.Response) ([]core.SearchResult, error) { } logrus.Tracef("Yandex search document size: %d", len(doc.Text())) - return results, err + return core.DeduplicateResults(results), err } func Search(query core.Query) ([]core.SearchResult, error) { @@ -77,7 +99,7 @@ func Search(query core.Query) ([]core.SearchResult, error) { } logrus.Debugf("Yandex URL built: %s", googleURL) - res, err := yandexRequest(googleURL) + res, err := yandexRequest(googleURL, query) if err != nil { return nil, err }