Stealth browser as option. Update default config

Support http and socks5 proxy and fix failure due to updates of DOM on Google SERP

.gitignore

@@ -24,3 +24,4 @@ logrus.txt
 logs.txt
 .release
 core/test/
+.aider*

README.md

@@ -1,18 +1,23 @@
 # OpenSERP (Search Engine Results Page)
+
 ![OpenSERP](/logo.png)
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/karust/openserp)](https://goreportcard.com/report/github.com/karust/openserp)
 [![Go Reference](https://pkg.go.dev/badge/github.com/karust/openserp.svg)](https://pkg.go.dev/github.com/karust/openserp)
 [![release](https://img.shields.io/github/release-pre/karust/openserp.svg)](https://github.com/karust/openserp/releases)
+
 <!-- ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/karust/openserp/latest) -->
+
 API access for search engines results if available isn't free.
 
 Using OpenSERP, you can get search results from **Google**, **Yandex**, **Baidu** via API or CLI!
 
 See [Docker](#docker) and [CLI](#cli) usage examples below ([search](#search), [images](#images)).
 
-## Docker usage  <a name="docker"></a> 🐳
-* Run API server:
+## Docker usage <a name="docker"></a> 🐳
+
+- Run API server:
+
 ```bash
 # Use prebuilt image
 docker run -p 127.0.0.1:7000:7000 -it karust/openserp serve -a 0.0.0.0 -p 7000
@@ -22,26 +27,32 @@ docker-compose up --build
 ```
 
 ### Request parameters
-| Param | Description                                                  |
-|-------|--------------------------------------------------------------|
-| text  | Text to search                                               |
-| lang  | Search pages in selected language (`EN`, `DE`, `RU`...)      |
-| date  | Date in `YYYYMMDD..YYYYMMDD` format (e.g. 20181010..20231010) |
-| file  | File extension to search  (e.g. `PDF`, `DOC`)                 |
-| site  | Search within a specific website                                 |
-| limit | Limit the number of results  
-| answers | Include google answers as negative rank indexes (e.g. `true`, `false`)
+
+| Param   | Description                                                            |
+| ------- | ---------------------------------------------------------------------- |
+| text    | Text to search                                                         |
+| lang    | Search pages in selected language (`EN`, `DE`, `RU`...)                |
+| date    | Date in `YYYYMMDD..YYYYMMDD` format (e.g. 20181010..20231010)          |
+| file    | File extension to search (e.g. `PDF`, `DOC`)                           |
+| site    | Search within a specific website                                       |
+| limit   | Limit the number of results                                            |
+| answers | Include google answers as negative rank indexes (e.g. `true`, `false`) |
 
 ### **Search**
-### *Example request*
+
+### _Example request_
+
 Get 20 **Google** results for `hello world`, only in English:
+
 ```
 GET http:/127.0.0.1:7000/google/search?lang=EN&limit=20&text=hello world
 ```
-You can replace `google` to `yandex` or `baidu` in query to change search engine.
-                                |
 
-### *Example response*
+You can replace `google` to `yandex` or `baidu` in query to change search engine.
+|
+
+### _Example response_
+
 ```JSON
 [
     {
@@ -53,39 +64,66 @@ You can replace `google` to `yandex` or `baidu` in query to change search engine
     },
 ]
 ```
+
 ### **Images** **[WIP]**
-### *Example request*
+
+### _Example request_
+
 Get 100 **Google** results for `golden puppy`:
+
 ```
 GET http://127.0.0.1:7000/google/image?text=golden puppy&limit=100
 ```
 
-
 ## CLI <a name="cli"></a> ⌨️
-* Use `-h` flag to see commands.
-* You can use `serve` command to serve API:
+
+- Use `-h` flag to see all commands.
+- You can use `serve` command to serve API:
+
 ```bash
-openserp serve 
+openserp serve
 ```
-* Or print results in CLI using `search` command:
+
+- Or print results in CLI using `search` command:
+
 ```bash
 openserp search google "how to get banned from google fast" # Change `google` to `yandex` or `baidu`
 ```
+
 As a result you should get JSON output containting search results:
+
 ```json
 [
- {
-  "rank": 1,
-  "url": "https://www.cyberoptik.net/blog/6-sure-fire-ways-to-get-banned-from-google/",
-  "title": "11 Sure-Fire Ways to Get Banned From Google | CyberOptik",
-  "description": "How To Get Banned From Google · 1. Cloaking: The Art of Deception · 2. Plagiarism: Because Originality is Overrated · 3. Keyword Stuffing: More is Always Better · 4 ...",
-  "ad": false
- },
+  {
+    "rank": 1,
+    "url": "https://www.cyberoptik.net/blog/6-sure-fire-ways-to-get-banned-from-google/",
+    "title": "11 Sure-Fire Ways to Get Banned From Google | CyberOptik",
+    "description": "How To Get Banned From Google · 1. Cloaking: The Art of Deception · 2. Plagiarism: Because Originality is Overrated · 3. Keyword Stuffing: More is Always Better · 4 ...",
+    "ad": false
+  }
 ]
- ```
+```
+
+### Proxy Support
+
+Both browser mode and raw mode support HTTP and SOCKS5 proxies with authentication:
+
+```bash
+# HTTP proxy with auth
+openserp search google "query" --proxy http://user:pass@127.0.0.1:8080
+
+# Serve with SOCKS5 proxy
+openserp serve --proxy socks5://127.0.0.1:1080
+
+# For HTTPS sites through HTTP proxy, use --insecure to ignore certificate errors
+openserp search google "query" --proxy http://127.0.0.1:8080 --insecure
+openserp search google "query" -x http://127.0.0.1:8080 -k
+```
+
+## License
 
- ## License
 This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details
 
 ## Bugs + Questions 👾
-If you have some issues/bugs/questions, feel free to open an issue.
+
+If you have some issues/bugs/questions, feel free to open an issue.

baidu/search.go

@@ -24,19 +24,11 @@ type imageDataJson struct {
 		Height      int
 		Width       int
 		IsCopyright int
-
-		URL []struct {
+		AdType      string `json:"adType"`
+		URL         []struct {
 			SourcePage string `json:"FromURL"`
 			Original   string `json:"ObjURL"`
 		} `json:"replaceUrl"`
-
-		// Versions []struct {
-		// 	Height        int
-		// 	Width         int
-		// 	ImgSourcePage string `json:"fromURL"`
-		// 	URL           string `json:"objURL"`
-		// 	Type          string
-		// } `json:"setList"`
 	}
 }
 
@@ -63,18 +55,12 @@ func (baid *Baidu) GetRateLimiter() *rate.Limiter {
 
 func (baid *Baidu) isCaptcha(page *rod.Page) bool {
 	_, err := page.Timeout(baid.GetSelectorTimeout()).Search("div.passMod_dialog-body")
-	if err != nil {
-		return false
-	}
-	return true
+	return err == nil
 }
 
 func (baid *Baidu) isTimeout(page *rod.Page) bool {
 	_, err := page.Timeout(baid.GetSelectorTimeout()).Search("button.timeout-button")
-	if err != nil {
-		return false
-	}
-	return true
+	return err == nil
 }
 
 func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
@@ -88,7 +74,10 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
 		return nil, err
 	}
 
-	page := baid.Navigate(url)
+	page, err := baid.Navigate(url)
+	if err != nil {
+		return nil, err
+	}
 
 	results, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd")
 	if err != nil {
@@ -152,7 +141,7 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
 		}
 	}
 
-	return searchResults, nil
+	return core.DeduplicateResults(searchResults), nil
 }
 
 func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
@@ -168,7 +157,11 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		}
 
 		// Get anti-crawler cookies first, then reload page
-		page := baid.Navigate(url)
+		page, err := baid.Navigate(url)
+		if err != nil {
+			return nil, err
+		}
+
 		if !baid.LeavePageOpen {
 			defer page.Close()
 		}
@@ -226,7 +219,15 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
 				Rank:        (searchPage * 30) + (i + 1),
 				URL:         img.URL[0].Original,
 				Title:       img.Title,
-				Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright)}
+				Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright),
+				Ad: func() bool {
+					if img.AdType != "0" {
+						return true
+					} else {
+						return false
+					}
+				}(),
+			}
 			searchResults = append(searchResults, res)
 		}
 
@@ -237,5 +238,5 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		}
 	}
 
-	return searchResults, nil
+	return core.DeduplicateResults(searchResults), nil
 }

baidu/search_raw.go

@@ -1,9 +1,12 @@
 package baidu
 
 import (
+	"crypto/tls"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/corpix/uarand"
@@ -11,8 +14,27 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func baiduRequest(searchURL string) (*http.Response, error) {
-	baseClient := &http.Client{}
+func baiduRequest(searchURL string, query core.Query) (*http.Response, error) {
+	// Create HTTP transport with proxy
+	transport := &http.Transport{}
+	if query.ProxyURL != "" {
+		proxyUrl, err := url.Parse(query.ProxyURL)
+		if err != nil {
+			return nil, err
+		}
+		transport.Proxy = http.ProxyURL(proxyUrl)
+	}
+
+	// Set insecure TLS
+	if query.Insecure {
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+	baseClient := &http.Client{
+		Transport: transport,
+		Timeout:   time.Second * 10,
+	}
+
 	req, err := http.NewRequest("GET", searchURL, nil)
 	if err != nil {
 		return nil, err
@@ -27,7 +49,7 @@ func baiduRequest(searchURL string) (*http.Response, error) {
 }
 
 func baiduResultParser(response *http.Response) ([]core.SearchResult, error) {
-	doc, err := goquery.NewDocumentFromResponse(response)
+	doc, err := goquery.NewDocumentFromReader(response.Body)
 	if err != nil {
 		return nil, err
 	}
@@ -79,7 +101,7 @@ func Search(query core.Query) ([]core.SearchResult, error) {
 	}
 	logrus.Debugf("Baidu URL built: %s", googleURL)
 
-	res, err := baiduRequest(googleURL)
+	res, err := baiduRequest(googleURL, query)
 	if err != nil {
 		return nil, err
 	}
@@ -91,5 +113,5 @@ func Search(query core.Query) ([]core.SearchResult, error) {
 	}
 	logrus.Debugf("Baidu Raw results : %v", results)
 
-	return results, nil
+	return core.DeduplicateResults(results), nil
 }

cmd/root.go

@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	version               = "0.3"
+	version               = "0.4.1"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )
@@ -40,6 +40,9 @@ type AppConfig struct {
 	IsDebug       bool   `mapstructure:"debug"`
 	IsVerbose     bool   `mapstructure:"verbose"`
 	IsRawRequests bool   `mapstructure:"raw_requests"`
+	ProxyURL      string `mapstructure:"proxy"`
+	Insecure      bool   `mapstructure:"insecure"`
+	IsStealth     bool   `mapstructure:"stealth"`
 }
 
 var config = Config{}
@@ -133,4 +136,6 @@ func init() {
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
 	RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
+	RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "x", "", "HTTP or Socks5 proxy URL (e.g. http://user:pass@127.0.0.1:8080)")
+	RootCmd.PersistentFlags().BoolVarP(&config.App.IsStealth, "stealth", "s", false, "Use stealth browser plugin")
 }

cmd/search.go

@@ -26,8 +26,10 @@ func search(cmd *cobra.Command, args []string) {
 	var err error
 	engineType := args[0]
 	query := core.Query{
-		Text:  args[1],
-		Limit: 10,
+		Text:     args[1],
+		Limit:    10,
+		ProxyURL: config.App.ProxyURL,
+		Insecure: config.App.Insecure,
 	}
 	results := []core.SearchResult{}
 
@@ -59,6 +61,9 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er
 		Timeout:             time.Second * time.Duration(config.App.Timeout),
 		LeavePageOpen:       config.App.IsLeaveHead,
 		CaptchaSolverApiKey: config.Config2Capcha.ApiKey,
+		ProxyURL:            config.App.ProxyURL,
+		Insecure:            config.App.Insecure,
+		UseStealth:          config.App.IsStealth,
 	}
 
 	if config.App.IsDebug {

cmd/serve.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/karust/openserp/baidu"
@@ -9,8 +10,48 @@ import (
 	"github.com/karust/openserp/yandex"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"golang.org/x/time/rate"
 )
 
+// rawEngine implements SearchEngine interface for raw HTTP requests
+type rawEngine struct {
+	name string
+}
+
+func (r *rawEngine) Search(q core.Query) ([]core.SearchResult, error) {
+	// Inject proxy settings from config
+	q.ProxyURL = config.App.ProxyURL
+	q.Insecure = config.App.Insecure
+
+	switch r.name {
+	case "google":
+		return google.Search(q)
+	case "yandex":
+		return yandex.Search(q)
+	case "baidu":
+		return baidu.Search(q)
+	default:
+		return nil, fmt.Errorf("unsupported engine: %s", r.name)
+	}
+}
+
+func (r *rawEngine) SearchImage(q core.Query) ([]core.SearchResult, error) {
+	return nil, fmt.Errorf("image search is not supported in raw mode for %s", r.name)
+}
+
+func (r *rawEngine) Name() string {
+	return r.name
+}
+
+func (r *rawEngine) IsInitialized() bool {
+	return true
+}
+
+func (r *rawEngine) GetRateLimiter() *rate.Limiter {
+	// Use default rate limiter for raw requests
+	return rate.NewLimiter(rate.Every(time.Second), 5)
+}
+
 var serveCMD = &cobra.Command{
 	Use:     "serve",
 	Aliases: []string{"listen"},
@@ -20,12 +61,26 @@ var serveCMD = &cobra.Command{
 }
 
 func serve(cmd *cobra.Command, args []string) {
+	if config.App.IsRawRequests {
+		logrus.Warn("Browserless results are very inconsistent or may not even work!")
+		serv := core.NewServer(config.App.Host, config.App.Port,
+			&rawEngine{name: "google"},
+			&rawEngine{name: "yandex"},
+			&rawEngine{name: "baidu"},
+		)
+		serv.Listen()
+		return
+	}
+
 	opts := core.BrowserOpts{
 		IsHeadless:          !config.App.IsBrowserHead, // Disable headless if browser head mode is set
 		IsLeakless:          config.App.IsLeakless,
 		Timeout:             time.Second * time.Duration(config.App.Timeout),
 		LeavePageOpen:       config.App.IsLeaveHead,
 		CaptchaSolverApiKey: config.Config2Capcha.ApiKey,
+		ProxyURL:            config.App.ProxyURL,
+		Insecure:            config.App.Insecure,
+		UseStealth:          config.App.IsBrowserHead,
 	}
 
 	if config.App.IsDebug {
@@ -35,6 +90,7 @@ func serve(cmd *cobra.Command, args []string) {
 	browser, err := core.NewBrowser(opts)
 	if err != nil {
 		logrus.Error(err)
+		return
 	}
 
 	yand := yandex.New(*browser, config.YandexConfig)

config.yaml

@@ -7,6 +7,8 @@ app:
   head: false
   leakless: false
   leave_head: false
+  stealth: false
+  insecure: false
 
 2captcha:
   apikey: "123123123123123"

core/browser.go

@@ -1,6 +1,8 @@
 package core
 
 import (
+	"fmt"
+	"net/url"
 	"time"
 
 	"github.com/go-rod/rod"
@@ -19,6 +21,9 @@ type BrowserOpts struct {
 	LeavePageOpen       bool          // Leave pages and browser open
 	WaitLoadTime        time.Duration // Time to wait till page loads
 	CaptchaSolverApiKey string        // 2Captcha api key
+	ProxyURL            string        // Proxy URL
+	Insecure            bool          // Allow insecure TLS connections
+	UseStealth          bool          // Use go-rod stealth plugin
 
 }
 
@@ -47,9 +52,33 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) {
 	path, has := launcher.LookPath()
 	logrus.Debug("Browser found: ", has)
 
+	// Create launcher
+	l := launcher.New().Bin(path).Leakless(opts.IsLeakless).Headless(opts.IsHeadless)
+
+	// Configure proxy if specified
+	if opts.ProxyURL != "" {
+		proxyUrl, err := url.Parse(opts.ProxyURL)
+		if err != nil {
+			return nil, fmt.Errorf("invalid proxy URL: %v", err)
+		}
+
+		// Make sure the proxy URL includes the scheme when passed to launcher
+		// This ensures proper handling of SOCKS5 proxies
+		proxyStr := proxyUrl.String()
+		logrus.Debugf("Setting up proxy: %s", proxyStr)
+		l = l.Proxy(proxyStr)
+
+		// Check if proxy has auth credentials
+		if proxyUrl.User != nil {
+			username := proxyUrl.User.Username()
+			logrus.Debugf("Using proxy authentication: %s:****", username)
+			// We'll handle auth in the Navigate method
+		}
+	}
+
 	var err error
 	b := Browser{BrowserOpts: opts}
-	b.browserAddr, err = launcher.New().Bin(path).Leakless(opts.IsLeakless).Headless(opts.IsHeadless).Launch()
+	b.browserAddr, err = l.Launch()
 
 	if opts.CaptchaSolverApiKey != "" {
 		b.CaptchaSolver = NewSolver(opts.CaptchaSolverApiKey)
@@ -69,20 +98,47 @@ func (b *Browser) IsInitialized() bool {
 }
 
 // Open URL
-func (b *Browser) Navigate(URL string) *rod.Page {
+func (b *Browser) Navigate(URL string) (*rod.Page, error) {
 	logrus.Debug("Navigate to: ", URL)
 
 	b.browser = rod.New().ControlURL(b.browserAddr)
 	b.browser.MustConnect()
 	b.browser.SetCookies(nil)
 
-	//page := b.browser.MustPage(URL)
-	page := stealth.MustPage(b.browser)
+	// Handle proxy authentication before any navigations
+	if b.ProxyURL != "" {
+		proxyUrl, _ := url.Parse(b.ProxyURL)
+
+		// Always ignore certificate errors when using proxies
+		// This fixes the ERR_CERT_AUTHORITY_INVALID error for SOCKS5 proxies
+		b.browser.MustIgnoreCertErrors(true)
+
+		if proxyUrl.User != nil {
+			username := proxyUrl.User.Username()
+			password, _ := proxyUrl.User.Password()
+			// Launch auth handler before any navigation occurs
+			go b.browser.MustHandleAuth(username, password)()
+		}
+	} else if b.Insecure {
+		// Still respect the insecure flag if no proxy is used
+		b.browser.MustIgnoreCertErrors(true)
+	}
+
+	var page *rod.Page
+	if b.UseStealth {
+		page = stealth.MustPage(b.browser)
+	} else {
+		page = b.browser.MustPage(URL)
+	}
+
 	page.MustEmulate(devices.Device{
-		//UserAgent:      uarand.GetRandom(),
 		AcceptLanguage: b.LanguageCode,
 	})
-	page.MustNavigate(URL)
+
+	err := page.Navigate(URL)
+	if err != nil {
+		return nil, err
+	}
 
 	wait := page.MustWaitRequestIdle()
 	// may cause bugs with google
@@ -90,10 +146,7 @@ func (b *Browser) Navigate(URL string) *rod.Page {
 		wait()
 	}
 
-	// Wait till page loads
-	//time.Sleep(b.WaitLoadTime)
-
-	return page
+	return page, nil
 }
 
 func (b *Browser) Close() error {

core/browser_test.go

@@ -38,13 +38,12 @@ func TestBot(t *testing.T) {
 		t.Fatalf("Error failed initializing browser: %s", err)
 	}
 
-	page := browser.Navigate("https://bot.sannysoft.com")
+	page, _ := browser.Navigate("https://bot.sannysoft.com")
 	page.MustScreenshotFullPage("./test/screenshot_bot.png")
 
-	page = browser.Navigate("https://www.whatismybrowser.com/")
+	page, _ = browser.Navigate("https://www.whatismybrowser.com/")
 	page.MustScreenshotFullPage("./test/screenshot_browser.png")
 
-	page = browser.Navigate("https://abrahamjuliot.github.io/creepjs/")
+	page, _ = browser.Navigate("https://abrahamjuliot.github.io/creepjs/")
 	page.MustScreenshotFullPage("./test/screenshot_creep.png")
-
 }

core/common.go

@@ -20,6 +20,26 @@ type SearchResult struct {
 	Ad          bool   `json:"ad"`
 }
 
+func DeduplicateResults(results []SearchResult) []SearchResult {
+	unique := make(map[string]bool)
+	var deduped []SearchResult
+
+	for _, result := range results {
+		if result.URL == "" {
+			continue
+		}
+		if !unique[result.URL] {
+			unique[result.URL] = true
+			deduped = append(deduped, result)
+		}
+	}
+
+	sort.Slice(deduped, func(i, j int) bool {
+		return deduped[i].Rank < deduped[j].Rank
+	})
+	return deduped
+}
+
 func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult {
 	searchResults := []SearchResult{}
 
@@ -41,6 +61,8 @@ type Query struct {
 	Site         string // Search site
 	Limit        int    // Limit the number of results
 	Answers      bool   // Include question and answers from SERP page to results with negative indexes
+	ProxyURL     string // Proxy URL for raw requests
+	Insecure     bool   // Allow insecure TLS connections
 }
 
 func (q Query) IsEmpty() bool {

google/search.go

@@ -70,7 +70,7 @@ func (gogl *Google) getTotalResults(page *rod.Page) (int, error) {
 func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas string) bool {
 	logrus.Debugf("Solve google Captcha: sitekey=%s, datas=%s, url=%s", sitekey, datas, page.MustInfo().URL)
 
-	resp, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, page.MustInfo().URL, datas)
+	resp, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, page.MustInfo().URL)
 	if err != nil {
 		logrus.Errorf("Error solving google captcha: %s", err)
 		return false
@@ -132,6 +132,7 @@ func (gogl *Google) acceptCookies(page *rod.Page) {
 	btnElms[3].Click(proto.InputMouseButtonLeft, 1)
 
 }
+
 func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	logrus.Tracef("Start Google search, query: %+v", query)
 
@@ -142,7 +143,11 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	if err != nil {
 		return nil, err
 	}
-	page := gogl.Navigate(url)
+	page, err := gogl.Navigate(url)
+	if err != nil {
+		return nil, err
+	}
+
 	defer gogl.close(page)
 	gogl.preparePage(page)
 
@@ -157,8 +162,8 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 		gogl.acceptCookies(page)
 	}
 
-	// Find all results
-	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid]")
+	// Find all results using stable attributes
+	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved]")
 	if err != nil {
 		logrus.Errorf("Cannot parse search results: %s", err)
 		return nil, core.ErrSearchTimeout
@@ -265,35 +270,56 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 				searchResults = append(searchResults, srchRes)
 			}
 			continue
-		} else if strings.Contains(attrs, "data-ved") && strings.Contains(attrs, "lang") {
-			// 3. Parse regular search results
-			// Get URL
-			link, err := resEl.Element("a")
+		} else if strings.Contains(attrs, "data-ved") {
+			// Parse regular search results
+			// Get title from h3
+			titleTag, err := resEl.Element("h3")
 			if err != nil {
 				continue
 			}
-			href, err := link.Property("href")
-			if err != nil {
-				logrus.Debug("No `href` tag found")
+			srchRes.Title, _ = titleTag.Text()
+
+			// Get URL from parent link of h3
+			link, err := titleTag.Parent()
+			if err == nil && link.MustMatches("a") {
+				href, _ := link.Property("href")
+				srchRes.URL = href.String()
 			}
-			srchRes.URL = href.String()
+
+			// Skip if URL is empty
+			if srchRes.URL == "" {
+				continue
+			}
+
+			// Get description using multiple fallback strategies
+			desc := ""
+			if descTag, err := resEl.Element("div[data-sncf='1'] div"); err == nil {
+				desc = descTag.MustText()
+			} else if descTag, err := resEl.Element("div.VwiC3b"); err == nil {
+				desc = descTag.MustText()
+			} else {
+				// Structural fallback
+				parent, err := titleTag.Parent()
+				if err == nil {
+					parent, err = parent.Parent()
+					if err == nil {
+						parent, err = parent.Parent()
+						if err == nil {
+							if descTag, err := parent.Next(); err == nil {
+								if descDiv, err := descTag.Element("div"); err == nil {
+									desc = descDiv.MustText()
+								}
+							}
+						}
+					}
+				}
+			}
+			srchRes.Description = desc
+
 			rank += 1
-
-			// Get title
-			titleTag, err := link.Element("h3")
-			if err != nil {
-				continue
-			}
-
-			srchRes.Title, err = titleTag.Text()
-			if err != nil {
-				logrus.Debug("Cannot extract text from title")
-			}
-
-			// Get description
-			text := resEl.MustText()
-			textSliced := strings.Split(text, "\n")
-			srchRes.Description = strings.Join(textSliced[4:], "\n")
+			srchRes.Rank = rank
+			searchResults = append(searchResults, srchRes)
+			continue
 
 		} else {
 			//fmt.Println(i, attrs)
@@ -304,7 +330,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 		searchResults = append(searchResults, srchRes)
 	}
 
-	return searchResults, nil
+	return core.DeduplicateResults(searchResults), nil
 }
 
 func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
@@ -316,17 +342,12 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		return nil, err
 	}
 
-	page := gogl.Navigate(url)
-	defer gogl.close(page)
+	page, err := gogl.Navigate(url)
+	if err != nil {
+		return nil, err
+	}
 
-	//// TODO: Case with cookie accept (appears with VPN)
-	// if page.MustInfo().URL != url {
-	// 	results, _ := page.Search("button[aria-label][jsaction]")
-	// 	if results != nil {
-	// 		//buttons, _ := results.All()
-	// 		//buttons[1].Click(proto.InputMouseButtonLeft, 1)
-	// 	}
-	// }
+	defer gogl.close(page)
 
 	for len(searchResultsMap) < query.Limit {
 		page.WaitLoad()
@@ -365,18 +386,19 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
 				continue
 			}
 
-			dataID, err := r.Attribute("data-id")
+			dataVed, err := r.Attribute("data-ved")
 			if err != nil {
+				logrus.Error("Cannot find `data-ved` attr")
 				continue
 			}
 
 			// If already have image with this ID
-			if _, ok := searchResultsMap[*dataID]; ok {
+			if _, ok := searchResultsMap[*dataVed]; ok {
 				continue
 			}
 
 			// Get URLs
-			link, err := r.Element("a[tabindex][role]")
+			link, err := r.Element("a:not([ping])")
 			if err != nil {
 				continue
 			}
@@ -411,7 +433,7 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
 				Title:       title,
 				Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
 			}
-			searchResultsMap[*dataID] = gR
+			searchResultsMap[*dataVed] = gR
 
 			r.Remove()
 		}

google/search_raw.go

@@ -1,8 +1,11 @@
 package google
 
 import (
+	"crypto/tls"
 	"net/http"
+	"net/url"
 	"strings"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/corpix/uarand"
@@ -10,8 +13,27 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func googleRequest(searchURL string) (*http.Response, error) {
-	baseClient := &http.Client{}
+func googleRequest(searchURL string, query core.Query) (*http.Response, error) {
+	// Create HTTP transport with proxy
+	transport := &http.Transport{}
+	if query.ProxyURL != "" {
+		proxyUrl, err := url.Parse(query.ProxyURL)
+		if err != nil {
+			return nil, err
+		}
+		transport.Proxy = http.ProxyURL(proxyUrl)
+	}
+
+	// Set insecure TLS
+	if query.Insecure {
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+	baseClient := &http.Client{
+		Transport: transport,
+		Timeout:   time.Second * 10,
+	}
+
 	req, err := http.NewRequest("GET", searchURL, nil)
 	if err != nil {
 		return nil, err
@@ -26,7 +48,7 @@ func googleRequest(searchURL string) (*http.Response, error) {
 }
 
 func googleResultParser(response *http.Response) ([]core.SearchResult, error) {
-	doc, err := goquery.NewDocumentFromResponse(response)
+	doc, err := goquery.NewDocumentFromReader(response.Body)
 	if err != nil {
 		return nil, err
 	}
@@ -34,23 +56,49 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) {
 	results := []core.SearchResult{}
 	rank := 1
 
-	// Get individual results
-	sel := doc.Find("div.g")
+	// Use data attributes instead of class names to find results
+	// Both old and new DOM have data-hveid and data-ved attributes
+	sel := doc.Find("div[data-hveid][data-ved]")
 
 	for i := range sel.Nodes {
 		item := sel.Eq(i)
 
-		// Find URL
-		linkTag := item.Find("a")
-		link, _ := linkTag.Attr("href")
+		// Skip items without an h3 element (which indicates a search result)
+		if item.Find("h3").Length() == 0 {
+			continue
+		}
+
+		// Find URL - look for the anchor that contains the h3 title
+		linkTag := item.Find("h3").Parent()
+		if linkTag.Is("a") == false {
+			linkTag = item.Find("h3").Closest("a")
+		}
+
+		link, exists := linkTag.Attr("href")
+		if !exists || link == "" || link == "#" {
+			continue
+		}
 		link = strings.Trim(link, " ")
 
-		// Find title
+		// Find title - this is inside the h3 element
 		titleTag := item.Find("h3")
 		title := titleTag.Text()
 
-		// Find description
-		descTag := item.Find(`div[data-sncf~="1"]`)
+		// Find description - find div with text content after the heading
+		// Using attribute selectors that match the description container
+		descTag := item.Find("div[data-sncf='1']").Find("div").First()
+		if descTag.Length() == 0 {
+			// Try another selector approach if the first one fails
+			descTag = item.Find("div.VwiC3b")
+			if descTag.Length() == 0 {
+				// As a last resort, look for any div after the title that might contain description
+				titleParent := titleTag.Parent()
+				if titleParent.Is("a") {
+					titleParent = titleParent.Parent().Parent()
+				}
+				descTag = titleParent.NextAll().First().Find("div").First()
+			}
+		}
 		desc := descTag.Text()
 
 		if link != "" && link != "#" {
@@ -67,7 +115,7 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) {
 	}
 
 	logrus.Tracef("Google search document size: %d", len(doc.Text()))
-	return results, err
+	return core.DeduplicateResults(results), err
 }
 
 func Search(query core.Query) ([]core.SearchResult, error) {
@@ -77,7 +125,7 @@ func Search(query core.Query) ([]core.SearchResult, error) {
 	}
 	logrus.Debugf("Google URL built: %s", googleURL)
 
-	res, err := googleRequest(googleURL)
+	res, err := googleRequest(googleURL, query)
 	if err != nil {
 		return nil, err
 	}

yandex/search.go

@@ -2,28 +2,36 @@ package yandex
 
 import (
 	"encoding/json"
+	"fmt"
+	"sort"
 	"time"
 
 	"github.com/go-rod/rod"
-	"github.com/go-rod/rod/lib/input"
 	"github.com/karust/openserp/core"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/time/rate"
 )
 
-type YandexImageData struct {
-	SerpItem struct {
-		Freshness string
-		Snippet   struct {
-			Title     string
-			Text      string
-			URL       string
-			Domain    string
-			ShopScore int
-		}
-		ImgHref string `json:"img_href"`
-		Pos     int
-	} `json:"serp-item"`
+type ImageEntity struct {
+	ID        string `json:"id"`
+	Rank      int    `json:"pos"`
+	Width     int    `json:"origWidth"`
+	Height    int    `json:"origHeight"`
+	Title     string `json:"alt"`
+	OrigURL   string `json:"origUrl"`
+	ThumbURL  string `json:"image"`
+	Freshness string `json:"freshnessCounter"`
+	IsGIF     bool   `json:"gifLabel"`
+}
+
+type ImageData struct {
+	InitalState struct {
+		SerpList struct {
+			Items struct {
+				Entities map[string]ImageEntity `json:"entities"`
+			} `json:"items"`
+		} `json:"serpList"`
+	} `json:"initialState"`
 }
 
 type Yandex struct {
@@ -52,27 +60,13 @@ func (yand *Yandex) GetRateLimiter() *rate.Limiter {
 
 func (yand *Yandex) isCaptcha(page *rod.Page) bool {
 	_, err := page.Timeout(yand.GetSelectorTimeout()).Search("form#checkbox-captcha-form")
-	if err != nil {
-		return false
-	}
-	return true
+	return err == nil
 }
 
 // Check if nothig is found
 func (yand *Yandex) isNoResults(page *rod.Page) bool {
-	noResFound := false
-
-	_, err := page.Timeout(yand.GetSelectorTimeout()).Search("div.EmptySearchResults-Title")
-	if err == nil {
-		noResFound = true
-	}
-
-	_, err = page.Timeout(yand.GetSelectorTimeout()).Search("div>div.RequestMeta-Message")
-	if err == nil {
-		noResFound = true
-	}
-
-	return noResFound
+	_, err := page.Timeout(yand.GetSelectorTimeout()).Search("div.Correction.SearchCorrection")
+	return err == nil
 }
 
 func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.SearchResult {
@@ -130,7 +124,10 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
 			return nil, err
 		}
 
-		page := yand.Navigate(url)
+		page, err := yand.Navigate(url)
+		if err != nil {
+			return nil, err
+		}
 
 		// Get all search results in page
 		searchRes, err := page.Timeout(yand.Timeout).Search("li.serp-item")
@@ -175,31 +172,36 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
 		time.Sleep(yand.pageSleep)
 	}
 
-	return allResults, nil
+	return core.DeduplicateResults(allResults), nil
 }
 
 func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
 	logrus.Tracef("Start Yandex image search, query: %+v", query)
 
-	searchResultsMap := map[string]core.SearchResult{}
-	url, err := BuildImageURL(query)
-	if err != nil {
-		return nil, err
-	}
+	searchResults := []core.SearchResult{}
 
-	page := yand.Navigate(url)
-	if !yand.LeavePageOpen {
-		defer page.Close()
-	}
+	searchPage := 0
+	for len(searchResults) < query.Limit {
+		url, err := BuildImageURL(query, searchPage)
+		if err != nil {
+			return nil, err
+		}
+		searchPage += 1
 
-	for len(searchResultsMap) < query.Limit {
-		page.WaitLoad()
-		page.Keyboard.Press(input.End)
-		page.WaitLoad()
-		time.Sleep(time.Duration(time.Second * 2))
+		page, err := yand.Navigate(url)
+		if err != nil {
+			return nil, err
+		}
 
-		// Get all search results in page
-		results, err := page.Timeout(yand.Timeout).Search("div.serp-item")
+		if !yand.LeavePageOpen {
+			defer page.Close()
+		}
+
+		//page.Keyboard.Press(input.End)
+		//page.WaitLoad()
+		//time.Sleep(time.Duration(time.Second * 2))
+
+		results, err := page.Timeout(yand.Timeout).Search("div[role='main'] div[data-state]")
 		if err != nil {
 			logrus.Errorf("Cannot find search results: %s", err)
 		}
@@ -208,45 +210,33 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		if results == nil {
 			if yand.isCaptcha(page) {
 				logrus.Errorf("Yandex captcha occurred during: %s", url)
-				return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha
+				return searchResults, core.ErrCaptcha
 			} else if yand.isNoResults(page) {
 				logrus.Errorf("No results found")
 			}
-			return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
+			return searchResults, core.ErrSearchTimeout
 		}
 
-		for i := 0; i < results.ResultCount; i++ {
-			r, err := results.Get(i, 1)
-			if err != nil {
-				logrus.Errorf("Cannot [%v] element from search result, [%v total]: %s", i, results.ResultCount, err)
-				return *core.ConvertSearchResultsMap(searchResultsMap), err
+		data, err := results.First.Attribute("data-state")
+		if err != nil {
+			return nil, err
+		}
+
+		var imgData ImageData
+		if err := json.Unmarshal([]byte(*data), &imgData); err != nil {
+			return nil, err
+		}
+
+		for id := range imgData.InitalState.SerpList.Items.Entities {
+			img := imgData.InitalState.SerpList.Items.Entities[id]
+			res := core.SearchResult{
+				Rank:        img.Rank + 1,
+				URL:         img.OrigURL,
+				Title:       img.Title,
+				Description: fmt.Sprintf("%dx%d, freshness:%s, thumb_url:%s", img.Height, img.Width, img.Freshness, img.ThumbURL),
 			}
 
-			dataAttr, err := r[0].Attribute("data-bem")
-			if err != nil {
-				continue
-			}
-
-			var data YandexImageData
-
-			err = json.Unmarshal([]byte(*dataAttr), &data)
-			if err != nil {
-				logrus.Errorf("Cannot unmarshal yandex image data: %v\nData: %v", err, *dataAttr)
-				continue
-			}
-
-			linkText := data.SerpItem.ImgHref
-			title := data.SerpItem.Snippet.Title
-			description := data.SerpItem.Snippet.Text
-
-			yR := core.SearchResult{
-				Rank:        (i + 1),
-				URL:         linkText,
-				Title:       title,
-				Description: description,
-			}
-
-			searchResultsMap[linkText+title] = yR
+			searchResults = append(searchResults, res)
 		}
 
 		if !yand.LeavePageOpen {
@@ -254,5 +244,9 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		}
 	}
 
-	return *core.ConvertSearchResultsMap(searchResultsMap), nil
+	sort.Slice(searchResults, func(i, j int) bool {
+		return searchResults[i].Rank < searchResults[j].Rank
+	})
+
+	return core.DeduplicateResults(searchResults), nil
 }

yandex/search_raw.go

@@ -1,8 +1,11 @@
 package yandex
 
 import (
+	"crypto/tls"
 	"net/http"
+	"net/url"
 	"strings"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/corpix/uarand"
@@ -10,8 +13,27 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func yandexRequest(searchURL string) (*http.Response, error) {
-	baseClient := &http.Client{}
+func yandexRequest(searchURL string, query core.Query) (*http.Response, error) {
+	// Create HTTP transport with proxy
+	transport := &http.Transport{}
+	if query.ProxyURL != "" {
+		proxyUrl, err := url.Parse(query.ProxyURL)
+		if err != nil {
+			return nil, err
+		}
+		transport.Proxy = http.ProxyURL(proxyUrl)
+	}
+
+	// Set insecure TLS
+	if query.Insecure {
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+	baseClient := &http.Client{
+		Transport: transport,
+		Timeout:   time.Second * 10,
+	}
+
 	req, err := http.NewRequest("GET", searchURL, nil)
 	if err != nil {
 		return nil, err
@@ -26,7 +48,7 @@ func yandexRequest(searchURL string) (*http.Response, error) {
 }
 
 func yandexResultParser(response *http.Response) ([]core.SearchResult, error) {
-	doc, err := goquery.NewDocumentFromResponse(response)
+	doc, err := goquery.NewDocumentFromReader(response.Body)
 	if err != nil {
 		return nil, err
 	}
@@ -67,7 +89,7 @@ func yandexResultParser(response *http.Response) ([]core.SearchResult, error) {
 	}
 
 	logrus.Tracef("Yandex search document size: %d", len(doc.Text()))
-	return results, err
+	return core.DeduplicateResults(results), err
 }
 
 func Search(query core.Query) ([]core.SearchResult, error) {
@@ -77,7 +99,7 @@ func Search(query core.Query) ([]core.SearchResult, error) {
 	}
 	logrus.Debugf("Yandex URL built: %s", googleURL)
 
-	res, err := yandexRequest(googleURL)
+	res, err := yandexRequest(googleURL, query)
 	if err != nil {
 		return nil, err
 	}

yandex/search_test.go

@@ -14,6 +14,19 @@ func init() {
 	browser, _ = core.NewBrowser(opts)
 }
 
+// func TestParseImgData(t *testing.T) {
+
+// 	jsonData, _ := os.ReadFile("./testImgData.json")
+// 	var obj ImageData
+// 	if err := json.Unmarshal(jsonData, &obj); err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	if (len(obj.InitalState.SerpList.Items.Entities)) != 30 {
+// 		t.Fail()
+// 	}
+// }
+
 func TestSearchYandex(t *testing.T) {
 
 	yand := New(*browser, core.SearchEngineOptions{})
@@ -32,13 +45,13 @@ func TestSearchYandex(t *testing.T) {
 func TestImageYandex(t *testing.T) {
 	yand := New(*browser, core.SearchEngineOptions{})
 
-	query := core.Query{Text: "furry tiger", Limit: 90}
+	query := core.Query{Text: "furry tiger", Limit: 30}
 	results, err := yand.SearchImage(query)
 	if err != nil {
 		t.Fatalf("Cannot [ImageYandex]: %s", err)
 	}
 
-	if len(results) < 90 {
+	if len(results) < 30 {
 		t.Fatalf("[ImageYandex] returned empty result")
 	}
 }

yandex/url.go

@@ -35,14 +35,14 @@ func BuildURL(q core.Query, page int) (string, error) {
 	}
 
 	if len(params.Get("text")) == 0 {
-		return "", errors.New("Empty query built")
+		return "", errors.New("empty query built")
 	}
 
 	base.RawQuery = params.Encode()
 	return base.String(), nil
 }
 
-func BuildImageURL(q core.Query) (string, error) {
+func BuildImageURL(q core.Query, page int) (string, error) {
 	// TODO: Add other parameters
 	base, _ := url.Parse(baseURL)
 	base.Path += "images/search/"
@@ -56,11 +56,11 @@ func BuildImageURL(q core.Query) (string, error) {
 		}
 
 		params.Add("text", text)
-		//params.Add("p", fmt.Sprint(page))
+		params.Add("p", fmt.Sprint(page))
 	}
 
 	if len(params.Get("text")) == 0 {
-		return "", errors.New("Empty query built")
+		return "", errors.New("empty query built")
 	}
 
 	if q.Site != "" {