Merge pull request #10 from PACHAKUTlQ/main

Support http and socks5 proxy and fix failure due to updates of DOM on Google SERP

.gitignore

@@ -24,3 +24,4 @@ logrus.txt
 logs.txt
 .release
 core/test/
+.aider*

README.md

@@ -1,18 +1,23 @@
 # OpenSERP (Search Engine Results Page)
+
 ![OpenSERP](/logo.png)
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/karust/openserp)](https://goreportcard.com/report/github.com/karust/openserp)
 [![Go Reference](https://pkg.go.dev/badge/github.com/karust/openserp.svg)](https://pkg.go.dev/github.com/karust/openserp)
 [![release](https://img.shields.io/github/release-pre/karust/openserp.svg)](https://github.com/karust/openserp/releases)
+
 <!-- ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/karust/openserp/latest) -->
+
 API access for search engines results if available isn't free.
 
 Using OpenSERP, you can get search results from **Google**, **Yandex**, **Baidu** via API or CLI!
 
 See [Docker](#docker) and [CLI](#cli) usage examples below ([search](#search), [images](#images)).
 
-## Docker usage  <a name="docker"></a> 🐳
-* Run API server:
+## Docker usage <a name="docker"></a> 🐳
+
+- Run API server:
+
 ```bash
 # Use prebuilt image
 docker run -p 127.0.0.1:7000:7000 -it karust/openserp serve -a 0.0.0.0 -p 7000
@@ -22,26 +27,32 @@ docker-compose up --build
 ```
 
 ### Request parameters
-| Param | Description                                                  |
-|-------|--------------------------------------------------------------|
-| text  | Text to search                                               |
-| lang  | Search pages in selected language (`EN`, `DE`, `RU`...)      |
-| date  | Date in `YYYYMMDD..YYYYMMDD` format (e.g. 20181010..20231010) |
-| file  | File extension to search  (e.g. `PDF`, `DOC`)                 |
-| site  | Search within a specific website                                 |
-| limit | Limit the number of results  
-| answers | Include google answers as negative rank indexes (e.g. `true`, `false`)
+
+| Param   | Description                                                            |
+| ------- | ---------------------------------------------------------------------- |
+| text    | Text to search                                                         |
+| lang    | Search pages in selected language (`EN`, `DE`, `RU`...)                |
+| date    | Date in `YYYYMMDD..YYYYMMDD` format (e.g. 20181010..20231010)          |
+| file    | File extension to search (e.g. `PDF`, `DOC`)                           |
+| site    | Search within a specific website                                       |
+| limit   | Limit the number of results                                            |
+| answers | Include google answers as negative rank indexes (e.g. `true`, `false`) |
 
 ### **Search**
-### *Example request*
+
+### _Example request_
+
 Get 20 **Google** results for `hello world`, only in English:
+
 ```
 GET http:/127.0.0.1:7000/google/search?lang=EN&limit=20&text=hello world
 ```
-You can replace `google` to `yandex` or `baidu` in query to change search engine.
-                                |
 
-### *Example response*
+You can replace `google` to `yandex` or `baidu` in query to change search engine.
+|
+
+### _Example response_
+
 ```JSON
 [
     {
@@ -53,39 +64,66 @@ You can replace `google` to `yandex` or `baidu` in query to change search engine
     },
 ]
 ```
+
 ### **Images** **[WIP]**
-### *Example request*
+
+### _Example request_
+
 Get 100 **Google** results for `golden puppy`:
+
 ```
 GET http://127.0.0.1:7000/google/image?text=golden puppy&limit=100
 ```
 
-
 ## CLI <a name="cli"></a> ⌨️
-* Use `-h` flag to see commands.
-* You can use `serve` command to serve API:
+
+- Use `-h` flag to see all commands.
+- You can use `serve` command to serve API:
+
 ```bash
-openserp serve 
+openserp serve
 ```
-* Or print results in CLI using `search` command:
+
+- Or print results in CLI using `search` command:
+
 ```bash
 openserp search google "how to get banned from google fast" # Change `google` to `yandex` or `baidu`
 ```
+
 As a result you should get JSON output containting search results:
+
 ```json
 [
- {
-  "rank": 1,
-  "url": "https://www.cyberoptik.net/blog/6-sure-fire-ways-to-get-banned-from-google/",
-  "title": "11 Sure-Fire Ways to Get Banned From Google | CyberOptik",
-  "description": "How To Get Banned From Google · 1. Cloaking: The Art of Deception · 2. Plagiarism: Because Originality is Overrated · 3. Keyword Stuffing: More is Always Better · 4 ...",
-  "ad": false
- },
+  {
+    "rank": 1,
+    "url": "https://www.cyberoptik.net/blog/6-sure-fire-ways-to-get-banned-from-google/",
+    "title": "11 Sure-Fire Ways to Get Banned From Google | CyberOptik",
+    "description": "How To Get Banned From Google · 1. Cloaking: The Art of Deception · 2. Plagiarism: Because Originality is Overrated · 3. Keyword Stuffing: More is Always Better · 4 ...",
+    "ad": false
+  }
 ]
- ```
+```
+
+### Proxy Support
+
+Both browser mode and raw mode support HTTP and SOCKS5 proxies with authentication:
+
+```bash
+# HTTP proxy with auth
+openserp search google "query" --proxy http://user:pass@127.0.0.1:8080
+
+# Serve with SOCKS5 proxy
+openserp serve --proxy socks5://127.0.0.1:1080
+
+# For HTTPS sites through HTTP proxy, use --insecure to ignore certificate errors
+openserp search google "query" --proxy http://127.0.0.1:8080 --insecure
+openserp search google "query" -x http://127.0.0.1:8080 -k
+```
+
+## License
 
- ## License
 This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details
 
 ## Bugs + Questions 👾
-If you have some issues/bugs/questions, feel free to open an issue.
+
+If you have some issues/bugs/questions, feel free to open an issue.

baidu/search.go

@@ -138,7 +138,7 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
 		}
 	}
 
-	return searchResults, nil
+	return core.DeduplicateResults(searchResults), nil
 }
 
 func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
@@ -231,5 +231,5 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		}
 	}
 
-	return searchResults, nil
+	return core.DeduplicateResults(searchResults), nil
 }

baidu/search_raw.go

@@ -1,9 +1,12 @@
 package baidu
 
 import (
+	"crypto/tls"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/corpix/uarand"
@@ -11,8 +14,27 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func baiduRequest(searchURL string) (*http.Response, error) {
-	baseClient := &http.Client{}
+func baiduRequest(searchURL string, query core.Query) (*http.Response, error) {
+	// Create HTTP transport with proxy
+	transport := &http.Transport{}
+	if query.ProxyURL != "" {
+		proxyUrl, err := url.Parse(query.ProxyURL)
+		if err != nil {
+			return nil, err
+		}
+		transport.Proxy = http.ProxyURL(proxyUrl)
+	}
+
+	// Set insecure TLS
+	if query.Insecure {
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+	baseClient := &http.Client{
+		Transport: transport,
+		Timeout:   time.Second * 10,
+	}
+
 	req, err := http.NewRequest("GET", searchURL, nil)
 	if err != nil {
 		return nil, err
@@ -27,7 +49,7 @@ func baiduRequest(searchURL string) (*http.Response, error) {
 }
 
 func baiduResultParser(response *http.Response) ([]core.SearchResult, error) {
-	doc, err := goquery.NewDocumentFromResponse(response)
+	doc, err := goquery.NewDocumentFromReader(response.Body)
 	if err != nil {
 		return nil, err
 	}
@@ -79,7 +101,7 @@ func Search(query core.Query) ([]core.SearchResult, error) {
 	}
 	logrus.Debugf("Baidu URL built: %s", googleURL)
 
-	res, err := baiduRequest(googleURL)
+	res, err := baiduRequest(googleURL, query)
 	if err != nil {
 		return nil, err
 	}
@@ -91,5 +113,5 @@ func Search(query core.Query) ([]core.SearchResult, error) {
 	}
 	logrus.Debugf("Baidu Raw results : %v", results)
 
-	return results, nil
+	return core.DeduplicateResults(results), nil
 }

cmd/root.go

@@ -40,6 +40,8 @@ type AppConfig struct {
 	IsDebug       bool   `mapstructure:"debug"`
 	IsVerbose     bool   `mapstructure:"verbose"`
 	IsRawRequests bool   `mapstructure:"raw_requests"`
+	ProxyURL      string `mapstructure:"proxy"`
+	Insecure      bool   `mapstructure:"insecure"`
 }
 
 var config = Config{}
@@ -133,4 +135,6 @@ func init() {
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
 	RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
+	RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "x", "", "HTTP or Socks5 proxy URL (e.g. http://user:pass@127.0.0.1:8080)")
+	RootCmd.PersistentFlags().BoolVarP(&config.App.Insecure, "insecure", "k", false, "Allow insecure TLS connections")
 }

cmd/search.go

@@ -26,8 +26,10 @@ func search(cmd *cobra.Command, args []string) {
 	var err error
 	engineType := args[0]
 	query := core.Query{
-		Text:  args[1],
-		Limit: 10,
+		Text:     args[1],
+		Limit:    10,
+		ProxyURL: config.App.ProxyURL,
+		Insecure: config.App.Insecure,
 	}
 	results := []core.SearchResult{}
 
@@ -59,6 +61,8 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er
 		Timeout:             time.Second * time.Duration(config.App.Timeout),
 		LeavePageOpen:       config.App.IsLeaveHead,
 		CaptchaSolverApiKey: config.Config2Capcha.ApiKey,
+		ProxyURL:            config.App.ProxyURL,
+		Insecure:            config.App.Insecure,
 	}
 
 	if config.App.IsDebug {

cmd/serve.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/karust/openserp/baidu"
@@ -9,8 +10,48 @@ import (
 	"github.com/karust/openserp/yandex"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"golang.org/x/time/rate"
 )
 
+// rawEngine implements SearchEngine interface for raw HTTP requests
+type rawEngine struct {
+	name string
+}
+
+func (r *rawEngine) Search(q core.Query) ([]core.SearchResult, error) {
+	// Inject proxy settings from config
+	q.ProxyURL = config.App.ProxyURL
+	q.Insecure = config.App.Insecure
+
+	switch r.name {
+	case "google":
+		return google.Search(q)
+	case "yandex":
+		return yandex.Search(q)
+	case "baidu":
+		return baidu.Search(q)
+	default:
+		return nil, fmt.Errorf("unsupported engine: %s", r.name)
+	}
+}
+
+func (r *rawEngine) SearchImage(q core.Query) ([]core.SearchResult, error) {
+	return nil, fmt.Errorf("image search is not supported in raw mode for %s", r.name)
+}
+
+func (r *rawEngine) Name() string {
+	return r.name
+}
+
+func (r *rawEngine) IsInitialized() bool {
+	return true
+}
+
+func (r *rawEngine) GetRateLimiter() *rate.Limiter {
+	// Use default rate limiter for raw requests
+	return rate.NewLimiter(rate.Every(time.Second), 5)
+}
+
 var serveCMD = &cobra.Command{
 	Use:     "serve",
 	Aliases: []string{"listen"},
@@ -20,12 +61,25 @@ var serveCMD = &cobra.Command{
 }
 
 func serve(cmd *cobra.Command, args []string) {
+	if config.App.IsRawRequests {
+		logrus.Warn("Browserless results are very inconsistent or may not even work!")
+		serv := core.NewServer(config.App.Host, config.App.Port,
+			&rawEngine{name: "google"},
+			&rawEngine{name: "yandex"},
+			&rawEngine{name: "baidu"},
+		)
+		serv.Listen()
+		return
+	}
+
 	opts := core.BrowserOpts{
 		IsHeadless:          !config.App.IsBrowserHead, // Disable headless if browser head mode is set
 		IsLeakless:          config.App.IsLeakless,
 		Timeout:             time.Second * time.Duration(config.App.Timeout),
 		LeavePageOpen:       config.App.IsLeaveHead,
 		CaptchaSolverApiKey: config.Config2Capcha.ApiKey,
+		ProxyURL:            config.App.ProxyURL,
+		Insecure:            config.App.Insecure,
 	}
 
 	if config.App.IsDebug {
@@ -35,6 +89,7 @@ func serve(cmd *cobra.Command, args []string) {
 	browser, err := core.NewBrowser(opts)
 	if err != nil {
 		logrus.Error(err)
+		return
 	}
 
 	yand := yandex.New(*browser, config.YandexConfig)

core/browser.go

@@ -1,6 +1,8 @@
 package core
 
 import (
+	"fmt"
+	"net/url"
 	"time"
 
 	"github.com/go-rod/rod"
@@ -19,6 +21,8 @@ type BrowserOpts struct {
 	LeavePageOpen       bool          // Leave pages and browser open
 	WaitLoadTime        time.Duration // Time to wait till page loads
 	CaptchaSolverApiKey string        // 2Captcha api key
+	ProxyURL            string        // Proxy URL
+	Insecure            bool          // Allow insecure TLS connections
 
 }
 
@@ -47,9 +51,33 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) {
 	path, has := launcher.LookPath()
 	logrus.Debug("Browser found: ", has)
 
+	// Create launcher
+	l := launcher.New().Bin(path).Leakless(opts.IsLeakless).Headless(opts.IsHeadless)
+
+	// Configure proxy if specified
+	if opts.ProxyURL != "" {
+		proxyUrl, err := url.Parse(opts.ProxyURL)
+		if err != nil {
+			return nil, fmt.Errorf("invalid proxy URL: %v", err)
+		}
+
+		// Make sure the proxy URL includes the scheme when passed to launcher
+		// This ensures proper handling of SOCKS5 proxies
+		proxyStr := proxyUrl.String()
+		logrus.Debugf("Setting up proxy: %s", proxyStr)
+		l = l.Proxy(proxyStr)
+
+		// Check if proxy has auth credentials
+		if proxyUrl.User != nil {
+			username := proxyUrl.User.Username()
+			logrus.Debugf("Using proxy authentication: %s:****", username)
+			// We'll handle auth in the Navigate method
+		}
+	}
+
 	var err error
 	b := Browser{BrowserOpts: opts}
-	b.browserAddr, err = launcher.New().Bin(path).Leakless(opts.IsLeakless).Headless(opts.IsHeadless).Launch()
+	b.browserAddr, err = l.Launch()
 
 	if opts.CaptchaSolverApiKey != "" {
 		b.CaptchaSolver = NewSolver(opts.CaptchaSolverApiKey)
@@ -76,10 +104,27 @@ func (b *Browser) Navigate(URL string) *rod.Page {
 	b.browser.MustConnect()
 	b.browser.SetCookies(nil)
 
-	//page := b.browser.MustPage(URL)
+	// Handle proxy authentication before any navigations
+	if b.ProxyURL != "" {
+		proxyUrl, _ := url.Parse(b.ProxyURL)
+
+		// Always ignore certificate errors when using proxies
+		// This fixes the ERR_CERT_AUTHORITY_INVALID error for SOCKS5 proxies
+		b.browser.MustIgnoreCertErrors(true)
+
+		if proxyUrl.User != nil {
+			username := proxyUrl.User.Username()
+			password, _ := proxyUrl.User.Password()
+			// Launch auth handler before any navigation occurs
+			go b.browser.MustHandleAuth(username, password)()
+		}
+	} else if b.Insecure {
+		// Still respect the insecure flag if no proxy is used
+		b.browser.MustIgnoreCertErrors(true)
+	}
+
 	page := stealth.MustPage(b.browser)
 	page.MustEmulate(devices.Device{
-		//UserAgent:      uarand.GetRandom(),
 		AcceptLanguage: b.LanguageCode,
 	})
 	page.MustNavigate(URL)
@@ -90,9 +135,6 @@ func (b *Browser) Navigate(URL string) *rod.Page {
 		wait()
 	}
 
-	// Wait till page loads
-	//time.Sleep(b.WaitLoadTime)
-
 	return page
 }
 

core/common.go

@@ -20,6 +20,26 @@ type SearchResult struct {
 	Ad          bool   `json:"ad"`
 }
 
+func DeduplicateResults(results []SearchResult) []SearchResult {
+	unique := make(map[string]bool)
+	var deduped []SearchResult
+
+	for _, result := range results {
+		if result.URL == "" {
+			continue
+		}
+		if !unique[result.URL] {
+			unique[result.URL] = true
+			deduped = append(deduped, result)
+		}
+	}
+
+	sort.Slice(deduped, func(i, j int) bool {
+		return deduped[i].Rank < deduped[j].Rank
+	})
+	return deduped
+}
+
 func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult {
 	searchResults := []SearchResult{}
 
@@ -41,6 +61,8 @@ type Query struct {
 	Site         string // Search site
 	Limit        int    // Limit the number of results
 	Answers      bool   // Include question and answers from SERP page to results with negative indexes
+	ProxyURL     string // Proxy URL for raw requests
+	Insecure     bool   // Allow insecure TLS connections
 }
 
 func (q Query) IsEmpty() bool {

google/search.go

@@ -132,6 +132,7 @@ func (gogl *Google) acceptCookies(page *rod.Page) {
 	btnElms[3].Click(proto.InputMouseButtonLeft, 1)
 
 }
+
 func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	logrus.Tracef("Start Google search, query: %+v", query)
 
@@ -157,8 +158,8 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 		gogl.acceptCookies(page)
 	}
 
-	// Find all results
-	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid]")
+	// Find all results using stable attributes
+	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved]")
 	if err != nil {
 		logrus.Errorf("Cannot parse search results: %s", err)
 		return nil, core.ErrSearchTimeout
@@ -265,35 +266,56 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 				searchResults = append(searchResults, srchRes)
 			}
 			continue
-		} else if strings.Contains(attrs, "data-ved") && strings.Contains(attrs, "lang") {
-			// 3. Parse regular search results
-			// Get URL
-			link, err := resEl.Element("a")
+		} else if strings.Contains(attrs, "data-ved") {
+			// Parse regular search results
+			// Get title from h3
+			titleTag, err := resEl.Element("h3")
 			if err != nil {
 				continue
 			}
-			href, err := link.Property("href")
-			if err != nil {
-				logrus.Debug("No `href` tag found")
+			srchRes.Title, _ = titleTag.Text()
+
+			// Get URL from parent link of h3
+			link, err := titleTag.Parent()
+			if err == nil && link.MustMatches("a") {
+				href, _ := link.Property("href")
+				srchRes.URL = href.String()
 			}
-			srchRes.URL = href.String()
+
+			// Skip if URL is empty
+			if srchRes.URL == "" {
+				continue
+			}
+
+			// Get description using multiple fallback strategies
+			desc := ""
+			if descTag, err := resEl.Element("div[data-sncf='1'] div"); err == nil {
+				desc = descTag.MustText()
+			} else if descTag, err := resEl.Element("div.VwiC3b"); err == nil {
+				desc = descTag.MustText()
+			} else {
+				// Structural fallback
+				parent, err := titleTag.Parent()
+				if err == nil {
+					parent, err = parent.Parent()
+					if err == nil {
+						parent, err = parent.Parent()
+						if err == nil {
+							if descTag, err := parent.Next(); err == nil {
+								if descDiv, err := descTag.Element("div"); err == nil {
+									desc = descDiv.MustText()
+								}
+							}
+						}
+					}
+				}
+			}
+			srchRes.Description = desc
+
 			rank += 1
-
-			// Get title
-			titleTag, err := link.Element("h3")
-			if err != nil {
-				continue
-			}
-
-			srchRes.Title, err = titleTag.Text()
-			if err != nil {
-				logrus.Debug("Cannot extract text from title")
-			}
-
-			// Get description
-			text := resEl.MustText()
-			textSliced := strings.Split(text, "\n")
-			srchRes.Description = strings.Join(textSliced[4:], "\n")
+			srchRes.Rank = rank
+			searchResults = append(searchResults, srchRes)
+			continue
 
 		} else {
 			//fmt.Println(i, attrs)
@@ -304,7 +326,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 		searchResults = append(searchResults, srchRes)
 	}
 
-	return searchResults, nil
+	return core.DeduplicateResults(searchResults), nil
 }
 
 func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
@@ -324,7 +346,7 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		page.Mouse.Scroll(0, 1000000, 1)
 		page.WaitLoad()
 
-		results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction][jsdata]")
+		results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]")
 		if err != nil {
 			logrus.Errorf("Cannot parse search results: %s", err)
 			return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout

google/search_raw.go

@@ -1,8 +1,11 @@
 package google
 
 import (
+	"crypto/tls"
 	"net/http"
+	"net/url"
 	"strings"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/corpix/uarand"
@@ -10,8 +13,27 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func googleRequest(searchURL string) (*http.Response, error) {
-	baseClient := &http.Client{}
+func googleRequest(searchURL string, query core.Query) (*http.Response, error) {
+	// Create HTTP transport with proxy
+	transport := &http.Transport{}
+	if query.ProxyURL != "" {
+		proxyUrl, err := url.Parse(query.ProxyURL)
+		if err != nil {
+			return nil, err
+		}
+		transport.Proxy = http.ProxyURL(proxyUrl)
+	}
+
+	// Set insecure TLS
+	if query.Insecure {
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+	baseClient := &http.Client{
+		Transport: transport,
+		Timeout:   time.Second * 10,
+	}
+
 	req, err := http.NewRequest("GET", searchURL, nil)
 	if err != nil {
 		return nil, err
@@ -26,7 +48,7 @@ func googleRequest(searchURL string) (*http.Response, error) {
 }
 
 func googleResultParser(response *http.Response) ([]core.SearchResult, error) {
-	doc, err := goquery.NewDocumentFromResponse(response)
+	doc, err := goquery.NewDocumentFromReader(response.Body)
 	if err != nil {
 		return nil, err
 	}
@@ -34,23 +56,49 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) {
 	results := []core.SearchResult{}
 	rank := 1
 
-	// Get individual results
-	sel := doc.Find("div.g")
+	// Use data attributes instead of class names to find results
+	// Both old and new DOM have data-hveid and data-ved attributes
+	sel := doc.Find("div[data-hveid][data-ved]")
 
 	for i := range sel.Nodes {
 		item := sel.Eq(i)
 
-		// Find URL
-		linkTag := item.Find("a")
-		link, _ := linkTag.Attr("href")
+		// Skip items without an h3 element (which indicates a search result)
+		if item.Find("h3").Length() == 0 {
+			continue
+		}
+
+		// Find URL - look for the anchor that contains the h3 title
+		linkTag := item.Find("h3").Parent()
+		if linkTag.Is("a") == false {
+			linkTag = item.Find("h3").Closest("a")
+		}
+
+		link, exists := linkTag.Attr("href")
+		if !exists || link == "" || link == "#" {
+			continue
+		}
 		link = strings.Trim(link, " ")
 
-		// Find title
+		// Find title - this is inside the h3 element
 		titleTag := item.Find("h3")
 		title := titleTag.Text()
 
-		// Find description
-		descTag := item.Find(`div[data-sncf~="1"]`)
+		// Find description - find div with text content after the heading
+		// Using attribute selectors that match the description container
+		descTag := item.Find("div[data-sncf='1']").Find("div").First()
+		if descTag.Length() == 0 {
+			// Try another selector approach if the first one fails
+			descTag = item.Find("div.VwiC3b")
+			if descTag.Length() == 0 {
+				// As a last resort, look for any div after the title that might contain description
+				titleParent := titleTag.Parent()
+				if titleParent.Is("a") {
+					titleParent = titleParent.Parent().Parent()
+				}
+				descTag = titleParent.NextAll().First().Find("div").First()
+			}
+		}
 		desc := descTag.Text()
 
 		if link != "" && link != "#" {
@@ -67,7 +115,7 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) {
 	}
 
 	logrus.Tracef("Google search document size: %d", len(doc.Text()))
-	return results, err
+	return core.DeduplicateResults(results), err
 }
 
 func Search(query core.Query) ([]core.SearchResult, error) {
@@ -77,7 +125,7 @@ func Search(query core.Query) ([]core.SearchResult, error) {
 	}
 	logrus.Debugf("Google URL built: %s", googleURL)
 
-	res, err := googleRequest(googleURL)
+	res, err := googleRequest(googleURL, query)
 	if err != nil {
 		return nil, err
 	}

yandex/search.go

@@ -169,7 +169,7 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
 		time.Sleep(yand.pageSleep)
 	}
 
-	return allResults, nil
+	return core.DeduplicateResults(allResults), nil
 }
 
 func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
@@ -242,5 +242,5 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		return searchResults[i].Rank < searchResults[j].Rank
 	})
 
-	return searchResults, nil
+	return core.DeduplicateResults(searchResults), nil
 }

yandex/search_raw.go

@@ -1,8 +1,11 @@
 package yandex
 
 import (
+	"crypto/tls"
 	"net/http"
+	"net/url"
 	"strings"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/corpix/uarand"
@@ -10,8 +13,27 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func yandexRequest(searchURL string) (*http.Response, error) {
-	baseClient := &http.Client{}
+func yandexRequest(searchURL string, query core.Query) (*http.Response, error) {
+	// Create HTTP transport with proxy
+	transport := &http.Transport{}
+	if query.ProxyURL != "" {
+		proxyUrl, err := url.Parse(query.ProxyURL)
+		if err != nil {
+			return nil, err
+		}
+		transport.Proxy = http.ProxyURL(proxyUrl)
+	}
+
+	// Set insecure TLS
+	if query.Insecure {
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+	baseClient := &http.Client{
+		Transport: transport,
+		Timeout:   time.Second * 10,
+	}
+
 	req, err := http.NewRequest("GET", searchURL, nil)
 	if err != nil {
 		return nil, err
@@ -26,7 +48,7 @@ func yandexRequest(searchURL string) (*http.Response, error) {
 }
 
 func yandexResultParser(response *http.Response) ([]core.SearchResult, error) {
-	doc, err := goquery.NewDocumentFromResponse(response)
+	doc, err := goquery.NewDocumentFromReader(response.Body)
 	if err != nil {
 		return nil, err
 	}
@@ -67,7 +89,7 @@ func yandexResultParser(response *http.Response) ([]core.SearchResult, error) {
 	}
 
 	logrus.Tracef("Yandex search document size: %d", len(doc.Text()))
-	return results, err
+	return core.DeduplicateResults(results), err
 }
 
 func Search(query core.Query) ([]core.SearchResult, error) {
@@ -77,7 +99,7 @@ func Search(query core.Query) ([]core.SearchResult, error) {
 	}
 	logrus.Debugf("Yandex URL built: %s", googleURL)
 
-	res, err := yandexRequest(googleURL)
+	res, err := yandexRequest(googleURL, query)
 	if err != nil {
 		return nil, err
 	}