Collects google ads/answers + captcha solver test

README.md

@@ -30,6 +30,7 @@ docker-compose up --build
 | file  | File extension to search  (e.g. `PDF`, `DOC`)                 |
 | site  | Search within a specific website                                 |
 | limit | Limit the number of results  
+| answers | Include google answers as negative rank indexes (e.g. `true`, `false`)
 
 ### **Search**
 ### *Example request*
@@ -47,27 +48,18 @@ You can replace `google` to `yandex` or `baidu` in query to change search engine
         "rank": 1,
         "url": "https://en.wikipedia.org/wiki/%22Hello,_World!%22_program",
         "title": "\"Hello, World!\" program",
-        "description": "A \"Hello, World!\" program is generally a computer program that ignores any input, and outputs or displays a message similar to \"Hello, World!\"."
+        "description": "A \"Hello, World!\" program is generally a computer program that ignores any input, and outputs or displays a message similar to \"Hello, World!\".",
+        "ad": false
     },
 ]
 ```
-### **Images**
+### **Images** **[WIP]**
 ### *Example request*
 Get 100 **Google** results for `golden puppy`:
 ```
 GET http://127.0.0.1:7000/google/image?text=golden puppy&limit=100
 ```
-### *Example response*
-```JSON
-[
-    {
-        "rank": 1,
-        "url": "https://en.wikipedia.org/wiki/%22Hello,_World!%22_program",
-        "title": "\"Hello, World!\" program",
-        "description": "A \"Hello, World!\" program is generally a computer program that ignores any input, and outputs or displays a message similar to \"Hello, World!\"."
-    },
-]
-```
+
 
 ## CLI <a name="cli"></a> ⌨️
 * Use `-h` flag to see commands.
@@ -86,7 +78,8 @@ As a result you should get JSON output containting search results:
   "rank": 1,
   "url": "https://www.cyberoptik.net/blog/6-sure-fire-ways-to-get-banned-from-google/",
   "title": "11 Sure-Fire Ways to Get Banned From Google | CyberOptik",
-  "description": "How To Get Banned From Google · 1. Cloaking: The Art of Deception · 2. Plagiarism: Because Originality is Overrated · 3. Keyword Stuffing: More is Always Better · 4 ..."
+  "description": "How To Get Banned From Google · 1. Cloaking: The Art of Deception · 2. Plagiarism: Because Originality is Overrated · 3. Keyword Stuffing: More is Always Better · 4 ...",
+  "ad": false
  },
 ]
  ```

cmd/root.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"errors"
 	"fmt"
 	"strings"
 
@@ -13,16 +12,21 @@ import (
 )
 
 const (
-	version               = "0.2.1"
+	version               = "0.3"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )
 
 type Config struct {
-	App          AppConfig                `mapstructure:"app"`
-	GoogleConfig core.SearchEngineOptions `mapstructure:"google"`
-	YandexConfig core.SearchEngineOptions `mapstructure:"yandex"`
-	BaiduConfig  core.SearchEngineOptions `mapstructure:"baidu"`
+	App           AppConfig                `mapstructure:"app"`
+	Config2Capcha Config2Captcha           `mapstructure:"2captcha"`
+	GoogleConfig  core.SearchEngineOptions `mapstructure:"google"`
+	YandexConfig  core.SearchEngineOptions `mapstructure:"yandex"`
+	BaiduConfig   core.SearchEngineOptions `mapstructure:"baidu"`
+}
+
+type Config2Captcha struct {
+	ApiKey string `mapstructure:"apikey"`
 }
 
 type AppConfig struct {
@@ -88,7 +92,7 @@ func initializeConfig(cmd *cobra.Command) error {
 	// 1. Config. Return an error if we cannot parse the config file.
 	err := v.ReadInConfig()
 	if err != nil {
-		err = errors.New(fmt.Sprintf("Cannot read config: %v", err))
+		err = fmt.Errorf("cannot read config: %v", err)
 		logrus.Warn(err)
 	}
 
@@ -107,7 +111,7 @@ func initializeConfig(cmd *cobra.Command) error {
 	// Dump Viper values to config struct
 	err = v.Unmarshal(&config)
 	if err != nil {
-		return errors.New(fmt.Sprintf("Cannot unmarshall config: %v", err))
+		return fmt.Errorf("cannot unmarshall config: %v", err)
 	}
 
 	if config.App.IsDebug {
@@ -128,4 +132,5 @@ func init() {
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search")
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
+	RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
 }

cmd/search.go

@@ -54,10 +54,11 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er
 	var engine core.SearchEngine
 
 	opts := core.BrowserOpts{
-		IsHeadless:    !config.App.IsBrowserHead, // Disable headless if browser head mode is set
-		IsLeakless:    config.App.IsLeakless,
-		Timeout:       time.Second * time.Duration(config.App.Timeout),
-		LeavePageOpen: config.App.IsLeaveHead,
+		IsHeadless:          !config.App.IsBrowserHead, // Disable headless if browser head mode is set
+		IsLeakless:          config.App.IsLeakless,
+		Timeout:             time.Second * time.Duration(config.App.Timeout),
+		LeavePageOpen:       config.App.IsLeaveHead,
+		CaptchaSolverApiKey: config.Config2Capcha.ApiKey,
 	}
 
 	if config.App.IsDebug {

cmd/serve.go

@@ -21,10 +21,11 @@ var serveCMD = &cobra.Command{
 
 func serve(cmd *cobra.Command, args []string) {
 	opts := core.BrowserOpts{
-		IsHeadless:    !config.App.IsBrowserHead, // Disable headless if browser head mode is set
-		IsLeakless:    config.App.IsLeakless,
-		Timeout:       time.Second * time.Duration(config.App.Timeout),
-		LeavePageOpen: config.App.IsLeaveHead,
+		IsHeadless:          !config.App.IsBrowserHead, // Disable headless if browser head mode is set
+		IsLeakless:          config.App.IsLeakless,
+		Timeout:             time.Second * time.Duration(config.App.Timeout),
+		LeavePageOpen:       config.App.IsLeaveHead,
+		CaptchaSolverApiKey: config.Config2Capcha.ApiKey,
 	}
 
 	if config.App.IsDebug {

config.yaml

@@ -6,15 +6,20 @@ app:
   timeout: 15
   head: false
   leakless: false
+  leave_head: false
+
+2captcha:
+  apikey: "123123123123123"
 
 google:
-  rate_requests: 4     # Number of requests per Minute
-  rate_burst: 2        # Number of non-ratelimited requests per Minute
+  rate_requests: 4 # Number of requests per Minute
+  rate_burst: 2 # Number of non-ratelimited requests per Minute
+  captcha: true
 
 yandex:
-  rate_requests: 4     
-  rate_burst: 2        
+  rate_requests: 4
+  rate_burst: 2
 
 baidu:
-  rate_requests: 4     
-  rate_burst: 2        
+  rate_requests: 4
+  rate_burst: 2

core/captcha.go

@@ -0,0 +1,28 @@
+package core
+
+import (
+	api2captcha "github.com/2captcha/2captcha-go"
+)
+
+type CaptchaSolver struct {
+	client *api2captcha.Client
+}
+
+func NewSolver(apikey string) *CaptchaSolver {
+	cs := CaptchaSolver{}
+	cs.client = api2captcha.NewClient(apikey)
+	return &cs
+}
+
+func (cs *CaptchaSolver) SolveReCaptcha2(sitekey, pageUrl, dataS string) (string, error) {
+	cap := api2captcha.ReCaptcha{
+		SiteKey:   sitekey,
+		Url:       pageUrl,
+		DataS:     dataS,
+		Invisible: false,
+		Action:    "verify",
+	}
+	req := cap.ToRequest()
+	req.SetProxy("HTTPS", "login:password@IP_address:PORT")
+	return cs.client.Solve(req)
+}

core/captcha_test.go

@@ -0,0 +1,20 @@
+package core
+
+import (
+	"testing"
+)
+
+var (
+	API_KEY = ""
+)
+
+func Test2Captcha(t *testing.T) {
+	solver := NewSolver(API_KEY)
+	sitekey := "6LfwuyUTAAAAAOAmoS0fdqijC2PbbdH4kjq62Y1b"
+	url := "https://www.google.com/sorry/index?continue=https://www.google.de/search%3Fhl%3DDE%26lr%3Dlang_de%26nfpr%3D1%26num%3D500%26pws%3D0%26q%3Dwhere%2Bwhy%2Beach&hl=DE&q=EgRegw55GObHiq4GIjDqmzFKayGXrS2-s9ooWfcskhpK8-6tIjWSaSvhxd3f5eAyUXj7lYq2DYLDXB8ASz0yAXJaAUM"
+	datas := "Ghk0n7ZQNDS0c7ES53eef_YBfSdfeXnyRD0p2OR0R4Dg91CUXKS_hio5Do6TpJ8sHhhOat_NymTASZGe1gqAjP7w9dSvhvRT7QXsrdziO3JPngLDSRzDdjT42GDcSbO0kzInlDPxe1yy2t4yifo9xHpMnlZU7pTVNTQUIXqOMLHAR-iERi6aoSQDQ4d-88-jW3LEinquxEut0OhHG2l2stwG9AnCmNvCsUNJda-H24saFlOh5csK9KNXeeQmpr6at52_skMIMiLXSlY56vYFVCRMkXLQdAM"
+	resp, err := solver.SolveReCaptcha2(sitekey, url, datas)
+	if err != nil || resp == "" {
+		t.Fatalf("Failed to solve recaptchaV2: %s", err)
+	}
+}

core/common.go

@@ -9,14 +9,15 @@ import (
 	"github.com/gofiber/fiber/v2"
 )
 
-var ErrCaptcha = errors.New("Captcha detected")
-var ErrSearchTimeout = errors.New("Timeout. Cannot find element on page")
+var ErrCaptcha = errors.New("captcha detected")
+var ErrSearchTimeout = errors.New("timeout. Cannot find element on page")
 
 type SearchResult struct {
 	Rank        int    `json:"rank"`
 	URL         string `json:"url"`
 	Title       string `json:"title"`
 	Description string `json:"description"`
+	Ad          bool   `json:"ad"`
 }
 
 func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult {
@@ -39,6 +40,7 @@ type Query struct {
 	Filetype     string // File extension to search.
 	Site         string // Search site
 	Limit        int    // Limit the number of results
+	Answers      bool   // Include question and answers from SERP page to results with negative indexes
 }
 
 func (q Query) IsEmpty() bool {
@@ -48,23 +50,27 @@ func (q Query) IsEmpty() bool {
 	return false
 }
 
-func (q *Query) InitFromContext(c *fiber.Ctx) error {
-	q.Text = c.Query("text")
-	q.LangCode = c.Query("lang")
-	q.DateInterval = c.Query("date")
-	q.Filetype = c.Query("file")
-	q.Site = c.Query("site")
+func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
+	searchQuery.Text = reqCtx.Query("text")
+	searchQuery.LangCode = reqCtx.Query("lang")
+	searchQuery.DateInterval = reqCtx.Query("date")
+	searchQuery.Filetype = reqCtx.Query("file")
+	searchQuery.Site = reqCtx.Query("site")
 
-	limit, err := strconv.Atoi(c.Query("limit", "25"))
+	limit, err := strconv.Atoi(reqCtx.Query("limit", "25"))
 	if err != nil {
 		return err
 	}
-	q.Limit = limit
+	searchQuery.Limit = limit
 
-	if q.IsEmpty() {
-		return errors.New("Query cannot be empty")
+	searchQuery.Answers, err = strconv.ParseBool(reqCtx.Query("answers", "0"))
+	if err != nil {
+		return err
 	}
 
+	if searchQuery.IsEmpty() {
+		return errors.New("Query cannot be empty")
+	}
 	return nil
 }
 
@@ -73,6 +79,7 @@ type SearchEngineOptions struct {
 	RateTime        int64 `mapstructure:"rate_seconds"`
 	RateBurst       int   `mapstructure:"rate_burst"`
 	SelectorTimeout int64 `mapstructure:"selector_timeout"` // CSS selector timeout in seconds
+	IsSolveCaptcha  bool  `mapstructure:"captcha"`
 }
 
 func (o *SearchEngineOptions) Init() {

core/server.go

@@ -2,7 +2,6 @@ package core
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"strings"
 
@@ -52,9 +51,9 @@ func NewServer(host string, port int, searchEngines ...SearchEngine) *Server {
 			if err != nil {
 				switch err {
 				case ErrCaptcha:
-					err = errors.New(fmt.Sprintf("Captcha found, please stop sending requests for a while\n%s", err))
+					err = fmt.Errorf("captcha found, please stop sending requests for a while\n%s", err)
 				case ErrSearchTimeout:
-					err = errors.New(fmt.Sprintf("%s", err))
+					err = fmt.Errorf("%s", err)
 				}
 
 				logrus.Errorf("Error during %s search: %s", locEngine.Name(), err)
@@ -87,9 +86,9 @@ func NewServer(host string, port int, searchEngines ...SearchEngine) *Server {
 			if err != nil {
 				switch err {
 				case ErrCaptcha:
-					err = errors.New(fmt.Sprintf("Captcha found, please stop sending requests for a while\n%s", err))
+					err = fmt.Errorf("captcha found, please stop sending requests for a while: %s", err)
 				case ErrSearchTimeout:
-					err = errors.New(fmt.Sprintf("%s", err))
+					err = fmt.Errorf("%s", err)
 				}
 
 				logrus.Errorf("Error during %s search: %s", locEngine.Name(), err)

go.mod

@@ -3,6 +3,7 @@ module github.com/karust/openserp
 go 1.20
 
 require (
+	github.com/2captcha/2captcha-go v1.1.4
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/corpix/uarand v0.2.0
 	github.com/go-rod/rod v0.113.3

go.sum

@@ -36,6 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/2captcha/2captcha-go v1.1.4 h1:Fm62VPvVhEHYQ8AI+/uquiTg41ml9f8ASjUkVuBvHcE=
+github.com/2captcha/2captcha-go v1.1.4/go.mod h1:hYOq+KVOq/0zAG6OTYW7Y313qDkHv58CcaOyjdBQSco=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=

google/search.go

@@ -6,6 +6,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/proto"
@@ -24,7 +25,6 @@ func New(browser core.Browser, opts core.SearchEngineOptions) *Google {
 	gogl := Google{Browser: browser}
 	opts.Init()
 	gogl.SearchEngineOptions = opts
-
 	gogl.rgxpGetDigits = regexp.MustCompile("\\d")
 	return &gogl
 }
@@ -67,7 +67,26 @@ func (gogl *Google) getTotalResults(page *rod.Page) (int, error) {
 	return total, nil
 }
 
-func (gogl *Google) isCaptcha(page *rod.Page) bool {
+func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas string) bool {
+	logrus.Debugf("Solve google Captcha: sitekey=%s, datas=%s, url=%s", sitekey, datas, page.MustInfo().URL)
+
+	resp, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, page.MustInfo().URL, datas)
+	if err != nil {
+		logrus.Errorf("Error solving google captcha: %s", err)
+		return false
+	}
+
+	logrus.Debug("Resp:", resp)
+	_, err = page.Eval(fmt.Sprintf(`;(() => { document.getElementById("g-recaptcha-response").innerHTML="%s"; submitCallback(); })();`, resp))
+	if err != nil {
+		logrus.Errorf("Error setting captcha response: %s", err)
+		return false
+	}
+
+	return true
+}
+
+func (gogl *Google) checkCaptcha(page *rod.Page) bool {
 	captchaDiv, err := page.Timeout(gogl.GetSelectorTimeout()).Search("div[data-sitekey]")
 	if err != nil {
 		return false
@@ -85,15 +104,34 @@ func (gogl *Google) isCaptcha(page *rod.Page) bool {
 		return false
 	}
 
-	logrus.Debug("Google Captcha:", *sitekey, *dataS)
+	if gogl.IsSolveCaptcha {
+		return !gogl.solveCaptcha(page, *sitekey, *dataS)
+	}
 	return true
 }
 
 func (gogl *Google) preparePage(page *rod.Page) {
 	// Remove "similar queries" lists
-	page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove());  })();")
+	_, err := page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove());  })();")
+	if err != nil {
+		logrus.Errorf("Error preparing the page: %s", err)
+	}
 }
 
+func (gogl *Google) acceptCookies(page *rod.Page) {
+	diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search("div[role='dialog'][aria-modal] button")
+	if err != nil {
+		logrus.Errorf("Cannot find cookie consent: %s", err)
+		return
+	}
+	btnElms, err := diaglogBtns.All()
+	if err != nil {
+		logrus.Errorf("Cannot get cookie consent buttons: %s", err)
+		return
+	}
+	btnElms[3].Click(proto.InputMouseButtonLeft, 1)
+
+}
 func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	logrus.Tracef("Start Google search, query: %+v", query)
 
@@ -109,19 +147,23 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	gogl.preparePage(page)
 
 	// Check first if there captcha
-	if gogl.isCaptcha(page) {
+	if gogl.checkCaptcha(page) {
 		logrus.Errorf("Google captcha occurred during: %s", url)
 		return nil, core.ErrCaptcha
 	}
 
+	// Accept cookie consent to get google answers
+	if query.Answers {
+		gogl.acceptCookies(page)
+	}
+
 	// Find all results
-	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][lang], div[data-surl][jsaction]")
+	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid]")
 	if err != nil {
 		logrus.Errorf("Cannot parse search results: %s", err)
 		return nil, core.ErrSearchTimeout
 	}
 
-	// Check why no results, maybe captcha?
 	if results == nil {
 		return nil, nil
 	}
@@ -132,47 +174,134 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	}
 	logrus.Infof("%d SERP results", totalResults)
 
-	resultElements, err := results.All()
+	searchResultElems, err := results.All()
 	if err != nil {
 		return nil, err
 	}
 
-	for i, r := range resultElements {
-		// Get URL
-		link, err := r.Element("a")
-		if err != nil {
+	rank := 0
+	for _, resEl := range searchResultElems {
+		srchRes := core.SearchResult{}
+
+		attrs := strings.Join(resEl.MustDescribe().Attributes, " ")
+
+		if strings.Contains(attrs, "data-text-ad") {
+			// 1. Parse ads
+
+			srchRes.Ad = true
+
+			// Get URL
+			link, err := resEl.Element("a")
+			if err != nil {
+				logrus.Debug("No link found")
+				continue
+			}
+			link.MoveMouseOut()
+
+			href, err := link.Property("href")
+			if err != nil {
+				logrus.Debug("No `href` tag found")
+				continue
+			}
+			srchRes.URL = href.String()
+
+			// Get title
+			srchRes.Title = link.MustText()
+
+			// Get description
+			text := resEl.MustText()
+			textSliced := strings.Split(text, "\n")
+			srchRes.Description = strings.Join(textSliced[4:], "\n")
+			rank += 1
+
+		} else if query.Answers && strings.Contains(attrs, "data-ulkwtsb") && !strings.Contains(attrs, "data-ispaa") {
+			// 2. Parse answer boxes
+			answerEls, err := resEl.Page().Search("div[data-hveid][data-ulkwtsb] div[data-q]")
+			if err != nil {
+				logrus.Debugf("Error while parsing answer box 1: %s", err.Error())
+				continue
+			}
+
+			answers, err := answerEls.All()
+			if err != nil {
+				logrus.Debugf("Error while parsing answer box 2: %s", err.Error())
+				continue
+			}
+
+			logrus.Infof("%d answers found", len(answers))
+
+			// Unvail answer contents
+			for _, answ := range answers {
+				answ.Click(proto.InputMouseButtonLeft, 1)
+				answ.Focus()
+				//answ.Page().WaitRepaint()
+			}
+			time.Sleep(time.Millisecond * 2000)
+
+			for i, answ := range answers {
+				answerText := strings.Split(answ.MustText(), "\n")
+				if len(answerText) < 2 {
+					logrus.Debugf("Short answer text: %s", answerText)
+					continue
+				}
+
+				// Get URL
+				link, err := answ.Element("a")
+				if err != nil {
+					logrus.Debug("No answer link found")
+					continue
+				}
+				link.MoveMouseOut()
+
+				href, err := link.Property("href")
+				if err != nil {
+					logrus.Debug("No answer `href` tag found")
+					continue
+				}
+				srchRes.URL = href.String()
+				srchRes.Title = answerText[0]
+				srchRes.Description = strings.Join(answerText[1:len(answerText)-2], "\n")
+				srchRes.Rank = -1 * (i + 1)
+				searchResults = append(searchResults, srchRes)
+			}
 			continue
-		}
-		linkText, err := link.Property("href")
-		if err != nil {
-			logrus.Error("No `href` tag found")
-		}
+		} else if strings.Contains(attrs, "data-ved") && strings.Contains(attrs, "lang") {
+			// 3. Parse regular search results
+			// Get URL
+			link, err := resEl.Element("a")
+			if err != nil {
+				continue
+			}
+			href, err := link.Property("href")
+			if err != nil {
+				logrus.Debug("No `href` tag found")
+			}
+			srchRes.URL = href.String()
+			rank += 1
 
-		// Get title
-		titleTag, err := link.Element("h3")
-		if err != nil {
-			logrus.Error("No `h3` tag found")
-			continue
-		}
+			// Get title
+			titleTag, err := link.Element("h3")
+			if err != nil {
+				continue
+			}
 
-		title, err := titleTag.Text()
-		if err != nil {
-			logrus.Error("Cannot extract text from title")
-			title = "No title"
-		}
+			srchRes.Title, err = titleTag.Text()
+			if err != nil {
+				logrus.Debug("Cannot extract text from title")
+			}
+
+			// Get description
+			text := resEl.MustText()
+			textSliced := strings.Split(text, "\n")
+			srchRes.Description = strings.Join(textSliced[4:], "\n")
 
-		// Get description
-		// doesn't catch all
-		descTag, err := r.Element(`div[data-sncf~="1"]`)
-		desc := ""
-		if err != nil {
-			logrus.Trace(`No description 'div[data-sncf~="1"]' tag found`)
 		} else {
-			desc = descTag.MustText()
+			//fmt.Println(i, attrs)
+			continue
 		}
 
-		gR := core.SearchResult{Rank: i + 1, URL: linkText.String(), Title: title, Description: desc}
-		searchResults = append(searchResults, gR)
+		srchRes.Rank = rank
+		searchResults = append(searchResults, srchRes)
 	}
 
 	return searchResults, nil
@@ -212,7 +341,7 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
 
 		// Check why no results
 		if results == nil {
-			if gogl.isCaptcha(page) {
+			if gogl.checkCaptcha(page) {
 				logrus.Errorf("Google captcha occurred during: %s", url)
 				return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha
 			}

google/search_test.go

@@ -10,7 +10,7 @@ import (
 var browser *core.Browser
 
 func init() {
-	opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 5, LeavePageOpen: true}
+	opts := core.BrowserOpts{IsHeadless: true, IsLeakless: false, Timeout: time.Second * 5, LeavePageOpen: false}
 	browser, _ = core.NewBrowser(opts)
 }