Collects google ads/answers + captcha solver test

Remove random UA generator
Additional google query params
2026-08-25 17:22:09 +08:00 · 2024-05-06 01:13:07 +03:00 · 2024-05-06 00:53:57 +03:00 · 2024-05-06 00:51:35 +03:00 · 2024-02-07 01:30:57 +03:00 · 2023-08-06 15:09:26 +03:00
22 changed files with 1009 additions and 144 deletions
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@ API access for search engines results if available isn't free.

 Using OpenSERP, you can get search results from **Google**, **Yandex**, **Baidu** via API or CLI!

-See [Docker](#docker) and [CLI](#cli) usage examples below.
+See [Docker](#docker) and [CLI](#cli) usage examples below ([search](#search), [images](#images)).

 ## Docker usage  <a name="docker"></a> 🐳
 * Run API server:
@@ -21,13 +21,6 @@ docker run -p 127.0.0.1:7000:7000 -it karust/openserp serve -a 0.0.0.0 -p 7000
 docker-compose up --build
 ```

-### *Example request*
-Get 20 **Google** results for `hello world`, only in English:
-```
-GET http:/127.0.0.1:7000/google/search?lang=EN&limit=20&text=hello world
-```
-You can replace `google` to `yandex` or `baidu` in query to change search engine.
-
 ### Request parameters
 | Param | Description                                                  |
 |-------|--------------------------------------------------------------|
@@ -36,7 +29,17 @@ You can replace `google` to `yandex` or `baidu` in query to change search engine
 | date  | Date in `YYYYMMDD..YYYYMMDD` format (e.g. 20181010..20231010) |
 | file  | File extension to search  (e.g. `PDF`, `DOC`)                 |
 | site  | Search within a specific website                                 |
-| limit | Limit the number of results                                  |
+| limit | Limit the number of results  
+| answers | Include google answers as negative rank indexes (e.g. `true`, `false`)
+
+### **Search**
+### *Example request*
+Get 20 **Google** results for `hello world`, only in English:
+```
+GET http:/127.0.0.1:7000/google/search?lang=EN&limit=20&text=hello world
+```
+You can replace `google` to `yandex` or `baidu` in query to change search engine.
+                                |

 ### *Example response*
 ```JSON
@@ -45,10 +48,18 @@ You can replace `google` to `yandex` or `baidu` in query to change search engine
        "rank": 1,
        "url": "https://en.wikipedia.org/wiki/%22Hello,_World!%22_program",
        "title": "\"Hello, World!\" program",
-        "description": "A \"Hello, World!\" program is generally a computer program that ignores any input, and outputs or displays a message similar to \"Hello, World!\"."
+        "description": "A \"Hello, World!\" program is generally a computer program that ignores any input, and outputs or displays a message similar to \"Hello, World!\".",
+        "ad": false
    },
 ]
 ```
+### **Images** **[WIP]**
+### *Example request*
+Get 100 **Google** results for `golden puppy`:
+```
+GET http://127.0.0.1:7000/google/image?text=golden puppy&limit=100
+```
+

 ## CLI <a name="cli"></a> ⌨️
 * Use `-h` flag to see commands.
@@ -67,7 +78,8 @@ As a result you should get JSON output containting search results:
  "rank": 1,
  "url": "https://www.cyberoptik.net/blog/6-sure-fire-ways-to-get-banned-from-google/",
  "title": "11 Sure-Fire Ways to Get Banned From Google | CyberOptik",
-  "description": "How To Get Banned From Google · 1. Cloaking: The Art of Deception · 2. Plagiarism: Because Originality is Overrated · 3. Keyword Stuffing: More is Always Better · 4 ..."
+  "description": "How To Get Banned From Google · 1. Cloaking: The Art of Deception · 2. Plagiarism: Because Originality is Overrated · 3. Keyword Stuffing: More is Always Better · 4 ...",
+  "ad": false
 },
 ]
 ```
--- a/baidu/all_test.go
+++ b/baidu/all_test.go
@@ -13,7 +13,7 @@ var testQuery = core.Query{Text: "go", Site: "tutorialspoint.com", DateInterval:
 func init() {
 	core.InitLogger(true, true)

-	opts := core.BrowserOpts{IsHeadless: true, IsLeakless: false, Timeout: time.Second * 2, WaitRequests: false}
+	opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 10}
 	browser, _ = core.NewBrowser(opts)
 }

@@ -30,16 +30,7 @@ func TestUrlBuild(t *testing.T) {
 	}
 }

-// func TestSearchRaw(t *testing.T) {
-// 	results, err := Search(testQuery)
-// 	if err != nil {
-// 		t.Fatal(err)
-// 	}
-
-// 	fmt.Println(results)
-// }
-
-func TestSearchBaidu(t *testing.T) {
+func TestSearch(t *testing.T) {
 	baid := New(*browser, core.SearchEngineOptions{})
 	results, err := baid.Search(testQuery)
 	if err != nil {
@@ -50,3 +41,31 @@ func TestSearchBaidu(t *testing.T) {
 		t.Fatal("No results got from Baidu search")
 	}
 }
+
+func TestImageUrlBuild(t *testing.T) {
+	query := core.Query{Text: "金毛猎犬"}
+
+	got, err := BuildImageURL(query, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want := "https://image.baidu.com/search/acjson?cl=2&fp=result&ie=utf-8&ipn=rj&oe=utf-8&pn=0&rn=30&tn=resultjson_com&word=%E9%87%91%E6%AF%9B%E7%8C%8E%E7%8A%AC"
+	if want != got {
+		t.Fatalf("Want: `%s`, Got `%s`", want, got)
+	}
+}
+
+func TestImageSearch(t *testing.T) {
+	baid := New(*browser, core.SearchEngineOptions{})
+
+	query := core.Query{Text: "each each data", Limit: 60}
+	results, err := baid.SearchImage(query)
+	if err != nil {
+		t.Fatalf("Cannot [ImageBaidu]: %s", err)
+	}
+
+	if len(results) < 60 {
+		t.Fatalf("[ImageBaidu] returned not full result")
+	}
+}
--- a/baidu/search.go
+++ b/baidu/search.go
@@ -1,6 +1,9 @@
 package baidu

 import (
+	"encoding/json"
+	"fmt"
+	"regexp"
 	"strings"

 	"github.com/go-rod/rod"
@@ -9,6 +12,34 @@ import (
 	"golang.org/x/time/rate"
 )

+type imageDataJson struct {
+	Query        string `json:"queryExt"`
+	TotalResults int    `json:"displayNum"`
+
+	Data []struct {
+		Title       string `json:"fromPageTitle"`
+		PictureDate string `json:"bdImgnewsDate"`
+		ThumbURL    string `json:"thumbURL"`
+		Type        string
+		Height      int
+		Width       int
+		IsCopyright int
+
+		URL []struct {
+			SourcePage string `json:"FromURL"`
+			Original   string `json:"ObjURL"`
+		} `json:"replaceUrl"`
+
+		// Versions []struct {
+		// 	Height        int
+		// 	Width         int
+		// 	ImgSourcePage string `json:"fromURL"`
+		// 	URL           string `json:"objURL"`
+		// 	Type          string
+		// } `json:"setList"`
+	}
+}
+
 type Baidu struct {
 	core.Browser
 	core.SearchEngineOptions
@@ -123,3 +154,88 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {

 	return searchResults, nil
 }
+
+func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
+	logrus.Tracef("Start Baidu Image search, query: %+v", query)
+
+	searchResults := []core.SearchResult{}
+	searchPage := 0
+
+	for len(searchResults) < query.Limit {
+		url, err := BuildImageURL(query, searchPage)
+		if err != nil {
+			return nil, err
+		}
+
+		// Get anti-crawler cookies first, then reload page
+		page := baid.Navigate(url)
+		if !baid.LeavePageOpen {
+			defer page.Close()
+		}
+		page.Reload()
+		page.WaitLoad()
+
+		result, err := page.Timeout(baid.Timeout).Search("body > pre")
+		if err != nil {
+			defer page.Close()
+			logrus.Errorf("Cannot parse search results: %s", err)
+			return nil, core.ErrSearchTimeout
+		}
+
+		// Check why no results, maybe captcha?
+		if result == nil {
+			defer page.Close()
+
+			if baid.isCaptcha(page) {
+				logrus.Errorf("Baidu captcha occurred during: %s", url)
+				return nil, core.ErrCaptcha
+			} else if baid.isTimeout(page) {
+				logrus.Errorf("Baidu timeout occurred during: %s", url)
+				return nil, core.ErrCaptcha
+			}
+			return nil, nil
+		}
+
+		jsonText, err := result.First.Text()
+		if err != nil {
+			return nil, err
+		}
+
+		var data imageDataJson
+
+		// Fix broken JSON
+		jsonText = strings.ReplaceAll(jsonText, `\'`, "'")
+		matchNewlines := regexp.MustCompile(`[\r\n\t]`)
+		escapeNewlines := func(s string) string {
+			return matchNewlines.ReplaceAllString(s, "\\n")
+		}
+		re := regexp.MustCompile(`"[^"\\]*(?:\\[\s\S][^"\\]*)*"`)
+		fixedJson := re.ReplaceAllStringFunc(jsonText, escapeNewlines)
+
+		err = json.Unmarshal([]byte(fixedJson), &data)
+		if err != nil {
+			logrus.Errorf("Cannot unmarshal JSON: %v\nData: %v", err, jsonText)
+			return nil, err
+		}
+
+		for i, img := range data.Data {
+			if len(img.URL) == 0 {
+				continue
+			}
+			res := core.SearchResult{
+				Rank:        (searchPage * 30) + (i + 1),
+				URL:         img.URL[0].Original,
+				Title:       img.Title,
+				Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright)}
+			searchResults = append(searchResults, res)
+		}
+
+		searchPage += 1
+
+		if !baid.LeavePageOpen {
+			page.Close()
+		}
+	}
+
+	return searchResults, nil
+}
--- a/baidu/url.go
+++ b/baidu/url.go
@@ -76,3 +76,32 @@ func BuildURL(q core.Query) (string, error) {
 	base.RawQuery = params.Encode()
 	return base.String(), nil
 }
+
+func BuildImageURL(q core.Query, pageNum int) (string, error) {
+	base, _ := url.Parse("https://image.baidu.com/")
+	base.Path += "search/acjson"
+
+	params := url.Values{}
+	params.Add("tn", "resultjson_com")
+	params.Add("cl", "2") // Cl = 2 indicates image search
+
+	if q.Text != "" {
+		params.Add("word", q.Text)
+	}
+
+	if len(params.Get("word")) == 0 {
+		return "", errors.New("Empty query built")
+	}
+
+	if q.Limit != 0 {
+		params.Add("rn", "30")                     // Results per page
+		params.Add("pn", strconv.Itoa(pageNum*30)) // Offset
+	}
+
+	params.Add("fp", "result")
+	params.Add("ipn", "rj")
+	params.Add("ie", "utf-8")
+	params.Add("oe", "utf-8")
+	base.RawQuery = params.Encode()
+	return base.String(), nil
+}
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -1,7 +1,6 @@
 package cmd

 import (
-	"errors"
 	"fmt"
 	"strings"

@@ -13,16 +12,21 @@ import (
 )

 const (
-	version               = "0.2.1"
+	version               = "0.3"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )

 type Config struct {
-	App          AppConfig                `mapstructure:"app"`
-	GoogleConfig core.SearchEngineOptions `mapstructure:"google"`
-	YandexConfig core.SearchEngineOptions `mapstructure:"yandex"`
-	BaiduConfig  core.SearchEngineOptions `mapstructure:"baidu"`
+	App           AppConfig                `mapstructure:"app"`
+	Config2Capcha Config2Captcha           `mapstructure:"2captcha"`
+	GoogleConfig  core.SearchEngineOptions `mapstructure:"google"`
+	YandexConfig  core.SearchEngineOptions `mapstructure:"yandex"`
+	BaiduConfig   core.SearchEngineOptions `mapstructure:"baidu"`
+}
+
+type Config2Captcha struct {
+	ApiKey string `mapstructure:"apikey"`
 }

 type AppConfig struct {
@@ -88,7 +92,7 @@ func initializeConfig(cmd *cobra.Command) error {
 	// 1. Config. Return an error if we cannot parse the config file.
 	err := v.ReadInConfig()
 	if err != nil {
-		err = errors.New(fmt.Sprintf("Cannot read config: %v", err))
+		err = fmt.Errorf("cannot read config: %v", err)
 		logrus.Warn(err)
 	}

@@ -107,7 +111,7 @@ func initializeConfig(cmd *cobra.Command) error {
 	// Dump Viper values to config struct
 	err = v.Unmarshal(&config)
 	if err != nil {
-		return errors.New(fmt.Sprintf("Cannot unmarshall config: %v", err))
+		return fmt.Errorf("cannot unmarshall config: %v", err)
 	}

 	if config.App.IsDebug {
@@ -128,4 +132,5 @@ func init() {
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search")
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
+	RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
 }
--- a/cmd/search.go
+++ b/cmd/search.go
@@ -54,10 +54,11 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er
 	var engine core.SearchEngine

 	opts := core.BrowserOpts{
-		IsHeadless:    !config.App.IsBrowserHead, // Disable headless if browser head mode is set
-		IsLeakless:    config.App.IsLeakless,
-		Timeout:       time.Second * time.Duration(config.App.Timeout),
-		LeavePageOpen: config.App.IsLeaveHead,
+		IsHeadless:          !config.App.IsBrowserHead, // Disable headless if browser head mode is set
+		IsLeakless:          config.App.IsLeakless,
+		Timeout:             time.Second * time.Duration(config.App.Timeout),
+		LeavePageOpen:       config.App.IsLeaveHead,
+		CaptchaSolverApiKey: config.Config2Capcha.ApiKey,
 	}

 	if config.App.IsDebug {
--- a/cmd/serve.go
+++ b/cmd/serve.go
@@ -21,10 +21,11 @@ var serveCMD = &cobra.Command{

 func serve(cmd *cobra.Command, args []string) {
 	opts := core.BrowserOpts{
-		IsHeadless:    !config.App.IsBrowserHead, // Disable headless if browser head mode is set
-		IsLeakless:    config.App.IsLeakless,
-		Timeout:       time.Second * time.Duration(config.App.Timeout),
-		LeavePageOpen: config.App.IsLeaveHead,
+		IsHeadless:          !config.App.IsBrowserHead, // Disable headless if browser head mode is set
+		IsLeakless:          config.App.IsLeakless,
+		Timeout:             time.Second * time.Duration(config.App.Timeout),
+		LeavePageOpen:       config.App.IsLeaveHead,
+		CaptchaSolverApiKey: config.Config2Capcha.ApiKey,
 	}

 	if config.App.IsDebug {
--- a/config.yaml
+++ b/config.yaml
@@ -6,15 +6,20 @@ app:
  timeout: 15
  head: false
  leakless: false
+  leave_head: false
+
+2captcha:
+  apikey: "123123123123123"

 google:
-  rate_requests: 4     # Number of requests per Minute
-  rate_burst: 2        # Number of non-ratelimited requests per Minute
+  rate_requests: 4 # Number of requests per Minute
+  rate_burst: 2 # Number of non-ratelimited requests per Minute
+  captcha: true

 yandex:
-  rate_requests: 4     
-  rate_burst: 2        
+  rate_requests: 4
+  rate_burst: 2

 baidu:
-  rate_requests: 4     
-  rate_burst: 2        
+  rate_requests: 4
+  rate_burst: 2
--- a/core/browser.go
+++ b/core/browser.go
@@ -3,7 +3,6 @@ package core
 import (
 	"time"

-	"github.com/corpix/uarand"
 	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/devices"
 	"github.com/go-rod/rod/lib/launcher"
@@ -12,13 +11,15 @@ import (
 )

 type BrowserOpts struct {
-	IsHeadless    bool          // Use browser interface
-	IsLeakless    bool          // Force to kill browser
-	Timeout       time.Duration // Timeout
-	LanguageCode  string
-	WaitRequests  bool          // Wait requests to complete after navigation
-	LeavePageOpen bool          // Leave pages and browser open
-	WaitLoadTime  time.Duration // Time to wait till page loads
+	IsHeadless          bool          // Use browser interface
+	IsLeakless          bool          // Force to kill browser
+	Timeout             time.Duration // Timeout
+	LanguageCode        string
+	WaitRequests        bool          // Wait requests to complete after navigation
+	LeavePageOpen       bool          // Leave pages and browser open
+	WaitLoadTime        time.Duration // Time to wait till page loads
+	CaptchaSolverApiKey string        // 2Captcha api key
+
 }

 // Initialize browser parameters with default values if they are not set
@@ -34,8 +35,9 @@ func (o *BrowserOpts) Check() {

 type Browser struct {
 	BrowserOpts
-	browserAddr string
-	browser     *rod.Browser
+	browserAddr   string
+	browser       *rod.Browser
+	CaptchaSolver *CaptchaSolver
 }

 func NewBrowser(opts BrowserOpts) (*Browser, error) {
@@ -49,6 +51,11 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) {
 	b := Browser{BrowserOpts: opts}
 	b.browserAddr, err = launcher.New().Bin(path).Leakless(opts.IsLeakless).Headless(opts.IsHeadless).Launch()

+	if opts.CaptchaSolverApiKey != "" {
+		b.CaptchaSolver = NewSolver(opts.CaptchaSolverApiKey)
+		logrus.Debug("Captcha solver initialized")
+	}
+
 	return &b, err
 }

@@ -69,22 +76,22 @@ func (b *Browser) Navigate(URL string) *rod.Page {
 	b.browser.MustConnect()
 	b.browser.SetCookies(nil)

+	//page := b.browser.MustPage(URL)
 	page := stealth.MustPage(b.browser)
-	wait := page.MustWaitRequestIdle()
+	page.MustEmulate(devices.Device{
+		//UserAgent:      uarand.GetRandom(),
+		AcceptLanguage: b.LanguageCode,
+	})
 	page.MustNavigate(URL)

-	// causes bugs in google
+	wait := page.MustWaitRequestIdle()
+	// may cause bugs with google
 	if b.WaitRequests {
 		wait()
 	}

-	page.MustEmulate(devices.Device{
-		UserAgent:      uarand.GetRandom(),
-		AcceptLanguage: b.LanguageCode,
-	})
-
 	// Wait till page loads
-	time.Sleep(b.WaitLoadTime)
+	//time.Sleep(b.WaitLoadTime)

 	return page
 }
--- a/core/captcha.go
+++ b/core/captcha.go
@@ -0,0 +1,28 @@
+package core
+
+import (
+	api2captcha "github.com/2captcha/2captcha-go"
+)
+
+type CaptchaSolver struct {
+	client *api2captcha.Client
+}
+
+func NewSolver(apikey string) *CaptchaSolver {
+	cs := CaptchaSolver{}
+	cs.client = api2captcha.NewClient(apikey)
+	return &cs
+}
+
+func (cs *CaptchaSolver) SolveReCaptcha2(sitekey, pageUrl, dataS string) (string, error) {
+	cap := api2captcha.ReCaptcha{
+		SiteKey:   sitekey,
+		Url:       pageUrl,
+		DataS:     dataS,
+		Invisible: false,
+		Action:    "verify",
+	}
+	req := cap.ToRequest()
+	req.SetProxy("HTTPS", "login:password@IP_address:PORT")
+	return cs.client.Solve(req)
+}
--- a/core/captcha_test.go
+++ b/core/captcha_test.go
@@ -0,0 +1,20 @@
+package core
+
+import (
+	"testing"
+)
+
+var (
+	API_KEY = ""
+)
+
+func Test2Captcha(t *testing.T) {
+	solver := NewSolver(API_KEY)
+	sitekey := "6LfwuyUTAAAAAOAmoS0fdqijC2PbbdH4kjq62Y1b"
+	url := "https://www.google.com/sorry/index?continue=https://www.google.de/search%3Fhl%3DDE%26lr%3Dlang_de%26nfpr%3D1%26num%3D500%26pws%3D0%26q%3Dwhere%2Bwhy%2Beach&hl=DE&q=EgRegw55GObHiq4GIjDqmzFKayGXrS2-s9ooWfcskhpK8-6tIjWSaSvhxd3f5eAyUXj7lYq2DYLDXB8ASz0yAXJaAUM"
+	datas := "Ghk0n7ZQNDS0c7ES53eef_YBfSdfeXnyRD0p2OR0R4Dg91CUXKS_hio5Do6TpJ8sHhhOat_NymTASZGe1gqAjP7w9dSvhvRT7QXsrdziO3JPngLDSRzDdjT42GDcSbO0kzInlDPxe1yy2t4yifo9xHpMnlZU7pTVNTQUIXqOMLHAR-iERi6aoSQDQ4d-88-jW3LEinquxEut0OhHG2l2stwG9AnCmNvCsUNJda-H24saFlOh5csK9KNXeeQmpr6at52_skMIMiLXSlY56vYFVCRMkXLQdAM"
+	resp, err := solver.SolveReCaptcha2(sitekey, url, datas)
+	if err != nil || resp == "" {
+		t.Fatalf("Failed to solve recaptchaV2: %s", err)
+	}
+}
--- a/core/common.go
+++ b/core/common.go
@@ -2,20 +2,35 @@ package core

 import (
 	"errors"
+	"sort"
 	"strconv"
 	"time"

 	"github.com/gofiber/fiber/v2"
 )

-var ErrCaptcha = errors.New("Captcha detected")
-var ErrSearchTimeout = errors.New("Timeout. Cannot find element on page")
+var ErrCaptcha = errors.New("captcha detected")
+var ErrSearchTimeout = errors.New("timeout. Cannot find element on page")

 type SearchResult struct {
 	Rank        int    `json:"rank"`
 	URL         string `json:"url"`
 	Title       string `json:"title"`
 	Description string `json:"description"`
+	Ad          bool   `json:"ad"`
+}
+
+func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult {
+	searchResults := []SearchResult{}
+
+	for _, v := range searchResultsMap {
+		searchResults = append(searchResults, v)
+	}
+
+	sort.Slice(searchResults, func(i, j int) bool {
+		return searchResults[i].Rank < searchResults[j].Rank
+	})
+	return &searchResults
 }

 type Query struct {
@@ -25,6 +40,7 @@ type Query struct {
 	Filetype     string // File extension to search.
 	Site         string // Search site
 	Limit        int    // Limit the number of results
+	Answers      bool   // Include question and answers from SERP page to results with negative indexes
 }

 func (q Query) IsEmpty() bool {
@@ -34,23 +50,27 @@ func (q Query) IsEmpty() bool {
 	return false
 }

-func (q *Query) InitFromContext(c *fiber.Ctx) error {
-	q.Text = c.Query("text")
-	q.LangCode = c.Query("lang")
-	q.DateInterval = c.Query("date")
-	q.Filetype = c.Query("file")
-	q.Site = c.Query("site")
+func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
+	searchQuery.Text = reqCtx.Query("text")
+	searchQuery.LangCode = reqCtx.Query("lang")
+	searchQuery.DateInterval = reqCtx.Query("date")
+	searchQuery.Filetype = reqCtx.Query("file")
+	searchQuery.Site = reqCtx.Query("site")

-	limit, err := strconv.Atoi(c.Query("limit", "25"))
+	limit, err := strconv.Atoi(reqCtx.Query("limit", "25"))
 	if err != nil {
 		return err
 	}
-	q.Limit = limit
+	searchQuery.Limit = limit

-	if q.IsEmpty() {
-		return errors.New("Query cannot be empty")
+	searchQuery.Answers, err = strconv.ParseBool(reqCtx.Query("answers", "0"))
+	if err != nil {
+		return err
 	}

+	if searchQuery.IsEmpty() {
+		return errors.New("Query cannot be empty")
+	}
 	return nil
 }

@@ -59,6 +79,7 @@ type SearchEngineOptions struct {
 	RateTime        int64 `mapstructure:"rate_seconds"`
 	RateBurst       int   `mapstructure:"rate_burst"`
 	SelectorTimeout int64 `mapstructure:"selector_timeout"` // CSS selector timeout in seconds
+	IsSolveCaptcha  bool  `mapstructure:"captcha"`
 }

 func (o *SearchEngineOptions) Init() {
--- a/core/server.go
+++ b/core/server.go
@@ -2,7 +2,6 @@ package core

 import (
 	"context"
-	"errors"
 	"fmt"
 	"strings"

@@ -13,6 +12,7 @@ import (

 type SearchEngine interface {
 	Search(Query) ([]SearchResult, error)
+	SearchImage(Query) ([]SearchResult, error)
 	IsInitialized() bool
 	Name() string
 	GetRateLimiter() *rate.Limiter
@@ -51,9 +51,44 @@ func NewServer(host string, port int, searchEngines ...SearchEngine) *Server {
 			if err != nil {
 				switch err {
 				case ErrCaptcha:
-					err = errors.New(fmt.Sprintf("Captcha found, please stop sending requests for a while\n%s", err))
+					err = fmt.Errorf("captcha found, please stop sending requests for a while\n%s", err)
 				case ErrSearchTimeout:
-					err = errors.New(fmt.Sprintf("Error: %s\nProbably need to update CSS selector", err))
+					err = fmt.Errorf("%s", err)
+				}
+
+				logrus.Errorf("Error during %s search: %s", locEngine.Name(), err)
+				return fiber.NewError(fiber.StatusServiceUnavailable, err.Error())
+			}
+
+			return c.JSON(res)
+		})
+
+		serv.app.Get(fmt.Sprintf("/%s/image", strings.ToLower(locEngine.Name())), func(c *fiber.Ctx) error {
+			q := Query{}
+			err := q.InitFromContext(c)
+			if err != nil {
+				logrus.Errorf("Error while setting %s query: %s", locEngine.Name(), err)
+				return err
+			}
+
+			err = limiter.Wait(context.Background())
+			if err != nil {
+				logrus.Errorf("Ratelimiter error during %s query: %s", locEngine.Name(), err)
+			}
+
+			res, err := locEngine.SearchImage(q)
+
+			if err != nil && len(res) > 0 {
+				c.Status(503)
+				return c.JSON(res)
+			}
+
+			if err != nil {
+				switch err {
+				case ErrCaptcha:
+					err = fmt.Errorf("captcha found, please stop sending requests for a while: %s", err)
+				case ErrSearchTimeout:
+					err = fmt.Errorf("%s", err)
 				}

 				logrus.Errorf("Error during %s search: %s", locEngine.Name(), err)
--- a/core/server_test.go
+++ b/core/server_test.go
@@ -27,6 +27,9 @@ func (SeMock) IsInitialized() bool {
 func (s SeMock) Search(q Query) (res []SearchResult, err error) {
 	return []SearchResult{{Title: s.EngineName}}, nil
 }
+func (s SeMock) SearchImage(q Query) (res []SearchResult, err error) {
+	return []SearchResult{{Title: s.EngineName}}, nil
+}
 func (s SeMock) GetRateLimiter() *rate.Limiter {
 	return nil
 }
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@ module github.com/karust/openserp
 go 1.20

 require (
+	github.com/2captcha/2captcha-go v1.1.4
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/corpix/uarand v0.2.0
 	github.com/go-rod/rod v0.113.3
--- a/go.sum
+++ b/go.sum
@@ -36,6 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/2captcha/2captcha-go v1.1.4 h1:Fm62VPvVhEHYQ8AI+/uquiTg41ml9f8ASjUkVuBvHcE=
+github.com/2captcha/2captcha-go v1.1.4/go.mod h1:hYOq+KVOq/0zAG6OTYW7Y313qDkHv58CcaOyjdBQSco=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
--- a/google/search.go
+++ b/google/search.go
@@ -2,11 +2,14 @@ package google

 import (
 	"errors"
+	"fmt"
 	"regexp"
 	"strconv"
 	"strings"
+	"time"

 	"github.com/go-rod/rod"
+	"github.com/go-rod/rod/lib/proto"
 	"github.com/karust/openserp/core"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/time/rate"
@@ -15,15 +18,14 @@ import (
 type Google struct {
 	core.Browser
 	core.SearchEngineOptions
-	findNumRgxp *regexp.Regexp
+	rgxpGetDigits *regexp.Regexp
 }

 func New(browser core.Browser, opts core.SearchEngineOptions) *Google {
 	gogl := Google{Browser: browser}
 	opts.Init()
 	gogl.SearchEngineOptions = opts
-
-	gogl.findNumRgxp = regexp.MustCompile("\\d")
+	gogl.rgxpGetDigits = regexp.MustCompile("\\d")
 	return &gogl
 }

@@ -36,41 +38,100 @@ func (gogl *Google) GetRateLimiter() *rate.Limiter {
 	return rate.NewLimiter(ratelimit, gogl.RateBurst)
 }

-func (gogl *Google) findTotalResults(page *rod.Page) (int, error) {
+func (gogl *Google) getTotalResults(page *rod.Page) (int, error) {
 	resultsStats, err := page.Timeout(gogl.GetSelectorTimeout()).Search("div#result-stats")
 	if err != nil {
 		return 0, errors.New("Result stats not found: " + err.Error())
 	}

-	stats, err := resultsStats.First.Text()
+	statsText, err := resultsStats.First.Text()
 	if err != nil {
 		return 0, errors.New("Cannot extract result stats text: " + err.Error())
 	}

-	// Escape moment with `seconds` and extract digits
-	allNums := gogl.findNumRgxp.FindAllString(stats[:len(stats)-15], -1)
-	stats = strings.Join(allNums, "")
+	if len(statsText) == 0 {
+		return 0, nil
+	}

-	total, err := strconv.Atoi(stats)
+	// Remove search time seconds info from the end
+	if len(statsText) > 15 {
+		statsText = statsText[:len(statsText)-15]
+	}
+
+	foundDigits := gogl.rgxpGetDigits.FindAllString(statsText, -1)
+	totalNum := strings.Join(foundDigits, "")
+	total, err := strconv.Atoi(totalNum)
 	if err != nil {
 		return 0, err
 	}
 	return total, nil
 }

-func (gogl *Google) isCaptcha(page *rod.Page) bool {
-	_, err := page.Timeout(gogl.GetSelectorTimeout()).Search("form#captcha-form")
+func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas string) bool {
+	logrus.Debugf("Solve google Captcha: sitekey=%s, datas=%s, url=%s", sitekey, datas, page.MustInfo().URL)
+
+	resp, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, page.MustInfo().URL, datas)
+	if err != nil {
+		logrus.Errorf("Error solving google captcha: %s", err)
+		return false
+	}
+
+	logrus.Debug("Resp:", resp)
+	_, err = page.Eval(fmt.Sprintf(`;(() => { document.getElementById("g-recaptcha-response").innerHTML="%s"; submitCallback(); })();`, resp))
+	if err != nil {
+		logrus.Errorf("Error setting captcha response: %s", err)
+		return false
+	}
+
+	return true
+}
+
+func (gogl *Google) checkCaptcha(page *rod.Page) bool {
+	captchaDiv, err := page.Timeout(gogl.GetSelectorTimeout()).Search("div[data-sitekey]")
 	if err != nil {
 		return false
 	}
+
+	sitekey, err := captchaDiv.First.Attribute("data-sitekey")
+	if err != nil {
+		logrus.Errorf("Cannot get Google captcha sitekey: %s", err)
+		return false
+	}
+
+	dataS, err := captchaDiv.First.Attribute("data-s")
+	if err != nil {
+		logrus.Errorf("Cannot get Google captcha datas: %s", err)
+		return false
+	}
+
+	if gogl.IsSolveCaptcha {
+		return !gogl.solveCaptcha(page, *sitekey, *dataS)
+	}
 	return true
 }

 func (gogl *Google) preparePage(page *rod.Page) {
 	// Remove "similar queries" lists
-	page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove());  })();")
+	_, err := page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove());  })();")
+	if err != nil {
+		logrus.Errorf("Error preparing the page: %s", err)
+	}
 }

+func (gogl *Google) acceptCookies(page *rod.Page) {
+	diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search("div[role='dialog'][aria-modal] button")
+	if err != nil {
+		logrus.Errorf("Cannot find cookie consent: %s", err)
+		return
+	}
+	btnElms, err := diaglogBtns.All()
+	if err != nil {
+		logrus.Errorf("Cannot get cookie consent buttons: %s", err)
+		return
+	}
+	btnElms[3].Click(proto.InputMouseButtonLeft, 1)
+
+}
 func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	logrus.Tracef("Start Google search, query: %+v", query)

@@ -81,83 +142,289 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	page := gogl.Navigate(url)
+	defer gogl.close(page)
 	gogl.preparePage(page)

-	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][lang], div[data-surl][jsaction]")
+	// Check first if there captcha
+	if gogl.checkCaptcha(page) {
+		logrus.Errorf("Google captcha occurred during: %s", url)
+		return nil, core.ErrCaptcha
+	}
+
+	// Accept cookie consent to get google answers
+	if query.Answers {
+		gogl.acceptCookies(page)
+	}
+
+	// Find all results
+	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid]")
 	if err != nil {
-		defer page.Close()
 		logrus.Errorf("Cannot parse search results: %s", err)
 		return nil, core.ErrSearchTimeout
 	}

-	// Check why no results, maybe captcha?
 	if results == nil {
-		defer page.Close()
-
-		if gogl.isCaptcha(page) {
-			logrus.Errorf("Google captcha occurred during: %s", url)
-			return nil, core.ErrCaptcha
-		}
-		return nil, err
+		return nil, nil
 	}

-	totalResults, err := gogl.findTotalResults(page)
+	totalResults, err := gogl.getTotalResults(page)
 	if err != nil {
 		logrus.Errorf("Error capturing total results: %v", err)
 	}
-	logrus.Infof("%d total results found", totalResults)
+	logrus.Infof("%d SERP results", totalResults)

-	resultElements, err := results.All()
+	searchResultElems, err := results.All()
 	if err != nil {
 		return nil, err
 	}

-	for i, r := range resultElements {
-		// Get URL
-		link, err := r.Element("a")
-		if err != nil {
+	rank := 0
+	for _, resEl := range searchResultElems {
+		srchRes := core.SearchResult{}
+
+		attrs := strings.Join(resEl.MustDescribe().Attributes, " ")
+
+		if strings.Contains(attrs, "data-text-ad") {
+			// 1. Parse ads
+
+			srchRes.Ad = true
+
+			// Get URL
+			link, err := resEl.Element("a")
+			if err != nil {
+				logrus.Debug("No link found")
+				continue
+			}
+			link.MoveMouseOut()
+
+			href, err := link.Property("href")
+			if err != nil {
+				logrus.Debug("No `href` tag found")
+				continue
+			}
+			srchRes.URL = href.String()
+
+			// Get title
+			srchRes.Title = link.MustText()
+
+			// Get description
+			text := resEl.MustText()
+			textSliced := strings.Split(text, "\n")
+			srchRes.Description = strings.Join(textSliced[4:], "\n")
+			rank += 1
+
+		} else if query.Answers && strings.Contains(attrs, "data-ulkwtsb") && !strings.Contains(attrs, "data-ispaa") {
+			// 2. Parse answer boxes
+			answerEls, err := resEl.Page().Search("div[data-hveid][data-ulkwtsb] div[data-q]")
+			if err != nil {
+				logrus.Debugf("Error while parsing answer box 1: %s", err.Error())
+				continue
+			}
+
+			answers, err := answerEls.All()
+			if err != nil {
+				logrus.Debugf("Error while parsing answer box 2: %s", err.Error())
+				continue
+			}
+
+			logrus.Infof("%d answers found", len(answers))
+
+			// Unvail answer contents
+			for _, answ := range answers {
+				answ.Click(proto.InputMouseButtonLeft, 1)
+				answ.Focus()
+				//answ.Page().WaitRepaint()
+			}
+			time.Sleep(time.Millisecond * 2000)
+
+			for i, answ := range answers {
+				answerText := strings.Split(answ.MustText(), "\n")
+				if len(answerText) < 2 {
+					logrus.Debugf("Short answer text: %s", answerText)
+					continue
+				}
+
+				// Get URL
+				link, err := answ.Element("a")
+				if err != nil {
+					logrus.Debug("No answer link found")
+					continue
+				}
+				link.MoveMouseOut()
+
+				href, err := link.Property("href")
+				if err != nil {
+					logrus.Debug("No answer `href` tag found")
+					continue
+				}
+				srchRes.URL = href.String()
+				srchRes.Title = answerText[0]
+				srchRes.Description = strings.Join(answerText[1:len(answerText)-2], "\n")
+				srchRes.Rank = -1 * (i + 1)
+				searchResults = append(searchResults, srchRes)
+			}
 			continue
-		}
-		linkText, err := link.Property("href")
-		if err != nil {
-			logrus.Error("No `href` tag found")
-		}
+		} else if strings.Contains(attrs, "data-ved") && strings.Contains(attrs, "lang") {
+			// 3. Parse regular search results
+			// Get URL
+			link, err := resEl.Element("a")
+			if err != nil {
+				continue
+			}
+			href, err := link.Property("href")
+			if err != nil {
+				logrus.Debug("No `href` tag found")
+			}
+			srchRes.URL = href.String()
+			rank += 1

-		// Get title
-		titleTag, err := link.Element("h3")
-		if err != nil {
-			logrus.Error("No `h3` tag found")
-			continue
-		}
+			// Get title
+			titleTag, err := link.Element("h3")
+			if err != nil {
+				continue
+			}

-		title, err := titleTag.Text()
-		if err != nil {
-			logrus.Error("Cannot extract text from title")
-			title = "No title"
-		}
+			srchRes.Title, err = titleTag.Text()
+			if err != nil {
+				logrus.Debug("Cannot extract text from title")
+			}
+
+			// Get description
+			text := resEl.MustText()
+			textSliced := strings.Split(text, "\n")
+			srchRes.Description = strings.Join(textSliced[4:], "\n")

-		// Get description
-		// doesn't catch all
-		descTag, err := r.Element(`div[data-sncf~="1"]`)
-		desc := ""
-		if err != nil {
-			logrus.Trace(`No description 'div[data-sncf~="1"]' tag found`)
 		} else {
-			desc = descTag.MustText()
+			//fmt.Println(i, attrs)
+			continue
 		}

-		gR := core.SearchResult{Rank: i + 1, URL: linkText.String(), Title: title, Description: desc}
-		searchResults = append(searchResults, gR)
-	}
-
-	if !gogl.Browser.LeavePageOpen {
-		err = page.Close()
-		if err != nil {
-			logrus.Error(err)
-		}
+		srchRes.Rank = rank
+		searchResults = append(searchResults, srchRes)
 	}

 	return searchResults, nil
 }
+
+func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
+	logrus.Tracef("Start Google Image search, query: %+v", query)
+
+	searchResultsMap := map[string]core.SearchResult{}
+	url, err := BuildImageURL(query)
+	if err != nil {
+		return nil, err
+	}
+
+	page := gogl.Navigate(url)
+	defer gogl.close(page)
+
+	//// TODO: Case with cookie accept (appears with VPN)
+	// if page.MustInfo().URL != url {
+	// 	results, _ := page.Search("button[aria-label][jsaction]")
+	// 	if results != nil {
+	// 		//buttons, _ := results.All()
+	// 		//buttons[1].Click(proto.InputMouseButtonLeft, 1)
+	// 	}
+	// }
+
+	for len(searchResultsMap) < query.Limit {
+		page.WaitLoad()
+		page.Mouse.Scroll(0, 1000000, 1)
+		page.WaitLoad()
+
+		results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]")
+		if err != nil {
+			logrus.Errorf("Cannot parse search results: %s", err)
+			return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
+		}
+
+		// Check why no results
+		if results == nil {
+			if gogl.checkCaptcha(page) {
+				logrus.Errorf("Google captcha occurred during: %s", url)
+				return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha
+			}
+			return *core.ConvertSearchResultsMap(searchResultsMap), err
+		}
+
+		resultElements, err := results.All()
+		if err != nil {
+			return *core.ConvertSearchResultsMap(searchResultsMap), err
+		}
+
+		if len(resultElements) < len(searchResultsMap) {
+			continue
+		}
+
+		for i, r := range resultElements {
+			// TODO: parse AF_initDataCallback to optimize instead of this?
+			err := r.Click(proto.InputMouseButtonRight, 1)
+			if err != nil {
+				logrus.Error("Error clicking")
+				continue
+			}
+
+			dataID, err := r.Attribute("data-id")
+			if err != nil {
+				continue
+			}
+
+			// If already have image with this ID
+			if _, ok := searchResultsMap[*dataID]; ok {
+				continue
+			}
+
+			// Get URLs
+			link, err := r.Element("a[tabindex][role]")
+			if err != nil {
+				continue
+			}
+
+			linkText, err := link.Property("href")
+			if err != nil {
+				logrus.Error("No `href` tag found")
+			}
+
+			imgSrc, err := parseSourceImageURL(linkText.String())
+			if err != nil {
+				logrus.Errorf("Cannot parse image href: %v", err)
+				continue
+			}
+
+			// Get title
+			titleTag, err := r.Element("h3")
+			if err != nil {
+				logrus.Error("No `h3` tag found")
+				continue
+			}
+
+			title, err := titleTag.Text()
+			if err != nil {
+				logrus.Error("Cannot extract text from title")
+				title = "No title"
+			}
+
+			gR := core.SearchResult{
+				Rank:        i + 1,
+				URL:         imgSrc.OriginalURL,
+				Title:       title,
+				Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
+			}
+			searchResultsMap[*dataID] = gR
+
+			r.Remove()
+		}
+	}
+
+	return *core.ConvertSearchResultsMap(searchResultsMap), nil
+}
+
+func (gogl *Google) close(page *rod.Page) {
+	if !gogl.Browser.LeavePageOpen {
+		err := page.Close()
+		if err != nil {
+			logrus.Error(err)
+		}
+	}
+}
--- a/google/search_test.go
+++ b/google/search_test.go
@@ -10,7 +10,7 @@ import (
 var browser *core.Browser

 func init() {
-	opts := core.BrowserOpts{IsHeadless: true, IsLeakless: true, Timeout: time.Second * 2, WaitRequests: false}
+	opts := core.BrowserOpts{IsHeadless: true, IsLeakless: false, Timeout: time.Second * 5, LeavePageOpen: false}
 	browser, _ = core.NewBrowser(opts)
 }

@@ -27,3 +27,41 @@ func TestSearchGoogle(t *testing.T) {
 		t.Fatalf("[SearchGoogle] returned empty result")
 	}
 }
+
+func TestParseSourceImageURL(t *testing.T) {
+	//href1 := `/imgres?imgurl=https%3A%2F%2Fupload.wikimedia.org%2Fwikipedia%2Fcommons%2F2%2F26%2FMarmota_marmota_Alpes2.jpg&amp;tbnid=Be_RycOe8xzlpM&amp;vet=12ahUKEwjkh6WzwIeAAxWV_yoKHRzHC9wQMygAegUIARD0AQ..i&amp;imgrefurl=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FAlpine_marmot&amp;docid=7miWbc2QiSw9uM&amp;w=801&amp;h=599&amp;q=alpine%20marmot&amp;ved=2ahUKEwjkh6WzwIeAAxWV_yoKHRzHC9wQMygAegUIARD0AQ`
+	href2 := `/imgres?imgurl=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fnaturerules1%2Fimages%2Ff%2Ff2%2F13d79d934ccf6f7919777fcb6dbb6e6c.jpg%2Frevision%2Flatest%3Fcb%3D20210218225522&tbnid=JxC8NUyBjdNbdM&vet=12ahUKEwiHrJnN1YeAAxXvEBAIHfRADAAQMygCegUIARD4AQ..i&imgrefurl=https%3A%2F%2Fnaturerules1.fandom.com%2Fwiki%2FAlpine_Marmot&docid=XXYeDjL67badNM&w=1600&h=1200&q=alpine%20marmot&ved=2ahUKEwiHrJnN1YeAAxXvEBAIHfRADAAQMygCegUIARD4AQ`
+	want := SourceImage{
+		OriginalURL: "https://static.wikia.nocookie.net/naturerules1/images/f/f2/13d79d934ccf6f7919777fcb6dbb6e6c.jpg/revision/latest?cb=20210218225522",
+		PageURL:     "https://naturerules1.fandom.com/wiki/Alpine_Marmot",
+		Width:       "1600",
+		Height:      "1200",
+	}
+
+	got, err := parseSourceImageURL(href2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if want != got {
+		t.Fatalf("Want: %v, Got: %v", want, got)
+	}
+}
+
+func TestImageSearch(t *testing.T) {
+	gogl := New(*browser, core.SearchEngineOptions{})
+	query := core.Query{Text: "Ferrari Testarossa", Limit: 77}
+	results, err := gogl.SearchImage(query)
+
+	if err != nil {
+		t.Fatalf("Cannot search images: %s", err)
+	}
+
+	if len(results) < 77 {
+		t.Fatalf("Returned not full result")
+	}
+
+	if results[0].URL == "" {
+		t.Fatalf("First result doesn't contain URL, %v+", results[0])
+	}
+}
--- a/google/url.go
+++ b/google/url.go
@@ -236,17 +236,78 @@ func BuildURL(q core.Query) (string, error) {

 		logrus.Tracef("Query text: %s", text)
 		params.Add("q", text)
+		params.Add("oq", text)
 	}

 	if len(params.Get("q")) == 0 {
-		return "", errors.New("Empty query built")
+		return "", errors.New("empty query built")
 	}

 	// Set search date range
 	if q.DateInterval != "" {
 		intervals := strings.Split(q.DateInterval, "..")
 		if len(intervals) != 2 {
-			return "", errors.New("Incorrect data interval provided")
+			return "", errors.New("incorrect data interval provided")
+		}
+
+		dataParam := fmt.Sprintf("cdr:1,cd_min:%s,cd_max:%s", intervals[0], intervals[1])
+		params.Add("tbs", dataParam)
+	}
+
+	// Limit number of results
+	if q.Limit != 0 {
+		params.Add("num", strconv.Itoa(q.Limit))
+	}
+
+	if q.LangCode != "" {
+		params.Add("hl", q.LangCode)
+		params.Add("lr", "lang_"+strings.ToLower(q.LangCode))
+	}
+
+	params.Add("pws", "0")  // Do not personalize earch results
+	params.Add("nfpr", "1") // Do not auto correct search queries
+	params.Add("sourceid", "chrome")
+	params.Add("ie", "UTF-8")
+
+	base.RawQuery = params.Encode()
+	return base.String(), nil
+}
+
+func BuildImageURL(q core.Query) (string, error) {
+	// TODO: Add new params
+	googleBase := GoogleDomains[strings.ToLower(q.LangCode)]
+	base, err := url.Parse(fmt.Sprintf("https://www.google.%s", googleBase))
+	if err != nil {
+		return "", err
+	}
+
+	base.Path += "search"
+	params := url.Values{}
+	params.Add("tbm", "isch") // Search images
+
+	// Set request text
+	if q.Text != "" || q.Site != "" || q.Filetype != "" {
+		text := q.Text
+		if q.Site != "" {
+			text += " site:" + q.Site
+		}
+		if q.Filetype != "" {
+			text += " filetype:" + q.Filetype
+		}
+
+		params.Add("q", text)
+		params.Add("oq", text)
+	}
+
+	if len(params.Get("q")) == 0 {
+		return "", errors.New("empty query built")
+	}
+
+	// Set search date range
+	if q.DateInterval != "" {
+		intervals := strings.Split(q.DateInterval, "..")
+		if len(intervals) != 2 {
+			return "", errors.New("incorrect data interval provided")
 		}

 		dataParam := fmt.Sprintf("cdr:1,cd_min:%s,cd_max:%s", intervals[0], intervals[1])
@@ -269,3 +330,52 @@ func BuildURL(q core.Query) (string, error) {
 	base.RawQuery = params.Encode()
 	return base.String(), nil
 }
+
+type SourceImage struct {
+	PageURL     string
+	OriginalURL string
+	Width       string
+	Height      string
+}
+
+func parseSourceImageURL(href string) (SourceImage, error) {
+	source := SourceImage{}
+
+	href = strings.ReplaceAll(href, ";", "&")
+	parsed, err := url.QueryUnescape(href)
+	if err != nil {
+		return source, err
+	}
+
+	u, err := url.Parse(parsed)
+	if err != nil {
+		return source, err
+	}
+
+	queryMap, err := url.ParseQuery(u.RawQuery)
+	if err != nil {
+		return source, err
+	}
+
+	val, ok := queryMap["h"]
+	if ok && len(val) > 0 {
+		source.Height = val[0]
+	}
+
+	val, ok = queryMap["w"]
+	if ok && len(val) > 0 {
+		source.Width = val[0]
+	}
+
+	val, ok = queryMap["imgrefurl"]
+	if ok && len(val) > 0 {
+		source.PageURL = val[0]
+	}
+
+	val, ok = queryMap["imgurl"]
+	if ok && len(val) > 0 {
+		source.OriginalURL = val[0]
+	}
+
+	return source, nil
+}
--- a/yandex/search.go
+++ b/yandex/search.go
@@ -1,14 +1,31 @@
 package yandex

 import (
+	"encoding/json"
 	"time"

 	"github.com/go-rod/rod"
+	"github.com/go-rod/rod/lib/input"
 	"github.com/karust/openserp/core"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/time/rate"
 )

+type YandexImageData struct {
+	SerpItem struct {
+		Freshness string
+		Snippet   struct {
+			Title     string
+			Text      string
+			URL       string
+			Domain    string
+			ShopScore int
+		}
+		ImgHref string `json:"img_href"`
+		Pos     int
+	} `json:"serp-item"`
+}
+
 type Yandex struct {
 	core.Browser
 	core.SearchEngineOptions
@@ -160,3 +177,82 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {

 	return allResults, nil
 }
+
+func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
+	logrus.Tracef("Start Yandex image search, query: %+v", query)
+
+	searchResultsMap := map[string]core.SearchResult{}
+	url, err := BuildImageURL(query)
+	if err != nil {
+		return nil, err
+	}
+
+	page := yand.Navigate(url)
+	if !yand.LeavePageOpen {
+		defer page.Close()
+	}
+
+	for len(searchResultsMap) < query.Limit {
+		page.WaitLoad()
+		page.Keyboard.Press(input.End)
+		page.WaitLoad()
+		time.Sleep(time.Duration(time.Second * 2))
+
+		// Get all search results in page
+		results, err := page.Timeout(yand.Timeout).Search("div.serp-item")
+		if err != nil {
+			logrus.Errorf("Cannot find search results: %s", err)
+		}
+
+		// Check why no results
+		if results == nil {
+			if yand.isCaptcha(page) {
+				logrus.Errorf("Yandex captcha occurred during: %s", url)
+				return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha
+			} else if yand.isNoResults(page) {
+				logrus.Errorf("No results found")
+			}
+			return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
+		}
+
+		for i := 0; i < results.ResultCount; i++ {
+			r, err := results.Get(i, 1)
+			if err != nil {
+				logrus.Errorf("Cannot [%v] element from search result, [%v total]: %s", i, results.ResultCount, err)
+				return *core.ConvertSearchResultsMap(searchResultsMap), err
+			}
+
+			dataAttr, err := r[0].Attribute("data-bem")
+			if err != nil {
+				continue
+			}
+
+			var data YandexImageData
+
+			err = json.Unmarshal([]byte(*dataAttr), &data)
+			if err != nil {
+				logrus.Errorf("Cannot unmarshal yandex image data: %v\nData: %v", err, *dataAttr)
+				continue
+			}
+
+			linkText := data.SerpItem.ImgHref
+			title := data.SerpItem.Snippet.Title
+			description := data.SerpItem.Snippet.Text
+
+			yR := core.SearchResult{
+				Rank:        (i + 1),
+				URL:         linkText,
+				Title:       title,
+				Description: description,
+			}
+
+			searchResultsMap[linkText+title] = yR
+		}
+
+		if !yand.LeavePageOpen {
+			page.Close()
+		}
+	}
+
+	return *core.ConvertSearchResultsMap(searchResultsMap), nil
+}
--- a/yandex/search_test.go
+++ b/yandex/search_test.go
@@ -10,7 +10,7 @@ import (
 var browser *core.Browser

 func init() {
-	opts := core.BrowserOpts{IsHeadless: true, IsLeakless: true, Timeout: time.Second * 2, WaitRequests: false}
+	opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 15, LeavePageOpen: true}
 	browser, _ = core.NewBrowser(opts)
 }

@@ -28,3 +28,17 @@ func TestSearchYandex(t *testing.T) {
 		t.Fatalf("[SearchYandex] returned empty result")
 	}
 }
+
+func TestImageYandex(t *testing.T) {
+	yand := New(*browser, core.SearchEngineOptions{})
+
+	query := core.Query{Text: "furry tiger", Limit: 90}
+	results, err := yand.SearchImage(query)
+	if err != nil {
+		t.Fatalf("Cannot [ImageYandex]: %s", err)
+	}
+
+	if len(results) < 90 {
+		t.Fatalf("[ImageYandex] returned empty result")
+	}
+}
--- a/yandex/url.go
+++ b/yandex/url.go
@@ -8,8 +8,10 @@ import (
 	"github.com/karust/openserp/core"
 )

+const baseURL = "https://www.yandex.com"
+
 func BuildURL(q core.Query, page int) (string, error) {
-	base, _ := url.Parse("https://www.yandex.com")
+	base, _ := url.Parse(baseURL)
 	base.Path += "search/"

 	params := url.Values{}
@@ -39,3 +41,36 @@ func BuildURL(q core.Query, page int) (string, error) {
 	base.RawQuery = params.Encode()
 	return base.String(), nil
 }
+
+func BuildImageURL(q core.Query) (string, error) {
+	// TODO: Add other parameters
+	base, _ := url.Parse(baseURL)
+	base.Path += "images/search/"
+
+	params := url.Values{}
+	if q.Text != "" {
+		text := q.Text
+
+		if q.DateInterval != "" {
+			text += " date:" + q.DateInterval
+		}
+
+		params.Add("text", text)
+		//params.Add("p", fmt.Sprint(page))
+	}
+
+	if len(params.Get("text")) == 0 {
+		return "", errors.New("Empty query built")
+	}
+
+	if q.Site != "" {
+		params.Add("site", q.Site)
+	}
+
+	if q.Filetype != "" {
+		params.Add("itype", q.Filetype)
+	}
+
+	base.RawQuery = params.Encode()
+	return base.String(), nil
+}
Author	SHA1	Message	Date
Rustem Kamalov	0a3c07b149	Collects google ads/answers + captcha solver test	2024-05-06 01:13:07 +03:00
Rustem Kamalov	482f725c10	Remove random UA generator	2024-05-06 00:53:57 +03:00
Rustem Kamalov	923b7c2443	Additional google query params	2024-05-06 00:51:35 +03:00
Rustem Kamalov	e2bdad75dd	Google: Fix total results bug, Fix captcha detection	2024-02-07 01:30:57 +03:00
Rustem Kamalov	b040b24a70	Readme update	2023-08-06 15:09:26 +03:00
Rustem Kamalov	a3455b592d	Add image API	2023-07-23 04:37:01 +03:00
Rustem Kamalov	29a7c2ca14	Yandex images scroll parsing	2023-07-23 04:35:48 +03:00
Rustem Kamalov	38183b804a	Google images scroll parsing	2023-07-23 04:35:28 +03:00
Rustem Kamalov	34074161a5	Baidu. Fix broken JSON parsing	2023-07-23 04:33:53 +03:00
Rustem Kamalov	e12fff8313	Google image search test	2023-07-12 02:17:04 +03:00
Rustem Kamalov	4d6921d007	Baidu [ImageUrlBuild] test fix	2023-07-11 18:53:42 +03:00
Rustem Kamalov	284aebbfb2	Baidu image search test	2023-07-08 01:13:56 +03:00
Rustem Kamalov	6eaab4a059	Yandex image search test	2023-07-05 18:42:14 +03:00