fix: Fix search crash (slice bounds out of range [4:1])

baidu/search.go

@@ -24,19 +24,11 @@ type imageDataJson struct {
 		Height      int
 		Width       int
 		IsCopyright int
-
+		AdType      string `json:"adType"`
-		URL []struct {
+		URL         []struct {
 			SourcePage string `json:"FromURL"`
 			Original   string `json:"ObjURL"`
 		} `json:"replaceUrl"`
-
-		// Versions []struct {
-		// 	Height        int
-		// 	Width         int
-		// 	ImgSourcePage string `json:"fromURL"`
-		// 	URL           string `json:"objURL"`
-		// 	Type          string
-		// } `json:"setList"`
 	}
 }
 
@@ -63,18 +55,12 @@ func (baid *Baidu) GetRateLimiter() *rate.Limiter {
 
 func (baid *Baidu) isCaptcha(page *rod.Page) bool {
 	_, err := page.Timeout(baid.GetSelectorTimeout()).Search("div.passMod_dialog-body")
-	if err != nil {
+	return err == nil
-		return false
-	}
-	return true
 }
 
 func (baid *Baidu) isTimeout(page *rod.Page) bool {
 	_, err := page.Timeout(baid.GetSelectorTimeout()).Search("button.timeout-button")
-	if err != nil {
+	return err == nil
-		return false
-	}
-	return true
 }
 
 func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
@@ -226,7 +212,15 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
 				Rank:        (searchPage * 30) + (i + 1),
 				URL:         img.URL[0].Original,
 				Title:       img.Title,
-				Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright)}
+				Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright),
+				Ad: func() bool {
+					if img.AdType != "0" {
+						return true
+					} else {
+						return false
+					}
+				}(),
+			}
 			searchResults = append(searchResults, res)
 		}
 

google/search.go

@@ -293,7 +293,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 			// Get description
 			text := resEl.MustText()
 			textSliced := strings.Split(text, "\n")
-			srchRes.Description = strings.Join(textSliced[4:], "\n")
+			srchRes.Description = strings.Join(textSliced[:], "\n")
 
 		} else {
 			//fmt.Println(i, attrs)
@@ -319,21 +319,12 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
 	page := gogl.Navigate(url)
 	defer gogl.close(page)
 
-	//// TODO: Case with cookie accept (appears with VPN)
-	// if page.MustInfo().URL != url {
-	// 	results, _ := page.Search("button[aria-label][jsaction]")
-	// 	if results != nil {
-	// 		//buttons, _ := results.All()
-	// 		//buttons[1].Click(proto.InputMouseButtonLeft, 1)
-	// 	}
-	// }
-
 	for len(searchResultsMap) < query.Limit {
 		page.WaitLoad()
 		page.Mouse.Scroll(0, 1000000, 1)
 		page.WaitLoad()
 
-		results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]")
+		results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction][jsdata]")
 		if err != nil {
 			logrus.Errorf("Cannot parse search results: %s", err)
 			return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
@@ -365,18 +356,19 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
 				continue
 			}
 
-			dataID, err := r.Attribute("data-id")
+			dataVed, err := r.Attribute("data-ved")
 			if err != nil {
+				logrus.Error("Cannot find `data-ved` attr")
 				continue
 			}
 
 			// If already have image with this ID
-			if _, ok := searchResultsMap[*dataID]; ok {
+			if _, ok := searchResultsMap[*dataVed]; ok {
 				continue
 			}
 
 			// Get URLs
-			link, err := r.Element("a[tabindex][role]")
+			link, err := r.Element("a:not([ping])")
 			if err != nil {
 				continue
 			}
@@ -411,7 +403,7 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
 				Title:       title,
 				Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
 			}
-			searchResultsMap[*dataID] = gR
+			searchResultsMap[*dataVed] = gR
 
 			r.Remove()
 		}

yandex/search.go

@@ -2,28 +2,36 @@ package yandex
 
 import (
 	"encoding/json"
+	"fmt"
+	"sort"
 	"time"
 
 	"github.com/go-rod/rod"
-	"github.com/go-rod/rod/lib/input"
 	"github.com/karust/openserp/core"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/time/rate"
 )
 
-type YandexImageData struct {
+type ImageEntity struct {
-	SerpItem struct {
+	ID        string `json:"id"`
-		Freshness string
+	Rank      int    `json:"pos"`
-		Snippet   struct {
+	Width     int    `json:"origWidth"`
-			Title     string
+	Height    int    `json:"origHeight"`
-			Text      string
+	Title     string `json:"alt"`
-			URL       string
+	OrigURL   string `json:"origUrl"`
-			Domain    string
+	ThumbURL  string `json:"image"`
-			ShopScore int
+	Freshness string `json:"freshnessCounter"`
-		}
+	IsGIF     bool   `json:"gifLabel"`
-		ImgHref string `json:"img_href"`
+}
-		Pos     int
+
-	} `json:"serp-item"`
+type ImageData struct {
+	InitalState struct {
+		SerpList struct {
+			Items struct {
+				Entities map[string]ImageEntity `json:"entities"`
+			} `json:"items"`
+		} `json:"serpList"`
+	} `json:"initialState"`
 }
 
 type Yandex struct {
@@ -52,27 +60,13 @@ func (yand *Yandex) GetRateLimiter() *rate.Limiter {
 
 func (yand *Yandex) isCaptcha(page *rod.Page) bool {
 	_, err := page.Timeout(yand.GetSelectorTimeout()).Search("form#checkbox-captcha-form")
-	if err != nil {
+	return err == nil
-		return false
-	}
-	return true
 }
 
 // Check if nothig is found
 func (yand *Yandex) isNoResults(page *rod.Page) bool {
-	noResFound := false
+	_, err := page.Timeout(yand.GetSelectorTimeout()).Search("div.Correction.SearchCorrection")
-
+	return err == nil
-	_, err := page.Timeout(yand.GetSelectorTimeout()).Search("div.EmptySearchResults-Title")
-	if err == nil {
-		noResFound = true
-	}
-
-	_, err = page.Timeout(yand.GetSelectorTimeout()).Search("div>div.RequestMeta-Message")
-	if err == nil {
-		noResFound = true
-	}
-
-	return noResFound
 }
 
 func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.SearchResult {
@@ -181,25 +175,27 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
 func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
 	logrus.Tracef("Start Yandex image search, query: %+v", query)
 
-	searchResultsMap := map[string]core.SearchResult{}
+	searchResults := []core.SearchResult{}
-	url, err := BuildImageURL(query)
-	if err != nil {
-		return nil, err
-	}
 
-	page := yand.Navigate(url)
+	searchPage := 0
-	if !yand.LeavePageOpen {
+	for len(searchResults) < query.Limit {
-		defer page.Close()
+		url, err := BuildImageURL(query, searchPage)
-	}
+		if err != nil {
+			return nil, err
+		}
+		searchPage += 1
 
-	for len(searchResultsMap) < query.Limit {
+		page := yand.Navigate(url)
-		page.WaitLoad()
-		page.Keyboard.Press(input.End)
-		page.WaitLoad()
-		time.Sleep(time.Duration(time.Second * 2))
 
-		// Get all search results in page
+		if !yand.LeavePageOpen {
-		results, err := page.Timeout(yand.Timeout).Search("div.serp-item")
+			defer page.Close()
+		}
+
+		//page.Keyboard.Press(input.End)
+		//page.WaitLoad()
+		//time.Sleep(time.Duration(time.Second * 2))
+
+		results, err := page.Timeout(yand.Timeout).Search("div[role='main'] div[data-state]")
 		if err != nil {
 			logrus.Errorf("Cannot find search results: %s", err)
 		}
@@ -208,45 +204,33 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		if results == nil {
 			if yand.isCaptcha(page) {
 				logrus.Errorf("Yandex captcha occurred during: %s", url)
-				return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha
+				return searchResults, core.ErrCaptcha
 			} else if yand.isNoResults(page) {
 				logrus.Errorf("No results found")
 			}
-			return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
+			return searchResults, core.ErrSearchTimeout
 		}
 
-		for i := 0; i < results.ResultCount; i++ {
+		data, err := results.First.Attribute("data-state")
-			r, err := results.Get(i, 1)
+		if err != nil {
-			if err != nil {
+			return nil, err
-				logrus.Errorf("Cannot [%v] element from search result, [%v total]: %s", i, results.ResultCount, err)
+		}
-				return *core.ConvertSearchResultsMap(searchResultsMap), err
+
+		var imgData ImageData
+		if err := json.Unmarshal([]byte(*data), &imgData); err != nil {
+			return nil, err
+		}
+
+		for id := range imgData.InitalState.SerpList.Items.Entities {
+			img := imgData.InitalState.SerpList.Items.Entities[id]
+			res := core.SearchResult{
+				Rank:        img.Rank + 1,
+				URL:         img.OrigURL,
+				Title:       img.Title,
+				Description: fmt.Sprintf("%dx%d, freshness:%s, thumb_url:%s", img.Height, img.Width, img.Freshness, img.ThumbURL),
 			}
 
-			dataAttr, err := r[0].Attribute("data-bem")
+			searchResults = append(searchResults, res)
-			if err != nil {
-				continue
-			}
-
-			var data YandexImageData
-
-			err = json.Unmarshal([]byte(*dataAttr), &data)
-			if err != nil {
-				logrus.Errorf("Cannot unmarshal yandex image data: %v\nData: %v", err, *dataAttr)
-				continue
-			}
-
-			linkText := data.SerpItem.ImgHref
-			title := data.SerpItem.Snippet.Title
-			description := data.SerpItem.Snippet.Text
-
-			yR := core.SearchResult{
-				Rank:        (i + 1),
-				URL:         linkText,
-				Title:       title,
-				Description: description,
-			}
-
-			searchResultsMap[linkText+title] = yR
 		}
 
 		if !yand.LeavePageOpen {
@@ -254,5 +238,9 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		}
 	}
 
-	return *core.ConvertSearchResultsMap(searchResultsMap), nil
+	sort.Slice(searchResults, func(i, j int) bool {
+		return searchResults[i].Rank < searchResults[j].Rank
+	})
+
+	return searchResults, nil
 }

yandex/search_test.go

@@ -14,6 +14,19 @@ func init() {
 	browser, _ = core.NewBrowser(opts)
 }
 
+// func TestParseImgData(t *testing.T) {
+
+// 	jsonData, _ := os.ReadFile("./testImgData.json")
+// 	var obj ImageData
+// 	if err := json.Unmarshal(jsonData, &obj); err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	if (len(obj.InitalState.SerpList.Items.Entities)) != 30 {
+// 		t.Fail()
+// 	}
+// }
+
 func TestSearchYandex(t *testing.T) {
 
 	yand := New(*browser, core.SearchEngineOptions{})
@@ -32,13 +45,13 @@ func TestSearchYandex(t *testing.T) {
 func TestImageYandex(t *testing.T) {
 	yand := New(*browser, core.SearchEngineOptions{})
 
-	query := core.Query{Text: "furry tiger", Limit: 90}
+	query := core.Query{Text: "furry tiger", Limit: 30}
 	results, err := yand.SearchImage(query)
 	if err != nil {
 		t.Fatalf("Cannot [ImageYandex]: %s", err)
 	}
 
-	if len(results) < 90 {
+	if len(results) < 30 {
 		t.Fatalf("[ImageYandex] returned empty result")
 	}
 }

yandex/url.go

@@ -35,14 +35,14 @@ func BuildURL(q core.Query, page int) (string, error) {
 	}
 
 	if len(params.Get("text")) == 0 {
-		return "", errors.New("Empty query built")
+		return "", errors.New("empty query built")
 	}
 
 	base.RawQuery = params.Encode()
 	return base.String(), nil
 }
 
-func BuildImageURL(q core.Query) (string, error) {
+func BuildImageURL(q core.Query, page int) (string, error) {
 	// TODO: Add other parameters
 	base, _ := url.Parse(baseURL)
 	base.Path += "images/search/"
@@ -56,11 +56,11 @@ func BuildImageURL(q core.Query) (string, error) {
 		}
 
 		params.Add("text", text)
-		//params.Add("p", fmt.Sprint(page))
+		params.Add("p", fmt.Sprint(page))
 	}
 
 	if len(params.Get("text")) == 0 {
-		return "", errors.New("Empty query built")
+		return "", errors.New("empty query built")
 	}
 
 	if q.Site != "" {