mirror of
https://github.com/karust/openserp.git
synced 2026-08-16 21:36:06 +08:00
431 lines
10 KiB
Go
431 lines
10 KiB
Go
package google
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/go-rod/rod"
|
|
"github.com/go-rod/rod/lib/proto"
|
|
"github.com/karust/openserp/core"
|
|
"github.com/sirupsen/logrus"
|
|
"golang.org/x/time/rate"
|
|
)
|
|
|
|
type Google struct {
|
|
core.Browser
|
|
core.SearchEngineOptions
|
|
rgxpGetDigits *regexp.Regexp
|
|
}
|
|
|
|
func New(browser core.Browser, opts core.SearchEngineOptions) *Google {
|
|
gogl := Google{Browser: browser}
|
|
opts.Init()
|
|
gogl.SearchEngineOptions = opts
|
|
gogl.rgxpGetDigits = regexp.MustCompile("\\d")
|
|
return &gogl
|
|
}
|
|
|
|
func (gogl *Google) Name() string {
|
|
return "google"
|
|
}
|
|
|
|
func (gogl *Google) GetRateLimiter() *rate.Limiter {
|
|
ratelimit := rate.Every(gogl.GetRatelimit())
|
|
return rate.NewLimiter(ratelimit, gogl.RateBurst)
|
|
}
|
|
|
|
func (gogl *Google) getTotalResults(page *rod.Page) (int, error) {
|
|
resultsStats, err := page.Timeout(gogl.GetSelectorTimeout()).Search("div#result-stats")
|
|
if err != nil {
|
|
return 0, errors.New("Result stats not found: " + err.Error())
|
|
}
|
|
|
|
statsText, err := resultsStats.First.Text()
|
|
if err != nil {
|
|
return 0, errors.New("Cannot extract result stats text: " + err.Error())
|
|
}
|
|
|
|
if len(statsText) == 0 {
|
|
return 0, nil
|
|
}
|
|
|
|
// Remove search time seconds info from the end
|
|
if len(statsText) > 15 {
|
|
statsText = statsText[:len(statsText)-15]
|
|
}
|
|
|
|
foundDigits := gogl.rgxpGetDigits.FindAllString(statsText, -1)
|
|
totalNum := strings.Join(foundDigits, "")
|
|
total, err := strconv.Atoi(totalNum)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return total, nil
|
|
}
|
|
|
|
func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas string) bool {
|
|
logrus.Debugf("Solve google Captcha: sitekey=%s, datas=%s, url=%s", sitekey, datas, page.MustInfo().URL)
|
|
|
|
resp, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, page.MustInfo().URL, datas)
|
|
if err != nil {
|
|
logrus.Errorf("Error solving google captcha: %s", err)
|
|
return false
|
|
}
|
|
|
|
logrus.Debug("Resp:", resp)
|
|
_, err = page.Eval(fmt.Sprintf(`;(() => { document.getElementById("g-recaptcha-response").innerHTML="%s"; submitCallback(); })();`, resp))
|
|
if err != nil {
|
|
logrus.Errorf("Error setting captcha response: %s", err)
|
|
return false
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func (gogl *Google) checkCaptcha(page *rod.Page) bool {
|
|
captchaDiv, err := page.Timeout(gogl.GetSelectorTimeout()).Search("div[data-sitekey]")
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
sitekey, err := captchaDiv.First.Attribute("data-sitekey")
|
|
if err != nil {
|
|
logrus.Errorf("Cannot get Google captcha sitekey: %s", err)
|
|
return false
|
|
}
|
|
|
|
dataS, err := captchaDiv.First.Attribute("data-s")
|
|
if err != nil {
|
|
logrus.Errorf("Cannot get Google captcha datas: %s", err)
|
|
return false
|
|
}
|
|
|
|
if gogl.IsSolveCaptcha {
|
|
return !gogl.solveCaptcha(page, *sitekey, *dataS)
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (gogl *Google) preparePage(page *rod.Page) {
|
|
// Remove "similar queries" lists
|
|
_, err := page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove()); })();")
|
|
if err != nil {
|
|
logrus.Errorf("Error preparing the page: %s", err)
|
|
}
|
|
}
|
|
|
|
func (gogl *Google) acceptCookies(page *rod.Page) {
|
|
diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search("div[role='dialog'][aria-modal] button")
|
|
if err != nil {
|
|
logrus.Errorf("Cannot find cookie consent: %s", err)
|
|
return
|
|
}
|
|
btnElms, err := diaglogBtns.All()
|
|
if err != nil {
|
|
logrus.Errorf("Cannot get cookie consent buttons: %s", err)
|
|
return
|
|
}
|
|
btnElms[3].Click(proto.InputMouseButtonLeft, 1)
|
|
|
|
}
|
|
func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
|
|
logrus.Tracef("Start Google search, query: %+v", query)
|
|
|
|
searchResults := []core.SearchResult{}
|
|
|
|
// Build URL from query struct to open in browser
|
|
url, err := BuildURL(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
page := gogl.Navigate(url)
|
|
defer gogl.close(page)
|
|
gogl.preparePage(page)
|
|
|
|
// Check first if there captcha
|
|
if gogl.checkCaptcha(page) {
|
|
logrus.Errorf("Google captcha occurred during: %s", url)
|
|
return nil, core.ErrCaptcha
|
|
}
|
|
|
|
// Accept cookie consent to get google answers
|
|
if query.Answers {
|
|
gogl.acceptCookies(page)
|
|
}
|
|
|
|
// Find all results
|
|
results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid]")
|
|
if err != nil {
|
|
logrus.Errorf("Cannot parse search results: %s", err)
|
|
return nil, core.ErrSearchTimeout
|
|
}
|
|
|
|
if results == nil {
|
|
return nil, nil
|
|
}
|
|
|
|
totalResults, err := gogl.getTotalResults(page)
|
|
if err != nil {
|
|
logrus.Errorf("Error capturing total results: %v", err)
|
|
}
|
|
logrus.Infof("%d SERP results", totalResults)
|
|
|
|
searchResultElems, err := results.All()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rank := 0
|
|
for _, resEl := range searchResultElems {
|
|
srchRes := core.SearchResult{}
|
|
|
|
attrs := strings.Join(resEl.MustDescribe().Attributes, " ")
|
|
|
|
if strings.Contains(attrs, "data-text-ad") {
|
|
// 1. Parse ads
|
|
|
|
srchRes.Ad = true
|
|
|
|
// Get URL
|
|
link, err := resEl.Element("a")
|
|
if err != nil {
|
|
logrus.Debug("No link found")
|
|
continue
|
|
}
|
|
link.MoveMouseOut()
|
|
|
|
href, err := link.Property("href")
|
|
if err != nil {
|
|
logrus.Debug("No `href` tag found")
|
|
continue
|
|
}
|
|
srchRes.URL = href.String()
|
|
|
|
// Get title
|
|
srchRes.Title = link.MustText()
|
|
|
|
// Get description
|
|
text := resEl.MustText()
|
|
textSliced := strings.Split(text, "\n")
|
|
srchRes.Description = strings.Join(textSliced[4:], "\n")
|
|
rank += 1
|
|
|
|
} else if query.Answers && strings.Contains(attrs, "data-ulkwtsb") && !strings.Contains(attrs, "data-ispaa") {
|
|
// 2. Parse answer boxes
|
|
answerEls, err := resEl.Page().Search("div[data-hveid][data-ulkwtsb] div[data-q]")
|
|
if err != nil {
|
|
logrus.Debugf("Error while parsing answer box 1: %s", err.Error())
|
|
continue
|
|
}
|
|
|
|
answers, err := answerEls.All()
|
|
if err != nil {
|
|
logrus.Debugf("Error while parsing answer box 2: %s", err.Error())
|
|
continue
|
|
}
|
|
|
|
logrus.Infof("%d answers found", len(answers))
|
|
|
|
// Unvail answer contents
|
|
for _, answ := range answers {
|
|
answ.Click(proto.InputMouseButtonLeft, 1)
|
|
answ.Focus()
|
|
//answ.Page().WaitRepaint()
|
|
}
|
|
time.Sleep(time.Millisecond * 2000)
|
|
|
|
for i, answ := range answers {
|
|
answerText := strings.Split(answ.MustText(), "\n")
|
|
if len(answerText) < 2 {
|
|
logrus.Debugf("Short answer text: %s", answerText)
|
|
continue
|
|
}
|
|
|
|
// Get URL
|
|
link, err := answ.Element("a")
|
|
if err != nil {
|
|
logrus.Debug("No answer link found")
|
|
continue
|
|
}
|
|
link.MoveMouseOut()
|
|
|
|
href, err := link.Property("href")
|
|
if err != nil {
|
|
logrus.Debug("No answer `href` tag found")
|
|
continue
|
|
}
|
|
srchRes.URL = href.String()
|
|
srchRes.Title = answerText[0]
|
|
srchRes.Description = strings.Join(answerText[1:len(answerText)-2], "\n")
|
|
srchRes.Rank = -1 * (i + 1)
|
|
searchResults = append(searchResults, srchRes)
|
|
}
|
|
continue
|
|
} else if strings.Contains(attrs, "data-ved") && strings.Contains(attrs, "lang") {
|
|
// 3. Parse regular search results
|
|
// Get URL
|
|
link, err := resEl.Element("a")
|
|
if err != nil {
|
|
continue
|
|
}
|
|
href, err := link.Property("href")
|
|
if err != nil {
|
|
logrus.Debug("No `href` tag found")
|
|
}
|
|
srchRes.URL = href.String()
|
|
rank += 1
|
|
|
|
// Get title
|
|
titleTag, err := link.Element("h3")
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
srchRes.Title, err = titleTag.Text()
|
|
if err != nil {
|
|
logrus.Debug("Cannot extract text from title")
|
|
}
|
|
|
|
// Get description
|
|
text := resEl.MustText()
|
|
textSliced := strings.Split(text, "\n")
|
|
srchRes.Description = strings.Join(textSliced[4:], "\n")
|
|
|
|
} else {
|
|
//fmt.Println(i, attrs)
|
|
continue
|
|
}
|
|
|
|
srchRes.Rank = rank
|
|
searchResults = append(searchResults, srchRes)
|
|
}
|
|
|
|
return searchResults, nil
|
|
}
|
|
|
|
func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
|
|
logrus.Tracef("Start Google Image search, query: %+v", query)
|
|
|
|
searchResultsMap := map[string]core.SearchResult{}
|
|
url, err := BuildImageURL(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
page := gogl.Navigate(url)
|
|
defer gogl.close(page)
|
|
|
|
//// TODO: Case with cookie accept (appears with VPN)
|
|
// if page.MustInfo().URL != url {
|
|
// results, _ := page.Search("button[aria-label][jsaction]")
|
|
// if results != nil {
|
|
// //buttons, _ := results.All()
|
|
// //buttons[1].Click(proto.InputMouseButtonLeft, 1)
|
|
// }
|
|
// }
|
|
|
|
for len(searchResultsMap) < query.Limit {
|
|
page.WaitLoad()
|
|
page.Mouse.Scroll(0, 1000000, 1)
|
|
page.WaitLoad()
|
|
|
|
results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]")
|
|
if err != nil {
|
|
logrus.Errorf("Cannot parse search results: %s", err)
|
|
return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
|
|
}
|
|
|
|
// Check why no results
|
|
if results == nil {
|
|
if gogl.checkCaptcha(page) {
|
|
logrus.Errorf("Google captcha occurred during: %s", url)
|
|
return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha
|
|
}
|
|
return *core.ConvertSearchResultsMap(searchResultsMap), err
|
|
}
|
|
|
|
resultElements, err := results.All()
|
|
if err != nil {
|
|
return *core.ConvertSearchResultsMap(searchResultsMap), err
|
|
}
|
|
|
|
if len(resultElements) < len(searchResultsMap) {
|
|
continue
|
|
}
|
|
|
|
for i, r := range resultElements {
|
|
// TODO: parse AF_initDataCallback to optimize instead of this?
|
|
err := r.Click(proto.InputMouseButtonRight, 1)
|
|
if err != nil {
|
|
logrus.Error("Error clicking")
|
|
continue
|
|
}
|
|
|
|
dataID, err := r.Attribute("data-id")
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
// If already have image with this ID
|
|
if _, ok := searchResultsMap[*dataID]; ok {
|
|
continue
|
|
}
|
|
|
|
// Get URLs
|
|
link, err := r.Element("a[tabindex][role]")
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
linkText, err := link.Property("href")
|
|
if err != nil {
|
|
logrus.Error("No `href` tag found")
|
|
}
|
|
|
|
imgSrc, err := parseSourceImageURL(linkText.String())
|
|
if err != nil {
|
|
logrus.Errorf("Cannot parse image href: %v", err)
|
|
continue
|
|
}
|
|
|
|
// Get title
|
|
titleTag, err := r.Element("h3")
|
|
if err != nil {
|
|
logrus.Error("No `h3` tag found")
|
|
continue
|
|
}
|
|
|
|
title, err := titleTag.Text()
|
|
if err != nil {
|
|
logrus.Error("Cannot extract text from title")
|
|
title = "No title"
|
|
}
|
|
|
|
gR := core.SearchResult{
|
|
Rank: i + 1,
|
|
URL: imgSrc.OriginalURL,
|
|
Title: title,
|
|
Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
|
|
}
|
|
searchResultsMap[*dataID] = gR
|
|
|
|
r.Remove()
|
|
}
|
|
}
|
|
|
|
return *core.ConvertSearchResultsMap(searchResultsMap), nil
|
|
}
|
|
|
|
func (gogl *Google) close(page *rod.Page) {
|
|
if !gogl.Browser.LeavePageOpen {
|
|
err := page.Close()
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
}
|