Settings via config file and environment variables

.gitignore

@@ -21,4 +21,6 @@ raw.go
 dev.md
 .gitignore
 logrus.txt
-logs.txt
+logs.txt
+.release
+core/test/

Dockerfile

@@ -17,6 +17,7 @@ RUN go build -o /app/openserp .
 FROM zenika/alpine-chrome:with-chromedriver
 
 COPY --from=builder /app/openserp /usr/local/bin/openserp
+ADD config.yaml /usr/src/app
 
 ENTRYPOINT ["openserp"]
 

README.md

@@ -3,14 +3,13 @@
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/karust/openserp)](https://goreportcard.com/report/github.com/karust/openserp)
 [![Go Reference](https://pkg.go.dev/badge/github.com/karust/openserp.svg)](https://pkg.go.dev/github.com/karust/openserp)
-<!-- ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/karust/openserp/latest)
-![GitHub release (by tag)](https://img.shields.io/github/downloads/karust/openserp/0.1/total) -->
-
+[![release](https://img.shields.io/github/release-pre/karust/openserp.svg)](https://github.com/karust/openserp/releases)
+<!-- ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/karust/openserp/latest) -->
 API access for search engines results if available isn't free.
 
 Using OpenSERP, you can get search results from **Google**, **Yandex**, **Baidu** via API or CLI!
 
-See [Docker](#docker), [CLI](#cli) usage examples below.
+See [Docker](#docker) and [CLI](#cli) usage examples below.
 
 ## Docker usage  <a name="docker"></a> 🐳
 * Run API server:
@@ -18,13 +17,13 @@ See [Docker](#docker), [CLI](#cli) usage examples below.
 # Use prebuilt image
 docker run -p 127.0.0.1:7000:7000 -it karust/openserp serve -a 0.0.0.0 -p 7000
 
-# Or
+# Or build one and run using docker-compose.yaml
 docker-compose up --build
 ```
 
 ### *Example request*
 Get 20 **Google** results for `hello world`, only in English:
-```JSON
+```
 GET http:/127.0.0.1:7000/google/search?lang=EN&limit=20&text=hello world
 ```
 You can replace `google` to `yandex` or `baidu` in query to change search engine.
@@ -36,14 +35,14 @@ You can replace `google` to `yandex` or `baidu` in query to change search engine
 | lang  | Search pages in selected language (`EN`, `DE`, `RU`...)      |
 | date  | Date in `YYYYMMDD..YYYYMMDD` format (e.g. 20181010..20231010) |
 | file  | File extension to search  (e.g. `PDF`, `DOC`)                 |
-| site  | Search only in selected site                                 |
+| site  | Search within a specific website                                 |
 | limit | Limit the number of results                                  |
 
 ### *Example response*
 ```JSON
 [
     {
-        "rank": 0,
+        "rank": 1,
         "url": "https://en.wikipedia.org/wiki/%22Hello,_World!%22_program",
         "title": "\"Hello, World!\" program",
         "description": "A \"Hello, World!\" program is generally a computer program that ignores any input, and outputs or displays a message similar to \"Hello, World!\"."
@@ -51,8 +50,6 @@ You can replace `google` to `yandex` or `baidu` in query to change search engine
 ]
 ```
 
-
-
 ## CLI <a name="cli"></a> ⌨️
 * Use `-h` flag to see commands.
 * You can use `serve` command to serve API:
@@ -61,13 +58,13 @@ openserp serve
 ```
 * Or print results in CLI using `search` command:
 ```bash
-openserp search google "how to get banned in google fast" # Change `google` to `yandex` or `baidu`
+openserp search google "how to get banned from google fast" # Change `google` to `yandex` or `baidu`
 ```
 As a result you should get JSON output containting search results:
 ```json
 [
  {
-  "rank": 0,
+  "rank": 1,
   "url": "https://www.cyberoptik.net/blog/6-sure-fire-ways-to-get-banned-from-google/",
   "title": "11 Sure-Fire Ways to Get Banned From Google | CyberOptik",
   "description": "How To Get Banned From Google · 1. Cloaking: The Art of Deception · 2. Plagiarism: Because Originality is Overrated · 3. Keyword Stuffing: More is Always Better · 4 ..."

baidu/all_test.go

@@ -40,7 +40,7 @@ func TestUrlBuild(t *testing.T) {
 // }
 
 func TestSearchBaidu(t *testing.T) {
-	baid := New(*browser)
+	baid := New(*browser, core.SearchEngineOptions{})
 	results, err := baid.Search(testQuery)
 	if err != nil {
 		t.Fatal(err)

baidu/search.go

@@ -2,26 +2,50 @@ package baidu
 
 import (
 	"strings"
-	"time"
 
+	"github.com/go-rod/rod"
 	"github.com/karust/openserp/core"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/time/rate"
 )
 
 type Baidu struct {
 	core.Browser
-	checkTimeout time.Duration
+	core.SearchEngineOptions
 }
 
-func New(browser core.Browser) *Baidu {
+func New(browser core.Browser, opts core.SearchEngineOptions) *Baidu {
 	baid := Baidu{Browser: browser}
-	baid.checkTimeout = time.Second * 2
+	opts.Init()
+	baid.SearchEngineOptions = opts
 	return &baid
 }
+
 func (baid *Baidu) Name() string {
 	return "baidu"
 }
 
+func (baid *Baidu) GetRateLimiter() *rate.Limiter {
+	ratelimit := rate.Every(baid.GetRatelimit())
+	return rate.NewLimiter(ratelimit, baid.RateBurst)
+}
+
+func (baid *Baidu) isCaptcha(page *rod.Page) bool {
+	_, err := page.Timeout(baid.GetSelectorTimeout()).Search("div.passMod_dialog-body")
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+func (baid *Baidu) isTimeout(page *rod.Page) bool {
+	_, err := page.Timeout(baid.GetSelectorTimeout()).Search("button.timeout-button")
+	if err != nil {
+		return false
+	}
+	return true
+}
+
 func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
 	logrus.Tracef("Start Baidu search, query: %+v", query)
 
@@ -37,7 +61,23 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
 
 	results, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd")
 	if err != nil {
-		return nil, err
+		defer page.Close()
+		logrus.Errorf("Cannot parse search results: %s", err)
+		return nil, core.ErrSearchTimeout
+	}
+
+	// Check why no results, maybe captcha?
+	if results == nil {
+		defer page.Close()
+
+		if baid.isCaptcha(page) {
+			logrus.Errorf("Baidu captcha occurred during: %s", url)
+			return nil, core.ErrCaptcha
+		} else if baid.isTimeout(page) {
+			logrus.Errorf("Baidu timeout occurred during: %s", url)
+			return nil, core.ErrCaptcha
+		}
+		return nil, nil
 	}
 
 	resultElements, err := results.All()

cmd/root.go

@@ -1,7 +1,9 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/karust/openserp/core"
 	"github.com/sirupsen/logrus"
@@ -11,42 +13,55 @@ import (
 )
 
 const (
-	version                    = "0.1.1"
-	defaultConfigFilename      = "config"
-	envPrefix                  = "OPENSERP"
-	replaceHyphenWithCamelCase = false
+	version               = "0.2.1"
+	defaultConfigFilename = "config"
+	envPrefix             = "OPENSERP"
 )
 
-type AppConfig struct {
-	Host          string
-	Port          int
-	Timeout       int
-	ConfigPath    string
-	IsBrowserHead bool `mapstructure:"head"`
-	IsLeaveHead   bool `mapstructure:"leave_head"`
-	IsLeakless    bool `mapstructure:"leakless"`
-	IsDebug       bool `mapstructure:"debug"`
-	IsVerbose     bool `mapstructure:"verbose"`
-	IsRawRequests bool `mapstructure:"raw_requests"`
+type Config struct {
+	App          AppConfig                `mapstructure:"app"`
+	GoogleConfig core.SearchEngineOptions `mapstructure:"google"`
+	YandexConfig core.SearchEngineOptions `mapstructure:"yandex"`
+	BaiduConfig  core.SearchEngineOptions `mapstructure:"baidu"`
 }
 
-var appConf = AppConfig{}
+type AppConfig struct {
+	Host          string `mapstructure:"host"`
+	Port          int    `mapstructure:"port"`
+	Timeout       int    `mapstructure:"timeout"`
+	ConfigPath    string `mapstructure:"config_path"`
+	IsBrowserHead bool   `mapstructure:"head"`
+	IsLeaveHead   bool   `mapstructure:"leave_head"`
+	IsLeakless    bool   `mapstructure:"leakless"`
+	IsDebug       bool   `mapstructure:"debug"`
+	IsVerbose     bool   `mapstructure:"verbose"`
+	IsRawRequests bool   `mapstructure:"raw_requests"`
+}
+
+var config = Config{}
 
 var RootCmd = &cobra.Command{
-	Use:     "openserp",
-	Short:   "Open SERP",
-	Long:    `Search via Google, Yandex and Baidu`,
-	Version: version,
+	Use:          "openserp",
+	Short:        "Open SERP",
+	Long:         `Get [Google, Yandex, Baidu] search engine results via API or CLI.`,
+	Version:      version,
+	SilenceUsage: true,
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-		core.InitLogger(appConf.IsVerbose, appConf.IsDebug)
+		core.InitLogger(config.App.IsVerbose, config.App.IsDebug)
+
 		err := initializeConfig(cmd)
-		logrus.Debugf("Config: %+v", appConf)
-		return err
+		if err != nil {
+			return err
+		}
+
+		logrus.Debugf("Final config: %+v", config)
+		return nil
 	},
+
 	// Run: func(cmd *cobra.Command, args []string) {
 	// 	// Working with OutOrStdout/OutOrStderr allows us to unit test our command easier
 	// 	//out := cmd.OutOrStdout()
-	// 	logrus.Trace("Config:", appConf)
+	// 	logrus.Trace("Config:", config)
 	// },
 }
 
@@ -54,14 +69,10 @@ var RootCmd = &cobra.Command{
 func bindFlags(cmd *cobra.Command, vpr *viper.Viper) {
 	cmd.Flags().VisitAll(func(flg *pflag.Flag) {
 		configName := "app." + flg.Name
-		//if replaceHyphenWithCamelCase {
-		//	configName = strings.ReplaceAll(f.Name, "-", "")
-		//}
 
 		// Apply viper config value to the flag if viper has a value
-		if !flg.Changed && vpr.IsSet(configName) {
-			val := vpr.Get(configName)
-			cmd.Flags().Set(flg.Name, fmt.Sprintf("%v", val))
+		if flg.Changed {
+			vpr.Set(configName, flg.Value)
 		}
 	})
 }
@@ -74,34 +85,47 @@ func initializeConfig(cmd *cobra.Command) error {
 	v.SetConfigName(defaultConfigFilename)
 	v.AddConfigPath(".")
 
-	// Return an error if we cannot parse the config file.
-	if err := v.ReadInConfig(); err != nil {
-		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
-			return err
+	// 1. Config. Return an error if we cannot parse the config file.
+	err := v.ReadInConfig()
+	if err != nil {
+		err = errors.New(fmt.Sprintf("Cannot read config: %v", err))
+		logrus.Warn(err)
+	}
+
+	// 2. Env. Bind environment variables to their equivalent keys with underscores
+	for _, key := range v.AllKeys() {
+		envKey := envPrefix + "_" + strings.ToUpper(strings.ReplaceAll(key, ".", "_"))
+		err := v.BindEnv(key, envKey)
+		if err != nil {
+			logrus.Errorf("Unable to bind ENV valye: %v", err)
 		}
 	}
 
-	v.SetEnvPrefix(envPrefix)
-
-	// Bind environment variables to their equivalent keys with underscores
-	//v.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
-	v.AutomaticEnv()
-
-	// Bind the current command's flags to viper
+	// 3. Cmd flags. Bind the current command's flags to viper
 	bindFlags(cmd, v)
 
+	// Dump Viper values to config struct
+	err = v.Unmarshal(&config)
+	if err != nil {
+		return errors.New(fmt.Sprintf("Cannot unmarshall config: %v", err))
+	}
+
+	if config.App.IsDebug {
+		logrus.Debug("Viper config:")
+		v.Debug()
+	}
 	return nil
 }
 
 func init() {
-	RootCmd.PersistentFlags().IntVarP(&appConf.Port, "port", "p", 7070, "Port number to run server")
-	RootCmd.PersistentFlags().StringVarP(&appConf.Host, "host", "a", "127.0.0.1", "Host address to run server")
-	RootCmd.PersistentFlags().IntVarP(&appConf.Timeout, "timeout", "t", 30, "Timeout to fail request")
-	RootCmd.PersistentFlags().StringVarP(&appConf.ConfigPath, "config", "c", "./config.yaml", "Configuration file path")
-	RootCmd.PersistentFlags().BoolVarP(&appConf.IsVerbose, "verbose", "v", false, "Use verbose output")
-	RootCmd.PersistentFlags().BoolVarP(&appConf.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser")
-	RootCmd.PersistentFlags().BoolVarP(&appConf.IsBrowserHead, "head", "", false, "Enable browser UI")
-	RootCmd.PersistentFlags().BoolVarP(&appConf.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search")
-	RootCmd.PersistentFlags().BoolVarP(&appConf.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
-	RootCmd.PersistentFlags().BoolVarP(&appConf.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
+	RootCmd.PersistentFlags().IntVarP(&config.App.Port, "port", "p", 7070, "Port number to run server")
+	RootCmd.PersistentFlags().StringVarP(&config.App.Host, "host", "a", "127.0.0.1", "Host address to run server")
+	RootCmd.PersistentFlags().IntVarP(&config.App.Timeout, "timeout", "t", 30, "Timeout to fail request")
+	RootCmd.PersistentFlags().StringVarP(&config.App.ConfigPath, "config", "c", "", "Configuration file path")
+	RootCmd.PersistentFlags().BoolVarP(&config.App.IsVerbose, "verbose", "v", false, "Use verbose output")
+	RootCmd.PersistentFlags().BoolVarP(&config.App.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser")
+	RootCmd.PersistentFlags().BoolVarP(&config.App.IsBrowserHead, "head", "", false, "Enable browser UI")
+	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search")
+	RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
+	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
 }

cmd/search.go

@@ -31,7 +31,7 @@ func search(cmd *cobra.Command, args []string) {
 	}
 	results := []core.SearchResult{}
 
-	if appConf.IsRawRequests {
+	if config.App.IsRawRequests {
 		results, err = searchRaw(engineType, query)
 	} else {
 		results, err = searchBrowser(engineType, query)
@@ -54,13 +54,13 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er
 	var engine core.SearchEngine
 
 	opts := core.BrowserOpts{
-		IsHeadless:    !appConf.IsBrowserHead, // Disable headless if browser head mode is set
-		IsLeakless:    appConf.IsLeakless,
-		Timeout:       time.Second * time.Duration(appConf.Timeout),
-		LeavePageOpen: appConf.IsLeaveHead,
+		IsHeadless:    !config.App.IsBrowserHead, // Disable headless if browser head mode is set
+		IsLeakless:    config.App.IsLeakless,
+		Timeout:       time.Second * time.Duration(config.App.Timeout),
+		LeavePageOpen: config.App.IsLeaveHead,
 	}
 
-	if appConf.IsDebug {
+	if config.App.IsDebug {
 		opts.IsHeadless = false
 	}
 
@@ -71,11 +71,11 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er
 
 	switch strings.ToLower(engineType) {
 	case "yandex":
-		engine = yandex.New(*browser)
+		engine = yandex.New(*browser, config.YandexConfig)
 	case "google":
-		engine = google.New(*browser)
+		engine = google.New(*browser, config.GoogleConfig)
 	case "baidu":
-		engine = baidu.New(*browser)
+		engine = baidu.New(*browser, config.BaiduConfig)
 	default:
 		logrus.Infof("No `%s` search engine found", engineType)
 	}

cmd/serve.go

@@ -21,13 +21,13 @@ var serveCMD = &cobra.Command{
 
 func serve(cmd *cobra.Command, args []string) {
 	opts := core.BrowserOpts{
-		IsHeadless:    !appConf.IsBrowserHead, // Disable headless if browser head mode is set
-		IsLeakless:    appConf.IsLeakless,
-		Timeout:       time.Second * time.Duration(appConf.Timeout),
-		LeavePageOpen: appConf.IsLeaveHead,
+		IsHeadless:    !config.App.IsBrowserHead, // Disable headless if browser head mode is set
+		IsLeakless:    config.App.IsLeakless,
+		Timeout:       time.Second * time.Duration(config.App.Timeout),
+		LeavePageOpen: config.App.IsLeaveHead,
 	}
 
-	if appConf.IsDebug {
+	if config.App.IsDebug {
 		opts.IsHeadless = false
 	}
 
@@ -36,11 +36,11 @@ func serve(cmd *cobra.Command, args []string) {
 		logrus.Error(err)
 	}
 
-	yand := yandex.New(*browser)
-	gogl := google.New(*browser)
-	baidu := baidu.New(*browser)
+	yand := yandex.New(*browser, config.YandexConfig)
+	gogl := google.New(*browser, config.GoogleConfig)
+	baidu := baidu.New(*browser, config.BaiduConfig)
 
-	serv := core.NewServer(appConf.Host, appConf.Port, gogl, yand, baidu)
+	serv := core.NewServer(config.App.Host, config.App.Port, gogl, yand, baidu)
 	serv.Listen()
 }
 

config.yaml

@@ -5,4 +5,16 @@ app:
   verbose: true
   timeout: 15
   head: false
-  leakless: true
+  leakless: false
+
+google:
+  rate_requests: 4     # Number of requests per Minute
+  rate_burst: 2        # Number of non-ratelimited requests per Minute
+
+yandex:
+  rate_requests: 4     
+  rate_burst: 2        
+
+baidu:
+  rate_requests: 4     
+  rate_burst: 2        

core/browser.go

@@ -16,18 +16,19 @@ type BrowserOpts struct {
 	IsLeakless    bool          // Force to kill browser
 	Timeout       time.Duration // Timeout
 	LanguageCode  string
-	WaitRequests  bool // Wait requests to complete after navigation
-	LeavePageOpen bool // Leave pages and browser open
+	WaitRequests  bool          // Wait requests to complete after navigation
+	LeavePageOpen bool          // Leave pages and browser open
+	WaitLoadTime  time.Duration // Time to wait till page loads
 }
 
 // Initialize browser parameters with default values if they are not set
-func (o *BrowserOpts) Init() {
+func (o *BrowserOpts) Check() {
 	if o.Timeout == 0 {
 		o.Timeout = time.Second * 30
 	}
 
-	if o.LanguageCode == "" {
-		o.LanguageCode = "en"
+	if o.WaitLoadTime == 0 {
+		o.WaitLoadTime = time.Second * 2
 	}
 }
 
@@ -38,7 +39,7 @@ type Browser struct {
 }
 
 func NewBrowser(opts BrowserOpts) (*Browser, error) {
-	opts.Init()
+	opts.Check()
 	logrus.Debugf("Browser options: %+v", opts)
 
 	path, has := launcher.LookPath()
@@ -66,11 +67,11 @@ func (b *Browser) Navigate(URL string) *rod.Page {
 
 	b.browser = rod.New().ControlURL(b.browserAddr)
 	b.browser.MustConnect()
-	//b.browser.SetCookies(nil)
+	b.browser.SetCookies(nil)
 
 	page := stealth.MustPage(b.browser)
 	wait := page.MustWaitRequestIdle()
-	page.Navigate(URL)
+	page.MustNavigate(URL)
 
 	// causes bugs in google
 	if b.WaitRequests {
@@ -83,7 +84,7 @@ func (b *Browser) Navigate(URL string) *rod.Page {
 	})
 
 	// Wait till page loads
-	time.Sleep(time.Second * 1)
+	time.Sleep(b.WaitLoadTime)
 
 	return page
 }

core/browser_test.go

@@ -27,3 +27,24 @@ func TestCreateBrowser(t *testing.T) {
 // 		t.Fatalf("Error failed initializing leakless browser: %s", err)
 // 	}
 // }
+
+// Manually observe test results for now
+func TestBot(t *testing.T) {
+
+	var err error
+	opts := BrowserOpts{IsHeadless: false, IsLeakless: true, LeavePageOpen: true}
+	browser, err = NewBrowser(opts)
+	if err != nil {
+		t.Fatalf("Error failed initializing browser: %s", err)
+	}
+
+	page := browser.Navigate("https://bot.sannysoft.com")
+	page.MustScreenshotFullPage("./test/screenshot_bot.png")
+
+	page = browser.Navigate("https://www.whatismybrowser.com/")
+	page.MustScreenshotFullPage("./test/screenshot_browser.png")
+
+	page = browser.Navigate("https://abrahamjuliot.github.io/creepjs/")
+	page.MustScreenshotFullPage("./test/screenshot_creep.png")
+
+}

core/common.go

@@ -3,10 +3,14 @@ package core
 import (
 	"errors"
 	"strconv"
+	"time"
 
 	"github.com/gofiber/fiber/v2"
 )
 
+var ErrCaptcha = errors.New("Captcha detected")
+var ErrSearchTimeout = errors.New("Timeout. Cannot find element on page")
+
 type SearchResult struct {
 	Rank        int    `json:"rank"`
 	URL         string `json:"url"`
@@ -49,3 +53,33 @@ func (q *Query) InitFromContext(c *fiber.Ctx) error {
 
 	return nil
 }
+
+type SearchEngineOptions struct {
+	RateRequests    int   `mapstructure:"rate_requests"`
+	RateTime        int64 `mapstructure:"rate_seconds"`
+	RateBurst       int   `mapstructure:"rate_burst"`
+	SelectorTimeout int64 `mapstructure:"selector_timeout"` // CSS selector timeout in seconds
+}
+
+func (o *SearchEngineOptions) Init() {
+	if o.RateRequests == 0 {
+		o.RateRequests = 6
+	}
+	if o.RateTime == 0 {
+		o.RateTime = 60
+	}
+	if o.RateBurst == 0 {
+		o.RateBurst = 1
+	}
+	if o.SelectorTimeout == 0 {
+		o.SelectorTimeout = 5
+	}
+}
+
+func (o *SearchEngineOptions) GetRatelimit() time.Duration {
+	return (time.Duration(o.RateTime) * time.Second) / time.Duration(o.RateRequests)
+}
+
+func (o *SearchEngineOptions) GetSelectorTimeout() time.Duration {
+	return time.Duration(o.SelectorTimeout) * time.Second
+}

core/server.go

@@ -1,17 +1,21 @@
 package core
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/time/rate"
 )
 
 type SearchEngine interface {
 	Search(Query) ([]SearchResult, error)
 	IsInitialized() bool
 	Name() string
+	GetRateLimiter() *rate.Limiter
 }
 
 type Server struct {
@@ -28,19 +32,32 @@ func NewServer(host string, port int, searchEngines ...SearchEngine) *Server {
 
 	for _, engine := range searchEngines {
 		locEngine := engine
+		limiter := engine.GetRateLimiter()
+
 		serv.app.Get(fmt.Sprintf("/%s/search", strings.ToLower(locEngine.Name())), func(c *fiber.Ctx) error {
 			q := Query{}
 			err := q.InitFromContext(c)
-
 			if err != nil {
 				logrus.Errorf("Error while setting %s query: %s", locEngine.Name(), err)
 				return err
 			}
 
+			err = limiter.Wait(context.Background())
+			if err != nil {
+				logrus.Errorf("Ratelimiter error during %s query: %s", locEngine.Name(), err)
+			}
+
 			res, err := locEngine.Search(q)
 			if err != nil {
+				switch err {
+				case ErrCaptcha:
+					err = errors.New(fmt.Sprintf("Captcha found, please stop sending requests for a while\n%s", err))
+				case ErrSearchTimeout:
+					err = errors.New(fmt.Sprintf("Error: %s\nProbably need to update CSS selector", err))
+				}
+
 				logrus.Errorf("Error during %s search: %s", locEngine.Name(), err)
-				return err
+				return fiber.NewError(fiber.StatusServiceUnavailable, err.Error())
 			}
 
 			return c.JSON(res)

core/server_test.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"testing"
 	"time"
+
+	"golang.org/x/time/rate"
 )
 
 var (
@@ -25,6 +27,9 @@ func (SeMock) IsInitialized() bool {
 func (s SeMock) Search(q Query) (res []SearchResult, err error) {
 	return []SearchResult{{Title: s.EngineName}}, nil
 }
+func (s SeMock) GetRateLimiter() *rate.Limiter {
+	return nil
+}
 
 func TestCreateServer(t *testing.T) {
 	se1 := SeMock{"mock_engine_1"}

docker-compose.yaml

@@ -3,7 +3,16 @@ version: '3'
 services:
   openserp:
     container_name: serp
-    image: openserp
+    build:
+      context: .
     ports: 
       - 7000:7000
-    command: serve -a 0.0.0.0 -p 7000
+    command: serve -l
+    #volumes:
+    #  - ./config.yaml:/usr/src/app/config.yaml
+    environment:
+      OPENSERP_APP_HOST: "0.0.0.0"
+      OPENSERP_APP_PORT: 7000
+      OPENSERP_BAIDU_RATE_REQUESTS: 6   # Number of requests per Minute
+      OPENSERP_BAIDU_RATE_BURST: 2      # Number of non-ratelimited requests per Minute
+    

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.16.0
+	golang.org/x/time v0.3.0
 )
 
 require (

go.sum

@@ -421,6 +421,8 @@ golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

google/search.go

@@ -5,34 +5,43 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/go-rod/rod"
 	"github.com/karust/openserp/core"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/time/rate"
 )
 
 type Google struct {
 	core.Browser
-	findNumRgxp  *regexp.Regexp
-	checkTimeout time.Duration
+	core.SearchEngineOptions
+	findNumRgxp *regexp.Regexp
 }
 
-func New(browser core.Browser) *Google {
+func New(browser core.Browser, opts core.SearchEngineOptions) *Google {
 	gogl := Google{Browser: browser}
-	gogl.checkTimeout = time.Second * 5
+	opts.Init()
+	gogl.SearchEngineOptions = opts
+
 	gogl.findNumRgxp = regexp.MustCompile("\\d")
 	return &gogl
 }
+
 func (gogl *Google) Name() string {
 	return "google"
 }
 
-func (gogl *Google) FindTotalResults(page *rod.Page) (int, error) {
-	resultsStats, err := page.Timeout(gogl.checkTimeout).Search("div#result-stats")
+func (gogl *Google) GetRateLimiter() *rate.Limiter {
+	ratelimit := rate.Every(gogl.GetRatelimit())
+	return rate.NewLimiter(ratelimit, gogl.RateBurst)
+}
+
+func (gogl *Google) findTotalResults(page *rod.Page) (int, error) {
+	resultsStats, err := page.Timeout(gogl.GetSelectorTimeout()).Search("div#result-stats")
 	if err != nil {
 		return 0, errors.New("Result stats not found: " + err.Error())
 	}
+
 	stats, err := resultsStats.First.Text()
 	if err != nil {
 		return 0, errors.New("Cannot extract result stats text: " + err.Error())
@@ -49,6 +58,14 @@ func (gogl *Google) FindTotalResults(page *rod.Page) (int, error) {
 	return total, nil
 }
 
+func (gogl *Google) isCaptcha(page *rod.Page) bool {
+	_, err := page.Timeout(gogl.GetSelectorTimeout()).Search("form#captcha-form")
+	if err != nil {
+		return false
+	}
+	return true
+}
+
 func (gogl *Google) preparePage(page *rod.Page) {
 	// Remove "similar queries" lists
 	page.Eval(";(() => { document.querySelectorAll(`div[data-initq]`).forEach( el => el.remove());  })();")
@@ -68,21 +85,30 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	page := gogl.Navigate(url)
 	gogl.preparePage(page)
 
-	totalResults, err := gogl.FindTotalResults(page)
+	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][lang], div[data-surl][jsaction]")
 	if err != nil {
-		return nil, err
-	}
-	logrus.Tracef("%d total results found", totalResults)
-
-	if totalResults == 0 {
-		return searchResults, nil
+		defer page.Close()
+		logrus.Errorf("Cannot parse search results: %s", err)
+		return nil, core.ErrSearchTimeout
 	}
 
-	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][lang]")
-	if err != nil {
+	// Check why no results, maybe captcha?
+	if results == nil {
+		defer page.Close()
+
+		if gogl.isCaptcha(page) {
+			logrus.Errorf("Google captcha occurred during: %s", url)
+			return nil, core.ErrCaptcha
+		}
 		return nil, err
 	}
 
+	totalResults, err := gogl.findTotalResults(page)
+	if err != nil {
+		logrus.Errorf("Error capturing total results: %v", err)
+	}
+	logrus.Infof("%d total results found", totalResults)
+
 	resultElements, err := results.All()
 	if err != nil {
 		return nil, err

google/search_test.go

@@ -15,7 +15,7 @@ func init() {
 }
 
 func TestSearchGoogle(t *testing.T) {
-	gogl := New(*browser)
+	gogl := New(*browser, core.SearchEngineOptions{})
 
 	query := core.Query{Text: "HEY", Limit: 10}
 	results, err := gogl.Search(query)

yandex/search.go

@@ -6,17 +6,20 @@ import (
 	"github.com/go-rod/rod"
 	"github.com/karust/openserp/core"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/time/rate"
 )
 
 type Yandex struct {
 	core.Browser
-	checkTimeout time.Duration // Timeout for secondary elements check
-	pageSleep    time.Duration // Sleep between pages
+	core.SearchEngineOptions
+	pageSleep time.Duration // Sleep between pages
 }
 
-func New(browser core.Browser) *Yandex {
+func New(browser core.Browser, opts core.SearchEngineOptions) *Yandex {
 	yand := Yandex{Browser: browser}
-	yand.checkTimeout = time.Second * 2
+	opts.Init()
+	yand.SearchEngineOptions = opts
+
 	yand.pageSleep = time.Second * 1
 	return &yand
 }
@@ -25,8 +28,13 @@ func (yand *Yandex) Name() string {
 	return "yandex"
 }
 
+func (yand *Yandex) GetRateLimiter() *rate.Limiter {
+	ratelimit := rate.Every(yand.GetRatelimit())
+	return rate.NewLimiter(ratelimit, yand.RateBurst)
+}
+
 func (yand *Yandex) isCaptcha(page *rod.Page) bool {
-	_, err := page.Timeout(yand.checkTimeout).Search("form#checkbox-captcha-form")
+	_, err := page.Timeout(yand.GetSelectorTimeout()).Search("form#checkbox-captcha-form")
 	if err != nil {
 		return false
 	}
@@ -37,12 +45,12 @@ func (yand *Yandex) isCaptcha(page *rod.Page) bool {
 func (yand *Yandex) isNoResults(page *rod.Page) bool {
 	noResFound := false
 
-	_, err := page.Timeout(yand.checkTimeout).Search("div.EmptySearchResults-Title")
+	_, err := page.Timeout(yand.GetSelectorTimeout()).Search("div.EmptySearchResults-Title")
 	if err == nil {
 		noResFound = true
 	}
 
-	_, err = page.Timeout(yand.checkTimeout).Search("div>div.RequestMeta-Message")
+	_, err = page.Timeout(yand.GetSelectorTimeout()).Search("div>div.RequestMeta-Message")
 	if err == nil {
 		noResFound = true
 	}
@@ -110,15 +118,20 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
 		// Get all search results in page
 		searchRes, err := page.Timeout(yand.Timeout).Search("li.serp-item")
 		if err != nil {
+			defer page.Close()
 			logrus.Errorf("Cannot parse search results: %s", err)
+			return nil, core.ErrSearchTimeout
 		}
 
 		// Check why no results, maybe captcha?
 		if searchRes == nil {
+			defer page.Close()
+
 			if yand.isNoResults(page) {
 				logrus.Errorf("No results found")
 			} else if yand.isCaptcha(page) {
 				logrus.Errorf("Yandex captcha occurred during: %s", url)
+				return nil, core.ErrCaptcha
 			}
 			break
 		}

yandex/search_test.go

@@ -16,7 +16,7 @@ func init() {
 
 func TestSearchYandex(t *testing.T) {
 
-	yand := New(*browser)
+	yand := New(*browser, core.SearchEngineOptions{})
 
 	query := core.Query{Text: "HEY", Limit: 10}
 	results, err := yand.Search(query)