zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-15 21:15:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Settings via config file and environment variables

Browse Source
This commit is contained in:
Rustem Kamalov
2023-06-30 22:49:58 +03:00
parent df972585c1
commit ffe33d6bc5
6 changed files with 115 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Dockerfile
View File

@@ -17,6 +17,7 @@ RUN go build -o /app/openserp .
FROM zenika/alpine-chrome:with-chromedriver
COPY --from=builder /app/openserp /usr/local/bin/openserp
ADD config.yaml /usr/src/app
ENTRYPOINT ["openserp"]

129
cmd/root.go
View File

@@ -1,7 +1,9 @@
package cmd
import (
"errors"
"fmt"
"strings"
"github.com/karust/openserp/core"
"github.com/sirupsen/logrus"
@@ -11,45 +13,55 @@ import (
)
const (
version = "0.2.1"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
replaceHyphenWithCamelCase = false
version = "0.2.1"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)
type AppConfig struct {
Host string
Port int
Timeout int
ConfigPath string
IsBrowserHead bool `mapstructure:"head"`
IsLeaveHead bool `mapstructure:"leave_head"`
IsLeakless bool `mapstructure:"leakless"`
IsDebug bool `mapstructure:"debug"`
IsVerbose bool `mapstructure:"verbose"`
IsRawRequests bool `mapstructure:"raw_requests"`
GoogleConfig core.SearchEngineOptions `mapstructure:"google"`
YandexConfig core.SearchEngineOptions `mapstructure:"yandex"`
BaiduConfig core.SearchEngineOptions `mapstructure:"baidu"`
type Config struct {
App AppConfig `mapstructure:"app"`
GoogleConfig core.SearchEngineOptions `mapstructure:"google"`
YandexConfig core.SearchEngineOptions `mapstructure:"yandex"`
BaiduConfig core.SearchEngineOptions `mapstructure:"baidu"`
}
var appConf = AppConfig{}
type AppConfig struct {
Host string `mapstructure:"host"`
Port int `mapstructure:"port"`
Timeout int `mapstructure:"timeout"`
ConfigPath string `mapstructure:"config_path"`
IsBrowserHead bool `mapstructure:"head"`
IsLeaveHead bool `mapstructure:"leave_head"`
IsLeakless bool `mapstructure:"leakless"`
IsDebug bool `mapstructure:"debug"`
IsVerbose bool `mapstructure:"verbose"`
IsRawRequests bool `mapstructure:"raw_requests"`
}
var config = Config{}
var RootCmd = &cobra.Command{
Use: "openserp",
Short: "Open SERP",
Long: `Search via Google, Yandex and Baidu`,
Version: version,
Use: "openserp",
Short: "Open SERP",
Long: `Get [Google, Yandex, Baidu] search engine results via API or CLI.`,
Version: version,
SilenceUsage: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
core.InitLogger(appConf.IsVerbose, appConf.IsDebug)
core.InitLogger(config.App.IsVerbose, config.App.IsDebug)
err := initializeConfig(cmd)
logrus.Debugf("Config: %+v", appConf)
return err
if err != nil {
return err
}
logrus.Debugf("Final config: %+v", config)
return nil
},
// Run: func(cmd *cobra.Command, args []string) {
// // Working with OutOrStdout/OutOrStderr allows us to unit test our command easier
// //out := cmd.OutOrStdout()
// logrus.Trace("Config:", appConf)
// logrus.Trace("Config:", config)
// },
}
@@ -57,14 +69,10 @@ var RootCmd = &cobra.Command{
func bindFlags(cmd *cobra.Command, vpr *viper.Viper) {
cmd.Flags().VisitAll(func(flg *pflag.Flag) {
configName := "app." + flg.Name
//if replaceHyphenWithCamelCase {
// configName = strings.ReplaceAll(f.Name, "-", "")
//}
// Apply viper config value to the flag if viper has a value
if !flg.Changed && vpr.IsSet(configName) {
val := vpr.Get(configName)
cmd.Flags().Set(flg.Name, fmt.Sprintf("%v", val))
if flg.Changed {
vpr.Set(configName, flg.Value)
}
})
}
@@ -77,34 +85,47 @@ func initializeConfig(cmd *cobra.Command) error {
v.SetConfigName(defaultConfigFilename)
v.AddConfigPath(".")
// Return an error if we cannot parse the config file.
if err := v.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
return err
// 1. Config. Return an error if we cannot parse the config file.
err := v.ReadInConfig()
if err != nil {
err = errors.New(fmt.Sprintf("Cannot read config: %v", err))
logrus.Warn(err)
}
// 2. Env. Bind environment variables to their equivalent keys with underscores
for _, key := range v.AllKeys() {
envKey := envPrefix + "_" + strings.ToUpper(strings.ReplaceAll(key, ".", "_"))
err := v.BindEnv(key, envKey)
if err != nil {
logrus.Errorf("Unable to bind ENV valye: %v", err)
}
}
v.SetEnvPrefix(envPrefix)
// Bind environment variables to their equivalent keys with underscores
//v.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
v.AutomaticEnv()
// Bind the current command's flags to viper
// 3. Cmd flags. Bind the current command's flags to viper
bindFlags(cmd, v)
// Dump Viper values to config struct
err = v.Unmarshal(&config)
if err != nil {
return errors.New(fmt.Sprintf("Cannot unmarshall config: %v", err))
}
if config.App.IsDebug {
logrus.Debug("Viper config:")
v.Debug()
}
return nil
}
func init() {
RootCmd.PersistentFlags().IntVarP(&appConf.Port, "port", "p", 7070, "Port number to run server")
RootCmd.PersistentFlags().StringVarP(&appConf.Host, "host", "a", "127.0.0.1", "Host address to run server")
RootCmd.PersistentFlags().IntVarP(&appConf.Timeout, "timeout", "t", 30, "Timeout to fail request")
RootCmd.PersistentFlags().StringVarP(&appConf.ConfigPath, "config", "c", "./config.yaml", "Configuration file path")
RootCmd.PersistentFlags().BoolVarP(&appConf.IsVerbose, "verbose", "v", false, "Use verbose output")
RootCmd.PersistentFlags().BoolVarP(&appConf.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser")
RootCmd.PersistentFlags().BoolVarP(&appConf.IsBrowserHead, "head", "", false, "Enable browser UI")
RootCmd.PersistentFlags().BoolVarP(&appConf.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search")
RootCmd.PersistentFlags().BoolVarP(&appConf.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
RootCmd.PersistentFlags().BoolVarP(&appConf.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
RootCmd.PersistentFlags().IntVarP(&config.App.Port, "port", "p", 7070, "Port number to run server")
RootCmd.PersistentFlags().StringVarP(&config.App.Host, "host", "a", "127.0.0.1", "Host address to run server")
RootCmd.PersistentFlags().IntVarP(&config.App.Timeout, "timeout", "t", 30, "Timeout to fail request")
RootCmd.PersistentFlags().StringVarP(&config.App.ConfigPath, "config", "c", "", "Configuration file path")
RootCmd.PersistentFlags().BoolVarP(&config.App.IsVerbose, "verbose", "v", false, "Use verbose output")
RootCmd.PersistentFlags().BoolVarP(&config.App.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser")
RootCmd.PersistentFlags().BoolVarP(&config.App.IsBrowserHead, "head", "", false, "Enable browser UI")
RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search")
RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
}

18
cmd/search.go
View File

@@ -31,7 +31,7 @@ func search(cmd *cobra.Command, args []string) {
}
results := []core.SearchResult{}
if appConf.IsRawRequests {
if config.App.IsRawRequests {
results, err = searchRaw(engineType, query)
} else {
results, err = searchBrowser(engineType, query)
@@ -54,13 +54,13 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er
var engine core.SearchEngine
opts := core.BrowserOpts{
IsHeadless: !appConf.IsBrowserHead, // Disable headless if browser head mode is set
IsLeakless: appConf.IsLeakless,
Timeout: time.Second * time.Duration(appConf.Timeout),
LeavePageOpen: appConf.IsLeaveHead,
IsHeadless: !config.App.IsBrowserHead, // Disable headless if browser head mode is set
IsLeakless: config.App.IsLeakless,
Timeout: time.Second * time.Duration(config.App.Timeout),
LeavePageOpen: config.App.IsLeaveHead,
}
if appConf.IsDebug {
if config.App.IsDebug {
opts.IsHeadless = false
}
@@ -71,11 +71,11 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er
switch strings.ToLower(engineType) {
case "yandex":
engine = yandex.New(*browser, appConf.YandexConfig)
engine = yandex.New(*browser, config.YandexConfig)
case "google":
engine = google.New(*browser, appConf.GoogleConfig)
engine = google.New(*browser, config.GoogleConfig)
case "baidu":
engine = baidu.New(*browser, appConf.BaiduConfig)
engine = baidu.New(*browser, config.BaiduConfig)
default:
logrus.Infof("No `%s` search engine found", engineType)
}

18
cmd/serve.go
View File

@@ -21,13 +21,13 @@ var serveCMD = &cobra.Command{
func serve(cmd *cobra.Command, args []string) {
opts := core.BrowserOpts{
IsHeadless: !appConf.IsBrowserHead, // Disable headless if browser head mode is set
IsLeakless: appConf.IsLeakless,
Timeout: time.Second * time.Duration(appConf.Timeout),
LeavePageOpen: appConf.IsLeaveHead,
IsHeadless: !config.App.IsBrowserHead, // Disable headless if browser head mode is set
IsLeakless: config.App.IsLeakless,
Timeout: time.Second * time.Duration(config.App.Timeout),
LeavePageOpen: config.App.IsLeaveHead,
}
if appConf.IsDebug {
if config.App.IsDebug {
opts.IsHeadless = false
}
@@ -36,11 +36,11 @@ func serve(cmd *cobra.Command, args []string) {
logrus.Error(err)
}
yand := yandex.New(*browser, appConf.YandexConfig)
gogl := google.New(*browser, appConf.GoogleConfig)
baidu := baidu.New(*browser, appConf.BaiduConfig)
yand := yandex.New(*browser, config.YandexConfig)
gogl := google.New(*browser, config.GoogleConfig)
baidu := baidu.New(*browser, config.BaiduConfig)
serv := core.NewServer(appConf.Host, appConf.Port, gogl, yand, baidu)
serv := core.NewServer(config.App.Host, config.App.Port, gogl, yand, baidu)
serv.Listen()
}

12
config.yaml
View File

@@ -6,3 +6,15 @@ app:
timeout: 15
head: false
leakless: false
google:
rate_requests: 4 # Number of requests per Minute
rate_burst: 2 # Number of non-ratelimited requests per Minute
yandex:
rate_requests: 4
rate_burst: 2
baidu:
rate_requests: 4
rate_burst: 2

10
docker-compose.yaml
View File

@@ -7,4 +7,12 @@ services:
context: .
ports:
- 7000:7000
command: serve -a 0.0.0.0 -p 7000 -v -l
command: serve -l
#volumes:
# - ./config.yaml:/usr/src/app/config.yaml
environment:
OPENSERP_APP_HOST: "0.0.0.0"
OPENSERP_APP_PORT: 7000
OPENSERP_BAIDU_RATE_REQUESTS: 6 # Number of requests per Minute
OPENSERP_BAIDU_RATE_BURST: 2 # Number of non-ratelimited requests per Minute