zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-09-08 20:46:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: propagate request context across search, retry, limiter, and browser navigation

Browse Source
This commit is contained in:
Rustem Kamalov
2026-04-20 05:53:13 +03:00
parent 0b7fbd3a45
commit e89b1a3a50
27 changed files with 424 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
google/search.go
+10 -5
View File
@@ -1,6 +1,7 @@
package google
import (
"context"
"errors"
"fmt"
"regexp"
@@ -163,7 +164,8 @@ func (gogl *Google) acceptCookies(page *rod.Page) {
// Search executes a Google web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
func (gogl *Google) Search(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.EnsureContext(ctx)
gogl.logger.Debug("Starting search, query: %+v", query)
searchResults := []core.SearchResult{}
@@ -173,7 +175,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
if err != nil {
return nil, err
}
page, err := gogl.Navigate(url)
page, err := gogl.Navigate(ctx, url)
if err != nil {
return nil, err
}
@@ -278,7 +280,9 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
}
//answ.Page().WaitRepaint()
}
time.Sleep(time.Millisecond * 2000)
if err := core.SleepContext(ctx, 2*time.Second); err != nil {
return nil, err
}
for i, answ := range answers {
answerText := strings.Split(answ.MustText(), "\n")
@@ -374,7 +378,8 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
// SearchImage executes a Google image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
func (gogl *Google) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.EnsureContext(ctx)
gogl.logger.Debug("Starting image search, query: %+v", query)
searchResultsMap := map[string]core.SearchResult{}
@@ -383,7 +388,7 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
return nil, err
}
page, err := gogl.Navigate(url)
page, err := gogl.Navigate(ctx, url)
if err != nil {
return nil, err
}
google/search_integration_test.go
+3 -2
View File
@@ -4,6 +4,7 @@
package google
import (
"context"
"testing"
"github.com/karust/openserp/core"
@@ -18,7 +19,7 @@ func TestSearchGoogle(t *testing.T) {
gogl := New(*browser, core.SearchEngineOptions{})
query := core.Query{Text: "golang programming", Limit: 10}
results, err := gogl.Search(query)
results, err := gogl.Search(context.Background(), query)
ithelper.HandleError(t, "google web search", err)
if len(results) == 0 {
@@ -39,7 +40,7 @@ func TestImageSearchGoogle(t *testing.T) {
gogl := New(*browser, core.SearchEngineOptions{})
query := core.Query{Text: "golden retriever puppy", Limit: 10}
results, err := gogl.SearchImage(query)
results, err := gogl.SearchImage(context.Background(), query)
ithelper.HandleError(t, "google image search", err)
if len(results) == 0 {
google/search_raw.go
+8 -4
View File
@@ -1,6 +1,7 @@
package google
import (
"context"
"net/http"
"strings"
@@ -10,13 +11,13 @@ import (
"github.com/sirupsen/logrus"
)
func googleRequest(searchURL string, query core.Query) (*http.Response, error) {
func googleRequest(ctx context.Context, searchURL string, query core.Query) (*http.Response, error) {
baseClient, err := core.NewRawHTTPClient(query)
if err != nil {
return nil, err
}
req, err := http.NewRequest("GET", searchURL, nil)
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
if err != nil {
return nil, err
}
@@ -100,17 +101,20 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) {
return core.DeduplicateResults(results), err
}
func Search(query core.Query) ([]core.SearchResult, error) {
func Search(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.EnsureContext(ctx)
googleURL, err := BuildURL(query)
if err != nil {
return nil, err
}
logrus.Debugf("Google URL built: %s", googleURL)
res, err := googleRequest(googleURL, query)
res, err := googleRequest(ctx, googleURL, query)
if err != nil {
return nil, err
}
defer res.Body.Close()
logrus.Debugf("Google Raw response: code=%d", res.StatusCode)
results, err := googleResultParser(res)