zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-15 21:15:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: Speed up Baidu. Fix raw client + reuse HTML parsers for raw search

Browse Source
This commit is contained in:
Rustem Kamalov
2026-05-08 00:00:33 +03:00
parent 471e5ad7fe
commit 88337cd7ce
19 changed files with 618 additions and 475 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
baidu/parse_html.go
View File

@@ -19,22 +19,61 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
}
func parseBaiduDocument(doc *goquery.Document) []core.SearchResult {
for _, selector := range baiduResultSelectors() {
if results := parseBaiduSelection(doc.Find(selector)); len(results) > 0 {
return results
}
}
return nil
}
func baiduResultSelectors() []string {
selectors := make([]string, 0, len(Selectors.ResultsAlt)+1)
selectors = append(selectors, Selectors.Results)
selectors = append(selectors, Selectors.ResultsAlt...)
return selectors
}
func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult {
var results []core.SearchResult
rank := 1
doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) {
linkTag := item.Find(Selectors.Link).First()
if linkTag.Length() == 0 {
sel.Each(func(_ int, item *goquery.Selection) {
// h3-first: organic results always carry a heading; this filters out
// non-result blocks that may share the wrapper class.
titleTag := item.Find("h3").First()
var (
title string
linkTag *goquery.Selection
)
if titleTag.Length() > 0 {
title = strings.TrimSpace(titleTag.Text())
if child := titleTag.Find("a[href]").First(); child.Length() > 0 {
linkTag = child
} else if closest := titleTag.Closest("a[href]"); closest.Length() > 0 {
linkTag = closest
}
}
if linkTag == nil || linkTag.Length() == 0 {
first := item.Find(Selectors.Link).First()
if first.Length() == 0 {
return
}
linkTag = first
}
if title == "" {
title = strings.TrimSpace(linkTag.Text())
}
if title == "" {
return
}
href, exists := linkTag.Attr("href")
if !exists || href == "" || href == "#" || !strings.HasPrefix(href, "http") {
if !exists {
return
}
title := strings.TrimSpace(linkTag.Text())
if title == "" {
href = strings.TrimSpace(href)
if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") {
return
}
@@ -42,6 +81,16 @@ func parseBaiduDocument(doc *goquery.Document) []core.SearchResult {
if descTag := item.Find(Selectors.Desc).First(); descTag.Length() > 0 {
desc = strings.TrimSpace(descTag.Text())
}
if desc == "" {
for _, alt := range Selectors.DescAlt {
if descTag := item.Find(alt).First(); descTag.Length() > 0 {
if t := strings.TrimSpace(descTag.Text()); t != "" {
desc = t
break
}
}
}
}
if desc == "" {
full := strings.TrimSpace(item.Text())
desc = strings.TrimSpace(strings.Replace(full, title, "", 1))
@@ -56,5 +105,12 @@ func parseBaiduDocument(doc *goquery.Document) []core.SearchResult {
rank++
})
return core.DeduplicateResults(results)
// Re-rank sequentially after dedup so callers get a clean 1..N sequence
// (dedup may drop intermediate ranks when the same URL appears in
// multiple Baidu result-card variants on the same SERP).
deduped := core.DeduplicateResults(results)
for i := range deduped {
deduped[i].Rank = i + 1
}
return deduped
}

53
baidu/parse_html_test.go
View File

@@ -51,3 +51,56 @@ func TestParseBaiduHTMLEmpty(t *testing.T) {
t.Fatalf("expected zero results for empty HTML, got %d", len(results))
}
}
func TestParseBaiduHTMLFallbackSelectors(t *testing.T) {
t.Parallel()
html := `
<div id="content_left">
<div class="result-op c-container">
<h3><a href="https://example.com/result">Fallback Title</a></h3>
<div class="summary-gap_3Jb4I">Fallback description</div>
</div>
</div>`
results, err := ParseHTML(bytes.NewReader([]byte(html)))
if err != nil {
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 1 {
t.Fatalf("expected 1 result, got %d", len(results))
}
if results[0].URL != "https://example.com/result" {
t.Fatalf("unexpected URL: %s", results[0].URL)
}
if results[0].Title != "Fallback Title" {
t.Fatalf("unexpected title: %s", results[0].Title)
}
if results[0].Description != "Fallback description" {
t.Fatalf("unexpected description: %s", results[0].Description)
}
}
func TestParseBaiduHTMLFallsBackWhenEarlierSelectorHasNoResult(t *testing.T) {
t.Parallel()
html := `
<div id="content_left">
<div class="result c-container"></div>
<div class="result-op c-container">
<h3><a href="https://example.com/parseable">Parseable Title</a></h3>
<div class="summary-gap_3Jb4I">Parseable description</div>
</div>
</div>`
results, err := ParseHTML(bytes.NewReader([]byte(html)))
if err != nil {
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 1 {
t.Fatalf("expected 1 result, got %d", len(results))
}
if results[0].URL != "https://example.com/parseable" {
t.Fatalf("unexpected URL: %s", results[0].URL)
}
}

115
baidu/search.go
View File

@@ -91,6 +91,9 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
scoped := *baid
scoped.logger = baid.logger.WithRequest(ctx)
if scoped.Browser.WaitLoadTime == 0 || scoped.Browser.WaitLoadTime > 250*time.Millisecond {
scoped.Browser.WaitLoadTime = 250 * time.Millisecond
}
baid = &scoped
baid.logger.Debug("Starting search, query: %+v", query)
@@ -101,8 +104,6 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
}
}()
searchResults := []core.SearchResult{}
// Build URL from query struct to open in browser
url, err := BuildURL(query)
if err != nil {
@@ -122,69 +123,63 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
}
}
resultElements, _, err := core.WaitForElements(
ctx,
page,
append([]string{Selectors.Results}, Selectors.ResultsAlt...),
baid.GetSelectorTimeout(),
)
searchResults, err := baid.waitForParsedSearchResults(ctx, page, url)
if err != nil {
if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
closePage()
return nil, blockErr
}
closePage()
baid.logger.Error("Cannot parse search results: %s", err)
return nil, core.ErrSearchTimeout
return nil, err
}
if len(resultElements) == 0 {
if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
closePage()
return nil, blockErr
}
closePage()
return nil, nil
}
for i, r := range resultElements {
// Get URL
link, err := r.Element(Selectors.Link)
if err != nil {
// Element detached mid-iteration (page navigated/refreshed): the rest
// of the slice is also stale, no point continuing.
if core.IsRodObjectNotFound(err) {
break
}
continue
}
linkText, err := link.Property("href")
if err != nil {
baid.logger.Debug("Missing href tag")
continue
}
// Get title
title, err := link.Text()
if err != nil {
baid.logger.Debug("Failed to extract title")
title = "No title"
}
// Get description
desc, err := r.Text()
if err != nil {
desc = ""
}
desc = strings.ReplaceAll(desc, title, "")
gR := core.SearchResult{Rank: query.Start + i + 1, URL: linkText.String(), Title: title, Description: desc}
searchResults = append(searchResults, gR)
}
closePage()
return core.DeduplicateResults(searchResults), nil
for i := range searchResults {
searchResults[i].Rank = query.Start + i + 1
}
return searchResults, nil
}
func (baid *Baidu) waitForParsedSearchResults(ctx context.Context, page *rod.Page, url string) ([]core.SearchResult, error) {
timeout := baid.GetSelectorTimeout()
if timeout <= 0 {
timeout = 5 * time.Second
}
deadline := time.Now().Add(timeout)
var sawResultContainer bool
var lastErr error
for {
html, err := page.HTML()
if err == nil {
results, parseErr := ParseHTML(strings.NewReader(html))
if parseErr == nil && len(results) > 0 {
return results, nil
}
lastErr = parseErr
} else {
lastErr = err
}
if core.HasAnySelector(page, baiduResultSelectors()) {
sawResultContainer = true
}
if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
return nil, blockErr
}
if !time.Now().Before(deadline) {
break
}
if err := core.SleepContext(ctx, 120*time.Millisecond); err != nil {
return nil, err
}
}
if sawResultContainer {
if lastErr != nil {
baid.logger.Debug("Baidu result containers found but HTML parsing failed: %v", lastErr)
} else {
baid.logger.Debug("Baidu result containers found but no parseable organic results")
}
return nil, core.ErrParser
}
return nil, nil
}
// SearchImage executes a Baidu image search and returns normalized image

113
baidu/search_raw.go
View File

@@ -1,97 +1,27 @@
package baidu
import (
"bytes"
"context"
"errors"
"fmt"
"net/http"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/corpix/uarand"
"github.com/karust/openserp/core"
"github.com/sirupsen/logrus"
)
func baiduRequest(ctx context.Context, searchURL string, query core.Query) (*http.Response, error) {
baseClient, err := core.NewRawHTTPClient(query)
func classifyBaiduRawHTML(body []byte) error {
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
if err != nil {
return nil, err
return err
}
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
if err != nil {
return nil, err
if doc.Find(Selectors.Captcha).Length() > 0 || doc.Find(Selectors.Timeout).Length() > 0 {
return core.ErrCaptcha
}
req.Header.Set("User-Agent", uarand.GetRandom())
core.SetAcceptLanguageHeader(req, query.LangCode)
res, err := baseClient.Do(req)
if err != nil {
return nil, err
if doc.Find("div.content_none, div.nors").Length() > 0 {
return core.ErrEmptyResult
}
return res, nil
}
func baiduResultParser(response *http.Response) ([]core.SearchResult, error) {
doc, err := goquery.NewDocumentFromReader(response.Body)
if err != nil {
return nil, err
}
results := []core.SearchResult{}
rank := 1
// Prefer organic result blocks from the main result column.
sel := doc.Find("#content_left .result.c-container")
if sel.Length() == 0 {
sel = doc.Find("div.c-container.new-pmd")
}
for i := range sel.Nodes {
item := sel.Eq(i)
// Find URL
titleTag := item.Find("h3").First()
if titleTag.Length() == 0 {
continue
}
linkTag := titleTag.Closest("a")
if linkTag.Length() == 0 {
linkTag = item.Find("a").First()
}
link, _ := linkTag.Attr("href")
link = strings.TrimSpace(link)
// Find title
title := strings.TrimSpace(titleTag.Text())
// Find description
descTag := item.Find(".c-abstract, .content-right_8Zs40, .summary-gap_3Jb4I").First()
desc := strings.TrimSpace(descTag.Text())
if desc == "" {
desc = strings.TrimSpace(item.Text())
}
desc = strings.ReplaceAll(desc, title, "")
desc = strings.TrimSpace(desc)
if link != "" && link != "#" && title != "" {
result := core.SearchResult{
Rank: rank,
URL: link,
Title: title,
Description: desc,
}
results = append(results, result)
rank++
}
}
logrus.WithField("document_size", len(doc.Text())).Trace(
fmt.Sprintf("Baidu search document size: %d", len(doc.Text())),
)
return results, err
return nil
}
func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -105,13 +35,13 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
}
}()
googleURL, err := BuildURL(query)
searchURL, err := BuildURL(query)
if err != nil {
return nil, err
}
core.WithRequest(ctx).WithField("url", googleURL).Debug(fmt.Sprintf("Baidu URL built: %s", googleURL))
core.WithRequest(ctx).WithField("url", searchURL).Debug(fmt.Sprintf("Baidu URL built: %s", searchURL))
res, err := baiduRequest(ctx, googleURL, query)
res, err := core.RawSearchRequest(ctx, searchURL, query)
if err != nil {
return nil, err
}
@@ -120,10 +50,25 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
fmt.Sprintf("Baidu Raw response: code=%d", res.StatusCode),
)
parsedResults, err := baiduResultParser(res)
body, err := core.ReadRawSearchBody(res)
if err != nil {
return nil, err
}
htmlStatus := classifyBaiduRawHTML(body)
if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) {
return nil, htmlStatus
}
parsedResults, err := ParseHTML(bytes.NewReader(body))
if err != nil {
return nil, err
}
if len(parsedResults) == 0 {
if errors.Is(htmlStatus, core.ErrEmptyResult) {
return []core.SearchResult{}, nil
}
return nil, fmt.Errorf("%w: baidu raw search returned no parseable results", core.ErrParser)
}
if query.Start > 0 {
for i := range parsedResults {
parsedResults[i].Rank = query.Start + i + 1

85
baidu/search_raw_test.go
View File

@@ -1,76 +1,69 @@
package baidu
import (
"errors"
"io"
"testing"
"github.com/karust/openserp/core"
"github.com/karust/openserp/testutil"
)
func TestBaiduResultParserSnapshots(t *testing.T) {
// TestBaiduParseHTMLFixtures covers the no-results and captcha fixtures.
// The happy path is covered in TestParseBaiduHTML; this file ensures the
// shared parser (used by both raw mode and the /baidu/parse endpoint) does
// not over-extract on captcha or empty SERPs.
func TestBaiduParseHTMLFixtures(t *testing.T) {
t.Parallel()
tests := []struct {
name string
fixture string
minResultCount int
maxResultCount int
wantZero bool
name string
fixture string
wantZero bool
}{
{
name: "search results",
fixture: "search_results.html",
minResultCount: 5,
maxResultCount: 30,
},
{
name: "no results",
fixture: "search_no_results.html",
wantZero: true,
},
{
name: "captcha page",
fixture: "search_captcha.html",
wantZero: true,
},
{name: "no results", fixture: "search_no_results.html", wantZero: true},
{name: "captcha page", fixture: "search_captcha.html", wantZero: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
results, err := baiduResultParser(testutil.ResponseFromFixture(t, tt.fixture))
results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body)
if err != nil {
t.Fatalf("baiduResultParser() error = %v", err)
t.Fatalf("ParseHTML() error = %v", err)
}
if tt.wantZero {
if len(results) != 0 {
t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results))
}
return
if tt.wantZero && len(results) != 0 {
t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results))
}
if len(results) < tt.minResultCount || len(results) > tt.maxResultCount {
t.Fatalf(
"unexpected result count for %s: got %d, want range [%d,%d]",
tt.fixture, len(results), tt.minResultCount, tt.maxResultCount,
)
}
testutil.AssertSequentialRanks(t, results)
testutil.AssertFirstResultFilled(t, results)
})
}
}
func TestBaiduResultParserEmptyHTML(t *testing.T) {
func TestBaiduClassifyRawHTML(t *testing.T) {
t.Parallel()
results, err := baiduResultParser(testutil.ResponseFromString(""))
if err != nil {
t.Fatalf("baiduResultParser() error = %v", err)
tests := []struct {
name string
fixture string
want error
}{
{name: "no results", fixture: "search_no_results.html", want: core.ErrEmptyResult},
{name: "captcha page", fixture: "search_captcha.html", want: core.ErrCaptcha},
}
if len(results) != 0 {
t.Fatalf("expected zero results for empty HTML, got %d", len(results))
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
body, err := io.ReadAll(testutil.ResponseFromFixture(t, tt.fixture).Body)
if err != nil {
t.Fatalf("read fixture body: %v", err)
}
err = classifyBaiduRawHTML(body)
if !errors.Is(err, tt.want) {
t.Fatalf("expected %v for %s, got %v", tt.want, tt.fixture, err)
}
})
}
}

8
baidu/selectors.go
View File

@@ -9,12 +9,16 @@ var Selectors = struct {
ImageJSONRoot []string
Link string
Desc string
// DescAlt are additional description containers tried when Desc misses.
// Baidu varies abstract markup across feature blocks (info cards, news rows).
DescAlt []string
}{
Captcha: "div.passMod_dialog-wrapper",
Timeout: "button.timeout-button",
Results: "div.c-container.new-pmd",
ResultsAlt: []string{"#content_left div.result.c-container", "#content_left div.result-op.c-container"},
Results: "#content_left div.result.c-container",
ResultsAlt: []string{"#content_left div.result-op.c-container", "div.c-container.new-pmd"},
ImageJSONRoot: []string{"body > pre", "pre"},
Link: "a",
Desc: "div.c-abstract",
DescAlt: []string{".content-right_8Zs40", ".summary-gap_3Jb4I"},
}

2
cmd/root.go
View File

@@ -16,7 +16,7 @@ import (
)
const (
version = "0.7.9"
version = "0.7.10"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)

67
core/http_client.go
View File

@@ -3,6 +3,7 @@ package core
import (
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
@@ -10,10 +11,11 @@ import (
"strings"
"time"
"github.com/corpix/uarand"
utls "github.com/refraction-networking/utls"
)
const rawHTTPTimeout = 10 * time.Second
const rawHTTPTimeout = 30 * time.Second
// SetAcceptLanguageHeader sets the Accept-Language header from a lang code.
// No-op when the code has no language subtag.
@@ -37,6 +39,53 @@ func DrainAndCloseResponse(resp *http.Response) {
_ = resp.Body.Close()
}
// RawSearchRequest builds and executes a raw-mode SERP HTTP GET. It uses the
// shared raw HTTP client (TLS fingerprinting, network usage tracking, proxy
// support), randomizes the User-Agent, and applies the Accept-Language header
// derived from the query locale. The caller owns the returned response and
// must drain/close it (see DrainAndCloseResponse).
func RawSearchRequest(ctx context.Context, searchURL string, query Query) (*http.Response, error) {
client, err := NewRawHTTPClient(query)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", uarand.GetRandom())
SetAcceptLanguageHeader(req, query.LangCode)
return client.Do(req)
}
func ReadRawSearchBody(resp *http.Response) ([]byte, error) {
if resp == nil {
return nil, fmt.Errorf("%w: nil raw search response", ErrEngineInternal)
}
if err := ClassifySearchHTTPStatus(resp.StatusCode); err != nil {
return nil, err
}
return io.ReadAll(resp.Body)
}
func ClassifySearchHTTPStatus(status int) error {
switch status {
case 0:
return nil
case http.StatusForbidden, http.StatusUnauthorized:
return ErrBlocked
case http.StatusTooManyRequests:
return ErrRateLimited
}
if status >= 500 {
return fmt.Errorf("%w: search engine returned HTTP %d", ErrBlocked, status)
}
if status < 200 || status >= 300 {
return fmt.Errorf("%w: search engine returned HTTP %d", ErrParser, status)
}
return nil
}
func NewRawHTTPClient(query Query) (*http.Client, error) {
transport, err := newRawTransport(query)
if err != nil {
@@ -87,9 +136,15 @@ func newRawTransport(query Query) (*http.Transport, error) {
config := &utls.Config{
ServerName: hostname,
InsecureSkipVerify: query.Insecure,
NextProtos: []string{"http/1.1"},
}
uconn := utls.UClient(rawConn, config, utls.HelloChrome_Auto)
if err := uconn.BuildHandshakeState(); err != nil {
rawConn.Close()
return nil, err
}
forceHTTP1ALPN(uconn)
if err := uconn.Handshake(); err != nil {
rawConn.Close()
return nil, err
@@ -101,6 +156,16 @@ func newRawTransport(query Query) (*http.Transport, error) {
return transport, nil
}
func forceHTTP1ALPN(conn *utls.UConn) {
for _, ext := range conn.Extensions {
if alpn, ok := ext.(*utls.ALPNExtension); ok {
alpn.AlpnProtocols = []string{"http/1.1"}
return
}
}
conn.Extensions = append(conn.Extensions, &utls.ALPNExtension{AlpnProtocols: []string{"http/1.1"}})
}
func dialNetworkUsageConn(ctx context.Context, network, addr string) (net.Conn, error) {
dialer := &net.Dialer{}
conn, err := dialer.DialContext(ctx, network, addr)

30
core/http_client_test.go
View File

@@ -36,6 +36,36 @@ func TestDrainAndCloseResponseDrainsAndCloses(t *testing.T) {
}
}
func TestClassifySearchHTTPStatus(t *testing.T) {
tests := []struct {
name string
status int
want error
}{
{name: "unknown browser status", status: 0, want: nil},
{name: "ok", status: http.StatusOK, want: nil},
{name: "blocked", status: http.StatusForbidden, want: ErrBlocked},
{name: "rate limited", status: http.StatusTooManyRequests, want: ErrRateLimited},
{name: "server error", status: http.StatusBadGateway, want: ErrBlocked},
{name: "unexpected status", status: http.StatusNotFound, want: ErrParser},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := ClassifySearchHTTPStatus(tt.status)
if tt.want == nil {
if err != nil {
t.Fatalf("expected nil error, got %v", err)
}
return
}
if !errors.Is(err, tt.want) {
t.Fatalf("expected %v, got %v", tt.want, err)
}
})
}
}
func TestDrainAndCloseResponseNilSafe(t *testing.T) {
DrainAndCloseResponse(nil)
DrainAndCloseResponse(&http.Response{})

28
core/proxy_test.go
View File

@@ -344,6 +344,34 @@ func TestNewRawHTTPClientSocks5hUsesProxyDNS(t *testing.T) {
}
}
func TestNewRawHTTPClientDirectTLSUsesHTTP1(t *testing.T) {
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(r.Proto))
}))
server.EnableHTTP2 = true
server.StartTLS()
defer server.Close()
client, err := NewRawHTTPClient(Query{Insecure: true})
if err != nil {
t.Fatalf("new raw http client: %v", err)
}
resp, err := client.Get(server.URL)
if err != nil {
t.Fatalf("expected direct TLS request to succeed, got %v", err)
}
defer DrainAndCloseResponse(resp)
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("read response body: %v", err)
}
if string(body) != "HTTP/1.1" {
t.Fatalf("expected raw client to use HTTP/1.1, got %q", string(body))
}
}
func TestClassifyProxyNetworkError(t *testing.T) {
tests := []struct {
name string

105
ecosia/search_raw.go
View File

@@ -1,84 +1,32 @@
package ecosia
import (
"bytes"
"context"
"errors"
"fmt"
"net/http"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/corpix/uarand"
"github.com/karust/openserp/core"
)
func ecosiaRequest(ctx context.Context, searchURL string, query core.Query) (*http.Response, error) {
baseClient, err := core.NewRawHTTPClient(query)
func classifyEcosiaRawHTML(body []byte) error {
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
if err != nil {
return nil, err
return err
}
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
if err != nil {
return nil, err
if strings.Contains(strings.ToLower(doc.Text()), "captcha") {
return core.ErrCaptcha
}
req.Header.Set("User-Agent", uarand.GetRandom())
core.SetAcceptLanguageHeader(req, query.LangCode)
res, err := baseClient.Do(req)
if err != nil {
return nil, err
if doc.Find("[data-test-id='web-no-results']").Length() > 0 ||
(doc.Find(Selectors.Mainline).Length() > 0 &&
doc.Find(Selectors.Result).Length() == 0 &&
doc.Find(Selectors.Ad).Length() == 0) {
return core.ErrEmptyResult
}
return res, nil
}
// resultParser parses an Ecosia SERP HTML response into search results
// using goquery (no browser required).
func resultParser(response *http.Response) ([]core.SearchResult, error) {
doc, err := goquery.NewDocumentFromReader(response.Body)
if err != nil {
return nil, err
}
var (
results []core.SearchResult
rank = 1
)
doc.Find(Selectors.Result).Each(func(_ int, s *goquery.Selection) {
href, ok := s.Find(Selectors.ResultLink).Attr("href")
if !ok || strings.TrimSpace(href) == "" {
return
}
var (
title = strings.TrimSpace(s.Find(Selectors.Title).Text())
desc = strings.TrimSpace(s.Find(Selectors.Desc).Text())
)
results = append(results, core.SearchResult{
Rank: rank,
URL: href,
Title: title,
Description: desc,
})
rank++
})
adRank := -1
doc.Find(Selectors.Ad).Each(func(_ int, s *goquery.Selection) {
href, ok := s.Find(Selectors.ResultLink).Attr("href")
if !ok || strings.TrimSpace(href) == "" {
return
}
var (
title = strings.TrimSpace(s.Find(Selectors.Title).Text())
desc = strings.TrimSpace(s.Find(Selectors.Desc).Text())
)
results = append(results, core.SearchResult{
Rank: adRank,
URL: href,
Title: title,
Description: desc,
Ad: true,
})
adRank--
})
return results, nil
return nil
}
// imageResultParser parses an Ecosia image SERP HTML response into search
@@ -144,7 +92,7 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
}
core.WithRequest(ctx).WithField("url", ecosiaURL).Debug(fmt.Sprintf("Ecosia URL built: %s", ecosiaURL))
res, err := ecosiaRequest(ctx, ecosiaURL, query)
res, err := core.RawSearchRequest(ctx, ecosiaURL, query)
if err != nil {
return nil, err
}
@@ -153,15 +101,36 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
fmt.Sprintf("Ecosia Raw response: code=%d", res.StatusCode),
)
parsedResults, err := resultParser(res)
body, err := core.ReadRawSearchBody(res)
if err != nil {
return nil, err
}
htmlStatus := classifyEcosiaRawHTML(body)
if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) {
return nil, htmlStatus
}
parsedResults, err := ParseHTML(bytes.NewReader(body))
if err != nil {
return nil, err
}
if len(parsedResults) == 0 {
if errors.Is(htmlStatus, core.ErrEmptyResult) {
return []core.SearchResult{}, nil
}
return nil, fmt.Errorf("%w: ecosia raw search returned no parseable results", core.ErrParser)
}
// Ecosia paginates by page index, not result offset, so re-rank from
// the page boundary rather than query.Start (off-grid offsets round down).
// Skip ads (rank<0) so organic ranks stay sequential from startRank.
organicIdx := 0
for i := range parsedResults {
parsedResults[i].Rank = startRank + i
if parsedResults[i].Ad {
continue
}
parsedResults[i].Rank = startRank + organicIdx
organicIdx++
}
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(

48
ecosia/search_raw_test.go
View File

@@ -1,44 +1,14 @@
package ecosia
import (
"errors"
"io"
"testing"
"github.com/karust/openserp/core"
"github.com/karust/openserp/testutil"
)
func TestEcosiaResultParser(t *testing.T) {
t.Parallel()
tests := []struct {
name string
fixture string
wantCount int
}{
{name: "search results", fixture: "search_results.html", wantCount: 10},
{name: "no results", fixture: "search_no_results.html", wantCount: 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
results, err := resultParser(testutil.ResponseFromFixture(t, tt.fixture))
if err != nil {
t.Fatalf("resultParser() error = %v", err)
}
if len(results) != tt.wantCount {
t.Fatalf("expected %d results for %s, got %d", tt.wantCount, tt.fixture, len(results))
}
if tt.wantCount == 0 {
return
}
testutil.AssertSequentialRanks(t, results)
testutil.AssertFirstResultFilled(t, results)
})
}
}
func TestEcosiaImageResultParser(t *testing.T) {
t.Parallel()
@@ -71,14 +41,16 @@ func TestEcosiaImageResultParser(t *testing.T) {
}
}
func TestEcosiaResultParserEmptyHTML(t *testing.T) {
func TestEcosiaClassifyRawHTML(t *testing.T) {
t.Parallel()
results, err := resultParser(testutil.ResponseFromString(""))
body, err := io.ReadAll(testutil.ResponseFromFixture(t, "search_no_results.html").Body)
if err != nil {
t.Fatalf("resultParser() error = %v", err)
t.Fatalf("read fixture body: %v", err)
}
if len(results) != 0 {
t.Fatalf("expected zero results for empty HTML, got %d", len(results))
err = classifyEcosiaRawHTML(body)
if !errors.Is(err, core.ErrEmptyResult) {
t.Fatalf("expected %v for search_no_results.html, got %v", core.ErrEmptyResult, err)
}
}

60
google/search_raw.go
View File

@@ -1,38 +1,18 @@
package google
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/corpix/uarand"
"github.com/karust/openserp/core"
"github.com/sirupsen/logrus"
)
func googleRequest(ctx context.Context, searchURL string, query core.Query) (*http.Response, error) {
baseClient, err := core.NewRawHTTPClient(query)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", uarand.GetRandom())
core.SetAcceptLanguageHeader(req, query.LangCode)
res, err := baseClient.Do(req)
if err != nil {
return nil, err
}
return res, nil
}
// ParseHTML parses a Google SERP HTML document and returns search results.
// It is the pure parser used by both raw HTTP search and parse endpoints.
func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
@@ -111,8 +91,21 @@ func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
return core.DeduplicateResults(results)
}
func googleResultParser(response *http.Response) ([]core.SearchResult, error) {
return ParseHTML(response.Body)
func classifyGoogleRawHTML(body []byte) error {
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
if err != nil {
return err
}
if doc.Find(Selectors.Captcha).Length() > 0 {
return core.ErrCaptcha
}
text := strings.ToLower(doc.Text())
if strings.Contains(text, "did not match any documents") ||
strings.Contains(text, "about 0 results") {
return core.ErrEmptyResult
}
return nil
}
func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -132,7 +125,7 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
}
core.WithRequest(ctx).WithField("url", googleURL).Debug(fmt.Sprintf("Google URL built: %s", googleURL))
res, err := googleRequest(ctx, googleURL, query)
res, err := core.RawSearchRequest(ctx, googleURL, query)
if err != nil {
return nil, err
}
@@ -141,10 +134,25 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
fmt.Sprintf("Google Raw response: code=%d", res.StatusCode),
)
parsedResults, err := googleResultParser(res)
body, err := core.ReadRawSearchBody(res)
if err != nil {
return nil, err
}
htmlStatus := classifyGoogleRawHTML(body)
if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) {
return nil, htmlStatus
}
parsedResults, err := ParseHTML(bytes.NewReader(body))
if err != nil {
return nil, err
}
if len(parsedResults) == 0 {
if errors.Is(htmlStatus, core.ErrEmptyResult) {
return []core.SearchResult{}, nil
}
return nil, fmt.Errorf("%w: google raw search returned no parseable results", core.ErrParser)
}
if query.Start > 0 {
for i := range parsedResults {

43
google/search_raw_test.go
View File

@@ -1,12 +1,15 @@
package google
import (
"errors"
"io"
"testing"
"github.com/karust/openserp/core"
"github.com/karust/openserp/testutil"
)
func TestGoogleResultParserSnapshots(t *testing.T) {
func TestGoogleParseHTMLFixtures(t *testing.T) {
t.Parallel()
tests := []struct {
@@ -38,9 +41,9 @@ func TestGoogleResultParserSnapshots(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
results, err := googleResultParser(testutil.ResponseFromFixture(t, tt.fixture))
results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body)
if err != nil {
t.Fatalf("googleResultParser() error = %v", err)
t.Fatalf("ParseHTML() error = %v", err)
}
if tt.wantZero {
@@ -63,14 +66,42 @@ func TestGoogleResultParserSnapshots(t *testing.T) {
}
}
func TestGoogleResultParserEmptyHTML(t *testing.T) {
func TestGoogleParseHTMLEmptyHTML(t *testing.T) {
t.Parallel()
results, err := googleResultParser(testutil.ResponseFromString(""))
results, err := ParseHTML(testutil.ResponseFromString("").Body)
if err != nil {
t.Fatalf("googleResultParser() error = %v", err)
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 0 {
t.Fatalf("expected zero results for empty HTML, got %d", len(results))
}
}
func TestGoogleClassifyRawHTML(t *testing.T) {
t.Parallel()
tests := []struct {
name string
fixture string
want error
}{
{name: "no results", fixture: "search_no_results.html", want: core.ErrEmptyResult},
{name: "captcha page", fixture: "search_captcha.html", want: core.ErrCaptcha},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
body, err := io.ReadAll(testutil.ResponseFromFixture(t, tt.fixture).Body)
if err != nil {
t.Fatalf("read fixture body: %v", err)
}
err = classifyGoogleRawHTML(body)
if !errors.Is(err, tt.want) {
t.Fatalf("expected %v for %s, got %v", tt.want, tt.fixture, err)
}
})
}
}

32
yandex/parse_html.go
View File

@@ -23,20 +23,35 @@ func parseYandexDocument(doc *goquery.Document) []core.SearchResult {
rank := 1
doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) {
linkTag := item.Find("a").First()
// Skip blocks without a result heading (filters out non-organic blocks
// that share the result-row container).
titleTag := item.Find(Selectors.Title).First()
if titleTag.Length() == 0 {
return
}
// Prefer the canonical organic-title link, then the closest <a> wrapping
// the title, then any <a> in the block.
linkTag := item.Find(Selectors.LinkPrimary).First()
if linkTag.Length() == 0 {
linkTag = titleTag.Closest("a")
}
if linkTag.Length() == 0 {
linkTag = item.Find(Selectors.Link).First()
}
if linkTag.Length() == 0 {
return
}
href, exists := linkTag.Attr("href")
if !exists || href == "" || href == "#" || strings.HasPrefix(href, "javascript:") {
if !exists {
return
}
href = strings.TrimSpace(href)
if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") {
return
}
titleTag := item.Find(Selectors.Title).First()
if titleTag.Length() == 0 {
return
}
title := strings.TrimSpace(titleTag.Text())
if title == "" {
return
@@ -46,6 +61,11 @@ func parseYandexDocument(doc *goquery.Document) []core.SearchResult {
if descTag := item.Find(Selectors.Desc).First(); descTag.Length() > 0 {
desc = strings.TrimSpace(descTag.Text())
}
if desc == "" {
if descTag := item.Find(Selectors.DescFallback).First(); descTag.Length() > 0 {
desc = strings.TrimSpace(descTag.Text())
}
}
results = append(results, core.SearchResult{
Rank: rank,

29
yandex/parse_html_test.go
View File

@@ -55,3 +55,32 @@ func TestParseYandexHTMLEmpty(t *testing.T) {
t.Fatalf("expected zero results for empty HTML, got %d", len(results))
}
}
func TestParseYandexHTMLFallbackSelectors(t *testing.T) {
t.Parallel()
html := `
<ul>
<li class="serp-item">
<h2><a href="https://example.com/result">Fallback Title</a></h2>
<div class="OrganicText">Fallback description</div>
</li>
</ul>`
results, err := ParseHTML(bytes.NewReader([]byte(html)))
if err != nil {
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 1 {
t.Fatalf("expected 1 result, got %d", len(results))
}
if results[0].URL != "https://example.com/result" {
t.Fatalf("unexpected URL: %s", results[0].URL)
}
if results[0].Title != "Fallback Title" {
t.Fatalf("unexpected title: %s", results[0].Title)
}
if results[0].Description != "Fallback description" {
t.Fatalf("unexpected description: %s", results[0].Description)
}
}

112
yandex/search_raw.go
View File

@@ -1,96 +1,27 @@
package yandex
import (
"bytes"
"context"
"errors"
"fmt"
"net/http"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/corpix/uarand"
"github.com/karust/openserp/core"
"github.com/sirupsen/logrus"
)
func yandexRequest(ctx context.Context, searchURL string, query core.Query) (*http.Response, error) {
baseClient, err := core.NewRawHTTPClient(query)
func classifyYandexRawHTML(body []byte) error {
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
if err != nil {
return nil, err
return err
}
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
if err != nil {
return nil, err
if doc.Find(Selectors.Captcha).Length() > 0 {
return core.ErrCaptcha
}
req.Header.Set("User-Agent", uarand.GetRandom())
core.SetAcceptLanguageHeader(req, query.LangCode)
res, err := baseClient.Do(req)
if err != nil {
return nil, err
if doc.Find(Selectors.NoResults).Length() > 0 {
return core.ErrEmptyResult
}
return res, nil
}
func yandexResultParser(response *http.Response) ([]core.SearchResult, error) {
doc, err := goquery.NewDocumentFromReader(response.Body)
if err != nil {
return nil, err
}
results := []core.SearchResult{}
rank := 1
// Prefer stable container + attributes and keep legacy fallback.
sel := doc.Find("#search-result > li[data-fast], li.serp-item")
for i := range sel.Nodes {
item := sel.Eq(i)
// Skip blocks without a result heading.
titleTag := item.Find("h2").First()
if titleTag.Length() == 0 {
continue
}
// Find URL
linkTag := item.Find("a.OrganicTitle-Link").First()
if linkTag.Length() == 0 {
linkTag = titleTag.Closest("a")
}
if linkTag.Length() == 0 {
linkTag = item.Find("a").First()
}
link, _ := linkTag.Attr("href")
link = strings.Trim(link, " ")
// Find title
title := strings.TrimSpace(titleTag.Text())
// Find description
descTag := item.Find(`span.OrganicTextContentSpan`).First()
if descTag.Length() == 0 {
descTag = item.Find("div.OrganicText").First()
}
desc := strings.TrimSpace(descTag.Text())
if link != "" && link != "#" && title != "" {
result := core.SearchResult{
Rank: rank,
URL: link,
Title: title,
Description: desc,
}
results = append(results, result)
rank++
}
}
logrus.WithField("document_size", len(doc.Text())).Trace(
fmt.Sprintf("Yandex search document size: %d", len(doc.Text())),
)
return core.DeduplicateResults(results), err
return nil
}
func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -109,13 +40,13 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
return nil, err
}
googleURL, err := BuildURL(query, startPage)
searchURL, err := BuildURL(query, startPage)
if err != nil {
return nil, err
}
core.WithRequest(ctx).WithField("url", googleURL).Debug(fmt.Sprintf("Yandex URL built: %s", googleURL))
core.WithRequest(ctx).WithField("url", searchURL).Debug(fmt.Sprintf("Yandex URL built: %s", searchURL))
res, err := yandexRequest(ctx, googleURL, query)
res, err := core.RawSearchRequest(ctx, searchURL, query)
if err != nil {
return nil, err
}
@@ -124,10 +55,25 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
fmt.Sprintf("Yandex Raw response: code=%d", res.StatusCode),
)
parsedResults, err := yandexResultParser(res)
body, err := core.ReadRawSearchBody(res)
if err != nil {
return nil, err
}
htmlStatus := classifyYandexRawHTML(body)
if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) {
return nil, htmlStatus
}
parsedResults, err := ParseHTML(bytes.NewReader(body))
if err != nil {
return nil, err
}
if len(parsedResults) == 0 {
if errors.Is(htmlStatus, core.ErrEmptyResult) {
return []core.SearchResult{}, nil
}
return nil, fmt.Errorf("%w: yandex raw search returned no parseable results", core.ErrParser)
}
if skipOnFirstPage > 0 {
if skipOnFirstPage >= len(parsedResults) {

85
yandex/search_raw_test.go
View File

@@ -1,76 +1,69 @@
package yandex
import (
"errors"
"io"
"testing"
"github.com/karust/openserp/core"
"github.com/karust/openserp/testutil"
)
func TestYandexResultParserSnapshots(t *testing.T) {
// TestYandexParseHTMLFixtures covers the no-results and captcha fixtures.
// The happy path is covered in TestParseYandexHTML; this file ensures the
// shared parser (used by both raw mode and the /yandex/parse endpoint) does
// not over-extract on captcha or empty SERPs.
func TestYandexParseHTMLFixtures(t *testing.T) {
t.Parallel()
tests := []struct {
name string
fixture string
minResultCount int
maxResultCount int
wantZero bool
name string
fixture string
wantZero bool
}{
{
name: "search results",
fixture: "search_results.html",
minResultCount: 1,
maxResultCount: 30,
},
{
name: "no results",
fixture: "search_no_results.html",
wantZero: true,
},
{
name: "captcha page",
fixture: "search_captcha.html",
wantZero: true,
},
{name: "no results", fixture: "search_no_results.html", wantZero: true},
{name: "captcha page", fixture: "search_captcha.html", wantZero: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
results, err := yandexResultParser(testutil.ResponseFromFixture(t, tt.fixture))
results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body)
if err != nil {
t.Fatalf("yandexResultParser() error = %v", err)
t.Fatalf("ParseHTML() error = %v", err)
}
if tt.wantZero {
if len(results) != 0 {
t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results))
}
return
if tt.wantZero && len(results) != 0 {
t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results))
}
if len(results) < tt.minResultCount || len(results) > tt.maxResultCount {
t.Fatalf(
"unexpected result count for %s: got %d, want range [%d,%d]",
tt.fixture, len(results), tt.minResultCount, tt.maxResultCount,
)
}
testutil.AssertSequentialRanks(t, results)
testutil.AssertFirstResultFilled(t, results)
})
}
}
func TestYandexResultParserEmptyHTML(t *testing.T) {
func TestYandexClassifyRawHTML(t *testing.T) {
t.Parallel()
results, err := yandexResultParser(testutil.ResponseFromString(""))
if err != nil {
t.Fatalf("yandexResultParser() error = %v", err)
tests := []struct {
name string
fixture string
want error
}{
{name: "no results", fixture: "search_no_results.html", want: core.ErrEmptyResult},
{name: "captcha page", fixture: "search_captcha.html", want: core.ErrCaptcha},
}
if len(results) != 0 {
t.Fatalf("expected zero results for empty HTML, got %d", len(results))
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
body, err := io.ReadAll(testutil.ResponseFromFixture(t, tt.fixture).Body)
if err != nil {
t.Fatalf("read fixture body: %v", err)
}
err = classifyYandexRawHTML(body)
if !errors.Is(err, tt.want) {
t.Fatalf("expected %v for %s, got %v", tt.want, tt.fixture, err)
}
})
}
}

6
yandex/selectors.go
View File

@@ -5,9 +5,13 @@ var Selectors = struct {
Captcha string
NoResults string
Results string
// LinkPrimary is preferred over a generic <a>; falls back to title.Closest("a")
// then the first <a> in the result block when absent.
LinkPrimary string
Link string
Title string
Desc string
DescFallback string
ImageItems string
ImageItemsAlt []string
ImageStateAll string
@@ -15,9 +19,11 @@ var Selectors = struct {
Captcha: "div.CheckboxCaptcha",
NoResults: "div.EmptySearchResults",
Results: "li[data-fast], li.serp-item",
LinkPrimary: "a.OrganicTitle-Link",
Link: "a",
Title: "h2",
Desc: "span.OrganicTextContentSpan",
DescFallback: "div.OrganicText",
ImageItems: "div[role='main'] div[data-state]",
ImageItemsAlt: []string{"div[data-state*='serpList']"},
ImageStateAll: "div[data-state]",