mirror of
https://github.com/karust/openserp.git
synced 2026-08-24 17:00:06 +08:00
122 lines
3.5 KiB
Go
122 lines
3.5 KiB
Go
package ecosia
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/karust/openserp/core"
|
|
)
|
|
|
|
// isCaptchaDoc reports whether a parsed Ecosia document is a Cloudflare
|
|
// challenge rather than a SERP. Prefers the hidden Turnstile input, then falls
|
|
// back to the shared challenge-page body phrases (see cfBodyMarkers).
|
|
func isCaptchaDoc(doc *goquery.Document) bool {
|
|
err := core.ClassifyChallengeDocument(doc, core.DocSignals{
|
|
CaptchaSelectors: []string{Selectors.Captcha},
|
|
CaptchaMarkers: cfBodyMarkers,
|
|
})
|
|
return errors.Is(err, core.ErrCaptcha)
|
|
}
|
|
|
|
func classifyEcosiaRawHTML(body []byte) error {
|
|
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return classifyEcosiaDocument(doc)
|
|
}
|
|
|
|
// imageResultParser parses an Ecosia image SERP HTML response into search
|
|
// results using goquery (no browser required).
|
|
func imageResultParser(response *http.Response) ([]core.SearchResult, error) {
|
|
doc, err := goquery.NewDocumentFromReader(response.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var (
|
|
results []core.SearchResult
|
|
rank = 1
|
|
)
|
|
doc.Find(Selectors.ImageResult).Each(func(_ int, s *goquery.Selection) {
|
|
link := s.Find(Selectors.ImageLink)
|
|
href, _ := link.Attr("href")
|
|
title, _ := link.Find("img").Attr("alt")
|
|
source := strings.TrimSpace(s.Find(Selectors.ImageSource).Text())
|
|
dims := strings.TrimSpace(s.Find(Selectors.ImageDims).Text())
|
|
if res, ok := assembleEcosiaImageRow(href, strings.TrimSpace(title), source, dims, rank); ok {
|
|
results = append(results, res)
|
|
rank++
|
|
}
|
|
})
|
|
return results, nil
|
|
}
|
|
|
|
func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
|
|
ctx = core.PrepareEngineContext(ctx, query, "ecosia")
|
|
|
|
pageNum, startRank, err := startPage(query.Start)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
ecosiaURL, err := BuildURL(query, pageNum)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
core.WithRequest(ctx).WithField("url", ecosiaURL).Debug(fmt.Sprintf("Ecosia URL built: %s", ecosiaURL))
|
|
|
|
res, err := core.RawSearchRequest(ctx, ecosiaURL, query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer core.DrainAndCloseResponse(res)
|
|
core.WithRequest(ctx).WithField("status_code", res.StatusCode).Debug(
|
|
fmt.Sprintf("Ecosia Raw response: code=%d", res.StatusCode),
|
|
)
|
|
|
|
body, err := core.ReadRawSearchBody(res)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
htmlStatus := classifyEcosiaRawHTML(body)
|
|
if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) {
|
|
return nil, htmlStatus
|
|
}
|
|
|
|
parsedResults, err := ParseHTML(bytes.NewReader(body))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(parsedResults) == 0 {
|
|
if errors.Is(htmlStatus, core.ErrEmptyResult) {
|
|
return []core.SearchResult{}, nil
|
|
}
|
|
return nil, fmt.Errorf("%w: ecosia raw search returned no parseable results", core.ErrParser)
|
|
}
|
|
|
|
// Ecosia paginates by page index, not result offset, so re-rank from
|
|
// the page boundary rather than query.Start (off-grid offsets round down).
|
|
// Skip ads so organic ranks stay sequential from startRank.
|
|
organicIdx := 0
|
|
for i := range parsedResults {
|
|
if parsedResults[i].Ad {
|
|
continue
|
|
}
|
|
parsedResults[i].Rank = startRank + organicIdx
|
|
organicIdx++
|
|
}
|
|
core.SetSeparatedAdAbsoluteRanks(parsedResults, pageNum*10)
|
|
|
|
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
|
|
fmt.Sprintf("Ecosia Raw results : %v", parsedResults),
|
|
)
|
|
|
|
deduped := core.StripResultFeatures(core.DeduplicateResults(parsedResults), query.Features)
|
|
return deduped, nil
|
|
}
|