mirror of
https://github.com/karust/openserp.git
synced 2026-08-15 13:14:16 +08:00
feat(api): v2 response shape - absolute rank, ad/organic split, slimmer fields
This commit is contained in:
16
README.md
16
README.md
@@ -106,7 +106,7 @@ Common parameters:
|
||||
| `date` | Date range | `20250101..20251231` |
|
||||
| `file` | File extension | `pdf`, `doc`, `xls` |
|
||||
| `site` | Site-specific search | `github.com` |
|
||||
| `limit` | Number of results, max 100 | `10`, `25`, `50` |
|
||||
| `limit` | Number of organic results, max 100. Ads may be returned in addition. | `10`, `25`, `50` |
|
||||
| `start` | Pagination offset | `0`, `10`, `20` |
|
||||
| `format` | Output format | `json`, `markdown`, `text`, `ndjson` |
|
||||
|
||||
@@ -130,7 +130,7 @@ Engine-specific parameters:
|
||||
"requested_at": "2026-04-25T22:27:52Z",
|
||||
"took_ms": 6410,
|
||||
"engines_failed": [],
|
||||
"version": "1.0"
|
||||
"version": "2.0"
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
@@ -143,22 +143,14 @@ Engine-specific parameters:
|
||||
"snippet": "Official Go documentation, tutorials, references, and release notes.",
|
||||
"domain": "go.dev",
|
||||
"favicon": "https://go.dev/favicon.ico",
|
||||
"is_ad": false,
|
||||
"position": {
|
||||
"absolute": 1,
|
||||
"page": 1,
|
||||
"on_page": 1
|
||||
"absolute": 1
|
||||
},
|
||||
"engine": "google",
|
||||
"domain_info": {
|
||||
"tld": "dev",
|
||||
"sld": "go",
|
||||
"is_gov": false,
|
||||
"is_edu": false,
|
||||
"is_social": false
|
||||
},
|
||||
"classification": {
|
||||
"content_type": "webpage"
|
||||
"category": ""
|
||||
}
|
||||
}
|
||||
],
|
||||
|
||||
@@ -37,8 +37,12 @@ func baiduResultSelectors() []string {
|
||||
func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult {
|
||||
var results []core.SearchResult
|
||||
rank := 1
|
||||
adRank := 1
|
||||
absoluteRank := 1
|
||||
|
||||
sel.Each(func(_ int, item *goquery.Selection) {
|
||||
isAd := baiduSelectionHasAdMarker(item)
|
||||
|
||||
// h3-first: organic results always carry a heading; this filters out
|
||||
// non-result blocks that may share the wrapper class.
|
||||
titleTag := item.Find("h3").First()
|
||||
@@ -96,21 +100,55 @@ func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult {
|
||||
desc = strings.TrimSpace(strings.Replace(full, title, "", 1))
|
||||
}
|
||||
|
||||
resultRank := rank
|
||||
if isAd {
|
||||
resultRank = adRank
|
||||
adRank++
|
||||
} else {
|
||||
rank++
|
||||
}
|
||||
|
||||
results = append(results, core.SearchResult{
|
||||
Rank: rank,
|
||||
URL: href,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Rank: resultRank,
|
||||
AbsoluteRank: absoluteRank,
|
||||
URL: href,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Ad: isAd,
|
||||
})
|
||||
rank++
|
||||
absoluteRank++
|
||||
})
|
||||
|
||||
// Re-rank sequentially after dedup so callers get a clean 1..N sequence
|
||||
// (dedup may drop intermediate ranks when the same URL appears in
|
||||
// multiple Baidu result-card variants on the same SERP).
|
||||
deduped := core.DeduplicateResults(results)
|
||||
organicIdx := 0
|
||||
for i := range deduped {
|
||||
deduped[i].Rank = i + 1
|
||||
if deduped[i].Ad {
|
||||
continue
|
||||
}
|
||||
organicIdx++
|
||||
deduped[i].Rank = organicIdx
|
||||
}
|
||||
return deduped
|
||||
}
|
||||
|
||||
func baiduSelectionHasAdMarker(item *goquery.Selection) bool {
|
||||
for _, selector := range Selectors.AdMarkers {
|
||||
if item.Is(selector) || item.Find(selector).Length() > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
isAd := false
|
||||
item.Find("span, i, em").EachWithBreak(func(_ int, marker *goquery.Selection) bool {
|
||||
text := strings.TrimSpace(marker.Text())
|
||||
if text == "广告" || text == "推广" || text == "商业推广" {
|
||||
isAd = true
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
return isAd
|
||||
}
|
||||
|
||||
@@ -104,3 +104,53 @@ func TestParseBaiduHTMLFallsBackWhenEarlierSelectorHasNoResult(t *testing.T) {
|
||||
t.Fatalf("unexpected URL: %s", results[0].URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBaiduHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
html := `
|
||||
<div id="content_left">
|
||||
<div class="result c-container" data-tuiguang="1">
|
||||
<h3><a href="https://ads.example.com">Sponsored Result</a></h3>
|
||||
<div class="c-abstract">Paid snippet</div>
|
||||
</div>
|
||||
<div class="result c-container">
|
||||
<h3><a href="https://organic.example.com/one">Organic One</a></h3>
|
||||
<div class="c-abstract">Organic snippet one</div>
|
||||
</div>
|
||||
<div class="result c-container">
|
||||
<h3><a href="https://organic.example.com/two">Organic Two</a></h3>
|
||||
<div class="c-abstract">Organic snippet two</div>
|
||||
</div>
|
||||
</div>`
|
||||
|
||||
results, err := ParseHTML(bytes.NewReader([]byte(html)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseHTML() error = %v", err)
|
||||
}
|
||||
if len(results) != 3 {
|
||||
t.Fatalf("expected 3 results, got %d", len(results))
|
||||
}
|
||||
|
||||
organicRank := 0
|
||||
adRank := 0
|
||||
for _, r := range results {
|
||||
if r.Ad {
|
||||
adRank++
|
||||
if r.Rank != adRank {
|
||||
t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
|
||||
}
|
||||
continue
|
||||
}
|
||||
organicRank++
|
||||
if r.Rank != organicRank {
|
||||
t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
|
||||
}
|
||||
}
|
||||
if organicRank != 2 {
|
||||
t.Fatalf("organic count = %d, want 2", organicRank)
|
||||
}
|
||||
if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
|
||||
t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,7 +112,13 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
|
||||
}
|
||||
|
||||
for i := range searchResults {
|
||||
searchResults[i].Rank = query.Start + i + 1
|
||||
if searchResults[i].AbsoluteRank > 0 {
|
||||
searchResults[i].AbsoluteRank += query.Start
|
||||
}
|
||||
if searchResults[i].Ad {
|
||||
continue
|
||||
}
|
||||
searchResults[i].Rank = query.Start + searchResults[i].Rank
|
||||
}
|
||||
return searchResults, nil
|
||||
}
|
||||
|
||||
@@ -69,7 +69,13 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
|
||||
}
|
||||
if query.Start > 0 {
|
||||
for i := range parsedResults {
|
||||
parsedResults[i].Rank = query.Start + i + 1
|
||||
if parsedResults[i].AbsoluteRank > 0 {
|
||||
parsedResults[i].AbsoluteRank += query.Start
|
||||
}
|
||||
if parsedResults[i].Ad {
|
||||
continue
|
||||
}
|
||||
parsedResults[i].Rank = query.Start + parsedResults[i].Rank
|
||||
}
|
||||
}
|
||||
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
|
||||
|
||||
@@ -6,6 +6,7 @@ var Selectors = struct {
|
||||
Timeout string
|
||||
Results string
|
||||
ResultsAlt []string
|
||||
AdMarkers []string
|
||||
ImageJSONRoot []string
|
||||
Link string
|
||||
Desc string
|
||||
@@ -17,6 +18,7 @@ var Selectors = struct {
|
||||
Timeout: "button.timeout-button",
|
||||
Results: "#content_left div.result.c-container",
|
||||
ResultsAlt: []string{"#content_left div.result-op.c-container", "div.c-container.new-pmd"},
|
||||
AdMarkers: []string{"[data-tuiguang]", "[data-click*='tuiguang']", ".ec-tuiguang", ".c-icon-bear-p"},
|
||||
ImageJSONRoot: []string{"body > pre", "pre"},
|
||||
Link: "a",
|
||||
Desc: "div.c-abstract",
|
||||
|
||||
@@ -22,6 +22,7 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
|
||||
func parseBingDocument(doc *goquery.Document) []core.SearchResult {
|
||||
var results []core.SearchResult
|
||||
rank := 1
|
||||
adRank := 1
|
||||
|
||||
doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) {
|
||||
titleTag := item.Find(Selectors.Title).First()
|
||||
@@ -69,17 +70,37 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult {
|
||||
}
|
||||
|
||||
results = append(results, core.SearchResult{
|
||||
Rank: -1,
|
||||
Rank: adRank,
|
||||
URL: link,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Ad: true,
|
||||
})
|
||||
adRank++
|
||||
})
|
||||
|
||||
setSeparatedAdAbsoluteRanks(results, 0)
|
||||
return core.DeduplicateResults(results)
|
||||
}
|
||||
|
||||
func setSeparatedAdAbsoluteRanks(results []core.SearchResult, start int) {
|
||||
adCount := 0
|
||||
for i := range results {
|
||||
if results[i].Ad {
|
||||
adCount++
|
||||
results[i].AbsoluteRank = start + results[i].Rank
|
||||
}
|
||||
}
|
||||
organicAbsoluteRank := start + adCount + 1
|
||||
for i := range results {
|
||||
if results[i].Ad {
|
||||
continue
|
||||
}
|
||||
results[i].AbsoluteRank = organicAbsoluteRank
|
||||
organicAbsoluteRank++
|
||||
}
|
||||
}
|
||||
|
||||
// descriptionFromItem extracts a description using the same 4-step fallback
|
||||
// chain as the rod-based browser parser.
|
||||
func descriptionFromItem(item *goquery.Selection, title string) string {
|
||||
|
||||
@@ -138,6 +138,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
|
||||
bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements))
|
||||
|
||||
rank := query.Start
|
||||
adRank := 1
|
||||
for _, result := range organicElements {
|
||||
srchRes := core.SearchResult{}
|
||||
|
||||
@@ -215,35 +216,17 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
|
||||
srchRes.Description, _ = descElem.Text()
|
||||
}
|
||||
|
||||
// Mark ads with negative rank
|
||||
srchRes.Rank = -1
|
||||
srchRes.Rank = adRank
|
||||
adRank++
|
||||
searchResults = append(searchResults, srchRes)
|
||||
}
|
||||
|
||||
setSeparatedAdAbsoluteRanks(searchResults, query.Start)
|
||||
|
||||
// Deduplicate results
|
||||
deduped := core.DeduplicateResults(searchResults)
|
||||
|
||||
// Trim to exact limit if necessary (only organic results, not ads)
|
||||
if query.Limit > 0 {
|
||||
organicResults := []core.SearchResult{}
|
||||
adResults := []core.SearchResult{}
|
||||
|
||||
for _, result := range deduped {
|
||||
if result.Ad {
|
||||
adResults = append(adResults, result)
|
||||
} else {
|
||||
organicResults = append(organicResults, result)
|
||||
}
|
||||
}
|
||||
|
||||
// Trim organic results to limit
|
||||
if len(organicResults) > query.Limit {
|
||||
organicResults = organicResults[:query.Limit]
|
||||
}
|
||||
|
||||
// Combine back: organic results + ads
|
||||
deduped = append(organicResults, adResults...)
|
||||
}
|
||||
deduped = core.LimitOrganicResults(deduped, query.Limit)
|
||||
|
||||
return deduped, nil
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
version = "0.7.10"
|
||||
version = "0.7.11"
|
||||
defaultConfigFilename = "config"
|
||||
envPrefix = "OPENSERP"
|
||||
)
|
||||
|
||||
@@ -95,9 +95,11 @@ func classifyProxyNetworkError(err error) error {
|
||||
|
||||
// SearchResult represents one normalized result item returned by any engine.
|
||||
type SearchResult struct {
|
||||
// Rank is a 1-based position in engine output. Some engines use negative
|
||||
// ranks for non-organic blocks such as ads or instant answers.
|
||||
// Rank is the 1-based position within this result type. For SEO callers,
|
||||
// organic rank must not be shifted by ads.
|
||||
Rank int `json:"rank"`
|
||||
// AbsoluteRank is the 1-based position in the mixed SERP stream.
|
||||
AbsoluteRank int `json:"absolute_rank,omitempty"`
|
||||
// URL is the canonical result URL.
|
||||
URL string `json:"url"`
|
||||
// Title is the result headline shown on the SERP.
|
||||
@@ -118,14 +120,15 @@ func DeduplicateResults(results []SearchResult) []SearchResult {
|
||||
if result.URL == "" {
|
||||
continue
|
||||
}
|
||||
if !unique[result.URL] {
|
||||
unique[result.URL] = true
|
||||
key := resultDedupKey(result)
|
||||
if !unique[key] {
|
||||
unique[key] = true
|
||||
deduped = append(deduped, result)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(deduped, func(i, j int) bool {
|
||||
return deduped[i].Rank < deduped[j].Rank
|
||||
return resultLess(deduped[i], deduped[j])
|
||||
})
|
||||
return deduped
|
||||
}
|
||||
@@ -140,11 +143,79 @@ func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]Search
|
||||
}
|
||||
|
||||
sort.Slice(searchResults, func(i, j int) bool {
|
||||
return searchResults[i].Rank < searchResults[j].Rank
|
||||
return resultLess(searchResults[i], searchResults[j])
|
||||
})
|
||||
return &searchResults
|
||||
}
|
||||
|
||||
// CountOrganicResults returns the number of non-ad results in a mixed SERP.
|
||||
func CountOrganicResults(results []SearchResult) int {
|
||||
count := 0
|
||||
for _, result := range results {
|
||||
if !result.Ad {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// LimitOrganicResults keeps all ads and at most limit non-ad results.
|
||||
func LimitOrganicResults(results []SearchResult, limit int) []SearchResult {
|
||||
if limit <= 0 {
|
||||
return results
|
||||
}
|
||||
out := make([]SearchResult, 0, len(results))
|
||||
organicCount := 0
|
||||
for _, result := range results {
|
||||
if result.Ad {
|
||||
out = append(out, result)
|
||||
continue
|
||||
}
|
||||
if organicCount >= limit {
|
||||
continue
|
||||
}
|
||||
organicCount++
|
||||
out = append(out, result)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func resultDedupKey(result SearchResult) string {
|
||||
resultType := "organic"
|
||||
if result.Ad {
|
||||
resultType = "ad"
|
||||
}
|
||||
return resultType + "\x00" + result.URL
|
||||
}
|
||||
|
||||
func resultLess(left, right SearchResult) bool {
|
||||
leftPos := resultSortPosition(left)
|
||||
rightPos := resultSortPosition(right)
|
||||
if leftPos != rightPos {
|
||||
return leftPos < rightPos
|
||||
}
|
||||
if left.Ad != right.Ad {
|
||||
return left.Ad
|
||||
}
|
||||
if left.Rank != right.Rank {
|
||||
return left.Rank < right.Rank
|
||||
}
|
||||
return left.URL < right.URL
|
||||
}
|
||||
|
||||
func resultSortPosition(result SearchResult) int {
|
||||
if result.AbsoluteRank > 0 {
|
||||
return result.AbsoluteRank
|
||||
}
|
||||
if result.Rank < 0 {
|
||||
return -result.Rank
|
||||
}
|
||||
if result.Rank > 0 {
|
||||
return result.Rank
|
||||
}
|
||||
return int(^uint(0) >> 1)
|
||||
}
|
||||
|
||||
// Query holds request parameters used by HTTP handlers and search engines.
|
||||
// Example minimal query: Query{Text: "golang", Limit: 10}.
|
||||
type Query struct {
|
||||
@@ -168,7 +239,7 @@ type Query struct {
|
||||
// For Google, false includes similar results and true hides them.
|
||||
Filter bool
|
||||
// Answers enables parsing answer modules when supported by the engine.
|
||||
// Such entries may be returned with negative rank values.
|
||||
// Such entries may be returned with non-positive internal rank values.
|
||||
Answers bool
|
||||
// ProxyURL is a direct proxy URL used by raw HTTP search paths.
|
||||
ProxyURL string
|
||||
|
||||
@@ -45,15 +45,9 @@ func EnrichDomainInfo(domain string) *DomainInfo {
|
||||
cfg := loadEnrichmentDomains()
|
||||
|
||||
info := &DomainInfo{
|
||||
TLD: tld,
|
||||
SLD: sld,
|
||||
IsGov: isGovTLD(domain, tld),
|
||||
IsEdu: isEduTLD(domain, tld),
|
||||
IsMil: isMilTLD(tld),
|
||||
IsNews: cfg.NewsDomains[domain],
|
||||
IsForum: cfg.ForumDomains[domain],
|
||||
IsMarketplace: cfg.MarketplaceDomains[domain],
|
||||
IsSocial: cfg.SocialDomains[domain],
|
||||
TLD: tld,
|
||||
SLD: sld,
|
||||
Category: domainCategory(domain, tld, cfg),
|
||||
}
|
||||
return info
|
||||
}
|
||||
@@ -67,6 +61,9 @@ func ClassifyURL(rawURL, domain string) *Classification {
|
||||
|
||||
contentType := classifyContentType(rawURL)
|
||||
sourceHint := classifySourceHint(domain)
|
||||
if contentType == "webpage" && sourceHint == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &Classification{
|
||||
ContentType: contentType,
|
||||
@@ -74,6 +71,27 @@ func ClassifyURL(rawURL, domain string) *Classification {
|
||||
}
|
||||
}
|
||||
|
||||
func domainCategory(domain, tld string, cfg enrichmentDomainsConfig) string {
|
||||
switch {
|
||||
case isGovTLD(domain, tld):
|
||||
return "gov"
|
||||
case isEduTLD(domain, tld):
|
||||
return "edu"
|
||||
case isMilTLD(tld):
|
||||
return "mil"
|
||||
case cfg.NewsDomains[domain]:
|
||||
return "news"
|
||||
case cfg.ForumDomains[domain]:
|
||||
return "forum"
|
||||
case cfg.MarketplaceDomains[domain]:
|
||||
return "marketplace"
|
||||
case cfg.SocialDomains[domain]:
|
||||
return "social"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// splitDomain returns (public suffix, registrable domain label).
|
||||
func splitDomain(domain string) (tld, sld string) {
|
||||
domain = normalizeDomain(domain)
|
||||
|
||||
@@ -22,9 +22,6 @@ func RenderMarkdown(env *Envelope) []byte {
|
||||
for i, r := range env.Results {
|
||||
fmt.Fprintf(&b, "## %d. %s\n\n", i+1, escapeMarkdown(r.Title))
|
||||
typeLabel := string(r.Type)
|
||||
if r.IsAd {
|
||||
typeLabel = "ad"
|
||||
}
|
||||
fmt.Fprintf(&b, "**%s** · %s\n\n", r.DisplayURL, typeLabel)
|
||||
if r.Snippet != "" {
|
||||
fmt.Fprintf(&b, "%s\n\n", r.Snippet)
|
||||
|
||||
@@ -11,13 +11,13 @@ type QueryEcho struct {
|
||||
|
||||
// ResponseMeta carries request-level metadata for observability and debugging.
|
||||
type ResponseMeta struct {
|
||||
RequestID string `json:"request_id"`
|
||||
RequestedAt string `json:"requested_at"`
|
||||
TookMs int64 `json:"took_ms"`
|
||||
EnginesResponded []string `json:"engines_responded,omitempty"`
|
||||
EnginesFailed []string `json:"engines_failed"`
|
||||
EngineErrors []EngineErrorDetail `json:"engine_errors,omitempty"`
|
||||
Version string `json:"version"`
|
||||
RequestID string `json:"request_id"`
|
||||
RequestedAt string `json:"requested_at"`
|
||||
TookMs int64 `json:"took_ms"`
|
||||
EnginesResponded []string `json:"engines_responded,omitempty"`
|
||||
EnginesFailed []string `json:"engines_failed"`
|
||||
EngineErrors []EngineErrorDetail `json:"engine_errors,omitempty"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// EngineErrorDetail is a client-facing, sanitized per-engine failure summary.
|
||||
@@ -34,7 +34,7 @@ type Pagination struct {
|
||||
NextStart int `json:"next_start"`
|
||||
}
|
||||
|
||||
// Envelope is the top-level v1 response wrapper for all search endpoints.
|
||||
// Envelope is the top-level v2 response wrapper for all search endpoints.
|
||||
type Envelope struct {
|
||||
Query QueryEcho `json:"query"`
|
||||
Meta ResponseMeta `json:"meta"`
|
||||
@@ -64,7 +64,7 @@ type ClusterOccurrence struct {
|
||||
ResultID string `json:"result_id"`
|
||||
}
|
||||
|
||||
// ImageEnvelope is the top-level v1 response wrapper for image search endpoints.
|
||||
// ImageEnvelope is the top-level v2 response wrapper for image search endpoints.
|
||||
type ImageEnvelope struct {
|
||||
Query QueryEcho `json:"query"`
|
||||
Meta ResponseMeta `json:"meta"`
|
||||
@@ -72,7 +72,7 @@ type ImageEnvelope struct {
|
||||
Pagination Pagination `json:"pagination"`
|
||||
}
|
||||
|
||||
const apiVersion = "1.0"
|
||||
const apiVersion = "2.0"
|
||||
|
||||
// NewEnvelope builds a fresh Envelope pre-filled with query echo and an open
|
||||
// meta block. Call Finalize before serializing.
|
||||
@@ -124,11 +124,21 @@ func (e *Envelope) Finalize(startedAt time.Time, q Query) {
|
||||
page := q.Start/limit + 1
|
||||
e.Pagination = Pagination{
|
||||
Page: page,
|
||||
HasMore: len(e.Results) >= limit,
|
||||
HasMore: countNonAdResults(e.Results) >= limit,
|
||||
NextStart: q.Start + limit,
|
||||
}
|
||||
}
|
||||
|
||||
func countNonAdResults(results []Result) int {
|
||||
count := 0
|
||||
for _, result := range results {
|
||||
if result.Type != ResultTypeAd {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// Finalize stamps the elapsed time and computes pagination fields.
|
||||
func (e *ImageEnvelope) Finalize(startedAt time.Time, q Query) {
|
||||
e.Meta.TookMs = time.Since(startedAt).Milliseconds()
|
||||
|
||||
@@ -24,7 +24,7 @@ type EnrichContext struct {
|
||||
Query Query
|
||||
}
|
||||
|
||||
// EnrichResult converts a raw engine result into the v1 Result shape.
|
||||
// EnrichResult converts a raw engine result into the v2 Result shape.
|
||||
func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
|
||||
normalizedURL := normalizeURL(raw.URL)
|
||||
domain := extractDomain(normalizedURL)
|
||||
@@ -42,17 +42,23 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
|
||||
if raw.Rank <= 0 && !raw.Ad {
|
||||
resultType = ResultTypeAnswerBox
|
||||
}
|
||||
|
||||
limit := ctx.Query.Limit
|
||||
if limit <= 0 {
|
||||
limit = 25
|
||||
rank := raw.Rank
|
||||
if rank < 0 {
|
||||
if raw.Ad {
|
||||
rank = -rank
|
||||
} else {
|
||||
rank = 0
|
||||
}
|
||||
}
|
||||
|
||||
absolute := computeResultPosition(raw, ctx.Query.Start)
|
||||
if absolute <= 0 {
|
||||
absolute = rank
|
||||
}
|
||||
page := ctx.Query.Start/limit + 1
|
||||
absolute, onPage := computeResultPosition(raw.Rank, ctx.Query.Start)
|
||||
|
||||
result := Result{
|
||||
ID: buildResultID(ctx.Engine, normalizedURL),
|
||||
Rank: raw.Rank,
|
||||
Rank: rank,
|
||||
Type: resultType,
|
||||
Title: raw.Title,
|
||||
URL: normalizedURL,
|
||||
@@ -60,13 +66,10 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
|
||||
Snippet: raw.Description,
|
||||
Domain: domain,
|
||||
Favicon: favicon,
|
||||
IsAd: raw.Ad,
|
||||
Position: Position{
|
||||
Absolute: absolute,
|
||||
Page: page,
|
||||
OnPage: onPage,
|
||||
},
|
||||
Engine: ctx.Engine,
|
||||
Engine: ctx.Engine,
|
||||
}
|
||||
if absolute > 0 {
|
||||
result.Position = &Position{Absolute: absolute}
|
||||
}
|
||||
|
||||
result.DomainInfo = EnrichDomainInfo(domain)
|
||||
@@ -75,7 +78,7 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
|
||||
return result
|
||||
}
|
||||
|
||||
// EnrichImageResult converts a raw engine result into the v1 ImageResult shape.
|
||||
// EnrichImageResult converts a raw engine result into the v2 ImageResult shape.
|
||||
func EnrichImageResult(raw SearchResult, ctx EnrichContext) ImageResult {
|
||||
imageURL := normalizeURL(raw.URL)
|
||||
meta := parseImageDescription(raw.Description)
|
||||
@@ -122,14 +125,22 @@ func shortMD5(value string) string {
|
||||
return hex.EncodeToString(h[:responseIDBytes])
|
||||
}
|
||||
|
||||
func computeResultPosition(rank, start int) (absolute, onPage int) {
|
||||
if rank <= 0 {
|
||||
return 0, 0
|
||||
func computeResultPosition(raw SearchResult, start int) int {
|
||||
if raw.AbsoluteRank > 0 {
|
||||
return raw.AbsoluteRank
|
||||
}
|
||||
|
||||
rank := raw.Rank
|
||||
if rank < 0 {
|
||||
rank = -rank
|
||||
}
|
||||
if rank == 0 {
|
||||
return 0
|
||||
}
|
||||
if start > 0 && rank > start {
|
||||
return rank, rank - start
|
||||
return rank
|
||||
}
|
||||
return start + rank, rank
|
||||
return start + rank
|
||||
}
|
||||
|
||||
// normalizeURL lowercases scheme+host, strips trailing slash, and removes
|
||||
|
||||
@@ -19,34 +19,29 @@ const (
|
||||
|
||||
// Position describes where a result sits in the overall result stream.
|
||||
type Position struct {
|
||||
// Absolute is the 1-based rank counting from the first result of the first page.
|
||||
// Absolute is the 1-based rank counting from the first result of the first page,
|
||||
// across both organic and ad blocks. Always emitted so SEO callers can plot
|
||||
// rank vs. on-page position without inferring it from the result order.
|
||||
Absolute int `json:"absolute"`
|
||||
// Page is the 1-based page number derived from start/limit.
|
||||
Page int `json:"page"`
|
||||
// OnPage is the 1-based rank within this page.
|
||||
OnPage int `json:"on_page"`
|
||||
}
|
||||
|
||||
// DomainInfo carries TLD-derived category signals for a result domain.
|
||||
type DomainInfo struct {
|
||||
TLD string `json:"tld"`
|
||||
SLD string `json:"sld"`
|
||||
IsGov bool `json:"is_gov"`
|
||||
IsEdu bool `json:"is_edu"`
|
||||
IsMil bool `json:"is_mil"`
|
||||
IsNews bool `json:"is_news"`
|
||||
IsForum bool `json:"is_forum"`
|
||||
IsMarketplace bool `json:"is_marketplace"`
|
||||
IsSocial bool `json:"is_social"`
|
||||
TLD string `json:"tld,omitempty"`
|
||||
SLD string `json:"sld,omitempty"`
|
||||
// Category is one of "gov", "edu", "mil", "news", "forum", "marketplace",
|
||||
// "social", or "" when the domain does not match any known category.
|
||||
Category string `json:"category"`
|
||||
}
|
||||
|
||||
// Classification holds URL-path heuristic hints for downstream consumers.
|
||||
type Classification struct {
|
||||
ContentType string `json:"content_type"`
|
||||
SourceHint string `json:"source_hint"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
SourceHint string `json:"source_hint,omitempty"`
|
||||
}
|
||||
|
||||
// Result is the v1 normalized result returned in every search response.
|
||||
// Result is the v2 normalized result returned in search responses. Optional
|
||||
// fields (Position, DomainInfo, Classification) are omitted when empty.
|
||||
type Result struct {
|
||||
ID string `json:"id"`
|
||||
Rank int `json:"rank"`
|
||||
@@ -57,8 +52,7 @@ type Result struct {
|
||||
Snippet string `json:"snippet"`
|
||||
Domain string `json:"domain"`
|
||||
Favicon string `json:"favicon"`
|
||||
IsAd bool `json:"is_ad"`
|
||||
Position Position `json:"position"`
|
||||
Position *Position `json:"position,omitempty"`
|
||||
Engine string `json:"engine"`
|
||||
DomainInfo *DomainInfo `json:"domain_info,omitempty"`
|
||||
Classification *Classification `json:"classification,omitempty"`
|
||||
@@ -78,7 +72,7 @@ type ImageSource struct {
|
||||
Domain string `json:"domain"`
|
||||
}
|
||||
|
||||
// ImageResult is the v1 shape for image search results.
|
||||
// ImageResult is the v2 shape for image search results.
|
||||
type ImageResult struct {
|
||||
ID string `json:"id"`
|
||||
Rank int `json:"rank"`
|
||||
|
||||
104
core/result_rank_test.go
Normal file
104
core/result_rank_test.go
Normal file
@@ -0,0 +1,104 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDeduplicateResultsOrdersByAbsoluteRank(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
results := DeduplicateResults([]SearchResult{
|
||||
{Rank: 1, AbsoluteRank: 3, URL: "https://organic.example.com/one"},
|
||||
{Rank: 1, AbsoluteRank: 1, URL: "https://ads.example.com/one", Ad: true},
|
||||
{Rank: 2, AbsoluteRank: 2, URL: "https://ads.example.com/two", Ad: true},
|
||||
{Rank: 2, AbsoluteRank: 4, URL: "https://organic.example.com/two"},
|
||||
})
|
||||
|
||||
wantRanks := []int{1, 2, 1, 2}
|
||||
wantAbsoluteRanks := []int{1, 2, 3, 4}
|
||||
if len(results) != len(wantRanks) {
|
||||
t.Fatalf("len(results) = %d, want %d", len(results), len(wantRanks))
|
||||
}
|
||||
for i, want := range wantRanks {
|
||||
if results[i].Rank != want {
|
||||
t.Fatalf("result %d rank = %d, want %d", i, results[i].Rank, want)
|
||||
}
|
||||
if results[i].AbsoluteRank != wantAbsoluteRanks[i] {
|
||||
t.Fatalf("result %d absolute rank = %d, want %d", i, results[i].AbsoluteRank, wantAbsoluteRanks[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeduplicateResultsKeepsAdAndOrganicForSameURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
results := DeduplicateResults([]SearchResult{
|
||||
{Rank: 1, AbsoluteRank: 1, URL: "https://example.com/page", Ad: true},
|
||||
{Rank: 1, AbsoluteRank: 2, URL: "https://example.com/page"},
|
||||
})
|
||||
|
||||
if len(results) != 2 {
|
||||
t.Fatalf("len(results) = %d, want 2", len(results))
|
||||
}
|
||||
if !results[0].Ad || results[1].Ad {
|
||||
t.Fatalf("expected ad and organic rows to be preserved separately: %+v", results)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLimitOrganicResultsDoesNotCountAds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
results := LimitOrganicResults([]SearchResult{
|
||||
{Rank: 1, URL: "https://ads.example.com/one", Ad: true},
|
||||
{Rank: 1, URL: "https://organic.example.com/one"},
|
||||
{Rank: 2, URL: "https://ads.example.com/two", Ad: true},
|
||||
{Rank: 2, URL: "https://organic.example.com/two"},
|
||||
}, 1)
|
||||
|
||||
if len(results) != 3 {
|
||||
t.Fatalf("len(results) = %d, want 3", len(results))
|
||||
}
|
||||
if CountOrganicResults(results) != 1 {
|
||||
t.Fatalf("organic count = %d, want 1", CountOrganicResults(results))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichResultUsesAdRankAndAbsolutePosition(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
result := EnrichResult(SearchResult{
|
||||
Rank: 1,
|
||||
AbsoluteRank: 2,
|
||||
URL: "https://ads.example.com/",
|
||||
Title: "Ad",
|
||||
Ad: true,
|
||||
}, EnrichContext{Engine: "google", Query: Query{Limit: 10}})
|
||||
|
||||
if result.Rank != 1 {
|
||||
t.Fatalf("rank = %d, want 1", result.Rank)
|
||||
}
|
||||
if result.Position == nil || result.Position.Absolute != 2 {
|
||||
t.Fatalf("unexpected position: %+v", result.Position)
|
||||
}
|
||||
if result.Type != ResultTypeAd {
|
||||
t.Fatalf("unexpected result type: %q", result.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvelopePaginationCountsOrganicResults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := NewEnvelope(Query{Text: "ads", Limit: 2}, "req-1", time.Now(), []string{"bing"})
|
||||
env.Results = []Result{
|
||||
{Rank: 1, Type: ResultTypeAd},
|
||||
{Rank: 2, Type: ResultTypeAd},
|
||||
{Rank: 1, Type: ResultTypeOrganic},
|
||||
}
|
||||
|
||||
env.Finalize(time.Now(), Query{Text: "ads", Limit: 2})
|
||||
|
||||
if env.Pagination.HasMore {
|
||||
t.Fatalf("has_more = true, want false when organic count is below limit: %+v", env.Pagination)
|
||||
}
|
||||
}
|
||||
@@ -1198,10 +1198,11 @@ func (s *Server) deduplicateMegaResults(results []MegaSearchResult) []MegaSearch
|
||||
if result.URL == "" {
|
||||
continue
|
||||
}
|
||||
key := NormalizeURLForClustering(result.URL)
|
||||
if key == "" {
|
||||
normalizedURL := NormalizeURLForClustering(result.URL)
|
||||
if normalizedURL == "" {
|
||||
continue
|
||||
}
|
||||
key := resultDedupKey(SearchResult{URL: normalizedURL, Ad: result.Ad})
|
||||
existing, exists := urlMap[key]
|
||||
if !exists {
|
||||
urlMap[key] = result
|
||||
@@ -1219,8 +1220,11 @@ func (s *Server) deduplicateMegaResults(results []MegaSearchResult) []MegaSearch
|
||||
}
|
||||
|
||||
sort.Slice(deduped, func(i, j int) bool {
|
||||
if deduped[i].Rank != deduped[j].Rank {
|
||||
return deduped[i].Rank < deduped[j].Rank
|
||||
if resultLess(deduped[i].SearchResult, deduped[j].SearchResult) {
|
||||
return true
|
||||
}
|
||||
if resultLess(deduped[j].SearchResult, deduped[i].SearchResult) {
|
||||
return false
|
||||
}
|
||||
if deduped[i].Engine != deduped[j].Engine {
|
||||
return deduped[i].Engine < deduped[j].Engine
|
||||
|
||||
@@ -2208,7 +2208,7 @@ func TestServerOptions_DisableCORSMiddleware(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedicatedEndpointReturnsV1Envelope(t *testing.T) {
|
||||
func TestDedicatedEndpointReturnsV2Envelope(t *testing.T) {
|
||||
engine := &engineMock{name: "google", initialized: true}
|
||||
opts := DefaultServerOptions()
|
||||
opts.Resilience.Retry.MaxRetries = 0
|
||||
@@ -2223,8 +2223,8 @@ func TestDedicatedEndpointReturnsV1Envelope(t *testing.T) {
|
||||
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
|
||||
t.Fatalf("decode envelope: %v", err)
|
||||
}
|
||||
if env.Meta.Version != "1.0" {
|
||||
t.Fatalf("expected meta.version=1.0, got %q", env.Meta.Version)
|
||||
if env.Meta.Version != "2.0" {
|
||||
t.Fatalf("expected meta.version=2.0, got %q", env.Meta.Version)
|
||||
}
|
||||
if env.Meta.RequestID == "" {
|
||||
t.Fatal("expected non-empty meta.request_id")
|
||||
@@ -2248,15 +2248,12 @@ func TestDedicatedEndpointReturnsV1Envelope(t *testing.T) {
|
||||
if r.Snippet == "" && r.Title == "" {
|
||||
t.Fatal("expected result to have title or snippet")
|
||||
}
|
||||
if r.Position.Page < 1 {
|
||||
t.Fatalf("expected position.page >= 1, got %d", r.Position.Page)
|
||||
}
|
||||
if env.Pagination.Page < 1 {
|
||||
t.Fatalf("expected pagination.page >= 1, got %d", env.Pagination.Page)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMegaSearchReturnsV1EnvelopeWithEnginesFailed(t *testing.T) {
|
||||
func TestMegaSearchReturnsV2EnvelopeWithEnginesFailed(t *testing.T) {
|
||||
good := &engineMock{name: "google", initialized: true}
|
||||
bad := &engineMock{
|
||||
name: "bing",
|
||||
@@ -2279,8 +2276,8 @@ func TestMegaSearchReturnsV1EnvelopeWithEnginesFailed(t *testing.T) {
|
||||
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
|
||||
t.Fatalf("decode envelope: %v", err)
|
||||
}
|
||||
if env.Meta.Version != "1.0" {
|
||||
t.Fatalf("expected meta.version=1.0, got %q", env.Meta.Version)
|
||||
if env.Meta.Version != "2.0" {
|
||||
t.Fatalf("expected meta.version=2.0, got %q", env.Meta.Version)
|
||||
}
|
||||
if len(env.Meta.EnginesFailed) == 0 {
|
||||
t.Fatal("expected engines_failed to contain bing")
|
||||
@@ -2446,6 +2443,49 @@ func TestFormatParamReturnsCorrectContentType(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONOmitsRedundantResultFields(t *testing.T) {
|
||||
engine := &engineMock{
|
||||
name: "google",
|
||||
initialized: true,
|
||||
searchFn: func(_ context.Context, _ Query) ([]SearchResult, error) {
|
||||
return []SearchResult{{
|
||||
Rank: 1,
|
||||
AbsoluteRank: 2,
|
||||
URL: "https://example.gov/page",
|
||||
Title: "Gov Result",
|
||||
Description: "Snippet",
|
||||
}}, nil
|
||||
},
|
||||
}
|
||||
opts := DefaultServerOptions()
|
||||
opts.Resilience.Retry.MaxRetries = 0
|
||||
srv := NewServerWithOptions("127.0.0.1", 7210, opts, engine)
|
||||
|
||||
resp := request(t, srv, "/google/search?text=clean-json")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", resp.StatusCode)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
payload := string(body)
|
||||
for _, omitted := range []string{`"is_ad"`, `"on_page"`, `"is_gov"`} {
|
||||
if strings.Contains(payload, omitted) {
|
||||
t.Fatalf("json payload should not contain %s: %s", omitted, payload)
|
||||
}
|
||||
}
|
||||
if strings.Contains(payload, `"position":{"absolute":2,"page"`) {
|
||||
t.Fatalf("json payload should not contain per-result page: %s", payload)
|
||||
}
|
||||
if !strings.Contains(payload, `"type":"organic"`) {
|
||||
t.Fatalf("json payload should keep result type: %s", payload)
|
||||
}
|
||||
if !strings.Contains(payload, `"category":"gov"`) {
|
||||
t.Fatalf("json payload should collapse domain category: %s", payload)
|
||||
}
|
||||
if !strings.Contains(payload, `"absolute":2`) {
|
||||
t.Fatalf("json payload should keep meaningful absolute position: %s", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatParamBypassesJSONCache(t *testing.T) {
|
||||
engine := &engineMock{name: "google", initialized: true}
|
||||
opts := DefaultServerOptions()
|
||||
@@ -2531,9 +2571,8 @@ func TestPaginatedPositionUsesAbsoluteRank(t *testing.T) {
|
||||
if len(env.Results) != 1 {
|
||||
t.Fatalf("expected one result, got %d", len(env.Results))
|
||||
}
|
||||
pos := env.Results[0].Position
|
||||
if pos.Absolute != 11 || pos.OnPage != 1 || pos.Page != 2 {
|
||||
t.Fatalf("unexpected position: %+v", pos)
|
||||
if env.Results[0].Position == nil || env.Results[0].Position.Absolute != 11 {
|
||||
t.Fatalf("expected absolute position 11, got %+v", env.Results[0].Position)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -83,7 +83,7 @@ Engine parsers return the older internal shape:
|
||||
- `Description`
|
||||
- `Ad`
|
||||
|
||||
HTTP handlers convert this into the public v1 response through `core/response_builder.go`.
|
||||
HTTP handlers convert this into the public v2 response through `core/response_builder.go`.
|
||||
|
||||
## HTTP Request Flow
|
||||
|
||||
@@ -118,7 +118,7 @@ HTTP request
|
||||
|
||||
## Public API Response
|
||||
|
||||
JSON endpoints return a v1 envelope.
|
||||
JSON endpoints return a v2 envelope.
|
||||
|
||||
Top-level fields:
|
||||
|
||||
@@ -192,7 +192,7 @@ Only JSON responses use the response cache. Cached JSON refreshes request-scoped
|
||||
|
||||
`core/enrichment_domain.go` derives:
|
||||
|
||||
- `domain_info`: public suffix, SLD, and category booleans
|
||||
- `domain_info`: public suffix, SLD, and collapsed category
|
||||
- `classification`: content type and known source hint
|
||||
|
||||
Public suffix parsing uses `golang.org/x/net/publicsuffix`.
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: OpenSERP API
|
||||
version: 1.0.0
|
||||
version: 2.0.0
|
||||
description: >
|
||||
OpenSERP provides dedicated and multi-engine search endpoints for Google, Yandex,
|
||||
Baidu, Bing, and DuckDuckGo. All responses are wrapped in a v1 envelope with
|
||||
Baidu, Bing, and DuckDuckGo. Search responses are wrapped in a v2 envelope with
|
||||
query echo, metadata, normalized results, and pagination. Invalid client input
|
||||
returns 400 with a machine-readable `reason` code.
|
||||
license:
|
||||
@@ -90,7 +90,7 @@ paths:
|
||||
requested_at: "2026-04-24T12:00:00Z"
|
||||
took_ms: 842
|
||||
engines_failed: []
|
||||
version: "1.0"
|
||||
version: "2.0"
|
||||
results:
|
||||
- id: s_a1b2c3d4e5f6a1b2
|
||||
rank: 1
|
||||
@@ -101,25 +101,13 @@ paths:
|
||||
snippet: Go is an open source programming language...
|
||||
domain: go.dev
|
||||
favicon: https://go.dev/favicon.ico
|
||||
is_ad: false
|
||||
position:
|
||||
absolute: 1
|
||||
page: 1
|
||||
on_page: 1
|
||||
engine: google
|
||||
domain_info:
|
||||
tld: dev
|
||||
sld: go
|
||||
is_gov: false
|
||||
is_edu: false
|
||||
is_mil: false
|
||||
is_news: false
|
||||
is_forum: false
|
||||
is_marketplace: false
|
||||
is_social: false
|
||||
classification:
|
||||
content_type: webpage
|
||||
source_hint: ""
|
||||
category: ""
|
||||
pagination:
|
||||
page: 1
|
||||
has_more: true
|
||||
@@ -612,7 +600,7 @@ components:
|
||||
name: limit
|
||||
in: query
|
||||
required: false
|
||||
description: Maximum results to return (1–100).
|
||||
description: Maximum organic results to return (1–100). Ads may be returned in addition.
|
||||
schema:
|
||||
type: integer
|
||||
minimum: 1
|
||||
@@ -690,7 +678,7 @@ components:
|
||||
in: query
|
||||
required: false
|
||||
description: >
|
||||
Output format. `json` (default) returns the full envelope. `markdown` returns a
|
||||
Output format. `json` (default) returns the envelope. `markdown` returns a
|
||||
Markdown document suitable for Slack/email. `text` returns a minimal plain-text
|
||||
block optimised for LLM context windows. `ndjson` returns one result object per
|
||||
line with no envelope. The `Accept` header is also checked
|
||||
@@ -993,7 +981,7 @@ components:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ErrorResponse"
|
||||
schemas:
|
||||
# ── v1 envelope ──────────────────────────────────────────────────
|
||||
# ── v2 envelope ──────────────────────────────────────────────────
|
||||
QueryEcho:
|
||||
type: object
|
||||
required: [text, engines_requested]
|
||||
@@ -1035,7 +1023,7 @@ components:
|
||||
$ref: "#/components/schemas/EngineErrorDetail"
|
||||
version:
|
||||
type: string
|
||||
example: "1.0"
|
||||
example: "2.0"
|
||||
EngineErrorDetail:
|
||||
type: object
|
||||
required: [engine, error]
|
||||
@@ -1067,21 +1055,18 @@ components:
|
||||
# ── Result ───────────────────────────────────────────────────────
|
||||
Position:
|
||||
type: object
|
||||
required: [absolute, page, on_page]
|
||||
required: [absolute]
|
||||
properties:
|
||||
absolute:
|
||||
type: integer
|
||||
description: 1-based rank counting from the first result of the first page.
|
||||
example: 1
|
||||
page:
|
||||
type: integer
|
||||
example: 1
|
||||
on_page:
|
||||
type: integer
|
||||
example: 1
|
||||
description: >
|
||||
1-based rank in the mixed SERP stream, across both organic and ad blocks.
|
||||
Always present so SEO callers can plot rank vs. on-page position without
|
||||
inferring it from result order.
|
||||
example: 2
|
||||
DomainInfo:
|
||||
type: object
|
||||
required: [tld, sld, is_gov, is_edu, is_mil, is_news, is_forum, is_marketplace, is_social]
|
||||
required: [category]
|
||||
properties:
|
||||
tld:
|
||||
type: string
|
||||
@@ -1089,23 +1074,12 @@ components:
|
||||
sld:
|
||||
type: string
|
||||
example: wikipedia
|
||||
is_gov:
|
||||
type: boolean
|
||||
is_edu:
|
||||
type: boolean
|
||||
is_mil:
|
||||
type: boolean
|
||||
is_news:
|
||||
type: boolean
|
||||
is_forum:
|
||||
type: boolean
|
||||
is_marketplace:
|
||||
type: boolean
|
||||
is_social:
|
||||
type: boolean
|
||||
category:
|
||||
type: string
|
||||
description: Empty string when the domain matches no known category.
|
||||
enum: ["", gov, edu, mil, news, forum, marketplace, social]
|
||||
Classification:
|
||||
type: object
|
||||
required: [content_type, source_hint]
|
||||
properties:
|
||||
content_type:
|
||||
type: string
|
||||
@@ -1117,8 +1091,8 @@ components:
|
||||
ResultType:
|
||||
type: string
|
||||
description: >
|
||||
SERP block type. New values require a minor version bump (`meta.version: "1.1"`).
|
||||
In v1.0, engines emit `organic`, `ad`, or `answer_box`; other enum values are
|
||||
SERP block type. New values require a minor version bump (`meta.version: "2.1"`).
|
||||
In v2.0, engines emit `organic`, `ad`, or `answer_box`; other enum values are
|
||||
reserved for future parser upgrades.
|
||||
enum:
|
||||
- organic
|
||||
@@ -1144,7 +1118,6 @@ components:
|
||||
- snippet
|
||||
- domain
|
||||
- favicon
|
||||
- is_ad
|
||||
- position
|
||||
- engine
|
||||
properties:
|
||||
@@ -1181,9 +1154,6 @@ components:
|
||||
type: string
|
||||
description: Constructed as `https://{domain}/favicon.ico`. Not probed.
|
||||
example: https://go.dev/favicon.ico
|
||||
is_ad:
|
||||
type: boolean
|
||||
example: false
|
||||
position:
|
||||
$ref: "#/components/schemas/Position"
|
||||
engine:
|
||||
|
||||
@@ -21,6 +21,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
|
||||
func parseDDGDocument(doc *goquery.Document) []core.SearchResult {
|
||||
var results []core.SearchResult
|
||||
rank := 1
|
||||
adRank := 1
|
||||
absoluteRank := 1
|
||||
|
||||
resultSel := firstMatchingSelector(doc, Selectors.Results)
|
||||
if resultSel == "" {
|
||||
@@ -40,32 +42,38 @@ func parseDDGDocument(doc *goquery.Document) []core.SearchResult {
|
||||
|
||||
desc := extractFirstText(item, Selectors.Desc)
|
||||
|
||||
isAd := false
|
||||
for _, sel := range Selectors.AdBadge {
|
||||
if item.Find(sel).Length() > 0 {
|
||||
isAd = true
|
||||
break
|
||||
}
|
||||
}
|
||||
isAd := duckduckgoSelectionHasAdMarker(item)
|
||||
|
||||
r := core.SearchResult{
|
||||
Rank: rank,
|
||||
URL: href,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Ad: isAd,
|
||||
Rank: rank,
|
||||
AbsoluteRank: absoluteRank,
|
||||
URL: href,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Ad: isAd,
|
||||
}
|
||||
if !isAd {
|
||||
rank++
|
||||
} else {
|
||||
r.Rank = -1
|
||||
r.Rank = adRank
|
||||
adRank++
|
||||
}
|
||||
results = append(results, r)
|
||||
absoluteRank++
|
||||
})
|
||||
|
||||
return core.DeduplicateResults(results)
|
||||
}
|
||||
|
||||
func duckduckgoSelectionHasAdMarker(item *goquery.Selection) bool {
|
||||
for _, sel := range Selectors.AdBadge {
|
||||
if item.Is(sel) || item.Find(sel).Length() > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// firstMatchingSelector returns the first selector from the list that matches
|
||||
// at least one element in the document.
|
||||
func firstMatchingSelector(doc *goquery.Document, selectors []string) string {
|
||||
|
||||
@@ -55,3 +55,51 @@ func TestParseDDGHTMLEmpty(t *testing.T) {
|
||||
t.Fatalf("expected zero results for empty HTML, got %d", len(results))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDDGHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
html := `
|
||||
<article data-testid="ad">
|
||||
<h2><a data-testid="result-title-a" href="https://ads.example.com">Sponsored Result</a></h2>
|
||||
<div data-result="snippet">Paid snippet</div>
|
||||
</article>
|
||||
<article data-testid="result">
|
||||
<h2><a data-testid="result-title-a" href="https://organic.example.com/one">Organic One</a></h2>
|
||||
<div data-result="snippet">Organic snippet one</div>
|
||||
</article>
|
||||
<article data-testid="result">
|
||||
<h2><a data-testid="result-title-a" href="https://organic.example.com/two">Organic Two</a></h2>
|
||||
<div data-result="snippet">Organic snippet two</div>
|
||||
</article>`
|
||||
|
||||
results, err := ParseHTML(bytes.NewReader([]byte(html)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseHTML() error = %v", err)
|
||||
}
|
||||
if len(results) != 3 {
|
||||
t.Fatalf("expected 3 results, got %d", len(results))
|
||||
}
|
||||
|
||||
organicRank := 0
|
||||
adRank := 0
|
||||
for _, r := range results {
|
||||
if r.Ad {
|
||||
adRank++
|
||||
if r.Rank != adRank {
|
||||
t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
|
||||
}
|
||||
continue
|
||||
}
|
||||
organicRank++
|
||||
if r.Rank != organicRank {
|
||||
t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
|
||||
}
|
||||
}
|
||||
if organicRank != 2 {
|
||||
t.Fatalf("organic count = %d, want 2", organicRank)
|
||||
}
|
||||
if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
|
||||
t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,8 +69,11 @@ func (ddg *DuckDuckGo) isNoResults(page *rod.Page) bool {
|
||||
|
||||
func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.SearchResult {
|
||||
searchResults := []core.SearchResult{}
|
||||
organicRank := pageNum * 10
|
||||
adRank := 1
|
||||
absoluteRank := pageNum*10 + 1
|
||||
|
||||
for i, r := range results {
|
||||
for _, r := range results {
|
||||
// Get URL - try multiple selectors
|
||||
var link *rod.Element
|
||||
var err error
|
||||
@@ -113,28 +116,47 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se
|
||||
desc := core.FirstNonEmptyText(r, Selectors.Desc...)
|
||||
|
||||
// Check if it's an ad
|
||||
isAd := false
|
||||
for _, selector := range Selectors.AdBadge {
|
||||
adIndicator, err := r.Element(selector)
|
||||
if err == nil && adIndicator != nil {
|
||||
isAd = true
|
||||
break
|
||||
}
|
||||
isAd := duckduckgoElementHasAdMarker(r)
|
||||
resultRank := 0
|
||||
if isAd {
|
||||
resultRank = adRank
|
||||
adRank++
|
||||
} else {
|
||||
organicRank++
|
||||
resultRank = organicRank
|
||||
}
|
||||
|
||||
result := core.SearchResult{
|
||||
Rank: (pageNum * 10) + (i + 1),
|
||||
URL: hrefStr,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Ad: isAd,
|
||||
Rank: resultRank,
|
||||
AbsoluteRank: absoluteRank,
|
||||
URL: hrefStr,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Ad: isAd,
|
||||
}
|
||||
searchResults = append(searchResults, result)
|
||||
absoluteRank++
|
||||
}
|
||||
|
||||
return searchResults
|
||||
}
|
||||
|
||||
func duckduckgoElementHasAdMarker(el *rod.Element) bool {
|
||||
if el == nil {
|
||||
return false
|
||||
}
|
||||
for _, selector := range Selectors.AdBadge {
|
||||
matches, err := el.Matches(selector)
|
||||
if err == nil && matches {
|
||||
return true
|
||||
}
|
||||
if adIndicator, err := el.Element(selector); err == nil && adIndicator != nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Search executes a DuckDuckGo web search and returns normalized search
|
||||
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
|
||||
func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
|
||||
@@ -193,13 +215,13 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []
|
||||
return false, nil
|
||||
}
|
||||
|
||||
for len(allResults) < query.Limit {
|
||||
for query.Limit <= 0 || core.CountOrganicResults(allResults) < query.Limit {
|
||||
done, err := fetchPage()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
searchPage++
|
||||
if done || len(allResults) >= query.Limit {
|
||||
if done || (query.Limit > 0 && core.CountOrganicResults(allResults) >= query.Limit) {
|
||||
break
|
||||
}
|
||||
if err := core.SleepContext(ctx, ddg.pageSleep); err != nil {
|
||||
@@ -211,9 +233,7 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []
|
||||
deduped := core.DeduplicateResults(allResults)
|
||||
|
||||
// Trim to exact limit if necessary
|
||||
if len(deduped) > query.Limit {
|
||||
deduped = deduped[:query.Limit]
|
||||
}
|
||||
deduped = core.LimitOrganicResults(deduped, query.Limit)
|
||||
|
||||
ddg.logger.Info("Search completed: %d results", len(deduped))
|
||||
return deduped, nil
|
||||
|
||||
@@ -43,9 +43,13 @@ var Selectors = struct {
|
||||
"anomaly",
|
||||
},
|
||||
Results: []string{
|
||||
"article[data-testid='result'], article[data-testid='ad'], li[data-layout='organic'], li[data-layout='ad'], div[data-testid='result'], div[data-testid='ad']",
|
||||
"article[data-testid='result']",
|
||||
"article[data-testid='ad']",
|
||||
"li[data-layout='organic']",
|
||||
"li[data-layout='ad']",
|
||||
"div[data-testid='result']",
|
||||
"div[data-testid='ad']",
|
||||
"div.result",
|
||||
},
|
||||
Title: []string{
|
||||
@@ -65,6 +69,9 @@ var Selectors = struct {
|
||||
"h3 a",
|
||||
},
|
||||
AdBadge: []string{
|
||||
"article[data-testid='ad']",
|
||||
"li[data-layout='ad']",
|
||||
"div[data-testid='ad']",
|
||||
"[data-testid='ad-badge']",
|
||||
".ad-badge",
|
||||
".result--ad",
|
||||
|
||||
@@ -21,6 +21,7 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
|
||||
func parseEcosiaDocument(doc *goquery.Document) []core.SearchResult {
|
||||
var results []core.SearchResult
|
||||
rank := 1
|
||||
adRank := 1
|
||||
|
||||
doc.Find(Selectors.Result).Each(func(_ int, item *goquery.Selection) {
|
||||
res, ok := parseEcosiaItem(item, rank, false)
|
||||
@@ -32,13 +33,15 @@ func parseEcosiaDocument(doc *goquery.Document) []core.SearchResult {
|
||||
})
|
||||
|
||||
doc.Find(Selectors.Ad).Each(func(_ int, item *goquery.Selection) {
|
||||
res, ok := parseEcosiaItem(item, -1, true)
|
||||
res, ok := parseEcosiaItem(item, adRank, true)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
results = append(results, res)
|
||||
adRank++
|
||||
})
|
||||
|
||||
setSeparatedAdAbsoluteRanks(results, 0)
|
||||
return core.DeduplicateResults(results)
|
||||
}
|
||||
|
||||
@@ -84,6 +87,24 @@ func parseEcosiaItem(item *goquery.Selection, rank int, ad bool) (core.SearchRes
|
||||
}, true
|
||||
}
|
||||
|
||||
func setSeparatedAdAbsoluteRanks(results []core.SearchResult, start int) {
|
||||
adCount := 0
|
||||
for i := range results {
|
||||
if results[i].Ad {
|
||||
adCount++
|
||||
results[i].AbsoluteRank = start + results[i].Rank
|
||||
}
|
||||
}
|
||||
organicAbsoluteRank := start + adCount + 1
|
||||
for i := range results {
|
||||
if results[i].Ad {
|
||||
continue
|
||||
}
|
||||
results[i].AbsoluteRank = organicAbsoluteRank
|
||||
organicAbsoluteRank++
|
||||
}
|
||||
}
|
||||
|
||||
// // parseEcosiaImageItem extracts a single image card from a goquery Selection.
|
||||
// func parseEcosiaImageItem(item *goquery.Selection, rank int) (core.SearchResult, bool) {
|
||||
// linkTag := item.Find(Selectors.ImageLink).First()
|
||||
|
||||
@@ -139,13 +139,13 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S
|
||||
}()
|
||||
|
||||
// nextRank counts up across pages for organic results; nextAdRank counts
|
||||
// down from -1 so ads keep unique, order-preserving negative ranks.
|
||||
// up within sponsored results so ad rank stays separate from SEO rank.
|
||||
all := []core.SearchResult{}
|
||||
pageNum, nextRank, err := startPage(query.Start)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nextAdRank := -1
|
||||
nextAdRank := 1
|
||||
// fetchPage loads one SERP page and appends parsed results.
|
||||
// Returns (done, error): done=true ends the outer loop without error.
|
||||
fetchPage := func() (bool, error) {
|
||||
@@ -192,13 +192,13 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S
|
||||
for _, r := range ads {
|
||||
if res, ok := e.parseResult(r, nextAdRank, true); ok {
|
||||
all = append(all, res)
|
||||
nextAdRank--
|
||||
nextAdRank++
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
for query.Limit <= 0 || len(all) < query.Limit {
|
||||
for query.Limit <= 0 || core.CountOrganicResults(all) < query.Limit {
|
||||
done, err := fetchPage()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -207,7 +207,7 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S
|
||||
if done {
|
||||
break
|
||||
}
|
||||
if query.Limit > 0 && len(all) >= query.Limit {
|
||||
if query.Limit > 0 && core.CountOrganicResults(all) >= query.Limit {
|
||||
break
|
||||
}
|
||||
if err := core.SleepContext(ctx, e.pageSleep); err != nil {
|
||||
@@ -215,13 +215,10 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S
|
||||
}
|
||||
}
|
||||
|
||||
setSeparatedAdAbsoluteRanks(all, query.Start)
|
||||
deduped := core.DeduplicateResults(all)
|
||||
if query.Limit > 0 {
|
||||
organic, ads := splitAds(deduped)
|
||||
if len(organic) > query.Limit {
|
||||
organic = organic[:query.Limit]
|
||||
}
|
||||
deduped = append(organic, ads...)
|
||||
deduped = core.LimitOrganicResults(deduped, query.Limit)
|
||||
}
|
||||
e.logger.Info("Search completed: %d results", len(deduped))
|
||||
return deduped, nil
|
||||
@@ -369,14 +366,3 @@ func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []c
|
||||
e.logger.Info("Image search completed: %d results", len(deduped))
|
||||
return deduped, nil
|
||||
}
|
||||
|
||||
func splitAds(in []core.SearchResult) (organic, ads []core.SearchResult) {
|
||||
for _, r := range in {
|
||||
if r.Ad {
|
||||
ads = append(ads, r)
|
||||
} else {
|
||||
organic = append(organic, r)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -121,7 +121,7 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
|
||||
|
||||
// Ecosia paginates by page index, not result offset, so re-rank from
|
||||
// the page boundary rather than query.Start (off-grid offsets round down).
|
||||
// Skip ads (rank<0) so organic ranks stay sequential from startRank.
|
||||
// Skip ads so organic ranks stay sequential from startRank.
|
||||
organicIdx := 0
|
||||
for i := range parsedResults {
|
||||
if parsedResults[i].Ad {
|
||||
@@ -130,6 +130,7 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
|
||||
parsedResults[i].Rank = startRank + organicIdx
|
||||
organicIdx++
|
||||
}
|
||||
setSeparatedAdAbsoluteRanks(parsedResults, pageNum*10)
|
||||
|
||||
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
|
||||
fmt.Sprintf("Ecosia Raw results : %v", parsedResults),
|
||||
|
||||
@@ -68,3 +68,53 @@ func TestParseHTMLNoResults(t *testing.T) {
|
||||
t.Fatalf("expected zero results, got %d", len(results))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
html := `
|
||||
<div>
|
||||
<div data-hveid="1" data-ved="ad" data-text-ad="1">
|
||||
<a href="https://ads.example.com"><h3>Sponsored Result</h3></a>
|
||||
<div data-sncf="1"><div>Paid snippet</div></div>
|
||||
</div>
|
||||
<div data-hveid="2" data-ved="organic1">
|
||||
<a href="https://organic.example.com/one"><h3>Organic One</h3></a>
|
||||
<div data-sncf="1"><div>Organic snippet one</div></div>
|
||||
</div>
|
||||
<div data-hveid="3" data-ved="organic2">
|
||||
<a href="https://organic.example.com/two"><h3>Organic Two</h3></a>
|
||||
<div data-sncf="1"><div>Organic snippet two</div></div>
|
||||
</div>
|
||||
</div>`
|
||||
|
||||
results, err := ParseHTML(bytes.NewReader([]byte(html)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseHTML() error = %v", err)
|
||||
}
|
||||
if len(results) != 3 {
|
||||
t.Fatalf("expected 3 results, got %d", len(results))
|
||||
}
|
||||
|
||||
organicRank := 0
|
||||
adRank := 0
|
||||
for _, r := range results {
|
||||
if r.Ad {
|
||||
adRank++
|
||||
if r.Rank != adRank {
|
||||
t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
|
||||
}
|
||||
continue
|
||||
}
|
||||
organicRank++
|
||||
if r.Rank != organicRank {
|
||||
t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
|
||||
}
|
||||
}
|
||||
if organicRank != 2 {
|
||||
t.Fatalf("organic count = %d, want 2", organicRank)
|
||||
}
|
||||
if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
|
||||
t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -236,6 +236,8 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
|
||||
gogl.logger.Info("Found %d total results", totalResults)
|
||||
|
||||
rank := query.Start
|
||||
adRank := 1
|
||||
absoluteRank := query.Start + 1
|
||||
for _, resEl := range searchResultElems {
|
||||
srchRes := core.SearchResult{}
|
||||
|
||||
@@ -285,7 +287,10 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
|
||||
} else {
|
||||
srchRes.Description = strings.TrimSpace(text)
|
||||
}
|
||||
rank += 1
|
||||
srchRes.Rank = adRank
|
||||
srchRes.AbsoluteRank = absoluteRank
|
||||
adRank++
|
||||
absoluteRank++
|
||||
|
||||
} else if isAnswerBox {
|
||||
// 2. Parse answer boxes
|
||||
@@ -406,6 +411,8 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
|
||||
|
||||
rank += 1
|
||||
srchRes.Rank = rank
|
||||
srchRes.AbsoluteRank = absoluteRank
|
||||
absoluteRank++
|
||||
searchResults = append(searchResults, srchRes)
|
||||
continue
|
||||
|
||||
@@ -413,7 +420,16 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
|
||||
continue
|
||||
}
|
||||
|
||||
srchRes.Rank = rank
|
||||
if srchRes.Ad && srchRes.Rank == 0 {
|
||||
srchRes.Rank = adRank
|
||||
adRank++
|
||||
} else if !srchRes.Ad {
|
||||
srchRes.Rank = rank
|
||||
}
|
||||
if srchRes.AbsoluteRank == 0 {
|
||||
srchRes.AbsoluteRank = absoluteRank
|
||||
absoluteRank++
|
||||
}
|
||||
searchResults = append(searchResults, srchRes)
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
|
||||
func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
|
||||
results := []core.SearchResult{}
|
||||
rank := 1
|
||||
adRank := 1
|
||||
absoluteRank := 1
|
||||
|
||||
// Use data attributes instead of class names to find results
|
||||
// Both old and new DOM have data-hveid and data-ved attributes
|
||||
@@ -55,6 +57,8 @@ func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
|
||||
titleTag := item.Find(Selectors.Title)
|
||||
title := titleTag.Text()
|
||||
|
||||
isAd := item.Is(Selectors.Ad) || item.Find(Selectors.Ad).Length() > 0
|
||||
|
||||
// Find description - find div with text content after the heading
|
||||
// Using attribute selectors that match the description container
|
||||
descTag := item.Find(Selectors.DescPrimary).First()
|
||||
@@ -73,15 +77,25 @@ func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
|
||||
desc := descTag.Text()
|
||||
|
||||
if link != "" && link != "#" {
|
||||
resultRank := rank
|
||||
if isAd {
|
||||
resultRank = adRank
|
||||
adRank++
|
||||
} else {
|
||||
rank++
|
||||
}
|
||||
|
||||
result := core.SearchResult{
|
||||
Rank: rank,
|
||||
URL: link,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Rank: resultRank,
|
||||
AbsoluteRank: absoluteRank,
|
||||
URL: link,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Ad: isAd,
|
||||
}
|
||||
|
||||
results = append(results, result)
|
||||
rank++
|
||||
absoluteRank++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,8 +167,18 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
|
||||
}
|
||||
|
||||
if query.Start > 0 {
|
||||
organicIdx := 0
|
||||
for i := range parsedResults {
|
||||
parsedResults[i].Rank = query.Start + i + 1
|
||||
if parsedResults[i].Ad {
|
||||
continue
|
||||
}
|
||||
organicIdx++
|
||||
parsedResults[i].Rank = query.Start + organicIdx
|
||||
}
|
||||
for i := range parsedResults {
|
||||
if parsedResults[i].AbsoluteRank > 0 {
|
||||
parsedResults[i].AbsoluteRank += query.Start
|
||||
}
|
||||
}
|
||||
}
|
||||
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
|
||||
|
||||
@@ -8,6 +8,7 @@ var Selectors = struct {
|
||||
ResultStats string
|
||||
CookieBtn string
|
||||
Results string
|
||||
Ad string
|
||||
Title string
|
||||
DescPrimary string
|
||||
DescFallback string
|
||||
@@ -24,6 +25,7 @@ var Selectors = struct {
|
||||
ResultStats: "div#result-stats",
|
||||
CookieBtn: "div[role='dialog'][aria-modal] button",
|
||||
Results: "div[data-hveid][data-ved]",
|
||||
Ad: "div[data-text-ad], [data-text-ad]",
|
||||
Title: "h3",
|
||||
DescPrimary: "div[data-sncf='1'] div",
|
||||
DescFallback: "div.VwiC3b",
|
||||
|
||||
@@ -2,6 +2,7 @@ package yandex
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
@@ -21,6 +22,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
|
||||
func parseYandexDocument(doc *goquery.Document) []core.SearchResult {
|
||||
var results []core.SearchResult
|
||||
rank := 1
|
||||
adRank := 1
|
||||
absoluteRank := 1
|
||||
|
||||
doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) {
|
||||
// Skip blocks without a result heading (filters out non-organic blocks
|
||||
@@ -51,6 +54,7 @@ func parseYandexDocument(doc *goquery.Document) []core.SearchResult {
|
||||
if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") {
|
||||
return
|
||||
}
|
||||
isAd := yandexSelectionHasAdMarker(item) || yandexURLLooksAd(href)
|
||||
|
||||
title := strings.TrimSpace(titleTag.Text())
|
||||
if title == "" {
|
||||
@@ -67,14 +71,84 @@ func parseYandexDocument(doc *goquery.Document) []core.SearchResult {
|
||||
}
|
||||
}
|
||||
|
||||
resultRank := rank
|
||||
if isAd {
|
||||
resultRank = adRank
|
||||
adRank++
|
||||
} else {
|
||||
rank++
|
||||
}
|
||||
|
||||
results = append(results, core.SearchResult{
|
||||
Rank: rank,
|
||||
URL: href,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Rank: resultRank,
|
||||
AbsoluteRank: absoluteRank,
|
||||
URL: href,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Ad: isAd,
|
||||
})
|
||||
rank++
|
||||
absoluteRank++
|
||||
})
|
||||
|
||||
return core.DeduplicateResults(results)
|
||||
}
|
||||
|
||||
func yandexSelectionHasAdMarker(item *goquery.Selection) bool {
|
||||
for _, selector := range Selectors.AdMarkers {
|
||||
if item.Is(selector) || item.Find(selector).Length() > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func yandexURLLooksAd(raw string) bool {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.TrimPrefix(strings.ToLower(u.Hostname()), "www.")
|
||||
return strings.HasPrefix(host, "yabs.yandex.") ||
|
||||
host == "an.yandex.ru" ||
|
||||
strings.HasSuffix(host, ".yandexadexchange.net")
|
||||
}
|
||||
|
||||
func skipOrganicResults(results []core.SearchResult, skip int) []core.SearchResult {
|
||||
if skip <= 0 {
|
||||
return results
|
||||
}
|
||||
out := results[:0]
|
||||
for _, result := range results {
|
||||
if !result.Ad && skip > 0 {
|
||||
skip--
|
||||
continue
|
||||
}
|
||||
out = append(out, result)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rebaseOrganicRanks(results []core.SearchResult, start int) {
|
||||
if start <= 0 {
|
||||
return
|
||||
}
|
||||
organicIdx := 0
|
||||
for i := range results {
|
||||
if results[i].Ad {
|
||||
continue
|
||||
}
|
||||
organicIdx++
|
||||
results[i].Rank = start + organicIdx
|
||||
}
|
||||
}
|
||||
|
||||
func offsetAbsoluteRanks(results []core.SearchResult, start int) {
|
||||
if start <= 0 {
|
||||
return
|
||||
}
|
||||
for i := range results {
|
||||
if results[i].AbsoluteRank > 0 {
|
||||
results[i].AbsoluteRank += start
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,3 +84,83 @@ func TestParseYandexHTMLFallbackSelectors(t *testing.T) {
|
||||
t.Fatalf("unexpected description: %s", results[0].Description)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseYandexHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
html := `
|
||||
<ul>
|
||||
<li class="serp-item" data-fast-name="direct">
|
||||
<h2><a href="https://ads.example.com">Sponsored Result</a></h2>
|
||||
<div class="OrganicText">Paid snippet</div>
|
||||
</li>
|
||||
<li class="serp-item">
|
||||
<h2><a href="https://organic.example.com/one">Organic One</a></h2>
|
||||
<div class="OrganicText">Organic snippet one</div>
|
||||
</li>
|
||||
<li class="serp-item">
|
||||
<h2><a href="https://organic.example.com/two">Organic Two</a></h2>
|
||||
<div class="OrganicText">Organic snippet two</div>
|
||||
</li>
|
||||
</ul>`
|
||||
|
||||
results, err := ParseHTML(bytes.NewReader([]byte(html)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseHTML() error = %v", err)
|
||||
}
|
||||
if len(results) != 3 {
|
||||
t.Fatalf("expected 3 results, got %d", len(results))
|
||||
}
|
||||
|
||||
organicRank := 0
|
||||
adRank := 0
|
||||
for _, r := range results {
|
||||
if r.Ad {
|
||||
adRank++
|
||||
if r.Rank != adRank {
|
||||
t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
|
||||
}
|
||||
continue
|
||||
}
|
||||
organicRank++
|
||||
if r.Rank != organicRank {
|
||||
t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
|
||||
}
|
||||
}
|
||||
if organicRank != 2 {
|
||||
t.Fatalf("organic count = %d, want 2", organicRank)
|
||||
}
|
||||
if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
|
||||
t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseYandexHTMLYabsURLIsAd(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
html := `
|
||||
<ul>
|
||||
<li class="serp-item">
|
||||
<h2><a href="https://yabs.yandex.ru/count/WuGejI_zOoVX2">WB Travel</a></h2>
|
||||
<div class="OrganicText">Paid snippet</div>
|
||||
</li>
|
||||
<li class="serp-item">
|
||||
<h2><a href="https://organic.example.com/one">Organic One</a></h2>
|
||||
<div class="OrganicText">Organic snippet one</div>
|
||||
</li>
|
||||
</ul>`
|
||||
|
||||
results, err := ParseHTML(bytes.NewReader([]byte(html)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseHTML() error = %v", err)
|
||||
}
|
||||
if len(results) != 2 {
|
||||
t.Fatalf("expected 2 results, got %d", len(results))
|
||||
}
|
||||
if !results[0].Ad || results[0].Rank != 1 {
|
||||
t.Fatalf("first result should be ad rank 1: %+v", results[0])
|
||||
}
|
||||
if results[1].Ad || results[1].Rank != 1 {
|
||||
t.Fatalf("second result should be organic rank 1: %+v", results[1])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,8 +71,11 @@ func (yand *Yandex) isNoResults(page *rod.Page) bool {
|
||||
|
||||
func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.SearchResult {
|
||||
searchResults := []core.SearchResult{}
|
||||
organicRank := pageNum * 10
|
||||
adRank := 1
|
||||
absoluteRank := pageNum*10 + 1
|
||||
|
||||
for i, r := range results {
|
||||
for _, r := range results {
|
||||
// Get URL
|
||||
link, err := r.Element(Selectors.Link)
|
||||
if err != nil {
|
||||
@@ -109,13 +112,48 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
|
||||
desc, _ = descTag.Text()
|
||||
}
|
||||
|
||||
r := core.SearchResult{Rank: (pageNum * 10) + (i + 1), URL: linkText.String(), Title: title, Description: desc}
|
||||
searchResults = append(searchResults, r)
|
||||
hrefStr := linkText.String()
|
||||
isAd := yandexElementHasAdMarker(r) || yandexURLLooksAd(hrefStr)
|
||||
resultRank := 0
|
||||
if isAd {
|
||||
resultRank = adRank
|
||||
adRank++
|
||||
} else {
|
||||
organicRank++
|
||||
resultRank = organicRank
|
||||
}
|
||||
|
||||
res := core.SearchResult{
|
||||
Rank: resultRank,
|
||||
AbsoluteRank: absoluteRank,
|
||||
URL: hrefStr,
|
||||
Title: title,
|
||||
Description: desc,
|
||||
Ad: isAd,
|
||||
}
|
||||
searchResults = append(searchResults, res)
|
||||
absoluteRank++
|
||||
}
|
||||
|
||||
return searchResults
|
||||
}
|
||||
|
||||
func yandexElementHasAdMarker(el *rod.Element) bool {
|
||||
if el == nil {
|
||||
return false
|
||||
}
|
||||
for _, selector := range Selectors.AdMarkers {
|
||||
matches, err := el.Matches(selector)
|
||||
if err == nil && matches {
|
||||
return true
|
||||
}
|
||||
if child, err := el.Element(selector); err == nil && child != nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (yand *Yandex) parseImageEntities(items rod.Elements) map[string]ImageEntity {
|
||||
entities := make(map[string]ImageEntity)
|
||||
for _, item := range items {
|
||||
@@ -193,17 +231,13 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor
|
||||
|
||||
r := yand.parseResults(elements, searchPage)
|
||||
if searchPage == startPage && skipOnFirstPage > 0 {
|
||||
if skipOnFirstPage >= len(r) {
|
||||
r = []core.SearchResult{}
|
||||
} else {
|
||||
r = r[skipOnFirstPage:]
|
||||
}
|
||||
r = skipOrganicResults(r, skipOnFirstPage)
|
||||
}
|
||||
allResults = append(allResults, r...)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
for len(allResults) < query.Limit {
|
||||
for query.Limit <= 0 || core.CountOrganicResults(allResults) < query.Limit {
|
||||
done, err := fetchPage()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -212,13 +246,16 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor
|
||||
if done {
|
||||
break
|
||||
}
|
||||
if query.Limit > 0 && core.CountOrganicResults(allResults) >= query.Limit {
|
||||
break
|
||||
}
|
||||
if err := core.SleepContext(ctx, yand.pageSleep); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
yand.logger.Info("Search completed: %d results", len(allResults))
|
||||
return core.DeduplicateResults(allResults), nil
|
||||
return core.LimitOrganicResults(core.DeduplicateResults(allResults), query.Limit), nil
|
||||
}
|
||||
|
||||
// SearchImage executes a Yandex image search and returns normalized image
|
||||
|
||||
@@ -74,17 +74,10 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
|
||||
}
|
||||
|
||||
if skipOnFirstPage > 0 {
|
||||
if skipOnFirstPage >= len(parsedResults) {
|
||||
parsedResults = []core.SearchResult{}
|
||||
} else {
|
||||
parsedResults = parsedResults[skipOnFirstPage:]
|
||||
}
|
||||
}
|
||||
if query.Start > 0 {
|
||||
for i := range parsedResults {
|
||||
parsedResults[i].Rank = query.Start + i + 1
|
||||
}
|
||||
parsedResults = skipOrganicResults(parsedResults, skipOnFirstPage)
|
||||
}
|
||||
rebaseOrganicRanks(parsedResults, query.Start)
|
||||
offsetAbsoluteRanks(parsedResults, startPage*10)
|
||||
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
|
||||
fmt.Sprintf("Yandex Raw results : %v", parsedResults),
|
||||
)
|
||||
|
||||
@@ -2,9 +2,10 @@ package yandex
|
||||
|
||||
// Selectors is the single source of truth for Yandex SERP CSS selectors.
|
||||
var Selectors = struct {
|
||||
Captcha string
|
||||
NoResults string
|
||||
Results string
|
||||
Captcha string
|
||||
NoResults string
|
||||
Results string
|
||||
AdMarkers []string
|
||||
// LinkPrimary is preferred over a generic <a>; falls back to title.Closest("a")
|
||||
// then the first <a> in the result block when absent.
|
||||
LinkPrimary string
|
||||
@@ -16,9 +17,18 @@ var Selectors = struct {
|
||||
ImageItemsAlt []string
|
||||
ImageStateAll string
|
||||
}{
|
||||
Captcha: "div.CheckboxCaptcha",
|
||||
NoResults: "div.EmptySearchResults",
|
||||
Results: "li[data-fast], li.serp-item",
|
||||
Captcha: "div.CheckboxCaptcha",
|
||||
NoResults: "div.EmptySearchResults",
|
||||
Results: "li[data-fast], li.serp-item",
|
||||
AdMarkers: []string{
|
||||
"[data-fast-name='direct']",
|
||||
"[data-fast-name='serp-adv']",
|
||||
"[data-bem*='serp-adv']",
|
||||
".serp-adv-item",
|
||||
".serp-adv__found",
|
||||
"[aria-label='Реклама']",
|
||||
"[title='Реклама']",
|
||||
},
|
||||
LinkPrimary: "a.OrganicTitle-Link",
|
||||
Link: "a",
|
||||
Title: "h2",
|
||||
|
||||
Reference in New Issue
Block a user