feat(api): v2 response shape - absolute rank, ad/organic split, slimmer fields

This commit is contained in:
Rustem Kamalov
2026-05-13 01:30:54 +03:00
parent 00630a5d31
commit b3ca4e0803
36 changed files with 972 additions and 279 deletions

View File

@@ -106,7 +106,7 @@ Common parameters:
| `date` | Date range | `20250101..20251231` |
| `file` | File extension | `pdf`, `doc`, `xls` |
| `site` | Site-specific search | `github.com` |
| `limit` | Number of results, max 100 | `10`, `25`, `50` |
| `limit` | Number of organic results, max 100. Ads may be returned in addition. | `10`, `25`, `50` |
| `start` | Pagination offset | `0`, `10`, `20` |
| `format` | Output format | `json`, `markdown`, `text`, `ndjson` |
@@ -130,7 +130,7 @@ Engine-specific parameters:
"requested_at": "2026-04-25T22:27:52Z",
"took_ms": 6410,
"engines_failed": [],
"version": "1.0"
"version": "2.0"
},
"results": [
{
@@ -143,22 +143,14 @@ Engine-specific parameters:
"snippet": "Official Go documentation, tutorials, references, and release notes.",
"domain": "go.dev",
"favicon": "https://go.dev/favicon.ico",
"is_ad": false,
"position": {
"absolute": 1,
"page": 1,
"on_page": 1
"absolute": 1
},
"engine": "google",
"domain_info": {
"tld": "dev",
"sld": "go",
"is_gov": false,
"is_edu": false,
"is_social": false
},
"classification": {
"content_type": "webpage"
"category": ""
}
}
],

View File

@@ -37,8 +37,12 @@ func baiduResultSelectors() []string {
func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult {
var results []core.SearchResult
rank := 1
adRank := 1
absoluteRank := 1
sel.Each(func(_ int, item *goquery.Selection) {
isAd := baiduSelectionHasAdMarker(item)
// h3-first: organic results always carry a heading; this filters out
// non-result blocks that may share the wrapper class.
titleTag := item.Find("h3").First()
@@ -96,21 +100,55 @@ func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult {
desc = strings.TrimSpace(strings.Replace(full, title, "", 1))
}
resultRank := rank
if isAd {
resultRank = adRank
adRank++
} else {
rank++
}
results = append(results, core.SearchResult{
Rank: rank,
URL: href,
Title: title,
Description: desc,
Rank: resultRank,
AbsoluteRank: absoluteRank,
URL: href,
Title: title,
Description: desc,
Ad: isAd,
})
rank++
absoluteRank++
})
// Re-rank sequentially after dedup so callers get a clean 1..N sequence
// (dedup may drop intermediate ranks when the same URL appears in
// multiple Baidu result-card variants on the same SERP).
deduped := core.DeduplicateResults(results)
organicIdx := 0
for i := range deduped {
deduped[i].Rank = i + 1
if deduped[i].Ad {
continue
}
organicIdx++
deduped[i].Rank = organicIdx
}
return deduped
}
func baiduSelectionHasAdMarker(item *goquery.Selection) bool {
for _, selector := range Selectors.AdMarkers {
if item.Is(selector) || item.Find(selector).Length() > 0 {
return true
}
}
isAd := false
item.Find("span, i, em").EachWithBreak(func(_ int, marker *goquery.Selection) bool {
text := strings.TrimSpace(marker.Text())
if text == "广告" || text == "推广" || text == "商业推广" {
isAd = true
return false
}
return true
})
return isAd
}

View File

@@ -104,3 +104,53 @@ func TestParseBaiduHTMLFallsBackWhenEarlierSelectorHasNoResult(t *testing.T) {
t.Fatalf("unexpected URL: %s", results[0].URL)
}
}
func TestParseBaiduHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
t.Parallel()
html := `
<div id="content_left">
<div class="result c-container" data-tuiguang="1">
<h3><a href="https://ads.example.com">Sponsored Result</a></h3>
<div class="c-abstract">Paid snippet</div>
</div>
<div class="result c-container">
<h3><a href="https://organic.example.com/one">Organic One</a></h3>
<div class="c-abstract">Organic snippet one</div>
</div>
<div class="result c-container">
<h3><a href="https://organic.example.com/two">Organic Two</a></h3>
<div class="c-abstract">Organic snippet two</div>
</div>
</div>`
results, err := ParseHTML(bytes.NewReader([]byte(html)))
if err != nil {
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 3 {
t.Fatalf("expected 3 results, got %d", len(results))
}
organicRank := 0
adRank := 0
for _, r := range results {
if r.Ad {
adRank++
if r.Rank != adRank {
t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
}
continue
}
organicRank++
if r.Rank != organicRank {
t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
}
}
if organicRank != 2 {
t.Fatalf("organic count = %d, want 2", organicRank)
}
if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
}
}

View File

@@ -112,7 +112,13 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
}
for i := range searchResults {
searchResults[i].Rank = query.Start + i + 1
if searchResults[i].AbsoluteRank > 0 {
searchResults[i].AbsoluteRank += query.Start
}
if searchResults[i].Ad {
continue
}
searchResults[i].Rank = query.Start + searchResults[i].Rank
}
return searchResults, nil
}

View File

@@ -69,7 +69,13 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
}
if query.Start > 0 {
for i := range parsedResults {
parsedResults[i].Rank = query.Start + i + 1
if parsedResults[i].AbsoluteRank > 0 {
parsedResults[i].AbsoluteRank += query.Start
}
if parsedResults[i].Ad {
continue
}
parsedResults[i].Rank = query.Start + parsedResults[i].Rank
}
}
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(

View File

@@ -6,6 +6,7 @@ var Selectors = struct {
Timeout string
Results string
ResultsAlt []string
AdMarkers []string
ImageJSONRoot []string
Link string
Desc string
@@ -17,6 +18,7 @@ var Selectors = struct {
Timeout: "button.timeout-button",
Results: "#content_left div.result.c-container",
ResultsAlt: []string{"#content_left div.result-op.c-container", "div.c-container.new-pmd"},
AdMarkers: []string{"[data-tuiguang]", "[data-click*='tuiguang']", ".ec-tuiguang", ".c-icon-bear-p"},
ImageJSONRoot: []string{"body > pre", "pre"},
Link: "a",
Desc: "div.c-abstract",

View File

@@ -22,6 +22,7 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
func parseBingDocument(doc *goquery.Document) []core.SearchResult {
var results []core.SearchResult
rank := 1
adRank := 1
doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) {
titleTag := item.Find(Selectors.Title).First()
@@ -69,17 +70,37 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult {
}
results = append(results, core.SearchResult{
Rank: -1,
Rank: adRank,
URL: link,
Title: title,
Description: desc,
Ad: true,
})
adRank++
})
setSeparatedAdAbsoluteRanks(results, 0)
return core.DeduplicateResults(results)
}
func setSeparatedAdAbsoluteRanks(results []core.SearchResult, start int) {
adCount := 0
for i := range results {
if results[i].Ad {
adCount++
results[i].AbsoluteRank = start + results[i].Rank
}
}
organicAbsoluteRank := start + adCount + 1
for i := range results {
if results[i].Ad {
continue
}
results[i].AbsoluteRank = organicAbsoluteRank
organicAbsoluteRank++
}
}
// descriptionFromItem extracts a description using the same 4-step fallback
// chain as the rod-based browser parser.
func descriptionFromItem(item *goquery.Selection, title string) string {

View File

@@ -138,6 +138,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements))
rank := query.Start
adRank := 1
for _, result := range organicElements {
srchRes := core.SearchResult{}
@@ -215,35 +216,17 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
srchRes.Description, _ = descElem.Text()
}
// Mark ads with negative rank
srchRes.Rank = -1
srchRes.Rank = adRank
adRank++
searchResults = append(searchResults, srchRes)
}
setSeparatedAdAbsoluteRanks(searchResults, query.Start)
// Deduplicate results
deduped := core.DeduplicateResults(searchResults)
// Trim to exact limit if necessary (only organic results, not ads)
if query.Limit > 0 {
organicResults := []core.SearchResult{}
adResults := []core.SearchResult{}
for _, result := range deduped {
if result.Ad {
adResults = append(adResults, result)
} else {
organicResults = append(organicResults, result)
}
}
// Trim organic results to limit
if len(organicResults) > query.Limit {
organicResults = organicResults[:query.Limit]
}
// Combine back: organic results + ads
deduped = append(organicResults, adResults...)
}
deduped = core.LimitOrganicResults(deduped, query.Limit)
return deduped, nil
}

View File

@@ -16,7 +16,7 @@ import (
)
const (
version = "0.7.10"
version = "0.7.11"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)

View File

@@ -95,9 +95,11 @@ func classifyProxyNetworkError(err error) error {
// SearchResult represents one normalized result item returned by any engine.
type SearchResult struct {
// Rank is a 1-based position in engine output. Some engines use negative
// ranks for non-organic blocks such as ads or instant answers.
// Rank is the 1-based position within this result type. For SEO callers,
// organic rank must not be shifted by ads.
Rank int `json:"rank"`
// AbsoluteRank is the 1-based position in the mixed SERP stream.
AbsoluteRank int `json:"absolute_rank,omitempty"`
// URL is the canonical result URL.
URL string `json:"url"`
// Title is the result headline shown on the SERP.
@@ -118,14 +120,15 @@ func DeduplicateResults(results []SearchResult) []SearchResult {
if result.URL == "" {
continue
}
if !unique[result.URL] {
unique[result.URL] = true
key := resultDedupKey(result)
if !unique[key] {
unique[key] = true
deduped = append(deduped, result)
}
}
sort.Slice(deduped, func(i, j int) bool {
return deduped[i].Rank < deduped[j].Rank
return resultLess(deduped[i], deduped[j])
})
return deduped
}
@@ -140,11 +143,79 @@ func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]Search
}
sort.Slice(searchResults, func(i, j int) bool {
return searchResults[i].Rank < searchResults[j].Rank
return resultLess(searchResults[i], searchResults[j])
})
return &searchResults
}
// CountOrganicResults returns the number of non-ad results in a mixed SERP.
func CountOrganicResults(results []SearchResult) int {
count := 0
for _, result := range results {
if !result.Ad {
count++
}
}
return count
}
// LimitOrganicResults keeps all ads and at most limit non-ad results.
func LimitOrganicResults(results []SearchResult, limit int) []SearchResult {
if limit <= 0 {
return results
}
out := make([]SearchResult, 0, len(results))
organicCount := 0
for _, result := range results {
if result.Ad {
out = append(out, result)
continue
}
if organicCount >= limit {
continue
}
organicCount++
out = append(out, result)
}
return out
}
func resultDedupKey(result SearchResult) string {
resultType := "organic"
if result.Ad {
resultType = "ad"
}
return resultType + "\x00" + result.URL
}
func resultLess(left, right SearchResult) bool {
leftPos := resultSortPosition(left)
rightPos := resultSortPosition(right)
if leftPos != rightPos {
return leftPos < rightPos
}
if left.Ad != right.Ad {
return left.Ad
}
if left.Rank != right.Rank {
return left.Rank < right.Rank
}
return left.URL < right.URL
}
func resultSortPosition(result SearchResult) int {
if result.AbsoluteRank > 0 {
return result.AbsoluteRank
}
if result.Rank < 0 {
return -result.Rank
}
if result.Rank > 0 {
return result.Rank
}
return int(^uint(0) >> 1)
}
// Query holds request parameters used by HTTP handlers and search engines.
// Example minimal query: Query{Text: "golang", Limit: 10}.
type Query struct {
@@ -168,7 +239,7 @@ type Query struct {
// For Google, false includes similar results and true hides them.
Filter bool
// Answers enables parsing answer modules when supported by the engine.
// Such entries may be returned with negative rank values.
// Such entries may be returned with non-positive internal rank values.
Answers bool
// ProxyURL is a direct proxy URL used by raw HTTP search paths.
ProxyURL string

View File

@@ -45,15 +45,9 @@ func EnrichDomainInfo(domain string) *DomainInfo {
cfg := loadEnrichmentDomains()
info := &DomainInfo{
TLD: tld,
SLD: sld,
IsGov: isGovTLD(domain, tld),
IsEdu: isEduTLD(domain, tld),
IsMil: isMilTLD(tld),
IsNews: cfg.NewsDomains[domain],
IsForum: cfg.ForumDomains[domain],
IsMarketplace: cfg.MarketplaceDomains[domain],
IsSocial: cfg.SocialDomains[domain],
TLD: tld,
SLD: sld,
Category: domainCategory(domain, tld, cfg),
}
return info
}
@@ -67,6 +61,9 @@ func ClassifyURL(rawURL, domain string) *Classification {
contentType := classifyContentType(rawURL)
sourceHint := classifySourceHint(domain)
if contentType == "webpage" && sourceHint == "" {
return nil
}
return &Classification{
ContentType: contentType,
@@ -74,6 +71,27 @@ func ClassifyURL(rawURL, domain string) *Classification {
}
}
func domainCategory(domain, tld string, cfg enrichmentDomainsConfig) string {
switch {
case isGovTLD(domain, tld):
return "gov"
case isEduTLD(domain, tld):
return "edu"
case isMilTLD(tld):
return "mil"
case cfg.NewsDomains[domain]:
return "news"
case cfg.ForumDomains[domain]:
return "forum"
case cfg.MarketplaceDomains[domain]:
return "marketplace"
case cfg.SocialDomains[domain]:
return "social"
default:
return ""
}
}
// splitDomain returns (public suffix, registrable domain label).
func splitDomain(domain string) (tld, sld string) {
domain = normalizeDomain(domain)

View File

@@ -22,9 +22,6 @@ func RenderMarkdown(env *Envelope) []byte {
for i, r := range env.Results {
fmt.Fprintf(&b, "## %d. %s\n\n", i+1, escapeMarkdown(r.Title))
typeLabel := string(r.Type)
if r.IsAd {
typeLabel = "ad"
}
fmt.Fprintf(&b, "**%s** · %s\n\n", r.DisplayURL, typeLabel)
if r.Snippet != "" {
fmt.Fprintf(&b, "%s\n\n", r.Snippet)

View File

@@ -11,13 +11,13 @@ type QueryEcho struct {
// ResponseMeta carries request-level metadata for observability and debugging.
type ResponseMeta struct {
RequestID string `json:"request_id"`
RequestedAt string `json:"requested_at"`
TookMs int64 `json:"took_ms"`
EnginesResponded []string `json:"engines_responded,omitempty"`
EnginesFailed []string `json:"engines_failed"`
EngineErrors []EngineErrorDetail `json:"engine_errors,omitempty"`
Version string `json:"version"`
RequestID string `json:"request_id"`
RequestedAt string `json:"requested_at"`
TookMs int64 `json:"took_ms"`
EnginesResponded []string `json:"engines_responded,omitempty"`
EnginesFailed []string `json:"engines_failed"`
EngineErrors []EngineErrorDetail `json:"engine_errors,omitempty"`
Version string `json:"version"`
}
// EngineErrorDetail is a client-facing, sanitized per-engine failure summary.
@@ -34,7 +34,7 @@ type Pagination struct {
NextStart int `json:"next_start"`
}
// Envelope is the top-level v1 response wrapper for all search endpoints.
// Envelope is the top-level v2 response wrapper for all search endpoints.
type Envelope struct {
Query QueryEcho `json:"query"`
Meta ResponseMeta `json:"meta"`
@@ -64,7 +64,7 @@ type ClusterOccurrence struct {
ResultID string `json:"result_id"`
}
// ImageEnvelope is the top-level v1 response wrapper for image search endpoints.
// ImageEnvelope is the top-level v2 response wrapper for image search endpoints.
type ImageEnvelope struct {
Query QueryEcho `json:"query"`
Meta ResponseMeta `json:"meta"`
@@ -72,7 +72,7 @@ type ImageEnvelope struct {
Pagination Pagination `json:"pagination"`
}
const apiVersion = "1.0"
const apiVersion = "2.0"
// NewEnvelope builds a fresh Envelope pre-filled with query echo and an open
// meta block. Call Finalize before serializing.
@@ -124,11 +124,21 @@ func (e *Envelope) Finalize(startedAt time.Time, q Query) {
page := q.Start/limit + 1
e.Pagination = Pagination{
Page: page,
HasMore: len(e.Results) >= limit,
HasMore: countNonAdResults(e.Results) >= limit,
NextStart: q.Start + limit,
}
}
func countNonAdResults(results []Result) int {
count := 0
for _, result := range results {
if result.Type != ResultTypeAd {
count++
}
}
return count
}
// Finalize stamps the elapsed time and computes pagination fields.
func (e *ImageEnvelope) Finalize(startedAt time.Time, q Query) {
e.Meta.TookMs = time.Since(startedAt).Milliseconds()

View File

@@ -24,7 +24,7 @@ type EnrichContext struct {
Query Query
}
// EnrichResult converts a raw engine result into the v1 Result shape.
// EnrichResult converts a raw engine result into the v2 Result shape.
func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
normalizedURL := normalizeURL(raw.URL)
domain := extractDomain(normalizedURL)
@@ -42,17 +42,23 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
if raw.Rank <= 0 && !raw.Ad {
resultType = ResultTypeAnswerBox
}
limit := ctx.Query.Limit
if limit <= 0 {
limit = 25
rank := raw.Rank
if rank < 0 {
if raw.Ad {
rank = -rank
} else {
rank = 0
}
}
absolute := computeResultPosition(raw, ctx.Query.Start)
if absolute <= 0 {
absolute = rank
}
page := ctx.Query.Start/limit + 1
absolute, onPage := computeResultPosition(raw.Rank, ctx.Query.Start)
result := Result{
ID: buildResultID(ctx.Engine, normalizedURL),
Rank: raw.Rank,
Rank: rank,
Type: resultType,
Title: raw.Title,
URL: normalizedURL,
@@ -60,13 +66,10 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
Snippet: raw.Description,
Domain: domain,
Favicon: favicon,
IsAd: raw.Ad,
Position: Position{
Absolute: absolute,
Page: page,
OnPage: onPage,
},
Engine: ctx.Engine,
Engine: ctx.Engine,
}
if absolute > 0 {
result.Position = &Position{Absolute: absolute}
}
result.DomainInfo = EnrichDomainInfo(domain)
@@ -75,7 +78,7 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
return result
}
// EnrichImageResult converts a raw engine result into the v1 ImageResult shape.
// EnrichImageResult converts a raw engine result into the v2 ImageResult shape.
func EnrichImageResult(raw SearchResult, ctx EnrichContext) ImageResult {
imageURL := normalizeURL(raw.URL)
meta := parseImageDescription(raw.Description)
@@ -122,14 +125,22 @@ func shortMD5(value string) string {
return hex.EncodeToString(h[:responseIDBytes])
}
func computeResultPosition(rank, start int) (absolute, onPage int) {
if rank <= 0 {
return 0, 0
func computeResultPosition(raw SearchResult, start int) int {
if raw.AbsoluteRank > 0 {
return raw.AbsoluteRank
}
rank := raw.Rank
if rank < 0 {
rank = -rank
}
if rank == 0 {
return 0
}
if start > 0 && rank > start {
return rank, rank - start
return rank
}
return start + rank, rank
return start + rank
}
// normalizeURL lowercases scheme+host, strips trailing slash, and removes

View File

@@ -19,34 +19,29 @@ const (
// Position describes where a result sits in the overall result stream.
type Position struct {
// Absolute is the 1-based rank counting from the first result of the first page.
// Absolute is the 1-based rank counting from the first result of the first page,
// across both organic and ad blocks. Always emitted so SEO callers can plot
// rank vs. on-page position without inferring it from the result order.
Absolute int `json:"absolute"`
// Page is the 1-based page number derived from start/limit.
Page int `json:"page"`
// OnPage is the 1-based rank within this page.
OnPage int `json:"on_page"`
}
// DomainInfo carries TLD-derived category signals for a result domain.
type DomainInfo struct {
TLD string `json:"tld"`
SLD string `json:"sld"`
IsGov bool `json:"is_gov"`
IsEdu bool `json:"is_edu"`
IsMil bool `json:"is_mil"`
IsNews bool `json:"is_news"`
IsForum bool `json:"is_forum"`
IsMarketplace bool `json:"is_marketplace"`
IsSocial bool `json:"is_social"`
TLD string `json:"tld,omitempty"`
SLD string `json:"sld,omitempty"`
// Category is one of "gov", "edu", "mil", "news", "forum", "marketplace",
// "social", or "" when the domain does not match any known category.
Category string `json:"category"`
}
// Classification holds URL-path heuristic hints for downstream consumers.
type Classification struct {
ContentType string `json:"content_type"`
SourceHint string `json:"source_hint"`
ContentType string `json:"content_type,omitempty"`
SourceHint string `json:"source_hint,omitempty"`
}
// Result is the v1 normalized result returned in every search response.
// Result is the v2 normalized result returned in search responses. Optional
// fields (Position, DomainInfo, Classification) are omitted when empty.
type Result struct {
ID string `json:"id"`
Rank int `json:"rank"`
@@ -57,8 +52,7 @@ type Result struct {
Snippet string `json:"snippet"`
Domain string `json:"domain"`
Favicon string `json:"favicon"`
IsAd bool `json:"is_ad"`
Position Position `json:"position"`
Position *Position `json:"position,omitempty"`
Engine string `json:"engine"`
DomainInfo *DomainInfo `json:"domain_info,omitempty"`
Classification *Classification `json:"classification,omitempty"`
@@ -78,7 +72,7 @@ type ImageSource struct {
Domain string `json:"domain"`
}
// ImageResult is the v1 shape for image search results.
// ImageResult is the v2 shape for image search results.
type ImageResult struct {
ID string `json:"id"`
Rank int `json:"rank"`

104
core/result_rank_test.go Normal file
View File

@@ -0,0 +1,104 @@
package core
import (
"testing"
"time"
)
func TestDeduplicateResultsOrdersByAbsoluteRank(t *testing.T) {
t.Parallel()
results := DeduplicateResults([]SearchResult{
{Rank: 1, AbsoluteRank: 3, URL: "https://organic.example.com/one"},
{Rank: 1, AbsoluteRank: 1, URL: "https://ads.example.com/one", Ad: true},
{Rank: 2, AbsoluteRank: 2, URL: "https://ads.example.com/two", Ad: true},
{Rank: 2, AbsoluteRank: 4, URL: "https://organic.example.com/two"},
})
wantRanks := []int{1, 2, 1, 2}
wantAbsoluteRanks := []int{1, 2, 3, 4}
if len(results) != len(wantRanks) {
t.Fatalf("len(results) = %d, want %d", len(results), len(wantRanks))
}
for i, want := range wantRanks {
if results[i].Rank != want {
t.Fatalf("result %d rank = %d, want %d", i, results[i].Rank, want)
}
if results[i].AbsoluteRank != wantAbsoluteRanks[i] {
t.Fatalf("result %d absolute rank = %d, want %d", i, results[i].AbsoluteRank, wantAbsoluteRanks[i])
}
}
}
func TestDeduplicateResultsKeepsAdAndOrganicForSameURL(t *testing.T) {
t.Parallel()
results := DeduplicateResults([]SearchResult{
{Rank: 1, AbsoluteRank: 1, URL: "https://example.com/page", Ad: true},
{Rank: 1, AbsoluteRank: 2, URL: "https://example.com/page"},
})
if len(results) != 2 {
t.Fatalf("len(results) = %d, want 2", len(results))
}
if !results[0].Ad || results[1].Ad {
t.Fatalf("expected ad and organic rows to be preserved separately: %+v", results)
}
}
func TestLimitOrganicResultsDoesNotCountAds(t *testing.T) {
t.Parallel()
results := LimitOrganicResults([]SearchResult{
{Rank: 1, URL: "https://ads.example.com/one", Ad: true},
{Rank: 1, URL: "https://organic.example.com/one"},
{Rank: 2, URL: "https://ads.example.com/two", Ad: true},
{Rank: 2, URL: "https://organic.example.com/two"},
}, 1)
if len(results) != 3 {
t.Fatalf("len(results) = %d, want 3", len(results))
}
if CountOrganicResults(results) != 1 {
t.Fatalf("organic count = %d, want 1", CountOrganicResults(results))
}
}
func TestEnrichResultUsesAdRankAndAbsolutePosition(t *testing.T) {
t.Parallel()
result := EnrichResult(SearchResult{
Rank: 1,
AbsoluteRank: 2,
URL: "https://ads.example.com/",
Title: "Ad",
Ad: true,
}, EnrichContext{Engine: "google", Query: Query{Limit: 10}})
if result.Rank != 1 {
t.Fatalf("rank = %d, want 1", result.Rank)
}
if result.Position == nil || result.Position.Absolute != 2 {
t.Fatalf("unexpected position: %+v", result.Position)
}
if result.Type != ResultTypeAd {
t.Fatalf("unexpected result type: %q", result.Type)
}
}
func TestEnvelopePaginationCountsOrganicResults(t *testing.T) {
t.Parallel()
env := NewEnvelope(Query{Text: "ads", Limit: 2}, "req-1", time.Now(), []string{"bing"})
env.Results = []Result{
{Rank: 1, Type: ResultTypeAd},
{Rank: 2, Type: ResultTypeAd},
{Rank: 1, Type: ResultTypeOrganic},
}
env.Finalize(time.Now(), Query{Text: "ads", Limit: 2})
if env.Pagination.HasMore {
t.Fatalf("has_more = true, want false when organic count is below limit: %+v", env.Pagination)
}
}

View File

@@ -1198,10 +1198,11 @@ func (s *Server) deduplicateMegaResults(results []MegaSearchResult) []MegaSearch
if result.URL == "" {
continue
}
key := NormalizeURLForClustering(result.URL)
if key == "" {
normalizedURL := NormalizeURLForClustering(result.URL)
if normalizedURL == "" {
continue
}
key := resultDedupKey(SearchResult{URL: normalizedURL, Ad: result.Ad})
existing, exists := urlMap[key]
if !exists {
urlMap[key] = result
@@ -1219,8 +1220,11 @@ func (s *Server) deduplicateMegaResults(results []MegaSearchResult) []MegaSearch
}
sort.Slice(deduped, func(i, j int) bool {
if deduped[i].Rank != deduped[j].Rank {
return deduped[i].Rank < deduped[j].Rank
if resultLess(deduped[i].SearchResult, deduped[j].SearchResult) {
return true
}
if resultLess(deduped[j].SearchResult, deduped[i].SearchResult) {
return false
}
if deduped[i].Engine != deduped[j].Engine {
return deduped[i].Engine < deduped[j].Engine

View File

@@ -2208,7 +2208,7 @@ func TestServerOptions_DisableCORSMiddleware(t *testing.T) {
}
}
func TestDedicatedEndpointReturnsV1Envelope(t *testing.T) {
func TestDedicatedEndpointReturnsV2Envelope(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()
opts.Resilience.Retry.MaxRetries = 0
@@ -2223,8 +2223,8 @@ func TestDedicatedEndpointReturnsV1Envelope(t *testing.T) {
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
t.Fatalf("decode envelope: %v", err)
}
if env.Meta.Version != "1.0" {
t.Fatalf("expected meta.version=1.0, got %q", env.Meta.Version)
if env.Meta.Version != "2.0" {
t.Fatalf("expected meta.version=2.0, got %q", env.Meta.Version)
}
if env.Meta.RequestID == "" {
t.Fatal("expected non-empty meta.request_id")
@@ -2248,15 +2248,12 @@ func TestDedicatedEndpointReturnsV1Envelope(t *testing.T) {
if r.Snippet == "" && r.Title == "" {
t.Fatal("expected result to have title or snippet")
}
if r.Position.Page < 1 {
t.Fatalf("expected position.page >= 1, got %d", r.Position.Page)
}
if env.Pagination.Page < 1 {
t.Fatalf("expected pagination.page >= 1, got %d", env.Pagination.Page)
}
}
func TestMegaSearchReturnsV1EnvelopeWithEnginesFailed(t *testing.T) {
func TestMegaSearchReturnsV2EnvelopeWithEnginesFailed(t *testing.T) {
good := &engineMock{name: "google", initialized: true}
bad := &engineMock{
name: "bing",
@@ -2279,8 +2276,8 @@ func TestMegaSearchReturnsV1EnvelopeWithEnginesFailed(t *testing.T) {
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
t.Fatalf("decode envelope: %v", err)
}
if env.Meta.Version != "1.0" {
t.Fatalf("expected meta.version=1.0, got %q", env.Meta.Version)
if env.Meta.Version != "2.0" {
t.Fatalf("expected meta.version=2.0, got %q", env.Meta.Version)
}
if len(env.Meta.EnginesFailed) == 0 {
t.Fatal("expected engines_failed to contain bing")
@@ -2446,6 +2443,49 @@ func TestFormatParamReturnsCorrectContentType(t *testing.T) {
}
}
func TestJSONOmitsRedundantResultFields(t *testing.T) {
engine := &engineMock{
name: "google",
initialized: true,
searchFn: func(_ context.Context, _ Query) ([]SearchResult, error) {
return []SearchResult{{
Rank: 1,
AbsoluteRank: 2,
URL: "https://example.gov/page",
Title: "Gov Result",
Description: "Snippet",
}}, nil
},
}
opts := DefaultServerOptions()
opts.Resilience.Retry.MaxRetries = 0
srv := NewServerWithOptions("127.0.0.1", 7210, opts, engine)
resp := request(t, srv, "/google/search?text=clean-json")
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
body, _ := io.ReadAll(resp.Body)
payload := string(body)
for _, omitted := range []string{`"is_ad"`, `"on_page"`, `"is_gov"`} {
if strings.Contains(payload, omitted) {
t.Fatalf("json payload should not contain %s: %s", omitted, payload)
}
}
if strings.Contains(payload, `"position":{"absolute":2,"page"`) {
t.Fatalf("json payload should not contain per-result page: %s", payload)
}
if !strings.Contains(payload, `"type":"organic"`) {
t.Fatalf("json payload should keep result type: %s", payload)
}
if !strings.Contains(payload, `"category":"gov"`) {
t.Fatalf("json payload should collapse domain category: %s", payload)
}
if !strings.Contains(payload, `"absolute":2`) {
t.Fatalf("json payload should keep meaningful absolute position: %s", payload)
}
}
func TestFormatParamBypassesJSONCache(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()
@@ -2531,9 +2571,8 @@ func TestPaginatedPositionUsesAbsoluteRank(t *testing.T) {
if len(env.Results) != 1 {
t.Fatalf("expected one result, got %d", len(env.Results))
}
pos := env.Results[0].Position
if pos.Absolute != 11 || pos.OnPage != 1 || pos.Page != 2 {
t.Fatalf("unexpected position: %+v", pos)
if env.Results[0].Position == nil || env.Results[0].Position.Absolute != 11 {
t.Fatalf("expected absolute position 11, got %+v", env.Results[0].Position)
}
}

View File

@@ -83,7 +83,7 @@ Engine parsers return the older internal shape:
- `Description`
- `Ad`
HTTP handlers convert this into the public v1 response through `core/response_builder.go`.
HTTP handlers convert this into the public v2 response through `core/response_builder.go`.
## HTTP Request Flow
@@ -118,7 +118,7 @@ HTTP request
## Public API Response
JSON endpoints return a v1 envelope.
JSON endpoints return a v2 envelope.
Top-level fields:
@@ -192,7 +192,7 @@ Only JSON responses use the response cache. Cached JSON refreshes request-scoped
`core/enrichment_domain.go` derives:
- `domain_info`: public suffix, SLD, and category booleans
- `domain_info`: public suffix, SLD, and collapsed category
- `classification`: content type and known source hint
Public suffix parsing uses `golang.org/x/net/publicsuffix`.

View File

@@ -1,10 +1,10 @@
openapi: 3.0.3
info:
title: OpenSERP API
version: 1.0.0
version: 2.0.0
description: >
OpenSERP provides dedicated and multi-engine search endpoints for Google, Yandex,
Baidu, Bing, and DuckDuckGo. All responses are wrapped in a v1 envelope with
Baidu, Bing, and DuckDuckGo. Search responses are wrapped in a v2 envelope with
query echo, metadata, normalized results, and pagination. Invalid client input
returns 400 with a machine-readable `reason` code.
license:
@@ -90,7 +90,7 @@ paths:
requested_at: "2026-04-24T12:00:00Z"
took_ms: 842
engines_failed: []
version: "1.0"
version: "2.0"
results:
- id: s_a1b2c3d4e5f6a1b2
rank: 1
@@ -101,25 +101,13 @@ paths:
snippet: Go is an open source programming language...
domain: go.dev
favicon: https://go.dev/favicon.ico
is_ad: false
position:
absolute: 1
page: 1
on_page: 1
engine: google
domain_info:
tld: dev
sld: go
is_gov: false
is_edu: false
is_mil: false
is_news: false
is_forum: false
is_marketplace: false
is_social: false
classification:
content_type: webpage
source_hint: ""
category: ""
pagination:
page: 1
has_more: true
@@ -612,7 +600,7 @@ components:
name: limit
in: query
required: false
description: Maximum results to return (1100).
description: Maximum organic results to return (1100). Ads may be returned in addition.
schema:
type: integer
minimum: 1
@@ -690,7 +678,7 @@ components:
in: query
required: false
description: >
Output format. `json` (default) returns the full envelope. `markdown` returns a
Output format. `json` (default) returns the envelope. `markdown` returns a
Markdown document suitable for Slack/email. `text` returns a minimal plain-text
block optimised for LLM context windows. `ndjson` returns one result object per
line with no envelope. The `Accept` header is also checked
@@ -993,7 +981,7 @@ components:
schema:
$ref: "#/components/schemas/ErrorResponse"
schemas:
# ── v1 envelope ──────────────────────────────────────────────────
# ── v2 envelope ──────────────────────────────────────────────────
QueryEcho:
type: object
required: [text, engines_requested]
@@ -1035,7 +1023,7 @@ components:
$ref: "#/components/schemas/EngineErrorDetail"
version:
type: string
example: "1.0"
example: "2.0"
EngineErrorDetail:
type: object
required: [engine, error]
@@ -1067,21 +1055,18 @@ components:
# ── Result ───────────────────────────────────────────────────────
Position:
type: object
required: [absolute, page, on_page]
required: [absolute]
properties:
absolute:
type: integer
description: 1-based rank counting from the first result of the first page.
example: 1
page:
type: integer
example: 1
on_page:
type: integer
example: 1
description: >
1-based rank in the mixed SERP stream, across both organic and ad blocks.
Always present so SEO callers can plot rank vs. on-page position without
inferring it from result order.
example: 2
DomainInfo:
type: object
required: [tld, sld, is_gov, is_edu, is_mil, is_news, is_forum, is_marketplace, is_social]
required: [category]
properties:
tld:
type: string
@@ -1089,23 +1074,12 @@ components:
sld:
type: string
example: wikipedia
is_gov:
type: boolean
is_edu:
type: boolean
is_mil:
type: boolean
is_news:
type: boolean
is_forum:
type: boolean
is_marketplace:
type: boolean
is_social:
type: boolean
category:
type: string
description: Empty string when the domain matches no known category.
enum: ["", gov, edu, mil, news, forum, marketplace, social]
Classification:
type: object
required: [content_type, source_hint]
properties:
content_type:
type: string
@@ -1117,8 +1091,8 @@ components:
ResultType:
type: string
description: >
SERP block type. New values require a minor version bump (`meta.version: "1.1"`).
In v1.0, engines emit `organic`, `ad`, or `answer_box`; other enum values are
SERP block type. New values require a minor version bump (`meta.version: "2.1"`).
In v2.0, engines emit `organic`, `ad`, or `answer_box`; other enum values are
reserved for future parser upgrades.
enum:
- organic
@@ -1144,7 +1118,6 @@ components:
- snippet
- domain
- favicon
- is_ad
- position
- engine
properties:
@@ -1181,9 +1154,6 @@ components:
type: string
description: Constructed as `https://{domain}/favicon.ico`. Not probed.
example: https://go.dev/favicon.ico
is_ad:
type: boolean
example: false
position:
$ref: "#/components/schemas/Position"
engine:

View File

@@ -21,6 +21,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
func parseDDGDocument(doc *goquery.Document) []core.SearchResult {
var results []core.SearchResult
rank := 1
adRank := 1
absoluteRank := 1
resultSel := firstMatchingSelector(doc, Selectors.Results)
if resultSel == "" {
@@ -40,32 +42,38 @@ func parseDDGDocument(doc *goquery.Document) []core.SearchResult {
desc := extractFirstText(item, Selectors.Desc)
isAd := false
for _, sel := range Selectors.AdBadge {
if item.Find(sel).Length() > 0 {
isAd = true
break
}
}
isAd := duckduckgoSelectionHasAdMarker(item)
r := core.SearchResult{
Rank: rank,
URL: href,
Title: title,
Description: desc,
Ad: isAd,
Rank: rank,
AbsoluteRank: absoluteRank,
URL: href,
Title: title,
Description: desc,
Ad: isAd,
}
if !isAd {
rank++
} else {
r.Rank = -1
r.Rank = adRank
adRank++
}
results = append(results, r)
absoluteRank++
})
return core.DeduplicateResults(results)
}
func duckduckgoSelectionHasAdMarker(item *goquery.Selection) bool {
for _, sel := range Selectors.AdBadge {
if item.Is(sel) || item.Find(sel).Length() > 0 {
return true
}
}
return false
}
// firstMatchingSelector returns the first selector from the list that matches
// at least one element in the document.
func firstMatchingSelector(doc *goquery.Document, selectors []string) string {

View File

@@ -55,3 +55,51 @@ func TestParseDDGHTMLEmpty(t *testing.T) {
t.Fatalf("expected zero results for empty HTML, got %d", len(results))
}
}
func TestParseDDGHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
t.Parallel()
html := `
<article data-testid="ad">
<h2><a data-testid="result-title-a" href="https://ads.example.com">Sponsored Result</a></h2>
<div data-result="snippet">Paid snippet</div>
</article>
<article data-testid="result">
<h2><a data-testid="result-title-a" href="https://organic.example.com/one">Organic One</a></h2>
<div data-result="snippet">Organic snippet one</div>
</article>
<article data-testid="result">
<h2><a data-testid="result-title-a" href="https://organic.example.com/two">Organic Two</a></h2>
<div data-result="snippet">Organic snippet two</div>
</article>`
results, err := ParseHTML(bytes.NewReader([]byte(html)))
if err != nil {
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 3 {
t.Fatalf("expected 3 results, got %d", len(results))
}
organicRank := 0
adRank := 0
for _, r := range results {
if r.Ad {
adRank++
if r.Rank != adRank {
t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
}
continue
}
organicRank++
if r.Rank != organicRank {
t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
}
}
if organicRank != 2 {
t.Fatalf("organic count = %d, want 2", organicRank)
}
if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
}
}

View File

@@ -69,8 +69,11 @@ func (ddg *DuckDuckGo) isNoResults(page *rod.Page) bool {
func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.SearchResult {
searchResults := []core.SearchResult{}
organicRank := pageNum * 10
adRank := 1
absoluteRank := pageNum*10 + 1
for i, r := range results {
for _, r := range results {
// Get URL - try multiple selectors
var link *rod.Element
var err error
@@ -113,28 +116,47 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se
desc := core.FirstNonEmptyText(r, Selectors.Desc...)
// Check if it's an ad
isAd := false
for _, selector := range Selectors.AdBadge {
adIndicator, err := r.Element(selector)
if err == nil && adIndicator != nil {
isAd = true
break
}
isAd := duckduckgoElementHasAdMarker(r)
resultRank := 0
if isAd {
resultRank = adRank
adRank++
} else {
organicRank++
resultRank = organicRank
}
result := core.SearchResult{
Rank: (pageNum * 10) + (i + 1),
URL: hrefStr,
Title: title,
Description: desc,
Ad: isAd,
Rank: resultRank,
AbsoluteRank: absoluteRank,
URL: hrefStr,
Title: title,
Description: desc,
Ad: isAd,
}
searchResults = append(searchResults, result)
absoluteRank++
}
return searchResults
}
func duckduckgoElementHasAdMarker(el *rod.Element) bool {
if el == nil {
return false
}
for _, selector := range Selectors.AdBadge {
matches, err := el.Matches(selector)
if err == nil && matches {
return true
}
if adIndicator, err := el.Element(selector); err == nil && adIndicator != nil {
return true
}
}
return false
}
// Search executes a DuckDuckGo web search and returns normalized search
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -193,13 +215,13 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []
return false, nil
}
for len(allResults) < query.Limit {
for query.Limit <= 0 || core.CountOrganicResults(allResults) < query.Limit {
done, err := fetchPage()
if err != nil {
return nil, err
}
searchPage++
if done || len(allResults) >= query.Limit {
if done || (query.Limit > 0 && core.CountOrganicResults(allResults) >= query.Limit) {
break
}
if err := core.SleepContext(ctx, ddg.pageSleep); err != nil {
@@ -211,9 +233,7 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []
deduped := core.DeduplicateResults(allResults)
// Trim to exact limit if necessary
if len(deduped) > query.Limit {
deduped = deduped[:query.Limit]
}
deduped = core.LimitOrganicResults(deduped, query.Limit)
ddg.logger.Info("Search completed: %d results", len(deduped))
return deduped, nil

View File

@@ -43,9 +43,13 @@ var Selectors = struct {
"anomaly",
},
Results: []string{
"article[data-testid='result'], article[data-testid='ad'], li[data-layout='organic'], li[data-layout='ad'], div[data-testid='result'], div[data-testid='ad']",
"article[data-testid='result']",
"article[data-testid='ad']",
"li[data-layout='organic']",
"li[data-layout='ad']",
"div[data-testid='result']",
"div[data-testid='ad']",
"div.result",
},
Title: []string{
@@ -65,6 +69,9 @@ var Selectors = struct {
"h3 a",
},
AdBadge: []string{
"article[data-testid='ad']",
"li[data-layout='ad']",
"div[data-testid='ad']",
"[data-testid='ad-badge']",
".ad-badge",
".result--ad",

View File

@@ -21,6 +21,7 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
func parseEcosiaDocument(doc *goquery.Document) []core.SearchResult {
var results []core.SearchResult
rank := 1
adRank := 1
doc.Find(Selectors.Result).Each(func(_ int, item *goquery.Selection) {
res, ok := parseEcosiaItem(item, rank, false)
@@ -32,13 +33,15 @@ func parseEcosiaDocument(doc *goquery.Document) []core.SearchResult {
})
doc.Find(Selectors.Ad).Each(func(_ int, item *goquery.Selection) {
res, ok := parseEcosiaItem(item, -1, true)
res, ok := parseEcosiaItem(item, adRank, true)
if !ok {
return
}
results = append(results, res)
adRank++
})
setSeparatedAdAbsoluteRanks(results, 0)
return core.DeduplicateResults(results)
}
@@ -84,6 +87,24 @@ func parseEcosiaItem(item *goquery.Selection, rank int, ad bool) (core.SearchRes
}, true
}
func setSeparatedAdAbsoluteRanks(results []core.SearchResult, start int) {
adCount := 0
for i := range results {
if results[i].Ad {
adCount++
results[i].AbsoluteRank = start + results[i].Rank
}
}
organicAbsoluteRank := start + adCount + 1
for i := range results {
if results[i].Ad {
continue
}
results[i].AbsoluteRank = organicAbsoluteRank
organicAbsoluteRank++
}
}
// // parseEcosiaImageItem extracts a single image card from a goquery Selection.
// func parseEcosiaImageItem(item *goquery.Selection, rank int) (core.SearchResult, bool) {
// linkTag := item.Find(Selectors.ImageLink).First()

View File

@@ -139,13 +139,13 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S
}()
// nextRank counts up across pages for organic results; nextAdRank counts
// down from -1 so ads keep unique, order-preserving negative ranks.
// up within sponsored results so ad rank stays separate from SEO rank.
all := []core.SearchResult{}
pageNum, nextRank, err := startPage(query.Start)
if err != nil {
return nil, err
}
nextAdRank := -1
nextAdRank := 1
// fetchPage loads one SERP page and appends parsed results.
// Returns (done, error): done=true ends the outer loop without error.
fetchPage := func() (bool, error) {
@@ -192,13 +192,13 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S
for _, r := range ads {
if res, ok := e.parseResult(r, nextAdRank, true); ok {
all = append(all, res)
nextAdRank--
nextAdRank++
}
}
return false, nil
}
for query.Limit <= 0 || len(all) < query.Limit {
for query.Limit <= 0 || core.CountOrganicResults(all) < query.Limit {
done, err := fetchPage()
if err != nil {
return nil, err
@@ -207,7 +207,7 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S
if done {
break
}
if query.Limit > 0 && len(all) >= query.Limit {
if query.Limit > 0 && core.CountOrganicResults(all) >= query.Limit {
break
}
if err := core.SleepContext(ctx, e.pageSleep); err != nil {
@@ -215,13 +215,10 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S
}
}
setSeparatedAdAbsoluteRanks(all, query.Start)
deduped := core.DeduplicateResults(all)
if query.Limit > 0 {
organic, ads := splitAds(deduped)
if len(organic) > query.Limit {
organic = organic[:query.Limit]
}
deduped = append(organic, ads...)
deduped = core.LimitOrganicResults(deduped, query.Limit)
}
e.logger.Info("Search completed: %d results", len(deduped))
return deduped, nil
@@ -369,14 +366,3 @@ func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []c
e.logger.Info("Image search completed: %d results", len(deduped))
return deduped, nil
}
func splitAds(in []core.SearchResult) (organic, ads []core.SearchResult) {
for _, r := range in {
if r.Ad {
ads = append(ads, r)
} else {
organic = append(organic, r)
}
}
return
}

View File

@@ -121,7 +121,7 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
// Ecosia paginates by page index, not result offset, so re-rank from
// the page boundary rather than query.Start (off-grid offsets round down).
// Skip ads (rank<0) so organic ranks stay sequential from startRank.
// Skip ads so organic ranks stay sequential from startRank.
organicIdx := 0
for i := range parsedResults {
if parsedResults[i].Ad {
@@ -130,6 +130,7 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
parsedResults[i].Rank = startRank + organicIdx
organicIdx++
}
setSeparatedAdAbsoluteRanks(parsedResults, pageNum*10)
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
fmt.Sprintf("Ecosia Raw results : %v", parsedResults),

View File

@@ -68,3 +68,53 @@ func TestParseHTMLNoResults(t *testing.T) {
t.Fatalf("expected zero results, got %d", len(results))
}
}
func TestParseHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
t.Parallel()
html := `
<div>
<div data-hveid="1" data-ved="ad" data-text-ad="1">
<a href="https://ads.example.com"><h3>Sponsored Result</h3></a>
<div data-sncf="1"><div>Paid snippet</div></div>
</div>
<div data-hveid="2" data-ved="organic1">
<a href="https://organic.example.com/one"><h3>Organic One</h3></a>
<div data-sncf="1"><div>Organic snippet one</div></div>
</div>
<div data-hveid="3" data-ved="organic2">
<a href="https://organic.example.com/two"><h3>Organic Two</h3></a>
<div data-sncf="1"><div>Organic snippet two</div></div>
</div>
</div>`
results, err := ParseHTML(bytes.NewReader([]byte(html)))
if err != nil {
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 3 {
t.Fatalf("expected 3 results, got %d", len(results))
}
organicRank := 0
adRank := 0
for _, r := range results {
if r.Ad {
adRank++
if r.Rank != adRank {
t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
}
continue
}
organicRank++
if r.Rank != organicRank {
t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
}
}
if organicRank != 2 {
t.Fatalf("organic count = %d, want 2", organicRank)
}
if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
}
}

View File

@@ -236,6 +236,8 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
gogl.logger.Info("Found %d total results", totalResults)
rank := query.Start
adRank := 1
absoluteRank := query.Start + 1
for _, resEl := range searchResultElems {
srchRes := core.SearchResult{}
@@ -285,7 +287,10 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
} else {
srchRes.Description = strings.TrimSpace(text)
}
rank += 1
srchRes.Rank = adRank
srchRes.AbsoluteRank = absoluteRank
adRank++
absoluteRank++
} else if isAnswerBox {
// 2. Parse answer boxes
@@ -406,6 +411,8 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
rank += 1
srchRes.Rank = rank
srchRes.AbsoluteRank = absoluteRank
absoluteRank++
searchResults = append(searchResults, srchRes)
continue
@@ -413,7 +420,16 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
continue
}
srchRes.Rank = rank
if srchRes.Ad && srchRes.Rank == 0 {
srchRes.Rank = adRank
adRank++
} else if !srchRes.Ad {
srchRes.Rank = rank
}
if srchRes.AbsoluteRank == 0 {
srchRes.AbsoluteRank = absoluteRank
absoluteRank++
}
searchResults = append(searchResults, srchRes)
}

View File

@@ -26,6 +26,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
results := []core.SearchResult{}
rank := 1
adRank := 1
absoluteRank := 1
// Use data attributes instead of class names to find results
// Both old and new DOM have data-hveid and data-ved attributes
@@ -55,6 +57,8 @@ func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
titleTag := item.Find(Selectors.Title)
title := titleTag.Text()
isAd := item.Is(Selectors.Ad) || item.Find(Selectors.Ad).Length() > 0
// Find description - find div with text content after the heading
// Using attribute selectors that match the description container
descTag := item.Find(Selectors.DescPrimary).First()
@@ -73,15 +77,25 @@ func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
desc := descTag.Text()
if link != "" && link != "#" {
resultRank := rank
if isAd {
resultRank = adRank
adRank++
} else {
rank++
}
result := core.SearchResult{
Rank: rank,
URL: link,
Title: title,
Description: desc,
Rank: resultRank,
AbsoluteRank: absoluteRank,
URL: link,
Title: title,
Description: desc,
Ad: isAd,
}
results = append(results, result)
rank++
absoluteRank++
}
}
@@ -153,8 +167,18 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
}
if query.Start > 0 {
organicIdx := 0
for i := range parsedResults {
parsedResults[i].Rank = query.Start + i + 1
if parsedResults[i].Ad {
continue
}
organicIdx++
parsedResults[i].Rank = query.Start + organicIdx
}
for i := range parsedResults {
if parsedResults[i].AbsoluteRank > 0 {
parsedResults[i].AbsoluteRank += query.Start
}
}
}
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(

View File

@@ -8,6 +8,7 @@ var Selectors = struct {
ResultStats string
CookieBtn string
Results string
Ad string
Title string
DescPrimary string
DescFallback string
@@ -24,6 +25,7 @@ var Selectors = struct {
ResultStats: "div#result-stats",
CookieBtn: "div[role='dialog'][aria-modal] button",
Results: "div[data-hveid][data-ved]",
Ad: "div[data-text-ad], [data-text-ad]",
Title: "h3",
DescPrimary: "div[data-sncf='1'] div",
DescFallback: "div.VwiC3b",

View File

@@ -2,6 +2,7 @@ package yandex
import (
"io"
"net/url"
"strings"
"github.com/PuerkitoBio/goquery"
@@ -21,6 +22,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
func parseYandexDocument(doc *goquery.Document) []core.SearchResult {
var results []core.SearchResult
rank := 1
adRank := 1
absoluteRank := 1
doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) {
// Skip blocks without a result heading (filters out non-organic blocks
@@ -51,6 +54,7 @@ func parseYandexDocument(doc *goquery.Document) []core.SearchResult {
if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") {
return
}
isAd := yandexSelectionHasAdMarker(item) || yandexURLLooksAd(href)
title := strings.TrimSpace(titleTag.Text())
if title == "" {
@@ -67,14 +71,84 @@ func parseYandexDocument(doc *goquery.Document) []core.SearchResult {
}
}
resultRank := rank
if isAd {
resultRank = adRank
adRank++
} else {
rank++
}
results = append(results, core.SearchResult{
Rank: rank,
URL: href,
Title: title,
Description: desc,
Rank: resultRank,
AbsoluteRank: absoluteRank,
URL: href,
Title: title,
Description: desc,
Ad: isAd,
})
rank++
absoluteRank++
})
return core.DeduplicateResults(results)
}
func yandexSelectionHasAdMarker(item *goquery.Selection) bool {
for _, selector := range Selectors.AdMarkers {
if item.Is(selector) || item.Find(selector).Length() > 0 {
return true
}
}
return false
}
func yandexURLLooksAd(raw string) bool {
u, err := url.Parse(raw)
if err != nil {
return false
}
host := strings.TrimPrefix(strings.ToLower(u.Hostname()), "www.")
return strings.HasPrefix(host, "yabs.yandex.") ||
host == "an.yandex.ru" ||
strings.HasSuffix(host, ".yandexadexchange.net")
}
func skipOrganicResults(results []core.SearchResult, skip int) []core.SearchResult {
if skip <= 0 {
return results
}
out := results[:0]
for _, result := range results {
if !result.Ad && skip > 0 {
skip--
continue
}
out = append(out, result)
}
return out
}
func rebaseOrganicRanks(results []core.SearchResult, start int) {
if start <= 0 {
return
}
organicIdx := 0
for i := range results {
if results[i].Ad {
continue
}
organicIdx++
results[i].Rank = start + organicIdx
}
}
func offsetAbsoluteRanks(results []core.SearchResult, start int) {
if start <= 0 {
return
}
for i := range results {
if results[i].AbsoluteRank > 0 {
results[i].AbsoluteRank += start
}
}
}

View File

@@ -84,3 +84,83 @@ func TestParseYandexHTMLFallbackSelectors(t *testing.T) {
t.Fatalf("unexpected description: %s", results[0].Description)
}
}
func TestParseYandexHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
t.Parallel()
html := `
<ul>
<li class="serp-item" data-fast-name="direct">
<h2><a href="https://ads.example.com">Sponsored Result</a></h2>
<div class="OrganicText">Paid snippet</div>
</li>
<li class="serp-item">
<h2><a href="https://organic.example.com/one">Organic One</a></h2>
<div class="OrganicText">Organic snippet one</div>
</li>
<li class="serp-item">
<h2><a href="https://organic.example.com/two">Organic Two</a></h2>
<div class="OrganicText">Organic snippet two</div>
</li>
</ul>`
results, err := ParseHTML(bytes.NewReader([]byte(html)))
if err != nil {
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 3 {
t.Fatalf("expected 3 results, got %d", len(results))
}
organicRank := 0
adRank := 0
for _, r := range results {
if r.Ad {
adRank++
if r.Rank != adRank {
t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
}
continue
}
organicRank++
if r.Rank != organicRank {
t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
}
}
if organicRank != 2 {
t.Fatalf("organic count = %d, want 2", organicRank)
}
if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
}
}
func TestParseYandexHTMLYabsURLIsAd(t *testing.T) {
t.Parallel()
html := `
<ul>
<li class="serp-item">
<h2><a href="https://yabs.yandex.ru/count/WuGejI_zOoVX2">WB Travel</a></h2>
<div class="OrganicText">Paid snippet</div>
</li>
<li class="serp-item">
<h2><a href="https://organic.example.com/one">Organic One</a></h2>
<div class="OrganicText">Organic snippet one</div>
</li>
</ul>`
results, err := ParseHTML(bytes.NewReader([]byte(html)))
if err != nil {
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 2 {
t.Fatalf("expected 2 results, got %d", len(results))
}
if !results[0].Ad || results[0].Rank != 1 {
t.Fatalf("first result should be ad rank 1: %+v", results[0])
}
if results[1].Ad || results[1].Rank != 1 {
t.Fatalf("second result should be organic rank 1: %+v", results[1])
}
}

View File

@@ -71,8 +71,11 @@ func (yand *Yandex) isNoResults(page *rod.Page) bool {
func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.SearchResult {
searchResults := []core.SearchResult{}
organicRank := pageNum * 10
adRank := 1
absoluteRank := pageNum*10 + 1
for i, r := range results {
for _, r := range results {
// Get URL
link, err := r.Element(Selectors.Link)
if err != nil {
@@ -109,13 +112,48 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
desc, _ = descTag.Text()
}
r := core.SearchResult{Rank: (pageNum * 10) + (i + 1), URL: linkText.String(), Title: title, Description: desc}
searchResults = append(searchResults, r)
hrefStr := linkText.String()
isAd := yandexElementHasAdMarker(r) || yandexURLLooksAd(hrefStr)
resultRank := 0
if isAd {
resultRank = adRank
adRank++
} else {
organicRank++
resultRank = organicRank
}
res := core.SearchResult{
Rank: resultRank,
AbsoluteRank: absoluteRank,
URL: hrefStr,
Title: title,
Description: desc,
Ad: isAd,
}
searchResults = append(searchResults, res)
absoluteRank++
}
return searchResults
}
func yandexElementHasAdMarker(el *rod.Element) bool {
if el == nil {
return false
}
for _, selector := range Selectors.AdMarkers {
matches, err := el.Matches(selector)
if err == nil && matches {
return true
}
if child, err := el.Element(selector); err == nil && child != nil {
return true
}
}
return false
}
func (yand *Yandex) parseImageEntities(items rod.Elements) map[string]ImageEntity {
entities := make(map[string]ImageEntity)
for _, item := range items {
@@ -193,17 +231,13 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor
r := yand.parseResults(elements, searchPage)
if searchPage == startPage && skipOnFirstPage > 0 {
if skipOnFirstPage >= len(r) {
r = []core.SearchResult{}
} else {
r = r[skipOnFirstPage:]
}
r = skipOrganicResults(r, skipOnFirstPage)
}
allResults = append(allResults, r...)
return false, nil
}
for len(allResults) < query.Limit {
for query.Limit <= 0 || core.CountOrganicResults(allResults) < query.Limit {
done, err := fetchPage()
if err != nil {
return nil, err
@@ -212,13 +246,16 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor
if done {
break
}
if query.Limit > 0 && core.CountOrganicResults(allResults) >= query.Limit {
break
}
if err := core.SleepContext(ctx, yand.pageSleep); err != nil {
return nil, err
}
}
yand.logger.Info("Search completed: %d results", len(allResults))
return core.DeduplicateResults(allResults), nil
return core.LimitOrganicResults(core.DeduplicateResults(allResults), query.Limit), nil
}
// SearchImage executes a Yandex image search and returns normalized image

View File

@@ -74,17 +74,10 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
}
if skipOnFirstPage > 0 {
if skipOnFirstPage >= len(parsedResults) {
parsedResults = []core.SearchResult{}
} else {
parsedResults = parsedResults[skipOnFirstPage:]
}
}
if query.Start > 0 {
for i := range parsedResults {
parsedResults[i].Rank = query.Start + i + 1
}
parsedResults = skipOrganicResults(parsedResults, skipOnFirstPage)
}
rebaseOrganicRanks(parsedResults, query.Start)
offsetAbsoluteRanks(parsedResults, startPage*10)
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
fmt.Sprintf("Yandex Raw results : %v", parsedResults),
)

View File

@@ -2,9 +2,10 @@ package yandex
// Selectors is the single source of truth for Yandex SERP CSS selectors.
var Selectors = struct {
Captcha string
NoResults string
Results string
Captcha string
NoResults string
Results string
AdMarkers []string
// LinkPrimary is preferred over a generic <a>; falls back to title.Closest("a")
// then the first <a> in the result block when absent.
LinkPrimary string
@@ -16,9 +17,18 @@ var Selectors = struct {
ImageItemsAlt []string
ImageStateAll string
}{
Captcha: "div.CheckboxCaptcha",
NoResults: "div.EmptySearchResults",
Results: "li[data-fast], li.serp-item",
Captcha: "div.CheckboxCaptcha",
NoResults: "div.EmptySearchResults",
Results: "li[data-fast], li.serp-item",
AdMarkers: []string{
"[data-fast-name='direct']",
"[data-fast-name='serp-adv']",
"[data-bem*='serp-adv']",
".serp-adv-item",
".serp-adv__found",
"[aria-label='Реклама']",
"[title='Реклама']",
},
LinkPrimary: "a.OrganicTitle-Link",
Link: "a",
Title: "h2",