feat(api): v2 response shape - absolute rank, ad/organic split, slimmer fields

google/parse_html_test.go

@@ -68,3 +68,53 @@ func TestParseHTMLNoResults(t *testing.T) {
 		t.Fatalf("expected zero results, got %d", len(results))
 	}
 }
+
+func TestParseHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
+	t.Parallel()
+
+	html := `
+<div>
+  <div data-hveid="1" data-ved="ad" data-text-ad="1">
+    <a href="https://ads.example.com"><h3>Sponsored Result</h3></a>
+    <div data-sncf="1"><div>Paid snippet</div></div>
+  </div>
+  <div data-hveid="2" data-ved="organic1">
+    <a href="https://organic.example.com/one"><h3>Organic One</h3></a>
+    <div data-sncf="1"><div>Organic snippet one</div></div>
+  </div>
+  <div data-hveid="3" data-ved="organic2">
+    <a href="https://organic.example.com/two"><h3>Organic Two</h3></a>
+    <div data-sncf="1"><div>Organic snippet two</div></div>
+  </div>
+</div>`
+
+	results, err := ParseHTML(bytes.NewReader([]byte(html)))
+	if err != nil {
+		t.Fatalf("ParseHTML() error = %v", err)
+	}
+	if len(results) != 3 {
+		t.Fatalf("expected 3 results, got %d", len(results))
+	}
+
+	organicRank := 0
+	adRank := 0
+	for _, r := range results {
+		if r.Ad {
+			adRank++
+			if r.Rank != adRank {
+				t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
+			}
+			continue
+		}
+		organicRank++
+		if r.Rank != organicRank {
+			t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
+		}
+	}
+	if organicRank != 2 {
+		t.Fatalf("organic count = %d, want 2", organicRank)
+	}
+	if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
+		t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
+	}
+}

google/search.go

@@ -236,6 +236,8 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
 	gogl.logger.Info("Found %d total results", totalResults)
 
 	rank := query.Start
+	adRank := 1
+	absoluteRank := query.Start + 1
 	for _, resEl := range searchResultElems {
 		srchRes := core.SearchResult{}
 
@@ -285,7 +287,10 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
 			} else {
 				srchRes.Description = strings.TrimSpace(text)
 			}
-			rank += 1
+			srchRes.Rank = adRank
+			srchRes.AbsoluteRank = absoluteRank
+			adRank++
+			absoluteRank++
 
 		} else if isAnswerBox {
 			// 2. Parse answer boxes
@@ -406,6 +411,8 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
 
 			rank += 1
 			srchRes.Rank = rank
+			srchRes.AbsoluteRank = absoluteRank
+			absoluteRank++
 			searchResults = append(searchResults, srchRes)
 			continue
 
@@ -413,7 +420,16 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
 			continue
 		}
 
-		srchRes.Rank = rank
+		if srchRes.Ad && srchRes.Rank == 0 {
+			srchRes.Rank = adRank
+			adRank++
+		} else if !srchRes.Ad {
+			srchRes.Rank = rank
+		}
+		if srchRes.AbsoluteRank == 0 {
+			srchRes.AbsoluteRank = absoluteRank
+			absoluteRank++
+		}
 		searchResults = append(searchResults, srchRes)
 	}
 

google/search_raw.go

@@ -26,6 +26,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
 func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
 	results := []core.SearchResult{}
 	rank := 1
+	adRank := 1
+	absoluteRank := 1
 
 	// Use data attributes instead of class names to find results
 	// Both old and new DOM have data-hveid and data-ved attributes
@@ -55,6 +57,8 @@ func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
 		titleTag := item.Find(Selectors.Title)
 		title := titleTag.Text()
 
+		isAd := item.Is(Selectors.Ad) || item.Find(Selectors.Ad).Length() > 0
+
 		// Find description - find div with text content after the heading
 		// Using attribute selectors that match the description container
 		descTag := item.Find(Selectors.DescPrimary).First()
@@ -73,15 +77,25 @@ func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
 		desc := descTag.Text()
 
 		if link != "" && link != "#" {
+			resultRank := rank
+			if isAd {
+				resultRank = adRank
+				adRank++
+			} else {
+				rank++
+			}
+
 			result := core.SearchResult{
-				Rank:        rank,
-				URL:         link,
-				Title:       title,
-				Description: desc,
+				Rank:         resultRank,
+				AbsoluteRank: absoluteRank,
+				URL:          link,
+				Title:        title,
+				Description:  desc,
+				Ad:           isAd,
 			}
 
 			results = append(results, result)
-			rank++
+			absoluteRank++
 		}
 	}
 
@@ -153,8 +167,18 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
 	}
 
 	if query.Start > 0 {
+		organicIdx := 0
 		for i := range parsedResults {
-			parsedResults[i].Rank = query.Start + i + 1
+			if parsedResults[i].Ad {
+				continue
+			}
+			organicIdx++
+			parsedResults[i].Rank = query.Start + organicIdx
+		}
+		for i := range parsedResults {
+			if parsedResults[i].AbsoluteRank > 0 {
+				parsedResults[i].AbsoluteRank += query.Start
+			}
 		}
 	}
 	core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(

google/selectors.go

@@ -8,6 +8,7 @@ var Selectors = struct {
 	ResultStats  string
 	CookieBtn    string
 	Results      string
+	Ad           string
 	Title        string
 	DescPrimary  string
 	DescFallback string
@@ -24,6 +25,7 @@ var Selectors = struct {
 	ResultStats:  "div#result-stats",
 	CookieBtn:    "div[role='dialog'][aria-modal] button",
 	Results:      "div[data-hveid][data-ved]",
+	Ad:           "div[data-text-ad], [data-text-ad]",
 	Title:        "h3",
 	DescPrimary:  "div[data-sncf='1'] div",
 	DescFallback: "div.VwiC3b",