openserp/bing/search.go

package bing

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/go-rod/rod"
	"github.com/go-rod/rod/lib/proto"
	"github.com/karust/openserp/core"
)

// Bing implements core.SearchEngine for Bing SERP pages.
type Bing struct {
	core.Browser
	core.SearchEngineOptions
	logger *core.EngineLogger
}

// New creates a Bing engine instance with browser/runtime options applied.
func New(browser core.Browser, opts core.SearchEngineOptions) *Bing {
	bing := Bing{Browser: browser}
	opts.Init()
	bing.SearchEngineOptions = opts
	bing.logger = core.NewEngineLogger("Bing")
	return &bing
}

// Name returns the stable engine identifier.
func (bing *Bing) Name() string {
	return "bing"
}

func (bing *Bing) getTotalResults(page *rod.Page) (int, error) {
	results, err := page.Timeout(bing.GetSelectorTimeout()).Elements(Selectors.Results)
	if err != nil {
		return 0, errors.New("Cannot find result elements: " + err.Error())
	}
	return len(results), nil
}

// checkCaptcha classifies the live page the same way classifyBingDocument
// (parse_html.go) classifies raw HTML, so /bing/search and /bing/parse agree.
func (bing *Bing) checkCaptcha(page *rod.Page) bool {
	if page == nil {
		return false
	}

	if info, err := page.Info(); err == nil {
		url := strings.ToLower(info.URL)
		if strings.Contains(url, "turing") || strings.Contains(url, "captcha") {
			return true
		}
	}

	for _, selector := range Selectors.Captcha {
		has, _, _ := page.Has(selector)
		if has {
			bing.logger.Debug("Captcha detected: %s", selector)
			return true
		}
	}

	return pageTextContainsAny(page, Selectors.CaptchaMarkers)
}

// checkNoResults reports whether the page text matches Bing's no-results
// phrasing, mirroring classifyBingDocument's text-marker check.
func (bing *Bing) checkNoResults(page *rod.Page) bool {
	return pageTextContainsAny(page, Selectors.NoResultsMarkers)
}

func pageTextContainsAny(page *rod.Page, markers []string) bool {
	if page == nil || len(markers) == 0 {
		return false
	}
	html, err := page.HTML()
	if err != nil {
		return false
	}
	text := strings.ToLower(html)
	for _, marker := range markers {
		if strings.Contains(text, marker) {
			return true
		}
	}
	return false
}

func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error {
	// Probe first so a banner-less SERP returns immediately instead of blocking
	// .Element for the full Timeout/10.
	if has, _, err := page.Has(Selectors.CookieBtn); err != nil || !has {
		return nil
	}
	consentBtn, err := page.Timeout(bing.Timeout / 10).Element(Selectors.CookieBtn)
	if err != nil {
		return nil
	}
	if err := consentBtn.Click(proto.InputMouseButtonLeft, 1); err != nil {
		bing.logger.Debug("Cookie consent click failed: %v", err)
	}

	return core.SleepContext(ctx, 500*time.Millisecond)
}

func bingElementMatches(el *rod.Element, selector string) bool {
	if el == nil {
		return false
	}
	matches, err := el.Matches(selector)
	return err == nil && matches
}

func (bing *Bing) parseResultElement(el *rod.Element, isAd bool, rank *core.RankState) (core.SearchResult, bool) {
	titleSelector := Selectors.Title
	if isAd {
		titleSelector = Selectors.AdTitle
	}

	titleElem, err := el.Element(titleSelector)
	if err != nil {
		bing.logger.Debug("Missing title")
		return core.SearchResult{}, false
	}
	href, err := titleElem.Property("href")
	if err != nil {
		bing.logger.Debug("Missing URL")
		return core.SearchResult{}, false
	}

	title := core.ElementAttribute(titleElem, "aria-label", "title")
	if title == "" {
		title = core.ElementText(titleElem)
	}
	if title == "" {
		title = core.FirstNonEmptyText(el, Selectors.TitleFallbacks...)
	}
	if title == "" {
		title = core.FirstNonEmptyAttribute(el, "aria-label", Selectors.TitleFallbacks...)
	}

	desc := core.FirstNonEmptyText(el, Selectors.DescPrimary, Selectors.DescFallback, Selectors.DescAny)
	if desc == "" {
		fullText, _ := el.Text()
		desc = core.NormalizeWhitespace(strings.Replace(fullText, title, "", 1))
	}

	return assembleBingRow(href.String(), title, desc, isAd, rank)
}

// Search executes a Bing web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
	ctx = core.PrepareEngineContext(ctx, query, bing.Name())
	scoped := *bing
	scoped.logger = bing.logger.WithRequest(ctx)
	bing = &scoped

	bing.logger.Debug("Starting search, query: %+v", query)

	searchResults := []core.SearchResult{}

	url, err := BuildURL(query)
	if err != nil {
		return nil, err
	}

	page, err := bing.Navigate(ctx, url)
	if err != nil {
		return nil, err
	}
	defer core.DeferClosePage(ctx, page, &bing.Browser)()

	if bing.checkCaptcha(page) {
		bing.logger.Error("Captcha detected: %s", url)
		return nil, core.ErrCaptcha
	}

	if err := bing.acceptCookies(ctx, page); err != nil {
		return nil, err
	}

	resultElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.ResultItems, Selectors.Results}, bing.GetSelectorTimeout())
	if err != nil {
		// Re-check captcha/no-results on timeout - Bing interstitials and
		// no-results pages can both render after WaitLoad.
		if bing.checkCaptcha(page) {
			bing.logger.Error("Captcha detected: %s", url)
			return nil, core.ErrCaptcha
		}
		if bing.checkNoResults(page) {
			return []core.SearchResult{}, nil
		}
		bing.logger.Error("Cannot parse organic results: %s", err)
		return nil, core.ErrSearchTimeout
	}

	totalResults, err := bing.getTotalResults(page)
	if err != nil {
		bing.logger.Debug("Failed to get total results: %v", err)
	}
	bing.logger.Info("Found %d organic result containers", totalResults)

	rank := core.NewRankStateAt(query.Start, query.Start+1)
	for _, result := range resultElements {
		isAd := bingElementMatches(result, Selectors.Ads)
		isOrganic := bingElementMatches(result, Selectors.Results)
		if !isAd && !isOrganic {
			continue
		}

		srchRes, ok := bing.parseResultElement(result, isAd, rank)
		if !ok {
			continue
		}
		searchResults = append(searchResults, srchRes)
	}

	// Deduplicate results
	deduped := core.DeduplicateResults(searchResults)

	deduped = core.LimitOrganicResults(deduped, query.Limit)

	if query.Features {
		deduped = core.AttachFeaturesToFirstResult(deduped, extractBingFeaturesFromPage(ctx, page))
	}
	return deduped, nil
}

// BingImageData represents metadata encoded in the image result `m` attribute.
type BingImageData struct {
	T      string `json:"t"`      // Title
	Desc   string `json:"desc"`   // Description
	IMGURL string `json:"imgurl"` // Original image URL
	W      int    `json:"w"`      // Width
	H      int    `json:"h"`      // Height
	PURL   string `json:"purl"`   // Page URL
	TURL   string `json:"turl"`   // Thumbnail URL
	MURL   string `json:"murl"`   // Image URL
}

func resolveImageLinkElement(container *rod.Element) (*rod.Element, error) {
	if container == nil {
		return nil, errors.New("nil image container")
	}
	if core.HasAttribute(container, "m") {
		return container, nil
	}
	return container.Element("a")
}

// SearchImage executes a Bing image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
	ctx = core.PrepareEngineContext(ctx, query, bing.Name())
	scoped := *bing
	scoped.logger = bing.logger.WithRequest(ctx)
	bing = &scoped

	bing.logger.Debug("Starting image search, query: %+v", query)

	searchResults := []core.SearchResult{}

	// Build Bing image search URL
	url, err := BuildImageURL(query)
	if err != nil {
		return nil, err
	}

	page, err := bing.Navigate(ctx, url)
	if err != nil {
		return nil, err
	}
	defer core.DeferClosePage(ctx, page, &bing.Browser)()

	// Check for captcha
	if bing.checkCaptcha(page) {
		bing.logger.Error("Captcha detected during image search: %s", url)
		return nil, core.ErrCaptcha
	}

	// Accept cookies if present
	if err := bing.acceptCookies(ctx, page); err != nil {
		return nil, err
	}

	imageContainers, _, err := core.WaitForElements(
		ctx,
		page,
		[]string{Selectors.ImageResults},
		bing.GetSelectorTimeout(),
	)
	if err != nil {
		if bing.checkCaptcha(page) {
			return nil, core.ErrCaptcha
		}
		bing.logger.Error("Cannot parse image results: %s", err)
		return nil, core.ErrSearchTimeout
	}

	if len(imageContainers) == 0 {
		return nil, errors.New("no image results found")
	}

	bing.logger.Info("Found %d image elements", len(imageContainers))

	rank := 0
	for _, c := range imageContainers {
		srchRes := core.SearchResult{}

		linkElem, err := resolveImageLinkElement(c)
		if err != nil {
			bing.logger.Debug("Missing image link element")
			continue
		}

		// Extract image metadata from m attribute (contains JSON)
		mAttr, err := linkElem.Attribute("m")
		if err != nil || mAttr == nil {
			bing.logger.Debug("Missing m attribute")
			continue
		}

		// Ensure we have valid JSON data to unmarshal
		jsonData := []byte(*mAttr)
		if len(jsonData) == 0 {
			bing.logger.Debug("Empty JSON data")
			continue
		}

		// Initialize imgData properly
		var imgData BingImageData
		err = json.Unmarshal(jsonData, &imgData)
		if err != nil {
			bing.logger.Debug("Failed to parse JSON: %s", err)
			continue
		}

		// Extract information from the parsed data
		srchRes.Title = imgData.T
		srchRes.URL = imgData.IMGURL
		srchRes.Description = fmt.Sprintf(
			"%s Source Page: %s, thumb_url:%s, %dx%d",
			imgData.Desc,
			imgData.PURL,
			imgData.TURL,
			imgData.W,
			imgData.H,
		)

		// Get the page URL
		if imgData.MURL != "" {
			srchRes.URL = imgData.MURL
		}

		rank++
		srchRes.Rank = rank

		searchResults = append(searchResults, srchRes)
		if query.Limit > 0 && len(searchResults) >= query.Limit {
			break
		}
	}

	return searchResults, nil
}