openserp/core/page_helpers.go

package core

import (
	"context"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/go-rod/rod"
)

// pollInterval is how often WaitForElements re-probes selectors while the
// page hydrates. Short enough to feel snappy, long enough not to hammer CDP.
const pollInterval = 120 * time.Millisecond

// WaitForElements probes the supplied CSS selectors until one returns at least
// one matching element or timeout elapses. It exists because rod's
// page.Search/Elements and the surrounding WaitLoad/WaitStable do not wait for
// a *specific* selector to hydrate — modern SPA SERPs (DDG, Bing, Google)
// regularly fire `load` and even reach DOM-stable before result rows render,
// causing parsers to see an empty page on the first probe and forcing the
// caller's retry layer to reload.
//
// The probe loop returns as soon as a selector matches, returning the matched
// elements and the selector that hit. On timeout it returns ErrSearchTimeout
// so callers can disambiguate between "no results" / "captcha" by inspecting
// the page directly.
func WaitForElements(ctx context.Context, page *rod.Page, selectors []string, timeout time.Duration) (rod.Elements, string, error) {
	if page == nil {
		return nil, "", ErrSearchTimeout
	}
	ctx = EnsureContext(ctx)
	if timeout <= 0 {
		timeout = 2 * time.Second
	}

	probe := func() (rod.Elements, string) {
		for _, selector := range selectors {
			elements, err := page.Elements(selector)
			if err != nil || len(elements) == 0 {
				continue
			}
			return elements, selector
		}
		return nil, ""
	}

	if elements, selector := probe(); len(elements) > 0 {
		return elements, selector, nil
	}

	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		if err := ctx.Err(); err != nil {
			return nil, "", err
		}
		if elements, selector := probe(); len(elements) > 0 {
			return elements, selector, nil
		}
		if err := SleepContext(ctx, pollInterval); err != nil {
			return nil, "", err
		}
	}
	return nil, "", ErrSearchTimeout
}

// DocSignals is the per-engine selector/text-marker config for
// ClassifyChallengeDocument.
type DocSignals struct {
	CaptchaSelectors []string
	CaptchaMarkers   []string
	EmptySelectors   []string
	EmptyMarkers     []string
}

// ClassifyChallengeDocument is the shared captcha/empty-result check behind
// every engine's classify*Document. Selectors are checked before markers
// since they're cheaper (no doc.Text() walk).
func ClassifyChallengeDocument(doc *goquery.Document, s DocSignals) error {
	if anySelectorMatches(doc, s.CaptchaSelectors) {
		return ErrCaptcha
	}
	var text string
	if len(s.CaptchaMarkers)+len(s.EmptyMarkers) > 0 {
		text = strings.ToLower(doc.Text())
	}
	if anyMarkerMatches(text, s.CaptchaMarkers) {
		return ErrCaptcha
	}
	if anySelectorMatches(doc, s.EmptySelectors) {
		return ErrEmptyResult
	}
	if anyMarkerMatches(text, s.EmptyMarkers) {
		return ErrEmptyResult
	}
	return nil
}

func anySelectorMatches(doc *goquery.Document, selectors []string) bool {
	for _, selector := range selectors {
		if doc.Find(selector).Length() > 0 {
			return true
		}
	}
	return false
}

func anyMarkerMatches(text string, markers []string) bool {
	for _, marker := range markers {
		if strings.Contains(text, marker) {
			return true
		}
	}
	return false
}

// HasAnySelector returns true if at least one of the supplied selectors
// currently matches in the page DOM. It does not wait — pair with
// WaitForElements when hydration may be in flight.
func HasAnySelector(page *rod.Page, selectors []string) bool {
	if page == nil {
		return false
	}
	for _, selector := range selectors {
		has, _, err := page.Has(selector)
		if err == nil && has {
			return true
		}
	}
	return false
}

// DeferClosePage returns a cleanup function that closes page unless the browser
// is configured to leave pages open for debugging.
func DeferClosePage(ctx context.Context, page *rod.Page, browser *Browser) func() {
	return func() {
		if browser != nil && browser.LeavePageOpen {
			return
		}
		if err := ClosePageWithTimeout(ctx, page, time.Second); err != nil {
			WithRequest(ctx).WithError(err).Debug("Page close error")
		}
	}
}

// HasAttribute reports whether el carries attr (regardless of value).
func HasAttribute(el *rod.Element, attr string) bool {
	if el == nil {
		return false
	}
	v, err := el.Attribute(attr)
	return err == nil && v != nil
}

// NormalizeWhitespace collapses runs of whitespace (newlines, source
// indentation) into single spaces and trims the result.
func NormalizeWhitespace(s string) string {
	return strings.Join(strings.Fields(s), " ")
}

// ElementText returns el's visible text, falling back to textContent (for nodes
// rod's Text() leaves empty), normalized. Empty string if el is nil or blank.
func ElementText(el *rod.Element) string {
	if el == nil {
		return ""
	}
	if text, err := el.Text(); err == nil {
		if normalized := NormalizeWhitespace(text); normalized != "" {
			return normalized
		}
	}
	if value, err := el.Property("textContent"); err == nil {
		return NormalizeWhitespace(value.String())
	}
	return ""
}

// ElementAttribute returns the first non-empty value among attrs on el,
// normalized. Empty string if el is nil or none are set.
func ElementAttribute(el *rod.Element, attrs ...string) string {
	if el == nil {
		return ""
	}
	for _, attr := range attrs {
		value, err := el.Attribute(attr)
		if err != nil || value == nil {
			continue
		}
		if normalized := NormalizeWhitespace(*value); normalized != "" {
			return normalized
		}
	}
	return ""
}

// FirstNonEmptyText returns the text (see ElementText) of the first selector
// under root that yields non-empty content. Empty string if none match.
func FirstNonEmptyText(root *rod.Element, selectors ...string) string {
	if root == nil {
		return ""
	}
	for _, selector := range selectors {
		el, err := root.Element(selector)
		if err != nil {
			continue
		}
		if text := ElementText(el); text != "" {
			return text
		}
	}
	return ""
}

// ClosestMatching walks up the ancestor chain (including el itself) and returns
// the first element matching selector, or nil if none is found within maxHops.
// rod has no native Closest helper, so this is a bounded walk used by parsers
// that need to recover a wrapping <a> from a nested title node.
func ClosestMatching(el *rod.Element, selector string, maxHops int) *rod.Element {
	if el == nil || selector == "" {
		return nil
	}
	current := el
	for hop := 0; hop <= maxHops; hop++ {
		if matches, err := current.Matches(selector); err == nil && matches {
			return current
		}
		parent, err := current.Parent()
		if err != nil || parent == nil {
			return nil
		}
		current = parent
	}
	return nil
}

// FirstNonEmptyAttribute returns the trimmed value of attr from the first
// selector under root whose attribute is non-empty.
func FirstNonEmptyAttribute(root *rod.Element, attr string, selectors ...string) string {
	if root == nil {
		return ""
	}
	for _, selector := range selectors {
		el, err := root.Element(selector)
		if err != nil {
			continue
		}
		value, err := el.Attribute(attr)
		if err != nil || value == nil {
			continue
		}
		if trimmed := strings.TrimSpace(*value); trimmed != "" {
			return trimmed
		}
	}
	return ""
}

// FeaturesFromPage renders a live rod page to HTML and runs a document-level
// feature extractor over it. Every engine's browser path shares this boilerplate
// (page.HTML -> goquery doc -> extract), so it lives here rather than being
// copied per engine. Returns nil on any rendering/parse error.
func FeaturesFromPage(page *rod.Page, extract func(*goquery.Document) []SerpFeature) []SerpFeature {
	if page == nil {
		return nil
	}
	doc, err := DocumentFromPage(page)
	if err != nil {
		return nil
	}
	return extract(doc)
}

// DocumentFromPage snapshots a live rod page to a goquery document, so the
// browser path can reuse the same parsing/classification rules as the raw
// HTTP path instead of reimplementing them against the rod API.
func DocumentFromPage(page *rod.Page) (*goquery.Document, error) {
	html, err := page.HTML()
	if err != nil {
		return nil, err
	}
	return goquery.NewDocumentFromReader(strings.NewReader(html))
}

// ClassifyFromPage snapshots page and runs classify over it, returning nil if
// the page can't be read (caller treats that as "not classified").
func ClassifyFromPage(page *rod.Page, classify func(*goquery.Document) error) error {
	doc, err := DocumentFromPage(page)
	if err != nil {
		return nil
	}
	return classify(doc)
}

const featureHydrationWait = 2000 * time.Millisecond

// FeaturesFromPageWithWait lets async feature modules render before snapshotting.
func FeaturesFromPageWithWait(ctx context.Context, page *rod.Page, extract func(*goquery.Document) []SerpFeature) []SerpFeature {
	if page != nil {
		_, _ = page.Eval(`() => window.scrollTo(0, document.body.scrollHeight)`)
		if err := SleepContext(ctx, featureHydrationWait); err != nil {
			return nil
		}
	}
	return FeaturesFromPage(page, extract)
}