diff --git a/google/search.go b/google/search.go index e0a29e4..78e27e3 100644 --- a/google/search.go +++ b/google/search.go @@ -157,8 +157,8 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { gogl.acceptCookies(page) } - // Find all results - results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid]") + // Find all results using stable attributes + results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved]") if err != nil { logrus.Errorf("Cannot parse search results: %s", err) return nil, core.ErrSearchTimeout @@ -265,35 +265,51 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { searchResults = append(searchResults, srchRes) } continue - } else if strings.Contains(attrs, "data-ved") && strings.Contains(attrs, "lang") { - // 3. Parse regular search results - // Get URL - link, err := resEl.Element("a") + } else if strings.Contains(attrs, "data-ved") { + // Parse regular search results + // Get title from h3 + titleTag, err := resEl.Element("h3") if err != nil { continue } - href, err := link.Property("href") - if err != nil { - logrus.Debug("No `href` tag found") + srchRes.Title, _ = titleTag.Text() + + // Get URL from parent link of h3 + link, err := titleTag.Parent() + if err == nil && link.MustMatches("a") { + href, _ := link.Property("href") + srchRes.URL = href.String() } - srchRes.URL = href.String() + + // Get description using multiple fallback strategies + desc := "" + if descTag, err := resEl.Element("div[data-sncf='1'] div"); err == nil { + desc = descTag.MustText() + } else if descTag, err := resEl.Element("div.VwiC3b"); err == nil { + desc = descTag.MustText() + } else { + // Structural fallback + parent, err := titleTag.Parent() + if err == nil { + parent, err = parent.Parent() + if err == nil { + parent, err = parent.Parent() + if err == nil { + if descTag, err := parent.Next(); err == nil { + if descDiv, err := descTag.Element("div"); err == nil { + desc = descDiv.MustText() + } + } + } + } + } + } + srchRes.Description = desc + rank += 1 - - // Get title - titleTag, err := link.Element("h3") - if err != nil { - continue - } - - srchRes.Title, err = titleTag.Text() - if err != nil { - logrus.Debug("Cannot extract text from title") - } - - // Get description - text := resEl.MustText() - textSliced := strings.Split(text, "\n") - srchRes.Description = strings.Join(textSliced[:], "\n") + srchRes.Rank = rank + searchResults = append(searchResults, srchRes) + continue } else { //fmt.Println(i, attrs) @@ -324,7 +340,7 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) { page.Mouse.Scroll(0, 1000000, 1) page.WaitLoad() - results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction][jsdata]") + results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]") if err != nil { logrus.Errorf("Cannot parse search results: %s", err) return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout diff --git a/google/search_raw.go b/google/search_raw.go index 287a355..59c15ff 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -33,6 +33,7 @@ func googleRequest(searchURL string, query core.Query) (*http.Response, error) { Transport: transport, Timeout: time.Second * 10, } + req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, err @@ -55,23 +56,49 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) { results := []core.SearchResult{} rank := 1 - // Get individual results - sel := doc.Find("div.g") + // Use data attributes instead of class names to find results + // Both old and new DOM have data-hveid and data-ved attributes + sel := doc.Find("div[data-hveid][data-ved]") for i := range sel.Nodes { item := sel.Eq(i) - // Find URL - linkTag := item.Find("a") - link, _ := linkTag.Attr("href") + // Skip items without an h3 element (which indicates a search result) + if item.Find("h3").Length() == 0 { + continue + } + + // Find URL - look for the anchor that contains the h3 title + linkTag := item.Find("h3").Parent() + if linkTag.Is("a") == false { + linkTag = item.Find("h3").Closest("a") + } + + link, exists := linkTag.Attr("href") + if !exists || link == "" || link == "#" { + continue + } link = strings.Trim(link, " ") - // Find title + // Find title - this is inside the h3 element titleTag := item.Find("h3") title := titleTag.Text() - // Find description - descTag := item.Find(`div[data-sncf~="1"]`) + // Find description - find div with text content after the heading + // Using attribute selectors that match the description container + descTag := item.Find("div[data-sncf='1']").Find("div").First() + if descTag.Length() == 0 { + // Try another selector approach if the first one fails + descTag = item.Find("div.VwiC3b") + if descTag.Length() == 0 { + // As a last resort, look for any div after the title that might contain description + titleParent := titleTag.Parent() + if titleParent.Is("a") { + titleParent = titleParent.Parent().Parent() + } + descTag = titleParent.NextAll().First().Find("div").First() + } + } desc := descTag.Text() if link != "" && link != "#" {