From 2b96a8330f0cf6b7150957ed175c5c1a23b9a232 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Thu, 28 May 2026 01:36:52 +0300 Subject: [PATCH] SERP features: AI summary, answer boxes, related searches/questions, etc - for supporting engines + other fixesa and improvements --- README.md | 9 +- baidu/features.go | 40 ++++ baidu/parse_html.go | 55 ++++- baidu/parse_html_test.go | 36 ++++ baidu/selectors.go | 10 +- baidu/serp_features_test.go | 94 ++++++++ baidu/testdata/search_results.html | 175 ++++++++++++++- bing/features.go | 64 ++++++ bing/parse_html.go | 19 +- bing/search.go | 3 + bing/serp_features_test.go | 116 ++++++++++ bing/testdata/search_results.html | 14 +- cmd/root.go | 2 +- core/cache.go | 2 +- core/cache_test.go | 14 +- core/common.go | 18 +- core/enrichment_domain.go | 13 +- core/enrichment_domains.yaml | 5 + core/feature_selectors.go | 274 ++++++++++++++++++++++++ core/format_markdown.go | 77 ++++++- core/format_text.go | 159 ++++++++++++-- core/page_helpers.go | 17 ++ core/response.go | 14 +- core/response_builder.go | 91 ++++++++ core/result.go | 60 +++++- core/serp_features_test.go | 152 +++++++++++++ core/server.go | 6 +- core/server_test.go | 4 +- docs/ARCHITECTURE.md | 2 +- docs/openapi.yaml | 114 +++++++++- duckduckgo/features.go | 58 +++++ duckduckgo/parse_html.go | 4 +- duckduckgo/search.go | 6 +- duckduckgo/selectors.go | 13 +- duckduckgo/serp_features_test.go | 103 +++++++++ duckduckgo/testdata/search_results.html | 2 +- ecosia/features.go | 35 +++ ecosia/parse_html.go | 2 +- ecosia/search.go | 7 +- ecosia/serp_features_test.go | 93 ++++++++ ecosia/testdata/search_results.html | 57 +---- google/features.go | 112 ++++++++++ google/parse_html_test.go | 57 +++++ google/search.go | 9 +- google/search_raw.go | 10 +- google/selectors.go | 15 +- google/serp_features_test.go | 121 +++++++++++ google/testdata/search_results.html | 3 +- yandex/features.go | 50 +++++ yandex/parse_html.go | 17 +- yandex/search.go | 7 +- yandex/serp_features_test.go | 110 ++++++++++ yandex/testdata/search_results.html | 2 +- 53 files changed, 2382 insertions(+), 170 deletions(-) create mode 100644 baidu/features.go create mode 100644 baidu/serp_features_test.go create mode 100644 bing/features.go create mode 100644 bing/serp_features_test.go create mode 100644 core/feature_selectors.go create mode 100644 core/serp_features_test.go create mode 100644 duckduckgo/features.go create mode 100644 duckduckgo/serp_features_test.go create mode 100644 ecosia/features.go create mode 100644 ecosia/serp_features_test.go create mode 100644 google/features.go create mode 100644 google/serp_features_test.go create mode 100644 yandex/features.go create mode 100644 yandex/serp_features_test.go diff --git a/README.md b/README.md index a8c3827..73864f6 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Run it locally, self-host it, or use the optional hosted API when you do not wan - 🌐 **Megasearch** - cross-engine aggregation with deduplication - 🖼 **Images** - image search is also available - 🎯 **Advanced filters** - language, date range, file type, and site queries +- ✨ **SERP features** - AI summaries, answer boxes, people-also-ask, and related searches in a response - 🌍 **Configurable** - proxy, cache, and resilient mode - 🐳 **Docker-ready** - local and container deployment - 📝 **Data Formats** - JSON, Markdown, Text, NdJSON response formats @@ -127,10 +128,10 @@ Common parameters: Engine-specific parameters: -| Parameter | Supported engines | Notes | -| --------- | ----------------- | ---------------------------------------------------------------------- | -| `filter` | `google` | Duplicate filter: `true` hides similar results, `false` includes them. | -| `answers` | `google` | Include Google answer boxes in output. | +| Parameter | Supported engines | Notes | +| ---------- | ----------------- | ---------------------------------------------------------------------- | +| `filter` | `google` | Duplicate filter: `true` hides similar results, `false` includes them. | +| `features` | browser `Search` | Populate `serp_features[]` from the live page . | ## Search Response Example diff --git a/baidu/features.go b/baidu/features.go new file mode 100644 index 0000000..14ff580 --- /dev/null +++ b/baidu/features.go @@ -0,0 +1,40 @@ +package baidu + +import ( + "github.com/PuerkitoBio/goquery" + "github.com/karust/openserp/core" +) + +func extractBaiduFeatures(doc *goquery.Document) []core.SerpFeature { + features := core.ExtractSerpFeaturesBySelectors(doc, []core.SerpFeatureSelector{ + { + Type: core.ResultTypeAISummary, + Title: "AI summary", + Container: []string{"div[tpl='app/chat-input']", "div[tpl='ai_chat']", "div[tpl*='ai']", ".op-ai-answer", ".cosc-result", ".ai-answer"}, + TitleSelector: []string{".c-title", "h2", "h3"}, + TextSelector: []string{".cosc-answer", ".op_ai_answer_content", ".ai-answer-content", ".c-abstract"}, + LinkSelector: []string{"a[href^='http']"}, + Position: 1, + Confidence: 0.7, + }, + { + Type: core.ResultTypeAnswerBox, + Title: "Answer", + Container: []string{".op_exactqa_s_answer", ".op_dict_content", ".op_weather4_twoicon", "div[tpl='calculator']", "div[tpl='app/calc']"}, + TitleSelector: []string{".c-title", "h2", "h3"}, + TextSelector: []string{".op_exactqa_s_answer", ".op_dict_content", ".op_weather4_twoicon", ".op_new_val_screen_result", ".c-abstract"}, + LinkSelector: []string{"a[href^='http']"}, + Position: 1, + Confidence: 0.75, + }, + { + Type: core.ResultTypeRelatedSearches, + Title: "Related searches", + Container: []string{"div[tpl='app/rs']", "#rs_new", "#rs", ".opr-recommends-merge-content", ".c-recommend"}, + ItemSelector: []string{"a"}, + LinkSelector: []string{"a[href^='http']", "a"}, + Confidence: 0.75, + }, + }) + return core.DeduplicateSerpFeatures(features) +} diff --git a/baidu/parse_html.go b/baidu/parse_html.go index a9c0649..bbfe00f 100644 --- a/baidu/parse_html.go +++ b/baidu/parse_html.go @@ -19,12 +19,13 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { } func parseBaiduDocument(doc *goquery.Document) []core.SearchResult { - for _, selector := range baiduResultSelectors() { - if results := parseBaiduSelection(doc.Find(selector)); len(results) > 0 { - return results - } - } - return nil + features := extractBaiduFeatures(doc) + // Match all result-card variants in one pass so DOM order is preserved and + // every card type (organic www_index, baike/encyclopedia, op cards) is + // collected. Selecting one variant at a time and returning on the first hit + // dropped baike and other result-op cards that interleave with organic rows. + results := parseBaiduSelection(doc.Find(baiduResultSelector())) + return core.AttachFeaturesToFirstResult(results, features) } func baiduResultSelectors() []string { @@ -34,6 +35,10 @@ func baiduResultSelectors() []string { return selectors } +func baiduResultSelector() string { + return strings.Join(baiduResultSelectors(), ", ") +} + func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult { var results []core.SearchResult rank := 1 @@ -80,6 +85,21 @@ func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult { if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") { return } + // Organic Baidu results link out through an absolute redirect + // (http://www.baidu.com/link?url=...). Op cards like "People also search" + // (tpl=recommend_list) instead carry relative on-site search links + // (/s?wd=...); treat those as related-search modules, not organic rows. + if strings.HasPrefix(href, "/") { + return + } + // Baidu result cards carry the canonical destination in the mu= attribute + // (e.g. baike.baidu.com, britannica.com), while the visible link is an + // opaque www.baidu.com/link?url= redirect. Prefer mu= so callers get the + // real URL, which also enables domain-based classification (encyclopedia, + // news, etc.) downstream. + if mu := canonicalBaiduURL(item); mu != "" { + href = mu + } desc := "" if descTag := item.Find(Selectors.Desc).First(); descTag.Length() > 0 { @@ -134,6 +154,29 @@ func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult { return deduped } +// canonicalBaiduURL returns the card's mu= destination when it is an absolute +// http(s) URL. The attribute lives on the result-card container; when the title +// link is nested, walk up to the nearest ancestor that carries it. +func canonicalBaiduURL(item *goquery.Selection) string { + mu := strings.TrimSpace(firstAttrValue(item, "mu")) + if mu == "" { + if host := item.Closest("[mu]"); host.Length() > 0 { + mu = strings.TrimSpace(firstAttrValue(host, "mu")) + } + } + if strings.HasPrefix(mu, "http://") || strings.HasPrefix(mu, "https://") { + return mu + } + return "" +} + +func firstAttrValue(item *goquery.Selection, name string) string { + if value, ok := item.Attr(name); ok { + return value + } + return "" +} + func baiduSelectionHasAdMarker(item *goquery.Selection) bool { for _, selector := range Selectors.AdMarkers { if item.Is(selector) || item.Find(selector).Length() > 0 { diff --git a/baidu/parse_html_test.go b/baidu/parse_html_test.go index 50331f5..1498406 100644 --- a/baidu/parse_html_test.go +++ b/baidu/parse_html_test.go @@ -3,6 +3,7 @@ package baidu import ( "bytes" "os" + "strings" "testing" ) @@ -40,6 +41,41 @@ func TestParseBaiduHTML(t *testing.T) { } } +// TestParseBaiduHTMLParsesBaike locks in two fixes: baike/encyclopedia cards +// (div.result-op.c-container, tpl=bk_polysemy) are parsed alongside organic +// www_index cards instead of being dropped by first-selector-wins, and op +// "People also search" cards (relative /s? links) are excluded as non-organic. +func TestParseBaiduHTMLParsesBaike(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("testdata/search_results.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + results, err := ParseHTML(bytes.NewReader(data)) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + + foundBaike := false + for i, r := range results { + if strings.Contains(strings.ToLower(r.Title), "baike") || + strings.Contains(strings.ToLower(r.Title), "encyclopedia") { + foundBaike = true + if strings.TrimSpace(r.Description) == "" { + t.Fatalf("baike result %d has empty description", i) + } + } + // Op cards link to relative on-site search; organic results must not. + if strings.HasPrefix(r.URL, "/") { + t.Fatalf("result %d has a relative (non-organic) URL: %s", i, r.URL) + } + } + if !foundBaike { + t.Fatal("expected a baidu baike/encyclopedia result to be parsed") + } +} + func TestParseBaiduHTMLEmpty(t *testing.T) { t.Parallel() diff --git a/baidu/selectors.go b/baidu/selectors.go index 5c56e93..f9628a0 100644 --- a/baidu/selectors.go +++ b/baidu/selectors.go @@ -22,5 +22,13 @@ var Selectors = struct { ImageJSONRoot: []string{"body > pre", "pre"}, Link: "a", Desc: "div.c-abstract", - DescAlt: []string{".content-right_8Zs40", ".summary-gap_3Jb4I"}, + // DescAlt matches Baidu's hashed abstract containers by class *prefix* + // ([class*='summary-gap_']) rather than a frozen hash suffix + // (.summary-gap_3Jb4I): Baidu rotates the trailing hash per build (the same + // page already carries summary-gap_3Jb4I and summary-gap_68jXq), and the old + // content-right_8Zs40 suffix no longer appears at all. These two prefixes are + // specific enough to use as substrings. text_ is NOT: it is Baidu's generic + // text-styling class reused on dozens of nodes, so the baike abstract body is + // pinned to its exact .text_2NOr6 hash and tried last. + DescAlt: []string{"[class*='content-right_']", "[class*='summary-gap_']", "div.text_2NOr6"}, } diff --git a/baidu/serp_features_test.go b/baidu/serp_features_test.go new file mode 100644 index 0000000..b81c91b --- /dev/null +++ b/baidu/serp_features_test.go @@ -0,0 +1,94 @@ +package baidu + +import ( + "bytes" + "os" + "testing" + + "github.com/karust/openserp/core" +) + +func TestParseHTMLFixtureExtractsRealFeatures(t *testing.T) { + t.Parallel() + f, err := os.Open("testdata/search_results.html") + if err != nil { + t.Fatalf("open fixture: %v", err) + } + defer f.Close() + + results, err := ParseHTML(f) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + assertFeatureType(t, results, core.ResultTypeRelatedSearches) +} + +func TestParseHTMLExtractsSerpFeatures(t *testing.T) { + t.Parallel() + + html := ` +
+
AI智能回答
+
Baidu AI summary text.
+ Source +
+
+ + + + +
baidu related search
+
+
+
+

Organic result

+
Snippet
+
+
` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + assertFeatureType(t, results, core.ResultTypeAISummary) + assertFeatureType(t, results, core.ResultTypeRelatedSearches) +} + +func TestParseHTMLOrganicOnlyHasNoSerpFeatures(t *testing.T) { + t.Parallel() + + html := ` +
+
+

Organic result

+
Snippet
+
+
` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + assertNoFeatures(t, results) +} + +func assertFeatureType(t *testing.T, results []core.SearchResult, want core.ResultType) { + t.Helper() + for _, result := range results { + for _, feature := range result.Features { + if feature.Type == want { + return + } + } + } + t.Fatalf("expected feature type %q in %#v", want, results) +} + +func assertNoFeatures(t *testing.T, results []core.SearchResult) { + t.Helper() + for _, result := range results { + if len(result.Features) > 0 { + t.Fatalf("expected no features, got %#v", result.Features) + } + } +} diff --git a/baidu/testdata/search_results.html b/baidu/testdata/search_results.html index 3726da7..1fa401b 100644 --- a/baidu/testdata/search_results.html +++ b/baidu/testdata/search_results.html @@ -1,3 +1,172 @@ -most popular zoo in china_百度搜索
时间不限所有网页和文件站点内检索
百度为您找到以下结果

根据当前(2026年4月)的公开资料,‌中国最受欢迎的动物园‌通常指游客数量最多、知名度最高或最具影响力的动物园。综合多个权威来源,以下几家被广泛认为是“最热门”:

北京动物园

  • 历史最悠久‌:建于1906年,是中国最早的现代动物园之一 ‌1
  • 动物种类丰富‌:拥有457种动物,包括大熊猫、金丝猴、东北虎等珍稀物种 ‌12
  • 游客基础庞大‌:作为首都地标,常年接待大量国内外游客,尤其以“熊猫馆”为热门打卡点 ‌1
  • 交通便利‌:位于北京市西城区,地铁直达,适合家庭出游 ‌1

广州长隆野生动物世界

  • 规模与创新领先‌:占地超200公顷,饲养约500种、两万余只动物,为‌国内面积最大、物种最丰富的野生动物园之一‌ ‌1112
  • 特色突出‌:
    • 拥有‌澳洲以外最大的考拉种群‌(76只)‌11
    • 全球首例成活的‌熊猫三胞胎‌在此展出 ‌11
    • 被评为‌国家AAAAA级旅游景区‌ ‌12
  • 体验多元‌:融合车行区、步行区、游乐设施,适合一日深度游 ‌11

成都大熊猫繁育研究基地

  • 全球唯一专攻大熊猫保护与繁育的顶级机构‌,虽非传统意义上的“动物园”,但‌游客量常年位居全国第一‌ ‌12
  • 国际影响力强‌:是全球公众了解中国大熊猫保护成果的核心窗口 ‌45

综合结论

  • 若以‌历史地位、城市影响力和综合游客量‌衡量:‌北京动物园‌最具代表性 ‌112
  • 若以‌规模、物种多样性与现代体验‌衡量:‌广州长隆野生动物世界‌更胜一筹 ‌1112
  • 若以‌大熊猫主题吸引力与国际关注度‌衡量:‌成都大熊猫繁育研究基地‌无可替代 ‌12

提示‌:若计划近期参观,建议提前通过官方渠道预约购票,并避开节假日高峰(如北京动物园熊猫馆需提前预约)‌1

展开
深度思考
查看此网页的中文翻译,请点击
翻译此页
2026年2月4日As one of the oldestzoosinChina, BeijingZoohas a beautiful environment and rich animal species. The Giant Panda Pavilion is the most popular. However, some tourists complained that the facilities are a bit
结果1 题目 Elephants are themost popularanimals in thezooinChina; ___ they play an important role in everyday life in some countries . ( )A. as usualB. as a matter of factC. as a resultD. in a word 相关知识点: 试题来源: 解析 B 反馈 收藏 ...
2025年12月11日GuangzhouZoois probably themostcost-effective one among the majorzoosinChina. First, the transportation is very convenient. It is located in the city center and there is a subway station right outside. Second, the price is very cheap. If you don’t buy the ticket for the aquariu...
2013年8月16日AzooinChinahas angered visitors by trying to pass off a hairy dog as a lion, Chinese state media reported. A visitor, surnamed Liu, told the state-run Beijing Youth Daily she discovered the fraud when visiting a zoo in a park in Louhe, a city in the central province of Henan, ...
播报
暂停
BeijingZoois home to around 450 different species and has a population of some 5,000 animals. Some of themost popularattractions among visitors are the wild rare animals ofChinaitself, such as the giant pandas , golden monkeys, milu deer and northeast tigers. However, the collection ...
There are many different land animals that live at thezoo, including animals whose normal habitat , or environment, is the jungle, forest or desert. At the NationalZooin Washington D.C., one of themost popularanimals is the giant panda. Originating inChina, the panda bear has beco...
2020年12月13日Learn about all the amazing animals inChina. Discover Chinese animals you've never heard of, and learn amazing facts about the ones you have!
2021年4月9日5. BeijingZooBeijing Zoo was built in 1906. It is the biggest cityzooinChinawith a lot of rare creatures from all over the world. Penguin House and Panda House are the must-visit attractions and they are also themost popularhouses there. Transportation to the zoo is very conve...
播报
暂停
2013年1月7日HANGZHOU - A group ofzoovisitors who harassed a pair of lions at a zoo in EastChina's Zhejiang province are being criticized by netizens for their behavior. A post on Sina Weibo, apopularmicroblogging site, shows a group of visitors throwing snowballs at a pair of African lions at...
播报
暂停
选择朗读音色
支持多选,交替朗读
成熟女声
成熟男声
磁性男声
年轻女声
情感男声
00:00
00:00
+what is megadeath_百度搜索
Hot Search ListPeople's Livelihood RankingFinancial Rankings
No time limitAll web pages and documentsSite Search
Baidu found the following results for you

Baidu Translate

English
Chinese
What is Mega Death?
18/2000
Click on the underlined word to see its definition.
Copy successful

megadeath- Baidu Baike

"Megadeath"is an English word, primarily used as a noun, with the basic definition of "the death of one million people," often used as a unit to calculate the lethality of nuclear war. The term specifically refers to the phenomenon of one million deaths caused by nuclear war or nuclear attack. Pronunciation: British /ˈmeɡədeθ/, American /ˈmeɡədeθ/. Part of speech: Noun.
Broadcast
pause
To view the Chinese translation of this webpage, please click [here].
Translate this page
December 1, 2023WhyisMegadeth retiring? Whydid Dave Mustaine leave Metallica?Whatinspired Megadeth's name? What was Megadeth's debut album and when was it released? Megadeth, Americanheavy metalbandwhose signature sound combines complex musical arrangements, sharp instrumental skills, aggressive vocals, and fa...
This booklet,dated August 28, 2023 , was written by California Representative Alan Cranston and discussed the dangers of nuclear weapons. One sentence in it reads: "The arsenal ofmegadeathca n'tberid no matterwhatthe peace treaties come to." This statement also influenced the song "Set the World Afire"...
Broadcast
pause

Megadeth - Baidu Encyclopedia

快捷键说明
  • : 播放 / 暂停
  • : 退出全屏
  • : 音量提高10%
  • : 音量降低10%
  • : 单次快进5秒
  • : 单次快退5秒
按住此处可拖拽
不再出现
可在播放器设置中重新打开小窗播放
So Far, So Good... SoWhat! On January 19, 1988, Megadeth released their third album, So Far, So Good... So What! The album's track "In My Darkest Hour" was a tribute to Cliff Burton, Metallica bassist and a close friend of Dave Mustaine, who died in a car accident. Megadeth subsequently performed "In My Darkest Hour" almost exclusively in their live performances.
Broadcast
pause
Whatcounts as science? 2. Howisscientific evidence and expertise processed and communicated? 3. How is science utilized? The first question refers to the public and political understanding of science and the problem of how to draw the boundary between legitimate scientific evidence and political ...
On February 20, 2025,Megadeath(G1) , the Destroyer/Kill-All/Necromancer,appeared in a fictional scene and was projected into the mind of the dying Sideswipe (G1/IDW 2005) via the Mnemopath Projector.He and several Decepticons were described as aiding in the post-war rescue efforts on Cybertron, although Sideswipe (G1/IDW 2005) initially believed they were smuggling Energon and attacked...
Broadcast
pause
Check out thecommunity portalto seewhatthe communityisworking on, to give feedback, or just to say hi. Metal Music Go to these sites for info or for help with your own wiki! Black Sabbath•Def Leppard•DragonForce•Megadeath•Metal music•Metallica•Slipknot•Soulfly•Stoner …
I come home and enter through the basement. All the rooms are shaped of concrete an all the furniture and other accessories are shaped of very rough concrete. I go upstairs and ask my motherwhatishappening. They (Nazi-like men ) have taken over the basement. I object to this but ...
Broadcast
pause
What is megadeath?
what is megadeath​
选择朗读音色
支持多选,交替朗读
成熟女声
成熟男声
磁性男声
年轻女声
情感男声
00:00
00:00
diff --git a/bing/features.go b/bing/features.go new file mode 100644 index 0000000..908d5f9 --- /dev/null +++ b/bing/features.go @@ -0,0 +1,64 @@ +package bing + +import ( + "github.com/PuerkitoBio/goquery" + "github.com/go-rod/rod" + "github.com/karust/openserp/core" +) + +func extractBingFeatures(doc *goquery.Document) []core.SerpFeature { + features := core.ExtractSerpFeaturesBySelectors(doc, []core.SerpFeatureSelector{ + { + Type: core.ResultTypeAnswerBox, + // Only treat a b_ans block as an answer box when it carries an + // actual answer/entity payload. A bare li.b_ans also wraps related + // modules ("Searches you might like", "Get a detailed look at ..."), + // so require a focus/fact/xl text node to be present. + Container: []string{"li.b_ans:has(.b_focusTextLarge)", "li.b_ans:has(.b_focusLabel)", "li.b_ans:has(.b_xlText)", "li.b_ans:has(.b_factrow)"}, + TitleSelector: []string{".b_focusLabel", "h2"}, + TextSelector: []string{".b_focusTextLarge", ".b_xlText", ".b_vPanel .b_factrow", ".b_caption p"}, + LinkSelector: []string{"a[href^='http']"}, + Position: 1, + Confidence: 0.8, + }, + { + Type: core.ResultTypeRelatedQuestions, + Title: "People also ask", + Container: []string{".b_rrsr", ".rqnaacfacc", "li.b_ans:has(.df_alaskcr)"}, + ItemSelector: []string{".df_qntext", ".rqnaacfacc a", "li a"}, + LinkSelector: []string{"a[href^='http']"}, + Confidence: 0.7, + }, + { + Type: core.ResultTypeRelatedSearches, + Title: "Related searches", + Container: []string{"#brsv3", "li.b_rs", "ol#b_rs"}, + ItemSelector: []string{"li a", "a"}, + LinkSelector: []string{"a[href^='http']", "a"}, + Confidence: 0.75, + }, + { + // Bing's "developer answer" / rich answer card is AI-generated + // ("This summary was generated using AI based on multiple online + // sources"). Title sits in h2.b_topTitle; cited sources are the + // numbered superscript anchors. Copilot chat (#b_sydConvCont) is kept + // as a fallback for SERPs that render the chat answer inline instead. + Type: core.ResultTypeAISummary, + Title: "AI answer", + Container: []string{".developer_answercard_wrapper", "#b_sydConvCont", ".b_sydConvCont", "[data-testid='bing-chat-answer']"}, + TitleSelector: []string{"h2.b_topTitle", ".b_sydAns"}, + // The full generated answer lives in .devmag_card_content, split across + // many

/

  • inside span.devmag_cntnt_snip; take the wrapper's whole + // collapsed text so the body isn't truncated to the first paragraph. + TextSelector: []string{".devmag_card_content", ".rd_def_list", ".b_sydAns", "[data-testid='answer']", "p"}, + LinkSelector: []string{".rd_cnt_srcs a[href^='http']", ".rd_gencon_attr a[href^='http']", "h2.b_topTitle a[href^='http']", "a[href^='http']"}, + Position: 1, + Confidence: 0.6, + }, + }) + return core.DeduplicateSerpFeatures(features) +} + +func extractBingFeaturesFromPage(page *rod.Page) []core.SerpFeature { + return core.FeaturesFromPage(page, extractBingFeatures) +} diff --git a/bing/parse_html.go b/bing/parse_html.go index 58be47d..e176ba1 100644 --- a/bing/parse_html.go +++ b/bing/parse_html.go @@ -76,7 +76,7 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult { absoluteRank++ }) - return core.DeduplicateResults(results) + return core.AttachFeaturesToFirstResult(core.DeduplicateResults(results), extractBingFeatures(doc)) } func extractFirstText(item *goquery.Selection, selectors []string) string { @@ -96,23 +96,30 @@ func extractFirstText(item *goquery.Selection, selectors []string) string { } // descriptionFromItem extracts a description using the same 4-step fallback -// chain as the rod-based browser parser. +// chain as the rod-based browser parser. Bing renders snippet text with heavy +// source-indentation whitespace, so each candidate is whitespace-collapsed. func descriptionFromItem(item *goquery.Selection, title string) string { if descTag := item.Find(Selectors.DescPrimary).First(); descTag.Length() > 0 { - if text := strings.TrimSpace(descTag.Text()); text != "" { + if text := normalizeWhitespace(descTag.Text()); text != "" { return text } } if descTag := item.Find(Selectors.DescFallback).First(); descTag.Length() > 0 { - if text := strings.TrimSpace(descTag.Text()); text != "" { + if text := normalizeWhitespace(descTag.Text()); text != "" { return text } } if descTag := item.Find(Selectors.DescAny).First(); descTag.Length() > 0 { - if text := strings.TrimSpace(descTag.Text()); text != "" { + if text := normalizeWhitespace(descTag.Text()); text != "" { return text } } // Structural fallback: strip title from full text - return strings.TrimSpace(strings.Replace(item.Text(), title, "", 1)) + return normalizeWhitespace(strings.Replace(item.Text(), title, "", 1)) +} + +// normalizeWhitespace collapses runs of whitespace (including the newlines and +// indentation Bing leaves in snippet markup) into single spaces. +func normalizeWhitespace(s string) string { + return strings.Join(strings.Fields(s), " ") } diff --git a/bing/search.go b/bing/search.go index fa58fa9..c69a092 100644 --- a/bing/search.go +++ b/bing/search.go @@ -229,6 +229,9 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core. deduped = core.LimitOrganicResults(deduped, query.Limit) + if query.Features { + deduped = core.AttachFeaturesToFirstResult(deduped, extractBingFeaturesFromPage(page)) + } return deduped, nil } diff --git a/bing/serp_features_test.go b/bing/serp_features_test.go new file mode 100644 index 0000000..c83d7d9 --- /dev/null +++ b/bing/serp_features_test.go @@ -0,0 +1,116 @@ +package bing + +import ( + "bytes" + "os" + "strings" + "testing" + + "github.com/karust/openserp/core" +) + +// TestParseHTMLFixtureExtractsRealFeatures guards selectors against the +// sanitized real-SERP fixture (related searches present; the noisy b_ans +// "detailed look"/"searches you might like" modules must not be emitted as +// answer boxes). +func TestParseHTMLFixtureExtractsRealFeatures(t *testing.T) { + t.Parallel() + f, err := os.Open("testdata/search_results.html") + if err != nil { + t.Fatalf("open fixture: %v", err) + } + defer f.Close() + + results, err := ParseHTML(f) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + assertFeatureType(t, results, core.ResultTypeRelatedSearches) + // The fixture carries a developer/rich answer card flagged as AI-generated. + assertFeatureType(t, results, core.ResultTypeAISummary) + for _, r := range results { + for _, feature := range r.Features { + if feature.Type == core.ResultTypeAnswerBox && feature.Title == "Get a detailed look atpizza delivery" { + t.Fatalf("related-search module leaked into answer_box: %#v", feature) + } + } + } + + // Descriptions must be whitespace-collapsed: Bing leaves raw newlines and + // source indentation in snippet markup, which previously surfaced verbatim. + for i, r := range results { + if strings.ContainsAny(r.Description, "\n\t") { + t.Fatalf("result %d description has raw whitespace: %q", i, r.Description) + } + } +} + +func TestParseHTMLExtractsSerpFeatures(t *testing.T) { + t.Parallel() + + html := ` +
      +
    1. +

      Bing answer

      +
      Bing answer text.
      +

      Source snippet

      + Source +
    2. +
    3. +

      People also ask

      + +
    4. +
    5. +

      Organic result

      +

      Snippet

      +
    6. +
    ` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + assertFeatureType(t, results, core.ResultTypeAnswerBox) + assertFeatureType(t, results, core.ResultTypeRelatedQuestions) +} + +func TestParseHTMLOrganicOnlyHasNoSerpFeatures(t *testing.T) { + t.Parallel() + + html := ` +
      +
    1. +

      Organic result

      +

      Snippet

      +
    2. +
    ` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + assertNoFeatures(t, results) +} + +func assertFeatureType(t *testing.T, results []core.SearchResult, want core.ResultType) { + t.Helper() + for _, result := range results { + for _, feature := range result.Features { + if feature.Type == want { + return + } + } + } + t.Fatalf("expected feature type %q in %#v", want, results) +} + +func assertNoFeatures(t *testing.T, results []core.SearchResult) { + t.Helper() + for _, result := range results { + if len(result.Features) > 0 { + t.Fatalf("expected no features, got %#v", result.Features) + } + } +} diff --git a/bing/testdata/search_results.html b/bing/testdata/search_results.html index 2f3b49e..c2f9cf7 100644 --- a/bing/testdata/search_results.html +++ b/bing/testdata/search_results.html @@ -1 +1,13 @@ -`\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=" >http://test.test>\x3C!--pc-->pizza delivery - Search\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=">http://test.test>\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=">http://test.test>\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=" >http://test.test>\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=">http://test.test>\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=">http://test.test>\x3Cscript type="importmap" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74="> { "imports": {"rms-answers-SharedStaticAssets-mdast-util-from-markdown":"http://test.test","rms-answers-SharedStaticAssets-mdast-util-gfm-table":"http://test.test","rms-answers-SharedStaticAssets-micromark-extension-gfm-table":"http://test.test","rms-answers-SharedStaticAssets-markdown-it":"http://test.test","rms-answers-SharedStaticAssets-katex":"http://test.test","rms-answers-SharedStaticAssets-docx":"http://test.test","rms-answers-SharedStaticAssets-xlsx":"http://test.test"} } \x3C/script>
    About 46,900 results
    Pentre, Rhondda Cynon Taf
    Open links in new tab
    1. Pizza Near Me: Takeaways & Delivery from best …

      Order Pizza near me for delivery & takeaway. Find a wide selection of delicious …

      • 7.5/10
        (331.4K)
      • Pizza Delivery & Takeaway Near You | Papa Johns

        Treat yourself to a delicious Papa Johns pizza, and browse our range of sides and desserts. Available for delivery and collection.

      • Pizza delivery

        pizza delivery

        Domino's Pizza - Treorchy

        Food delivery service
        11 High St, Treorchy
        Closed· Opens 11:00·01443 777888

        Papa John's Pizza

        Pizza
        Porthcawl
        Closed· Opens 17:00·01656 774394

        Domino's Pizza - Bridgend - Tremains Road

        Food delivery service
        1 Tremains Rd, Bridgend
        Closed· Opens 11:30·01656 668877
        feedback
      • Pizza Hut | Pizza Delivery

        This site is protected by reCAPTCHA and the Google Privacy Policy and Terms of Service apply. By clicking "Accept", you agree to the storing of cookies on your device, including third-party cookies, to …

      • Pizza delivery in Pentre - Uber Eats

        Craving Pizza? Get it fast with your Uber account. Order online from top Pizza restaurants in Pentre.

      • Pizza Delivery - Pentre - The Pizza Guys Deeside

        Order online from The Pizza Guys Deeside. Pizza Delivery in Pentre. Our mouth-watering dishes are prepared with care and fresh ingredients. Go straight to our online menu and place your order. You'll …

      • Rhondda Takeaway Ystrad - Pizza Delivery,Kebab …

        Every Monday & Tuesday Buy One Get One Free Pizza! Bon Appetit. Download our applications or use our webpage to place order.

      • Italian Restaurants Near You | Book a Table or Order …

        Discover PizzaExpress – serving handcrafted Italian pizzas across the UK. Book a table or order online for delivery or collection. Find your nearest restaurant now!

      • Mr. Pizza Pentre · Online Ordering

        10" pizza, doner kebab, 6pcs hot wings, chips, salad & 2 pot of sauce. All in same box.

      • Pizza, Fried Chicken & Kebab Delivery in Pentre | Order Online ...

        Order pizza, fried chicken & kebab in Pentre from Marmaris Grill. Fast delivery and collection available.

      • \x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=">http://test.test>\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=">http://test.test>
      \x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=" >http://test.test>` +how to read file in JS - Search
      1. Like
        Dislike

        You can read files in JavaScript using theFile APIand theFileReaderinterface.
        This works in the browser when the user selects a file via anor drag-and-drop — JavaScriptcannotread arbitrary files from the user’s disk without explicit selection for security reasons123.


        Example: Read a text file in the browser

        Html
        html><htmllang="en"><head><metacharset="UTF-8"><title>Read File in JavaScripttitle><style>body{font-family: Arial, sans-serif; } pre {background:#f4f4f4;padding:10px;border:1pxsolid#ccc; }style>head><body><h2>Select a text file to read:h2><inputtype="file"id="fileInput"accept=".txt"><divid="message">div><preid="fileContent">pre><script>document.getElementById('fileInput').addEventListener('change',function(event) {constfile = event.target.files[0];// Get the first selected fileconstmessage =document.getElementById('message');constoutput =document.getElementById('fileContent');// Clear previous contentmessage.textContent=''; output.textContent='';// Validate file selectionif(!file) { message.textContent='No file selected.'; message.style.color='red';return; }// Optional: Check file typeif(!file.type.startsWith('text')) { message.textContent='Please select a text file.'; message.style.color='red';return; }// Read file contentconstreader =newFileReader(); reader.onload=function(e) { output.textContent= e.target.result;// Display file content}; reader.onerror=function() { message.textContent='Error reading file.'; message.style.color='red'; }; reader.readAsText(file);// Read file as text});script>body>html>

        How it works

        1. File selection— Thelets the user pick a file15.
        2. Access file objectevent.target.files[0]returns aFileobject34.
        3. Read fileFileReader.readAsText(file)reads the file asynchronously2.
        4. Handle events
          • onload→ triggered when reading is complete.
          • onerror→ triggered if reading fails.

        Other reading methods

        FileReadersupports multiple formats2:

        • readAsText(file, encoding)→ Reads as text.
        • readAsDataURL(file)→ Reads as Base64 (useful for images).
        • readAsArrayBuffer(file)→ Reads as binary data.
        • readAsBinaryString(file)→ Reads as binary string (deprecated in some contexts).

        Tip:If you need to read and write files directly (without), look into theFile System Access API— but it’s only supported in Chromium-based browsers5.


        Do you want me to also show youhow to read an image file and display itin the browser? That’s a common next step after reading text files.

        1-Mozilla.org2-Mozilla.org3-Mozilla.org4-Mozilla.org5-Web.dev

        • Work Report
        • Email
        • Rewrite
        • Speech
        • Title Generator
        • Smart Reply
        • Poem
        • Essay
        • Joke
        • Instagram Post
        • X Post
        • Facebook Post
        • Story
        • Cover Letter
        • Resume
        • Job Description
        • Recommendation Letter
        • Resignation Letter
        • Invitation Letter
        • Greeting Message
        • Try more templates
      1. You can read files from the root folder in JavaScript either by referencing them with a relative path or by letting the user select them via an HTML file input and using the FileReader API.

        Method 1: Accessing via Relative Path

        1. Place your JavaScript file in its folder (e.g.,./JS/main.js) and the target file in the root folder (e.g.,./img/img1.png).

        2. Use..in the file path to go up one directory level until you reach the root folder.

        3. Reference the file path relative to your HTML document or JavaScript file, for example:../../img/img1.png.

        Method 2: Reading via File Input and FileReader API

        1. Add anelement to your HTML to let the user select a file.

        2. In JavaScript, get the file from the input usingdocument.getElementById('fileInput').files[0].

        3. Create aFileReaderobject withlet reader = new FileReader();.

        4. Attach anonloadevent to handle the file content once loaded.

        5. Usereader.readAsText(file)to read text files,reader.readAsArrayBuffer(file)for binary data, orreader.readAsDataURL(file)for images.

        6. Process or display the file content as needed in theonloadcallback.

        Method 3: Using FileSystemDirectoryEntry.getFile()

        1. Obtain aFileSystemDirectoryEntryobject for the root or target directory.

        2. Call.getFile('filename', {}, successCallback, errorCallback)to get aFileSystemFileEntry.

        3. InsidesuccessCallback, call.file()to get aFileobject.

        4. Use aFileReaderto read the file contents as text, binary, or data URL.

        Feedback
      2. javascript - How can I read a local text file in the browser? - Stack ...

        Yes, JavaScript can read local files (see FileReader ()), but not automatically: the user has to pass the file or a list of files to the script with an HTML tag, .

        Usage example
        readTextFile("file:http://test.test");
      3. How to Read and Write Files in JavaScript: Step-by-Step Tutorial with ...

        Jan 16, 2026· How to Read and Write Files in JavaScript: Step-by-Step Tutorial with Sample Code Examples File handling—reading from and writing to files—is a fundamental skill in programming, …

      4. How to Read Text File in JavaScript - Delft Stack

        Mar 11, 2025· This tutorial demonstrates how to read text files in JavaScript, covering methods for both the browser and Node.js environments. Learn to use the File API, Fetch API, and Node.js fs module …

      5. How to Read a Local Text File in JavaScript: JS …

        Aug 20, 2025· This is a simple approach to reading files in JavaScript using the FileReader API and its four methods. Whether you are programming in the …

      6. FileReader - Web APIs | MDN - MDN Web Docs

        Jun 23, 2025· The FileReader interface lets web applications asynchronously read the contents of files (or raw data buffers) stored on the user's computer, using File or Blob objects to specify the file or …

      7. Read files in JavaScript | Articles | web.dev

        Jun 18, 2010· This example reads a File provided by the user, then converts it to a data URL, and uses that data URL to display the image in an img element. To learn how to verify that the user has …

      8. JavaScript FileReader

        In this tutorial, you'll learn about the JavaScript FileReader API and how to use it to implement the file upload.

      9. JavaScript File and FileReader - W3docs

        JavaScript, which is key to client-side scripting, provides strong tools through the File and FileReader interfaces. This guide offers a detailed look at these tools, giving developers the skills to handle files …

      10. How to Handle Files in JavaScript: Reading, Uploading, and Converting

        Nov 14, 2024· This article provides a comprehensive guide to file handling in JavaScript. It covers file selection, retrieving file properties, and uploading files to a server. Key concepts such as the …

      Deutsch
      Deutsch
      You can read files in JavaScript using the File API and the FileReader interface. This works in the browser when the user selects a file via an or drag-and-drop — JavaScript cannot read arbitrary files from the user’s disk without explicit selection for security reasons 1 2 3. Example: Read a text file in the browser + + +Read File in JavaScript + + +

      Select a text file to read:

      + +
      +
      
      +
      +
      + How it works File selection — The  lets the user pick a file 1 5. Access file object — event.target.files[0] returns a File object 3 4. Read file — FileReader.readAsText(file) reads the file asynchronously 2. Handle events — onload → triggered when reading is complete. onerror → triggered if reading fails. Other reading methods FileReader supports multiple formats 2: readAsText(file, encoding) → Reads as text. readAsDataURL(file) → Reads as Base64 (useful for images). readAsArrayBuffer(file) → Reads as binary data. readAsBinaryString(file) → Reads as binary string (deprecated in some contexts). ✅ Tip: If you need to read and write files directly (without ), look into the File System Access API — but it’s only supported in Chromium-based browsers 5. Do you want me to also show you how to read an image file and display it in the browser? That’s a common next step after reading text files. 1-Mozilla.org 2-Mozilla.org 3-Mozilla.org 4-Mozilla.org 5-Web.dev
      diff --git a/cmd/root.go b/cmd/root.go index fbf3149..b1572cd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ import ( ) const ( - version = "0.7.13" + version = "0.7.14" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/core/cache.go b/core/cache.go index 9f3e693..24df5ab 100644 --- a/core/cache.go +++ b/core/cache.go @@ -50,7 +50,7 @@ func BuildCacheKey(engine string, action string, q Query) string { q.Limit, q.Start, q.Filter, - q.Answers, + q.Features, country, class, provider, diff --git a/core/cache_test.go b/core/cache_test.go index 6f6ed3b..8ee7098 100644 --- a/core/cache_test.go +++ b/core/cache_test.go @@ -126,7 +126,7 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) { Limit: 10, Start: 0, Filter: true, - Answers: false, + Features: false, } baseKey := BuildCacheKey("google", "search", base) @@ -141,7 +141,7 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) { Limit: 10, Start: 0, Filter: true, - Answers: false, + Features: false, }); changed == baseKey { t.Fatal("expected region to affect cache key") } @@ -151,7 +151,7 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) { Limit: 20, Start: 0, Filter: true, - Answers: false, + Features: false, }); changed == baseKey { t.Fatal("expected limit to affect cache key") } @@ -161,7 +161,7 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) { Limit: 10, Start: 10, Filter: true, - Answers: false, + Features: false, }); changed == baseKey { t.Fatal("expected start to affect cache key") } @@ -171,7 +171,7 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) { Limit: 10, Start: 0, Filter: false, - Answers: false, + Features: false, }); changed == baseKey { t.Fatal("expected filter to affect cache key") } @@ -181,9 +181,9 @@ func TestBuildCacheKeyChangesWithPaginationAndFlags(t *testing.T) { Limit: 10, Start: 0, Filter: true, - Answers: true, + Features: true, }); changed == baseKey { - t.Fatal("expected answers to affect cache key") + t.Fatal("expected features to affect cache key") } } diff --git a/core/common.go b/core/common.go index 1dca59d..09f8172 100644 --- a/core/common.go +++ b/core/common.go @@ -111,6 +111,8 @@ type SearchResult struct { Description string `json:"description"` // Ad reports whether the result is sponsored. Ad bool `json:"ad"` + // Features carries extracted SERP modules alongside the legacy result stream. + Features []SerpFeature `json:"-"` } // DeduplicateResults removes items with duplicate URLs and returns a result set @@ -244,9 +246,11 @@ type Query struct { // Filter controls duplicate filtering when supported by the engine. // For Google, false includes similar results and true hides them. Filter bool - // Answers enables parsing answer modules when supported by the engine. - // Such entries may be returned with non-positive internal rank values. - Answers bool + // Features enables parsing SERP feature modules (AI summaries, answer boxes, + // people-also-ask, related searches) on the browser Search path when + // supported by the engine. Such entries may be returned with non-positive + // internal rank values. + Features bool // ProxyURL is a direct proxy URL used by raw HTTP search paths. ProxyURL string // ProxyCountry identifies the proxy market country for cache/error metadata. @@ -273,9 +277,9 @@ func (q Query) String() string { maskedProxyURL = MaskProxyURL(q.ProxyURL) } return fmt.Sprintf( - "{Text:%s LangCode:%s Region:%s DateInterval:%s Filetype:%s Site:%s Limit:%d Start:%d Filter:%t Answers:%t ProxyURL:%s ProxyCountry:%s ProxyClass:%s ProxyProvider:%s ProxySessionID:%s ProxyOverride:%s Insecure:%t}", + "{Text:%s LangCode:%s Region:%s DateInterval:%s Filetype:%s Site:%s Limit:%d Start:%d Filter:%t Features:%t ProxyURL:%s ProxyCountry:%s ProxyClass:%s ProxyProvider:%s ProxySessionID:%s ProxyOverride:%s Insecure:%t}", q.Text, q.LangCode, q.Region, q.DateInterval, q.Filetype, q.Site, - q.Limit, q.Start, q.Filter, q.Answers, + q.Limit, q.Start, q.Filter, q.Features, maskedProxyURL, q.ProxyCountry, q.ProxyClass, q.ProxyProvider, q.ProxySessionID, q.ProxyOverride, q.Insecure, ) @@ -340,9 +344,9 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error { return errInvalidParam(fmt.Sprintf("filter: %v", err)) } - searchQuery.Answers, err = strconv.ParseBool(reqCtx.Query("answers", "0")) + searchQuery.Features, err = strconv.ParseBool(reqCtx.Query("features", "0")) if err != nil { - return errInvalidParam(fmt.Sprintf("answers: %v", err)) + return errInvalidParam(fmt.Sprintf("features: %v", err)) } searchQuery.ProxyOverride, err = NormalizeProxyRequestOverride(reqCtx.Get("X-Use-Proxy")) diff --git a/core/enrichment_domain.go b/core/enrichment_domain.go index 0729664..606891e 100644 --- a/core/enrichment_domain.go +++ b/core/enrichment_domain.go @@ -158,9 +158,20 @@ func classifyContentType(rawURL string) string { func classifySourceHint(domain string) string { cfg := loadEnrichmentDomains() - if hint, ok := cfg.DomainSourceHints[normalizeDomain(domain)]; ok { + domain = normalizeDomain(domain) + if hint, ok := cfg.DomainSourceHints[domain]; ok { return hint } + // Fall back to the registrable domain so subdomain hosts (e.g. + // megadeth.fandom.com, zh.m.wikipedia.org) match a hint keyed on the + // registrable domain (fandom.com, wikipedia.org). + if _, sld := splitDomain(domain); sld != "" { + if registrable, err := publicsuffix.EffectiveTLDPlusOne(domain); err == nil { + if hint, ok := cfg.DomainSourceHints[registrable]; ok { + return hint + } + } + } return "" } diff --git a/core/enrichment_domains.yaml b/core/enrichment_domains.yaml index 56aad43..84233ad 100644 --- a/core/enrichment_domains.yaml +++ b/core/enrichment_domains.yaml @@ -1,6 +1,11 @@ domain_source_hints: wikipedia.org: encyclopedia en.wikipedia.org: encyclopedia + zh.wikipedia.org: encyclopedia + baike.baidu.com: encyclopedia + wiki.mbalib.com: encyclopedia + britannica.com: encyclopedia + fandom.com: encyclopedia github.com: code_repository gitlab.com: code_repository stackoverflow.com: qa_forum diff --git a/core/feature_selectors.go b/core/feature_selectors.go new file mode 100644 index 0000000..e7b2678 --- /dev/null +++ b/core/feature_selectors.go @@ -0,0 +1,274 @@ +package core + +import ( + "strings" + + "github.com/PuerkitoBio/goquery" + "golang.org/x/net/html" + "golang.org/x/net/html/atom" +) + +// blockLevelTags are HTML elements whose boundaries should become line breaks +// when flattening a feature container to text, so block content (headings, +// paragraphs, list items, code blocks) does not fuse into the neighbouring text. +// div/section are deliberately excluded: some engines (e.g. Google's streaming +// AI Overview) wrap every word in its own
      , which would otherwise put each +// word on its own line. Structure there comes from p/h*/li/br instead. +var blockLevelTags = map[atom.Atom]bool{ + atom.P: true, atom.Br: true, atom.Li: true, + atom.Tr: true, atom.Pre: true, + atom.H1: true, atom.H2: true, atom.H3: true, atom.H4: true, atom.H5: true, atom.H6: true, + atom.Blockquote: true, +} + +// blockAwareText flattens a selection to text while inserting line breaks at +// block-element boundaries, then collapses horizontal whitespace per line and +// drops blank lines. The result keeps logical structure (one line per heading/ +// paragraph/list item) instead of fusing words across element edges, which is +// what goquery's raw .Text() does. +func blockAwareText(sel *goquery.Selection) string { + var sb strings.Builder + for _, node := range sel.Nodes { + writeNodeText(&sb, node) + } + lines := strings.Split(sb.String(), "\n") + cleaned := make([]string, 0, len(lines)) + for _, line := range lines { + if line = cleanFeatureText(line); line != "" { + cleaned = append(cleaned, line) + } + } + return strings.Join(cleaned, "\n") +} + +func isASCIISpace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\f' || b == '\v' +} + +func writeNodeText(sb *strings.Builder, node *html.Node) { + switch node.Type { + case html.TextNode: + // Collapse whitespace inside the text node (including source-formatting + // newlines) to single spaces, so only the block-boundary breaks inserted + // below survive. Preserve a single leading/trailing space so adjacent + // inline fragments ("global " + "fetch()") keep their word gap. + text := node.Data + collapsed := strings.Join(strings.Fields(text), " ") + if collapsed == "" { + return + } + if len(text) > 0 && isASCIISpace(text[0]) { + sb.WriteByte(' ') + } + sb.WriteString(collapsed) + if len(text) > 0 && isASCIISpace(text[len(text)-1]) { + sb.WriteByte(' ') + } + return + case html.ElementNode: + if node.DataAtom == atom.Script || node.DataAtom == atom.Style { + return + } + block := blockLevelTags[node.DataAtom] + if block { + sb.WriteByte('\n') + } + for child := node.FirstChild; child != nil; child = child.NextSibling { + writeNodeText(sb, child) + } + if block { + sb.WriteByte('\n') + } + default: + for child := node.FirstChild; child != nil; child = child.NextSibling { + writeNodeText(sb, child) + } + } +} + +// SerpFeatureSelector describes one engine-native SERP module shape. +type SerpFeatureSelector struct { + Type ResultType + Title string + Container []string + TitleSelector []string + TextSelector []string + ItemSelector []string + LinkSelector []string + Position int + Confidence float64 + // SingleMatch emits at most one feature for this spec: the first container + // node (across Container selectors, in order) that yields content. Use it + // for modules whose container selector also matches nested sub-panels, which + // would otherwise fragment one logical module into many features. + SingleMatch bool +} + +// ExtractSerpFeaturesBySelectors converts engine-native SERP module markup into +// normalized features. It is intentionally conservative: a matched container is +// emitted only when it yields text, grouped items, or source links. +func ExtractSerpFeaturesBySelectors(doc *goquery.Document, specs []SerpFeatureSelector) []SerpFeature { + var features []SerpFeature + for _, spec := range specs { + matched := false + for _, selector := range spec.Container { + if spec.SingleMatch && matched { + break + } + doc.Find(selector).EachWithBreak(func(_ int, container *goquery.Selection) bool { + feature := SerpFeature{ + Type: spec.Type, + Title: firstNonEmpty(spec.Title, firstSelectedText(container, spec.TitleSelector)), + Text: firstSelectedText(container, spec.TextSelector), + Items: selectedFeatureItems(container, spec.ItemSelector), + Links: selectedFeatureLinks(container, spec.LinkSelector), + Confidence: spec.Confidence, + } + if spec.Position > 0 { + feature.Position = &Position{Absolute: spec.Position} + } + if feature.Text == "" && len(feature.Items) == 0 && len(feature.Links) == 0 { + return true + } + features = append(features, feature) + matched = true + // Stop after the first content-bearing container when SingleMatch. + return !spec.SingleMatch + }) + } + } + return DeduplicateSerpFeatures(features) +} + +// AttachFeaturesToFirstResult keeps ParseHTML signatures unchanged while +// letting server response building split features onto the new top-level field. +func AttachFeaturesToFirstResult(results []SearchResult, features []SerpFeature) []SearchResult { + if len(features) == 0 { + return results + } + if len(results) == 0 { + return []SearchResult{{Features: features}} + } + results[0].Features = append(results[0].Features, features...) + return results +} + +// DeduplicateSerpFeatures removes duplicate modules emitted by overlapping +// selectors while preserving original order. +func DeduplicateSerpFeatures(features []SerpFeature) []SerpFeature { + seen := map[string]struct{}{} + unique := make([]SerpFeature, 0, len(features)) + for _, feature := range features { + key := serpFeatureKey(feature) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + unique = append(unique, feature) + } + return unique +} + +func firstSelectedText(container *goquery.Selection, selectors []string) string { + for _, selector := range selectors { + var text string + container.Find(selector).EachWithBreak(func(_ int, item *goquery.Selection) bool { + text = blockAwareText(item) + return text == "" + }) + if text != "" { + return text + } + } + return "" +} + +func selectedFeatureItems(container *goquery.Selection, selectors []string) []FeatureItem { + var items []FeatureItem + for _, selector := range selectors { + container.Find(selector).Each(func(_ int, item *goquery.Selection) { + text := cleanFeatureText(item.Text()) + title := firstAttr(item, "data-q", "data-title", "aria-label", "title") + // Some modules (e.g. Google PAA) carry the question in an attribute + // and render the answer lazily, so the element text can be empty. + if text == "" { + text = cleanFeatureText(title) + } + if text == "" { + return + } + link := firstAttr(item, "href", "data-url", "data-link") + items = append(items, FeatureItem{ + Title: strings.TrimSpace(title), + Text: text, + Link: strings.TrimSpace(link), + }) + }) + if len(items) > 0 { + break + } + } + return items +} + +func selectedFeatureLinks(container *goquery.Selection, selectors []string) []FeatureLink { + var links []FeatureLink + for _, selector := range selectors { + container.Find(selector).Each(func(_ int, item *goquery.Selection) { + href := strings.TrimSpace(firstAttr(item, "href", "data-url", "data-link")) + if href == "" { + return + } + title := cleanFeatureText(firstAttr(item, "data-title", "aria-label", "title")) + if title == "" { + title = cleanFeatureText(item.Text()) + } + links = append(links, FeatureLink{Title: title, URL: href}) + }) + if len(links) > 0 { + break + } + } + return links +} + +func firstAttr(item *goquery.Selection, names ...string) string { + for _, name := range names { + value, ok := item.Attr(name) + if ok && strings.TrimSpace(value) != "" { + return value + } + } + return "" +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func cleanFeatureText(value string) string { + return strings.Join(strings.Fields(value), " ") +} + +func serpFeatureKey(feature SerpFeature) string { + firstLink := "" + if len(feature.Links) > 0 { + firstLink = feature.Links[0].URL + } + firstItem := "" + if len(feature.Items) > 0 { + firstItem = feature.Items[0].Text + "|" + feature.Items[0].Link + } + return strings.Join([]string{ + string(feature.Type), + strings.ToLower(cleanFeatureText(feature.Title)), + strings.ToLower(cleanFeatureText(feature.Text)), + strings.ToLower(firstItem), + strings.ToLower(firstLink), + }, "|") +} diff --git a/core/format_markdown.go b/core/format_markdown.go index eda9dfe..0cd362c 100644 --- a/core/format_markdown.go +++ b/core/format_markdown.go @@ -12,23 +12,30 @@ func RenderMarkdown(env *Envelope) []byte { enginesStr := strings.Join(env.Query.EnginesRequested, ", ") fmt.Fprintf(&b, "# Search results for %q\n\n", env.Query.Text) - fmt.Fprintf(&b, "**Query:** %s · **Engines:** %s · **Took:** %dms\n\n", + fmt.Fprintf(&b, "**Query:** %s - **Engines:** %s - **Took:** %dms\n\n", env.Query.Text, enginesStr, env.Meta.TookMs) if len(env.Meta.EnginesFailed) > 0 { - fmt.Fprintf(&b, "> ⚠️ Engines that failed: %s\n\n", strings.Join(env.Meta.EnginesFailed, ", ")) + fmt.Fprintf(&b, "> Engines that failed: %s\n\n", strings.Join(env.Meta.EnginesFailed, ", ")) } + renderMarkdownFeatures(&b, env.SerpFeatures, featureRenderOrderBeforeResults()) + + if len(env.Results) > 0 { + b.WriteString("## Results\n\n") + } for i, r := range env.Results { - fmt.Fprintf(&b, "## %d. %s\n\n", i+1, escapeMarkdown(r.Title)) + fmt.Fprintf(&b, "### %d. %s\n\n", i+1, escapeMarkdown(r.Title)) typeLabel := string(r.Type) - fmt.Fprintf(&b, "**%s** · %s\n\n", r.DisplayURL, typeLabel) + fmt.Fprintf(&b, "**%s** - %s\n\n", r.DisplayURL, typeLabel) if r.Snippet != "" { fmt.Fprintf(&b, "%s\n\n", r.Snippet) } - fmt.Fprintf(&b, "→ %s\n\n", r.URL) + fmt.Fprintf(&b, "-> %s\n\n", r.URL) } + renderMarkdownFeatures(&b, env.SerpFeatures, featureRenderOrderAfterResults()) + return []byte(b.String()) } @@ -38,19 +45,73 @@ func RenderMarkdownImage(env *ImageEnvelope) []byte { enginesStr := strings.Join(env.Query.EnginesRequested, ", ") fmt.Fprintf(&b, "# Image results for %q\n\n", env.Query.Text) - fmt.Fprintf(&b, "**Query:** %s · **Engines:** %s · **Took:** %dms\n\n", + fmt.Fprintf(&b, "**Query:** %s - **Engines:** %s - **Took:** %dms\n\n", env.Query.Text, enginesStr, env.Meta.TookMs) for i, r := range env.Results { fmt.Fprintf(&b, "## %d. %s\n\n", i+1, escapeMarkdown(r.Title)) fmt.Fprintf(&b, "**Source:** %s\n\n", r.Source.Domain) - fmt.Fprintf(&b, "→ Image: %s\n", r.Image.URL) - fmt.Fprintf(&b, "→ Page: %s\n\n", r.Source.PageURL) + fmt.Fprintf(&b, "-> Image: %s\n", r.Image.URL) + fmt.Fprintf(&b, "-> Page: %s\n\n", r.Source.PageURL) } return []byte(b.String()) } +func renderMarkdownFeatures(b *strings.Builder, features []SerpFeature, order []ResultType) { + for _, featureType := range order { + for _, feature := range features { + if feature.Type != featureType { + continue + } + renderMarkdownFeature(b, feature) + } + } +} + +func renderMarkdownFeature(b *strings.Builder, feature SerpFeature) { + heading := featureHeading(feature) + if feature.Type == ResultTypeKnowledgePanel && feature.Title != "" { + heading += " - " + feature.Title + } + fmt.Fprintf(b, "## %s\n\n", heading) + if feature.Type == ResultTypeFeaturedSnippet && feature.Text != "" { + fmt.Fprintf(b, "> %s\n", feature.Text) + if len(feature.Links) > 0 { + fmt.Fprintf(b, "> - [%s](%s)\n", escapeMarkdown(feature.Links[0].Title), feature.Links[0].URL) + } + b.WriteString("\n") + return + } + if feature.Text != "" { + fmt.Fprintf(b, "%s\n\n", feature.Text) + } + if len(feature.Items) > 0 { + for _, item := range feature.Items { + switch { + case item.Title != "" && item.Text != "": + fmt.Fprintf(b, "- **%s** - %s\n", escapeMarkdown(item.Title), item.Text) + case item.Text != "": + fmt.Fprintf(b, "- %s\n", item.Text) + case item.Title != "": + fmt.Fprintf(b, "- %s\n", escapeMarkdown(item.Title)) + } + } + b.WriteString("\n") + } + if len(feature.Links) > 0 { + b.WriteString("Sources:\n") + for _, link := range feature.Links { + title := link.Title + if title == "" { + title = link.URL + } + fmt.Fprintf(b, "- [%s](%s)\n", escapeMarkdown(title), link.URL) + } + b.WriteString("\n") + } +} + func escapeMarkdown(s string) string { replacer := strings.NewReplacer( "*", `\*`, diff --git a/core/format_text.go b/core/format_text.go index 8118aca..ef17617 100644 --- a/core/format_text.go +++ b/core/format_text.go @@ -21,6 +21,11 @@ func RenderText(env *Envelope) []byte { } b.WriteString("\n") + renderTextFeatures(&b, env.SerpFeatures, featureRenderOrderBeforeResults()) + + if len(env.Results) > 0 { + b.WriteString("Results\n\n") + } for i, r := range env.Results { fmt.Fprintf(&b, "[%d] %s (%s)\n", i+1, r.Title, r.Domain) if r.Snippet != "" { @@ -29,6 +34,8 @@ func RenderText(env *Envelope) []byte { fmt.Fprintf(&b, "URL: %s\n\n", r.URL) } + renderTextFeatures(&b, env.SerpFeatures, featureRenderOrderAfterResults()) + return []byte(b.String()) } @@ -47,17 +54,14 @@ func RenderTextImage(env *ImageEnvelope) []byte { return []byte(b.String()) } -// RenderNDJSON formats an Envelope as newline-delimited JSON (one Result per line). -// The envelope meta is omitted from the body; clients should read response headers. +// RenderNDJSON formats an Envelope as newline-delimited JSON. func RenderNDJSON(env *Envelope) []byte { var b strings.Builder for _, r := range env.Results { - data, err := json.Marshal(r) - if err != nil { - continue - } - b.Write(data) - b.WriteByte('\n') + writeNDJSONLine(&b, "result", r) + } + for _, feature := range env.SerpFeatures { + writeNDJSONLine(&b, "feature", feature) } return []byte(b.String()) } @@ -66,12 +70,139 @@ func RenderNDJSON(env *Envelope) []byte { func RenderNDJSONImage(env *ImageEnvelope) []byte { var b strings.Builder for _, r := range env.Results { - data, err := json.Marshal(r) - if err != nil { - continue - } - b.Write(data) - b.WriteByte('\n') + writeNDJSONLine(&b, "result", r) } return []byte(b.String()) } + +func renderTextFeatures(b *strings.Builder, features []SerpFeature, order []ResultType) { + for _, featureType := range order { + for _, feature := range features { + if feature.Type != featureType { + continue + } + renderTextFeature(b, feature) + } + } +} + +func renderTextFeature(b *strings.Builder, feature SerpFeature) { + heading := featureHeading(feature) + if feature.Type == ResultTypeKnowledgePanel && feature.Title != "" { + heading += " - " + feature.Title + } + fmt.Fprintf(b, "%s\n", heading) + if feature.Text != "" { + fmt.Fprintf(b, "%s", feature.Text) + if len(feature.Links) == 1 { + fmt.Fprintf(b, " (source: %s)", feature.Links[0].URL) + } + b.WriteString("\n") + } + for _, item := range feature.Items { + switch { + case item.Title != "" && item.Text != "": + fmt.Fprintf(b, "- %s - %s\n", item.Title, item.Text) + case item.Text != "": + fmt.Fprintf(b, "- %s\n", item.Text) + case item.Title != "": + fmt.Fprintf(b, "- %s\n", item.Title) + } + } + if len(feature.Links) > 1 { + b.WriteString("Sources:\n") + for _, link := range feature.Links { + fmt.Fprintf(b, "- %s\n", link.URL) + } + } + b.WriteString("\n") +} + +func writeNDJSONLine(b *strings.Builder, kind string, value any) { + data, err := json.Marshal(value) + if err != nil { + return + } + var object map[string]any + if err := json.Unmarshal(data, &object); err != nil { + return + } + object["kind"] = kind + data, err = json.Marshal(object) + if err != nil { + return + } + b.Write(data) + b.WriteByte('\n') +} + +// featureRenderOrderBeforeResults lists the feature sections rendered above the +// results list, in fixed order (spec: AI summary -> answer box -> featured +// snippet -> PAA -> related questions -> knowledge panel -> results -> ...). +func featureRenderOrderBeforeResults() []ResultType { + return []ResultType{ + ResultTypeAISummary, + ResultTypeAnswerBox, + ResultTypeFeaturedSnippet, + ResultTypePeopleAlsoAsk, + ResultTypeRelatedQuestions, + ResultTypeKnowledgePanel, + } +} + +// featureRenderOrderAfterResults lists the feature sections rendered below the +// results list (related searches and the module gallery), in fixed order. +func featureRenderOrderAfterResults() []ResultType { + return []ResultType{ + ResultTypeRelatedSearches, + ResultTypeNews, + ResultTypeVideo, + ResultTypeVideos, + ResultTypeShopping, + ResultTypeImagesInline, + ResultTypeLocal, + ResultTypeSitelinks, + ResultTypeCalculator, + ResultTypeWeather, + ResultTypeDictionary, + } +} + +func featureHeading(feature SerpFeature) string { + switch feature.Type { + case ResultTypeAISummary: + return "AI summary" + case ResultTypeAnswerBox: + return "Answer box" + case ResultTypeFeaturedSnippet: + return "Featured snippet" + case ResultTypePeopleAlsoAsk: + return "People also ask" + case ResultTypeRelatedQuestions: + return "Related questions" + case ResultTypeKnowledgePanel: + return "Knowledge panel" + case ResultTypeRelatedSearches: + return "Related searches" + case ResultTypeNews: + return "News" + case ResultTypeVideo, ResultTypeVideos: + return "Videos" + case ResultTypeShopping: + return "Shopping" + case ResultTypeImagesInline: + return "Images" + case ResultTypeLocal: + return "Local pack" + case ResultTypeSitelinks: + return "Sitelinks" + case ResultTypeCalculator: + return "Calculator" + case ResultTypeWeather: + return "Weather" + case ResultTypeDictionary: + return "Dictionary" + default: + return strings.ReplaceAll(string(feature.Type), "_", " ") + } +} diff --git a/core/page_helpers.go b/core/page_helpers.go index 2b644a6..d6e81f0 100644 --- a/core/page_helpers.go +++ b/core/page_helpers.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "github.com/PuerkitoBio/goquery" "github.com/go-rod/rod" ) @@ -166,3 +167,19 @@ func FirstNonEmptyAttribute(root *rod.Element, attr string, selectors ...string) } return "" } + +// FeaturesFromPage renders a live rod page to HTML and runs a document-level +// feature extractor over it. Every engine's browser path shares this boilerplate +// (page.HTML -> goquery doc -> extract), so it lives here rather than being +// copied per engine. Returns nil on any rendering/parse error. +func FeaturesFromPage(page *rod.Page, extract func(*goquery.Document) []SerpFeature) []SerpFeature { + html, err := page.HTML() + if err != nil { + return nil + } + doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) + if err != nil { + return nil + } + return extract(doc) +} diff --git a/core/response.go b/core/response.go index 4d313f9..f5af1c8 100644 --- a/core/response.go +++ b/core/response.go @@ -37,10 +37,11 @@ type Pagination struct { // Envelope is the top-level v2 response wrapper for all search endpoints. type Envelope struct { - Query QueryEcho `json:"query"` - Meta ResponseMeta `json:"meta"` - Results []Result `json:"results"` - Pagination Pagination `json:"pagination"` + Query QueryEcho `json:"query"` + Meta ResponseMeta `json:"meta"` + Results []Result `json:"results"` + SerpFeatures []SerpFeature `json:"serp_features"` + Pagination Pagination `json:"pagination"` // Clusters is only populated by /mega/search (see clusters.go). Clusters *[]Cluster `json:"clusters,omitempty"` } @@ -91,8 +92,9 @@ func NewEnvelope(q Query, requestID string, startedAt time.Time, engines []strin EnginesFailed: []string{}, Version: apiVersion, }, - Results: []Result{}, - Pagination: Pagination{}, + Results: []Result{}, + SerpFeatures: []SerpFeature{}, + Pagination: Pagination{}, } } diff --git a/core/response_builder.go b/core/response_builder.go index 928b037..62ffb45 100644 --- a/core/response_builder.go +++ b/core/response_builder.go @@ -9,6 +9,7 @@ import ( "regexp" "strconv" "strings" + "time" ) const responseIDBytes = 8 @@ -84,6 +85,96 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result { return result } +// AppendEnrichedSearchResult preserves the legacy results[] surface while +// copying any extracted SERP features onto the top-level feature surface. +func AppendEnrichedSearchResult(env *Envelope, raw SearchResult, ctx EnrichContext, extractedAt time.Time) { + var sourceResultID string + if raw.URL != "" || raw.Title != "" || raw.Description != "" || raw.Rank != 0 { + result := EnrichResult(raw, ctx) + env.Results = append(env.Results, result) + sourceResultID = result.ID + } + + for _, rawFeature := range raw.Features { + env.SerpFeatures = append(env.SerpFeatures, EnrichSerpFeature(rawFeature, ctx.Engine, sourceResultID, extractedAt)) + } + if len(raw.Features) == 0 && sourceResultID != "" && shouldMirrorResultAsFeature(raw.Type) { + result := env.Results[len(env.Results)-1] + env.SerpFeatures = append(env.SerpFeatures, EnrichSerpFeature(SerpFeature{ + Type: result.Type, + Title: result.Title, + Text: result.Snippet, + Position: result.Position, + Links: []FeatureLink{{ + Title: result.Title, + URL: result.URL, + }}, + }, ctx.Engine, sourceResultID, extractedAt)) + } +} + +// EnrichSerpFeature stamps a raw feature with stable public fields. +func EnrichSerpFeature(raw SerpFeature, engine string, sourceResultID string, extractedAt time.Time) SerpFeature { + feature := raw + feature.Engine = engine + if feature.SourceResultIDs == nil { + feature.SourceResultIDs = []string{} + } + if sourceResultID != "" && !containsString(feature.SourceResultIDs, sourceResultID) { + feature.SourceResultIDs = append(feature.SourceResultIDs, sourceResultID) + } + for i := range feature.Links { + feature.Links[i].URL = normalizeURL(feature.Links[i].URL) + } + for i := range feature.Items { + feature.Items[i].Link = normalizeURL(feature.Items[i].Link) + } + if feature.ID == "" { + feature.ID = buildFeatureID(feature) + } + if feature.ExtractedAt == "" { + feature.ExtractedAt = extractedAt.UTC().Format(time.RFC3339) + } + return feature +} + +func buildFeatureID(feature SerpFeature) string { + primaryLink := "" + if len(feature.Links) > 0 { + primaryLink = feature.Links[0].URL + } + if primaryLink == "" && len(feature.Items) > 0 { + primaryLink = feature.Items[0].Link + } + key := strings.Join([]string{ + feature.Engine, + string(feature.Type), + strings.ToLower(strings.TrimSpace(feature.Title)), + strings.ToLower(strings.TrimSpace(feature.Text)), + primaryLink, + }, "|") + return "f_" + shortMD5(key) +} + +func containsString(values []string, needle string) bool { + for _, value := range values { + if value == needle { + return true + } + } + return false +} + +func shouldMirrorResultAsFeature(t ResultType) bool { + switch t { + case ResultTypeAnswerBox, ResultTypeFeaturedSnippet, ResultTypeKnowledgePanel, + ResultTypePeopleAlsoAsk, ResultTypeLocal: + return true + default: + return false + } +} + // EnrichImageResult converts a raw engine result into the v2 ImageResult shape. func EnrichImageResult(raw SearchResult, ctx EnrichContext) ImageResult { imageURL := normalizeURL(raw.URL) diff --git a/core/result.go b/core/result.go index e0d0a4f..1fd2346 100644 --- a/core/result.go +++ b/core/result.go @@ -4,17 +4,26 @@ package core type ResultType string const ( - ResultTypeOrganic ResultType = "organic" - ResultTypeAd ResultType = "ad" - ResultTypeFeaturedSnippet ResultType = "featured_snippet" - ResultTypeKnowledgePanel ResultType = "knowledge_panel" - ResultTypePeopleAlsoAsk ResultType = "people_also_ask" - ResultTypeVideo ResultType = "video" - ResultTypeImage ResultType = "image" - ResultTypeNews ResultType = "news" - ResultTypeShopping ResultType = "shopping" - ResultTypeLocal ResultType = "local" - ResultTypeAnswerBox ResultType = "answer_box" + ResultTypeOrganic ResultType = "organic" + ResultTypeAd ResultType = "ad" + ResultTypeFeaturedSnippet ResultType = "featured_snippet" + ResultTypeKnowledgePanel ResultType = "knowledge_panel" + ResultTypePeopleAlsoAsk ResultType = "people_also_ask" + ResultTypeVideo ResultType = "video" + ResultTypeImage ResultType = "image" + ResultTypeNews ResultType = "news" + ResultTypeShopping ResultType = "shopping" + ResultTypeLocal ResultType = "local" + ResultTypeAnswerBox ResultType = "answer_box" + ResultTypeAISummary ResultType = "ai_summary" + ResultTypeRelatedQuestions ResultType = "related_questions" + ResultTypeRelatedSearches ResultType = "related_searches" + ResultTypeSitelinks ResultType = "sitelinks" + ResultTypeVideos ResultType = "videos" + ResultTypeImagesInline ResultType = "images_inline" + ResultTypeCalculator ResultType = "calculator" + ResultTypeWeather ResultType = "weather" + ResultTypeDictionary ResultType = "dictionary" ) // Position describes where a result sits in the overall result stream. @@ -40,6 +49,35 @@ type Classification struct { SourceHint string `json:"source_hint,omitempty"` } +// FeatureItem is one child entry inside a grouped SERP feature. +type FeatureItem struct { + Title string `json:"title,omitempty"` + Text string `json:"text,omitempty"` + Link string `json:"link,omitempty"` +} + +// FeatureLink is a source or citation associated with a SERP feature. +type FeatureLink struct { + Title string `json:"title,omitempty"` + URL string `json:"url,omitempty"` +} + +// SerpFeature is a normalized non-organic SERP module surfaced separately +// from rankable results. +type SerpFeature struct { + ID string `json:"id"` + Engine string `json:"engine"` + Type ResultType `json:"type"` + Title string `json:"title,omitempty"` + Text string `json:"text,omitempty"` + Items []FeatureItem `json:"items,omitempty"` + Links []FeatureLink `json:"links,omitempty"` + SourceResultIDs []string `json:"source_result_ids,omitempty"` + Position *Position `json:"position,omitempty"` + Confidence float64 `json:"confidence,omitempty"` + ExtractedAt string `json:"extracted_at"` +} + // Result is the v2 normalized result returned in search responses. Optional // fields (Position, DomainInfo, Classification) are omitted when empty. type Result struct { diff --git a/core/serp_features_test.go b/core/serp_features_test.go new file mode 100644 index 0000000..d153f83 --- /dev/null +++ b/core/serp_features_test.go @@ -0,0 +1,152 @@ +package core + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestEnvelopeAlwaysIncludesSerpFeatures(t *testing.T) { + env := NewEnvelope(Query{Text: "golang"}, "req-1", time.Unix(0, 0), []string{"google"}) + + data, err := json.Marshal(env) + if err != nil { + t.Fatalf("marshal envelope: %v", err) + } + if !strings.Contains(string(data), `"serp_features":[]`) { + t.Fatalf("expected empty serp_features array in JSON, got %s", data) + } +} + +func TestAppendEnrichedSearchResultAddsFeatureSurface(t *testing.T) { + env := NewEnvelope(Query{Text: "weather"}, "req-1", time.Unix(0, 0), []string{"google"}) + raw := SearchResult{ + Rank: 1, + URL: "https://example.com/weather", + Title: "Weather result", + Description: "Organic snippet", + Features: []SerpFeature{{ + Type: ResultTypeAnswerBox, + Text: "72 F and sunny", + Links: []FeatureLink{{ + Title: "Weather source", + URL: "https://example.com/weather", + }}, + Position: &Position{Absolute: 1}, + Confidence: 0.9, + }}, + } + + AppendEnrichedSearchResult(env, raw, EnrichContext{Engine: "google", Query: Query{}}, time.Unix(0, 0)) + + if len(env.Results) != 1 { + t.Fatalf("expected 1 result, got %d", len(env.Results)) + } + if len(env.SerpFeatures) != 1 { + t.Fatalf("expected 1 feature, got %d", len(env.SerpFeatures)) + } + feature := env.SerpFeatures[0] + if feature.ID == "" || !strings.HasPrefix(feature.ID, "f_") { + t.Fatalf("expected stable feature ID, got %q", feature.ID) + } + if feature.Engine != "google" { + t.Fatalf("feature engine = %q, want google", feature.Engine) + } + if feature.ExtractedAt != "1970-01-01T00:00:00Z" { + t.Fatalf("feature extracted_at = %q", feature.ExtractedAt) + } + if len(feature.SourceResultIDs) != 1 || feature.SourceResultIDs[0] != env.Results[0].ID { + t.Fatalf("expected feature to reference result ID %q, got %#v", env.Results[0].ID, feature.SourceResultIDs) + } +} + +func TestAppendEnrichedSearchResultMirrorsExistingAnswerResultAsFeature(t *testing.T) { + env := NewEnvelope(Query{Text: "weather"}, "req-1", time.Unix(0, 0), []string{"google"}) + raw := SearchResult{ + Rank: -1, + Type: ResultTypeAnswerBox, + URL: "https://example.com/weather", + Title: "Weather", + Description: "72 F and sunny", + } + + AppendEnrichedSearchResult(env, raw, EnrichContext{Engine: "google", Query: Query{}}, time.Unix(0, 0)) + + if len(env.Results) != 1 { + t.Fatalf("expected existing result to be preserved, got %d results", len(env.Results)) + } + if len(env.SerpFeatures) != 1 { + t.Fatalf("expected mirrored feature, got %d", len(env.SerpFeatures)) + } + if env.SerpFeatures[0].Type != ResultTypeAnswerBox { + t.Fatalf("mirrored feature type = %q", env.SerpFeatures[0].Type) + } + if env.SerpFeatures[0].Text != "72 F and sunny" { + t.Fatalf("mirrored feature text = %q", env.SerpFeatures[0].Text) + } + if len(env.SerpFeatures[0].SourceResultIDs) != 1 || env.SerpFeatures[0].SourceResultIDs[0] != env.Results[0].ID { + t.Fatalf("mirrored feature did not reference result: %#v", env.SerpFeatures[0].SourceResultIDs) + } +} + +func TestRenderersIncludeSerpFeatures(t *testing.T) { + env := NewEnvelope(Query{Text: "openserp"}, "req-1", time.Unix(0, 0), []string{"google"}) + AppendEnrichedSearchResult(env, SearchResult{ + Rank: 1, + URL: "https://example.com/result", + Title: "Organic result", + Description: "Snippet", + Features: []SerpFeature{ + { + Type: ResultTypeAISummary, + Text: "OpenSERP is a search API.", + Links: []FeatureLink{{ + Title: "Example", + URL: "https://example.com/source", + }}, + Position: &Position{Absolute: 1}, + Confidence: 0.95, + }, + { + Type: ResultTypeRelatedSearches, + Items: []FeatureItem{ + {Text: "openserp cloud"}, + {Text: "serp api"}, + }, + }, + }, + }, EnrichContext{Engine: "google", Query: Query{}}, time.Unix(0, 0)) + env.Finalize(time.Unix(0, 0), Query{Text: "openserp", Limit: 10}) + + markdown := string(RenderMarkdown(env)) + if !strings.Contains(markdown, "## AI summary") || !strings.Contains(markdown, "Sources:") { + t.Fatalf("markdown missing AI summary feature section:\n%s", markdown) + } + if !strings.Contains(markdown, "## Results") { + t.Fatalf("markdown missing Results section:\n%s", markdown) + } + if !strings.Contains(markdown, "## Related searches") { + t.Fatalf("markdown missing related searches section:\n%s", markdown) + } + // Spec order: AI summary -> ... -> Results -> Related searches. + if aiIdx, resIdx, relIdx := strings.Index(markdown, "## AI summary"), strings.Index(markdown, "## Results"), strings.Index(markdown, "## Related searches"); !(aiIdx < resIdx && resIdx < relIdx) { + t.Fatalf("markdown section order wrong: ai=%d results=%d related=%d\n%s", aiIdx, resIdx, relIdx, markdown) + } + + text := string(RenderText(env)) + if !strings.Contains(text, "AI summary\n") || !strings.Contains(text, "Related searches\n") { + t.Fatalf("text missing feature sections:\n%s", text) + } + + lines := strings.Split(strings.TrimSpace(string(RenderNDJSON(env))), "\n") + if len(lines) != 3 { + t.Fatalf("expected 3 ndjson lines, got %d: %v", len(lines), lines) + } + if !strings.Contains(lines[0], `"kind":"result"`) { + t.Fatalf("first ndjson line should be a result, got %s", lines[0]) + } + if !strings.Contains(lines[1], `"kind":"feature"`) || !strings.Contains(lines[2], `"kind":"feature"`) { + t.Fatalf("feature ndjson lines missing kind tag: %v", lines) + } +} diff --git a/core/server.go b/core/server.go index d92cff6..307d18b 100644 --- a/core/server.go +++ b/core/server.go @@ -331,7 +331,7 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm } ectx := EnrichContext{Engine: usedEngine, Query: q} for _, r := range res { - env.Results = append(env.Results, EnrichResult(r, ectx)) + AppendEnrichedSearchResult(env, r, ectx, startedAt) } env.Finalize(startedAt, q) @@ -382,7 +382,7 @@ func (s *Server) handleParseEndpoint(c *fiber.Ctx, parser HTMLParser) error { env := NewEnvelope(q, requestID, startedAt, []string{parser.Name()}) ectx := EnrichContext{Engine: parser.Name(), Query: q} for _, r := range results { - env.Results = append(env.Results, EnrichResult(r, ectx)) + AppendEnrichedSearchResult(env, r, ectx, startedAt) } env.Finalize(startedAt, q) @@ -1021,7 +1021,7 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string) error { env.Meta.EngineErrors = engineErrors for _, r := range webResults { ectx := EnrichContext{Engine: r.Engine, Query: q} - env.Results = append(env.Results, EnrichResult(r.SearchResult, ectx)) + AppendEnrichedSearchResult(env, r.SearchResult, ectx, startedAt) } env.Finalize(startedAt, q) diff --git a/core/server_test.go b/core/server_test.go index 29ed304..7ea18df 100644 --- a/core/server_test.go +++ b/core/server_test.go @@ -280,8 +280,8 @@ func TestInvalidQueryParametersReturnJSONError(t *testing.T) { reason: ReasonInvalidParam, }, { - name: "invalid answers flag on mega endpoint", - path: "/mega/search?text=golang&answers=notabool", + name: "invalid features flag on mega endpoint", + path: "/mega/search?text=golang&features=notabool", message: "invalid syntax", reason: ReasonInvalidParam, }, diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 608ec3e..8482f5b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -71,7 +71,7 @@ All engines implement: ### `core.Query` -Parsed from query parameters (`text`, `lang`, `region`, `date`, `file`, `site`, `limit`, `start`, `filter`, `answers`) and the `X-Use-Proxy` request header. At least one of `text`, `site`, or `file` must be non-empty. +Parsed from query parameters (`text`, `lang`, `region`, `date`, `file`, `site`, `limit`, `start`, `filter`, `features`) and the `X-Use-Proxy` request header. At least one of `text`, `site`, or `file` must be non-empty. ### Internal `core.SearchResult` diff --git a/docs/openapi.yaml b/docs/openapi.yaml index da0b93b..4d1b0a9 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -46,7 +46,7 @@ paths: - $ref: "#/components/parameters/LimitQuery" - $ref: "#/components/parameters/StartQuery" - $ref: "#/components/parameters/FilterQuery" - - $ref: "#/components/parameters/AnswersQuery" + - $ref: "#/components/parameters/FeaturesQuery" - $ref: "#/components/parameters/FormatQuery" - $ref: "#/components/parameters/UseProxyHeader" - $ref: "#/components/parameters/ProxyURLHeader" @@ -110,6 +110,19 @@ paths: tld: dev sld: go category: "" + serp_features: + - id: f_a1b2c3d4e5f6a1b2 + engine: google + type: ai_summary + text: Go is an open source programming language used for fast, reliable services. + links: + - title: The Go Programming Language + url: https://go.dev/ + source_result_ids: [s_a1b2c3d4e5f6a1b2] + position: + absolute: 1 + confidence: 0.95 + extracted_at: "2026-04-24T12:00:00Z" pagination: page: 1 has_more: true @@ -155,7 +168,7 @@ paths: - $ref: "#/components/parameters/LimitQuery" - $ref: "#/components/parameters/StartQuery" - $ref: "#/components/parameters/FilterQuery" - - $ref: "#/components/parameters/AnswersQuery" + - $ref: "#/components/parameters/FeaturesQuery" - $ref: "#/components/parameters/FormatQuery" - $ref: "#/components/parameters/UseProxyHeader" - $ref: "#/components/parameters/ProxyURLHeader" @@ -307,7 +320,7 @@ paths: - $ref: "#/components/parameters/LimitQuery" - $ref: "#/components/parameters/StartQuery" - $ref: "#/components/parameters/FilterQuery" - - $ref: "#/components/parameters/AnswersQuery" + - $ref: "#/components/parameters/FeaturesQuery" - $ref: "#/components/parameters/EnginesQuery" - $ref: "#/components/parameters/MegaModeQuery" - $ref: "#/components/parameters/MegaDedupeQuery" @@ -380,7 +393,7 @@ paths: - $ref: "#/components/parameters/LimitQuery" - $ref: "#/components/parameters/StartQuery" - $ref: "#/components/parameters/FilterQuery" - - $ref: "#/components/parameters/AnswersQuery" + - $ref: "#/components/parameters/FeaturesQuery" - $ref: "#/components/parameters/EnginesQuery" - $ref: "#/components/parameters/MegaModeQuery" - $ref: "#/components/parameters/MegaDedupeQuery" @@ -645,11 +658,14 @@ components: schema: type: boolean default: true - AnswersQuery: - name: answers + FeaturesQuery: + name: features in: query required: false - description: Include answer box style results when supported. + description: > + Populate the top-level serp_features array (AI summaries, answer boxes, + people-also-ask, related searches) from the live browser search when + supported by the engine. schema: type: boolean default: false @@ -1129,6 +1145,15 @@ components: - shopping - local - answer_box + - ai_summary + - related_questions + - related_searches + - sitelinks + - videos + - images_inline + - calculator + - weather + - dictionary Result: type: object required: @@ -1186,6 +1211,72 @@ components: $ref: "#/components/schemas/DomainInfo" classification: $ref: "#/components/schemas/Classification" + FeatureItem: + type: object + properties: + title: + type: string + example: "What is OpenSERP?" + text: + type: string + example: "OpenSERP is an open-source SERP API." + link: + type: string + example: https://openserp.org/ + FeatureLink: + type: object + properties: + title: + type: string + example: OpenSERP + url: + type: string + example: https://openserp.org/ + SerpFeature: + type: object + required: [id, engine, type, extracted_at] + properties: + id: + type: string + description: Stable identifier prefixed with `f_`. + example: f_a1b2c3d4e5f6a1b2 + engine: + type: string + example: google + type: + $ref: "#/components/schemas/ResultType" + title: + type: string + example: OpenSERP + text: + type: string + description: Primary human-readable feature content. + example: OpenSERP is an open-source SERP API. + items: + type: array + items: + $ref: "#/components/schemas/FeatureItem" + links: + type: array + items: + $ref: "#/components/schemas/FeatureLink" + source_result_ids: + type: array + items: + type: string + example: [s_a1b2c3d4e5f6a1b2] + position: + $ref: "#/components/schemas/Position" + confidence: + type: number + format: float + minimum: 0 + maximum: 1 + example: 0.95 + extracted_at: + type: string + format: date-time + example: "2026-04-24T12:00:00Z" # ── Image result ───────────────────────────────────────────────── ImageData: type: object @@ -1289,7 +1380,7 @@ components: # ── Envelopes ───────────────────────────────────────────────────── SearchEnvelope: type: object - required: [query, meta, results, pagination] + required: [query, meta, results, serp_features, pagination] properties: query: $ref: "#/components/schemas/QueryEcho" @@ -1299,6 +1390,13 @@ components: type: array items: $ref: "#/components/schemas/Result" + serp_features: + type: array + description: > + Non-organic SERP modules such as AI summaries, answer boxes, + related questions, related searches, and knowledge panels. + items: + $ref: "#/components/schemas/SerpFeature" pagination: $ref: "#/components/schemas/Pagination" MegaSearchEnvelope: diff --git a/duckduckgo/features.go b/duckduckgo/features.go new file mode 100644 index 0000000..cdb7d51 --- /dev/null +++ b/duckduckgo/features.go @@ -0,0 +1,58 @@ +package duckduckgo + +import ( + "github.com/PuerkitoBio/goquery" + "github.com/go-rod/rod" + "github.com/karust/openserp/core" +) + +func extractDDGFeatures(doc *goquery.Document) []core.SerpFeature { + features := core.ExtractSerpFeaturesBySelectors(doc, []core.SerpFeatureSelector{ + { + // wikinlp is DDG's AI-assisted "DuckAssist" summary + // (li[data-layout='wikinlp']). The full multi-section answer lives in + // duckassist-expanded-answer-content; duckassist-answer-content is only + // the collapsed teaser. Prefer the expanded wrapper and take its whole + // collapsed text so the body isn't truncated to the teaser. + Type: core.ResultTypeAISummary, + Title: "Instant Answer", + Container: []string{"li[data-layout='wikinlp'] div.react-module", "[data-react-module-id='wikinlp']"}, + TitleSelector: []string{"h2", "h3", ".module__title"}, + TextSelector: []string{"[data-testid='duckassist-expanded-answer-content']", "[data-testid='duckassist-answer-content']", ".module__text", "p"}, + LinkSelector: []string{"[data-testid='duckassist-expanded-answer-content'] a[href^='http']", "[data-testid='duckassist-answer-content'] a[href^='http']", "a[href^='http']"}, + Position: 1, + Confidence: 0.7, + }, + { + Type: core.ResultTypeAnswerBox, + Title: "Answer", + Container: []string{"#zero_click_wrapper", ".zci", ".zci--answer", ".result--answer", "li[data-layout='about'] .module--about", ".module--about"}, + TitleSelector: []string{".module__title__sub", "h1", "h2", ".zci__title"}, + TextSelector: []string{".js-about-item-abstr", ".module__text", ".zci__result", ".zci__body", ".result__snippet"}, + LinkSelector: []string{"a.module__more-at[href^='http']", "a[href^='http']"}, + Position: 1, + Confidence: 0.8, + }, + { + Type: core.ResultTypeRelatedQuestions, + Title: "Related questions", + Container: []string{"[data-testid='related-questions']", ".related-questions", ".module--questions"}, + ItemSelector: []string{"a", "button"}, + LinkSelector: []string{"a[href^='http']", "a"}, + Confidence: 0.7, + }, + { + Type: core.ResultTypeRelatedSearches, + Title: "Related searches", + Container: []string{"[data-testid='related-searches']", ".related-searches", ".result__related"}, + ItemSelector: []string{"a"}, + LinkSelector: []string{"a[href^='http']", "a"}, + Confidence: 0.75, + }, + }) + return core.DeduplicateSerpFeatures(features) +} + +func extractDDGFeaturesFromPage(page *rod.Page) []core.SerpFeature { + return core.FeaturesFromPage(page, extractDDGFeatures) +} diff --git a/duckduckgo/parse_html.go b/duckduckgo/parse_html.go index 6ef35a8..1c6b806 100644 --- a/duckduckgo/parse_html.go +++ b/duckduckgo/parse_html.go @@ -26,7 +26,7 @@ func parseDDGDocument(doc *goquery.Document) []core.SearchResult { resultSel := firstMatchingSelector(doc, Selectors.Results) if resultSel == "" { - return results + return core.AttachFeaturesToFirstResult(results, extractDDGFeatures(doc)) } doc.Find(resultSel).Each(func(_ int, item *goquery.Selection) { @@ -62,7 +62,7 @@ func parseDDGDocument(doc *goquery.Document) []core.SearchResult { absoluteRank++ }) - return core.DeduplicateResults(results) + return core.AttachFeaturesToFirstResult(core.DeduplicateResults(results), extractDDGFeatures(doc)) } func duckduckgoSelectionHasAdMarker(item *goquery.Selection) bool { diff --git a/duckduckgo/search.go b/duckduckgo/search.go index 8cef4c3..a80832f 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -174,6 +174,7 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results [] }() allResults := []core.SearchResult{} + var pageFeatures []core.SerpFeature searchPage := 0 // fetchPage loads one SERP page and appends parsed results. @@ -211,6 +212,9 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results [] return false, core.ErrSearchTimeout } + if query.Features && searchPage == 0 { + pageFeatures = extractDDGFeaturesFromPage(page) + } allResults = append(allResults, r...) return false, nil } @@ -236,7 +240,7 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results [] deduped = core.LimitOrganicResults(deduped, query.Limit) ddg.logger.Info("Search completed: %d results", len(deduped)) - return deduped, nil + return core.AttachFeaturesToFirstResult(deduped, pageFeatures), nil } // SearchImage executes a DuckDuckGo image search and returns normalized image diff --git a/duckduckgo/selectors.go b/duckduckgo/selectors.go index c86d0dc..50b7af4 100644 --- a/duckduckgo/selectors.go +++ b/duckduckgo/selectors.go @@ -42,14 +42,21 @@ var Selectors = struct { "unusual traffic", "anomaly", }, + // Results selectors target the canonical result card. The data-testid cards + // are the innermost result element; selecting them avoids double-counting the + // li[data-layout] wrapper that encloses each one (which previously inflated + // ranks to 1,3,5,...). li[data-layout] is kept only as a fallback for older + // markup that lacks data-testid. wikinlp/about layouts are deliberately + // excluded here — they are instant-answer modules surfaced as serp_features. Results: []string{ - "article[data-testid='result'], article[data-testid='ad'], li[data-layout='organic'], li[data-layout='ad'], div[data-testid='result'], div[data-testid='ad']", + "article[data-testid='result'], article[data-testid='ad'], div[data-testid='result'], div[data-testid='ad']", "article[data-testid='result']", "article[data-testid='ad']", - "li[data-layout='organic']", - "li[data-layout='ad']", "div[data-testid='result']", "div[data-testid='ad']", + "li[data-layout='organic'], li[data-layout='ad']", + "li[data-layout='organic']", + "li[data-layout='ad']", "div.result", }, Title: []string{ diff --git a/duckduckgo/serp_features_test.go b/duckduckgo/serp_features_test.go new file mode 100644 index 0000000..510db46 --- /dev/null +++ b/duckduckgo/serp_features_test.go @@ -0,0 +1,103 @@ +package duckduckgo + +import ( + "bytes" + "os" + "testing" + + "github.com/karust/openserp/core" +) + +func TestParseHTMLFixtureExtractsRealFeatures(t *testing.T) { + t.Parallel() + f, err := os.Open("testdata/search_results.html") + if err != nil { + t.Fatalf("open fixture: %v", err) + } + defer f.Close() + + results, err := ParseHTML(f) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + assertFeatureType(t, results, core.ResultTypeRelatedSearches) + // wikinlp/about instant-answer modules are surfaced as features, not results. + assertFeatureType(t, results, core.ResultTypeAISummary) + + // Organic ranks must be a contiguous 1..N sequence. The combined result + // selector previously matched both the li[data-layout] wrapper and the inner + // article[data-testid], counting each result twice and leaving rank gaps. + rank := 0 + for i, r := range results { + if r.Ad { + continue + } + rank++ + if r.Rank != rank { + t.Fatalf("organic rank gap at index %d: got %d, want %d", i, r.Rank, rank) + } + } +} + +func TestParseHTMLExtractsSerpFeatures(t *testing.T) { + t.Parallel() + + html := ` +
      +

      DuckDuckGo answer

      +
      DuckDuckGo instant answer text.
      + Source +
      +
      + related one + related two +
      +` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + assertFeatureType(t, results, core.ResultTypeAnswerBox) + assertFeatureType(t, results, core.ResultTypeRelatedSearches) +} + +func TestParseHTMLOrganicOnlyHasNoSerpFeatures(t *testing.T) { + t.Parallel() + + html := ` +` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + assertNoFeatures(t, results) +} + +func assertFeatureType(t *testing.T, results []core.SearchResult, want core.ResultType) { + t.Helper() + for _, result := range results { + for _, feature := range result.Features { + if feature.Type == want { + return + } + } + } + t.Fatalf("expected feature type %q in %#v", want, results) +} + +func assertNoFeatures(t *testing.T, results []core.SearchResult) { + t.Helper() + for _, result := range results { + if len(result.Features) > 0 { + t.Fatalf("expected no features, got %#v", result.Features) + } + } +} diff --git a/duckduckgo/testdata/search_results.html b/duckduckgo/testdata/search_results.html index f86589b..b63ca4d 100644 --- a/duckduckgo/testdata/search_results.html +++ b/duckduckgo/testdata/search_results.html @@ -1 +1 @@ -open serp at DuckDuckGo
      1. OpenSERPis an API and CLI for accessing search engine results from Google, Yandex, Baidu, Bing, and DuckDuckGo. A developer-friendly alternative to paidSERPAPI services! Official website:openserp.org 💡OpenSerpis free andopen-source. Only links listed in this repository and on the official website are associated with the project.
        1. Request parameters
        2. Search
        3. Example request
        4. Get 20 Google results for hello world, only in English: You can replace google to yandex or baidu in query to change search engine. |
        5. Example response
        6. Images
        7. Example request
        8. Get 100 Google results for golden puppy:
        9. Example response
        See more on github.com
      2. Generate answer foropen serp
      Custom date rangeX
      +how to fetch in javascript at DuckDuckGo
      1. To fetch data in JavaScript, use thefetch()method, which takes a URL as an argument and returns a Promise that resolves to a Response object. You can then handle the response using methods like.json()to extract the data you need.

        MozillaMedium

        Fetching Data in JavaScript

        To fetch data in JavaScript, you can use thefetch()method. This method is designed to make HTTP requests to a specified URL and handle the responses.

        Basic Usage of fetch()

        Thefetch()method takes a URL as its first argument and returns a Promise that resolves to a Response object. Here’s a simple example:

        javascript
        fetch('http://test.test').then(response=>response.json()).then(data=>console.log(data)).catch(error=>console.error('Error:', error));

        Key Steps in Using fetch()

        1. Make a Request: Call thefetch()method with the desired URL.
        2. Handle the Response: Use the.then()method to process the Response object.
          • Use.json()to convert the response body to JSON format.
        3. Error Handling: Use.catch()to handle any errors that may occur during the fetch operation.

        Auto-generated based on listed sources. May contain inaccuracies.

        Was this helpful?
      2. Aug 20, 2025Using theFetchAPI TheFetchAPI provides aJavaScriptinterface for making HTTP requests and processing the responses.Fetchis the modern replacement for XMLHttpRequest: unlike XMLHttpRequest, which uses callbacks,Fetchis promise-based and is integrated with features of the modern web such as service workers and Cross-Origin Resource Sharing (CORS). With theFetchAPI, you make a request ...
      3. 1 day agoTheFetchAPI has emerged as the standard for handling HTTP requests inJavaScript, replacing the older XMLHttpRequest with a more modern, promise-based interface. Unlike XMLHttpRequest,Fetchis designed to work seamlessly with promises and async/await, making asynchronous code easier to read and maintain.
      4. TheFetchAPI is a modernJavaScriptinterface for making network requests, primarily designed to replace the older XMLHttpRequest. It provides a more straightforward and flexible way to handle HTTP requests, making it easier for developers to work with APIs andfetchdata from servers.
      5. Oct 15, 2025Practical examples of usingFetchinreal-world projects Understanding theFetchAPI Before usingFetch, it's important to understand what it does. TheFetchAPI is a built-inJavaScriptfeature that lets you make asynchronous HTTP requests, meaning your code canfetchdata from a server in the background without freezing the rest of the page.
      6. Feb 6, 2025With an understanding of the syntax for using theFetchAPI, you can now move on to usingfetch() on a real API. Step 2 — UsingFetchtoget Data from an API The following code samples will be based on the JSONPlaceholder API. Using the API, you will get ten users and display them on the page usingJavaScript.
      7. The Modern JavaScript Tutorial

        http://test.test› fetch

        Otherwise, if afetchfails, or the response has non-200 status, we just return null in the resulting array. Please note: .then call is attached directly tofetch, so that when we have the response, it doesn't wait for other fetches, but starts to read .json () immediately.
      1. JavaScript

        This kind of functionality was previously achieved using XMLHttpRequest. Fetch provides a better alternative that can be easily used by other technologies such as Service Workers. Fetch also provides a single logical place to define other HTTP-related concepts such as CORS and extensions to HTTP.

        More at MDN Web Docs
        Source:MDN Web Docs
        Was this helpful?
      Custom date rangeX
      diff --git a/ecosia/features.go b/ecosia/features.go new file mode 100644 index 0000000..454981a --- /dev/null +++ b/ecosia/features.go @@ -0,0 +1,35 @@ +package ecosia + +import ( + "github.com/PuerkitoBio/goquery" + "github.com/go-rod/rod" + "github.com/karust/openserp/core" +) + +func extractEcosiaFeatures(doc *goquery.Document) []core.SerpFeature { + features := core.ExtractSerpFeaturesBySelectors(doc, []core.SerpFeatureSelector{ + { + Type: core.ResultTypeAnswerBox, + Title: "Answer", + Container: []string{"[data-test-id='instant-answer']", "[data-test-id='answer-box']", ".instant-answer"}, + TitleSelector: []string{"h2", "[data-test-id='instant-answer-title']"}, + TextSelector: []string{"[data-test-id='instant-answer-description']", "[data-test-id='answer-box-description']", ".instant-answer__description", "p"}, + LinkSelector: []string{"a[href^='http']"}, + Position: 1, + Confidence: 0.8, + }, + { + Type: core.ResultTypeRelatedSearches, + Title: "Related searches", + Container: []string{"[data-test-id='web-related-queries']", ".related-queries__bottom", "[data-test-id='related-searches']"}, + ItemSelector: []string{"a"}, + LinkSelector: []string{"a[href^='http']", "a"}, + Confidence: 0.8, + }, + }) + return core.DeduplicateSerpFeatures(features) +} + +func extractEcosiaFeaturesFromPage(page *rod.Page) []core.SerpFeature { + return core.FeaturesFromPage(page, extractEcosiaFeatures) +} diff --git a/ecosia/parse_html.go b/ecosia/parse_html.go index a363bc2..1f77f57 100644 --- a/ecosia/parse_html.go +++ b/ecosia/parse_html.go @@ -42,7 +42,7 @@ func parseEcosiaDocument(doc *goquery.Document) []core.SearchResult { }) setSeparatedAdAbsoluteRanks(results, 0) - return core.DeduplicateResults(results) + return core.AttachFeaturesToFirstResult(core.DeduplicateResults(results), extractEcosiaFeatures(doc)) } func parseEcosiaItem(item *goquery.Selection, rank int, ad bool) (core.SearchResult, bool) { diff --git a/ecosia/search.go b/ecosia/search.go index e9fcca8..9885b4f 100644 --- a/ecosia/search.go +++ b/ecosia/search.go @@ -141,10 +141,12 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S // nextRank counts up across pages for organic results; nextAdRank counts // up within sponsored results so ad rank stays separate from SEO rank. all := []core.SearchResult{} + var pageFeatures []core.SerpFeature pageNum, nextRank, err := startPage(query.Start) if err != nil { return nil, err } + firstPage := pageNum nextAdRank := 1 // fetchPage loads one SERP page and appends parsed results. // Returns (done, error): done=true ends the outer loop without error. @@ -195,6 +197,9 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S nextAdRank++ } } + if query.Features && pageNum == firstPage { + pageFeatures = extractEcosiaFeaturesFromPage(page) + } return false, nil } @@ -221,7 +226,7 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S deduped = core.LimitOrganicResults(deduped, query.Limit) } e.logger.Info("Search completed: %d results", len(deduped)) - return deduped, nil + return core.AttachFeaturesToFirstResult(deduped, pageFeatures), nil } // parseImageResult extracts a single image card into a SearchResult, diff --git a/ecosia/serp_features_test.go b/ecosia/serp_features_test.go new file mode 100644 index 0000000..f58a154 --- /dev/null +++ b/ecosia/serp_features_test.go @@ -0,0 +1,93 @@ +package ecosia + +import ( + "bytes" + "os" + "testing" + + "github.com/karust/openserp/core" +) + +func TestParseHTMLFixtureExtractsRealFeatures(t *testing.T) { + t.Parallel() + f, err := os.Open("testdata/search_results.html") + if err != nil { + t.Fatalf("open fixture: %v", err) + } + defer f.Close() + + results, err := ParseHTML(f) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + assertFeatureType(t, results, core.ResultTypeRelatedSearches) +} + +func TestParseHTMLExtractsSerpFeatures(t *testing.T) { + t.Parallel() + + html := ` +
      +

      Answer

      +

      Ecosia answer text.

      +
      +
      + trees +
      +
      + +
      ` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + assertFeatureType(t, results, core.ResultTypeAnswerBox) + assertFeatureType(t, results, core.ResultTypeRelatedSearches) +} + +func TestParseHTMLOrganicOnlyHasNoSerpFeatures(t *testing.T) { + t.Parallel() + + html := ` +
      + +
      ` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + assertNoFeatures(t, results) +} + +func assertFeatureType(t *testing.T, results []core.SearchResult, want core.ResultType) { + t.Helper() + for _, result := range results { + for _, feature := range result.Features { + if feature.Type == want { + return + } + } + } + t.Fatalf("expected feature type %q in %#v", want, results) +} + +func assertNoFeatures(t *testing.T, results []core.SearchResult) { + t.Helper() + for _, result := range results { + if len(result.Features) > 0 { + t.Fatalf("expected no features, got %#v", result.Features) + } + } +} diff --git a/ecosia/testdata/search_results.html b/ecosia/testdata/search_results.html index 1ed9a0f..21b7394 100644 --- a/ecosia/testdata/search_results.html +++ b/ecosia/testdata/search_results.html @@ -1,56 +1 @@ - - - - - - - - - -llm tool use - Ecosia - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      Skip to main content

      Search

      Large language model

      Type of machine learning model

      A large language model (LLM) is a neural network trained on a vast amount of text for natural language processing tasks, especially language generation. LLMs can generate, summarize, translate and parse text in many contexts, and are a foundational technology behind modern chatbots. Biased or inaccurate training data c… Wikipedia


      - - - - - - +llm tool use - Ecosia
      Skip to main content

      Search

      Large language model

      Type of machine learning model

      A large language model (LLM) is a neural network trained on a vast amount of text for natural language processing tasks, especially language generation. LLMs can generate, summarize, translate and parse text in many contexts, and are a foundational technology behind modern chatbots. Biased or inaccurate training data c…Wikipedia


      diff --git a/google/features.go b/google/features.go new file mode 100644 index 0000000..c755302 --- /dev/null +++ b/google/features.go @@ -0,0 +1,112 @@ +package google + +import ( + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/go-rod/rod" + "github.com/karust/openserp/core" +) + +func extractGoogleFeatures(doc *goquery.Document) []core.SerpFeature { + features := core.ExtractSerpFeaturesBySelectors(doc, []core.SerpFeatureSelector{ + { + // Google's AI Overview prose renders into the main-col streaming + // container; the text is fragmented across span[data-subtree]// + // nodes, so the text selector takes the container's whole + // collapsed text to reconstruct the answer. The older data-mcpr/ + // data-rsoextract containers are kept as fallbacks for other layouts. + Type: core.ResultTypeAISummary, + Title: "AI Overview", + Container: []string{"div[data-container-id='main-col'][data-sfc-root='c']", "div[data-mcpr]", "div[aria-label*='AI Overview']", "div[data-rsoextract]"}, + TitleSelector: []string{"[role='heading']", "h2", "h3"}, + TextSelector: []string{"div[data-streaming-container]", "div[data-sncf='1']", "[data-attrid*='description']"}, + LinkSelector: []string{"a[href^='http']"}, + Position: 1, + Confidence: 0.75, + // Emit a single AI Overview: the main-col container yields the prose; + // the data-mcpr fallback otherwise also matches and sweeps embedded CSS. + SingleMatch: true, + }, + { + Type: core.ResultTypePeopleAlsoAsk, + Title: "People also ask", + // div[data-initq] is the single outer PAA module. jsname='yEVEwb' + // also matches inner expandable sub-panels (one per question), so + // using it as a container fragments the module into N features; + // keep it only as a fallback when data-initq is absent. + Container: []string{"div[data-initq]", "div[jsname='yEVEwb']"}, + ItemSelector: []string{"div.related-question-pair[data-q]", "div[data-q]"}, + LinkSelector: []string{"a[href^='http']"}, + Position: 1, + Confidence: 0.8, + SingleMatch: true, + }, + { + Type: core.ResultTypeRelatedSearches, + Title: "Related searches", + // Scope to the dedicated related-search footer modules only. The + // main #rso results container also carries data-async-context, so a + // query:-prefix match there yields navigation chips, not searches. + Container: []string{"div[jsname='yEVEwb'][role='navigation']", "div[data-abe='1']"}, + ItemSelector: []string{"a[href*='/search?']"}, + LinkSelector: []string{"a[href*='/search?']"}, + Confidence: 0.6, + }, + }) + features = filterGooglePlaceholders(features) + return core.DeduplicateSerpFeatures(features) +} + +// googlePlaceholderText flags AI-overview text that Google renders when no +// summary exists ("An AI Overview is not available...") and bare expander +// labels ("Show more") so we don't emit empty/false-positive features. +var googlePlaceholderText = []string{ + "ai overview is not available", + "an ai overview is not available for this search", +} + +func filterGooglePlaceholders(features []core.SerpFeature) []core.SerpFeature { + kept := features[:0] + for _, feature := range features { + if feature.Type == core.ResultTypeAISummary && isGooglePlaceholder(feature) { + continue + } + kept = append(kept, feature) + } + return kept +} + +func isGooglePlaceholder(feature core.SerpFeature) bool { + text := strings.ToLower(strings.TrimSpace(feature.Text)) + if text == "" || text == "show more" || text == "show less" { + return true + } + // A fallback container can wrap an inline