mirror of
https://github.com/karust/openserp.git
synced 2026-08-16 05:16:02 +08:00
fix api response format issues
This commit is contained in:
@@ -287,10 +287,19 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
|
||||
continue
|
||||
}
|
||||
res := core.SearchResult{
|
||||
Rank: (searchPage * 30) + (i + 1),
|
||||
URL: img.URL[0].Original,
|
||||
Title: img.Title,
|
||||
Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright),
|
||||
Rank: (searchPage * 30) + (i + 1),
|
||||
URL: img.URL[0].Original,
|
||||
Title: img.Title,
|
||||
Description: fmt.Sprintf(
|
||||
"Source Page: %s, thumb_url:%s, %dx%d, date:%v, type:%v, copyright:%v",
|
||||
img.URL[0].SourcePage,
|
||||
img.ThumbURL,
|
||||
img.Width,
|
||||
img.Height,
|
||||
img.PictureDate,
|
||||
img.Type,
|
||||
img.IsCopyright,
|
||||
),
|
||||
Ad: func() bool {
|
||||
if img.AdType != "0" {
|
||||
return true
|
||||
|
||||
@@ -382,12 +382,14 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
|
||||
// Extract information from the parsed data
|
||||
srchRes.Title = imgData.T
|
||||
srchRes.URL = imgData.IMGURL
|
||||
srchRes.Description = imgData.Desc
|
||||
|
||||
// Add dimensions to description if available
|
||||
if imgData.W > 0 && imgData.H > 0 {
|
||||
srchRes.Description += fmt.Sprintf(" (%dx%d)", imgData.W, imgData.H)
|
||||
}
|
||||
srchRes.Description = fmt.Sprintf(
|
||||
"%s Source Page: %s, thumb_url:%s, %dx%d",
|
||||
imgData.Desc,
|
||||
imgData.PURL,
|
||||
imgData.TURL,
|
||||
imgData.W,
|
||||
imgData.H,
|
||||
)
|
||||
|
||||
// Get the page URL
|
||||
if imgData.MURL != "" {
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
version = "0.6.7"
|
||||
version = "0.7.0"
|
||||
defaultConfigFilename = "config"
|
||||
envPrefix = "OPENSERP"
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"sort"
|
||||
)
|
||||
@@ -17,12 +17,12 @@ func BuildClusters(results []Result, enginesQueried int) []Cluster {
|
||||
}
|
||||
|
||||
type clusterAccum struct {
|
||||
occurrences []ClusterOccurrence
|
||||
scoreSum float64
|
||||
bestRank int
|
||||
title string
|
||||
occurrences []ClusterOccurrence
|
||||
scoreSum float64
|
||||
bestRank int
|
||||
title string
|
||||
canonicalURL string
|
||||
domain string
|
||||
domain string
|
||||
}
|
||||
|
||||
// Group by result ID (which is derived from normalized URL + engine).
|
||||
@@ -95,8 +95,8 @@ func BuildClusters(results []Result, enginesQueried int) []Cluster {
|
||||
}
|
||||
|
||||
func buildClusterID(normalizedURL string) string {
|
||||
h := sha256.Sum256([]byte(normalizedURL))
|
||||
return "c_" + hex.EncodeToString(h[:12])
|
||||
h := md5.Sum([]byte(normalizedURL))
|
||||
return "c_" + hex.EncodeToString(h[:responseIDBytes])
|
||||
}
|
||||
|
||||
func roundScore(s float64) float64 {
|
||||
|
||||
@@ -1,7 +1,37 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/net/publicsuffix"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
//go:embed enrichment_domains.yaml
|
||||
var defaultEnrichmentDomainsYAML []byte
|
||||
|
||||
type enrichmentDomainsFile struct {
|
||||
DomainSourceHints map[string]string `yaml:"domain_source_hints"`
|
||||
NewsDomains []string `yaml:"news_domains"`
|
||||
ForumDomains []string `yaml:"forum_domains"`
|
||||
MarketplaceDomains []string `yaml:"marketplace_domains"`
|
||||
SocialDomains []string `yaml:"social_domains"`
|
||||
}
|
||||
|
||||
type enrichmentDomainsConfig struct {
|
||||
DomainSourceHints map[string]string
|
||||
NewsDomains map[string]bool
|
||||
ForumDomains map[string]bool
|
||||
MarketplaceDomains map[string]bool
|
||||
SocialDomains map[string]bool
|
||||
}
|
||||
|
||||
var (
|
||||
enrichmentDomainsOnce sync.Once
|
||||
enrichmentDomains enrichmentDomainsConfig
|
||||
)
|
||||
|
||||
// EnrichDomainInfo derives TLD/category signals from a bare hostname.
|
||||
@@ -10,7 +40,9 @@ func EnrichDomainInfo(domain string) *DomainInfo {
|
||||
return nil
|
||||
}
|
||||
|
||||
domain = normalizeDomain(domain)
|
||||
tld, sld := splitDomain(domain)
|
||||
cfg := loadEnrichmentDomains()
|
||||
|
||||
info := &DomainInfo{
|
||||
TLD: tld,
|
||||
@@ -18,16 +50,16 @@ func EnrichDomainInfo(domain string) *DomainInfo {
|
||||
IsGov: isGovTLD(domain, tld),
|
||||
IsEdu: isEduTLD(domain, tld),
|
||||
IsMil: isMilTLD(tld),
|
||||
IsNews: newsDomains[domain],
|
||||
IsForum: forumDomains[domain],
|
||||
IsMarketplace: marketplaceDomains[domain],
|
||||
IsSocial: socialDomains[domain],
|
||||
IsNews: cfg.NewsDomains[domain],
|
||||
IsForum: cfg.ForumDomains[domain],
|
||||
IsMarketplace: cfg.MarketplaceDomains[domain],
|
||||
IsSocial: cfg.SocialDomains[domain],
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// ClassifyURL returns a rough content-type and source hint derived from the
|
||||
// URL path alone — no network calls.
|
||||
// URL path alone; no network calls.
|
||||
func ClassifyURL(rawURL, domain string) *Classification {
|
||||
if rawURL == "" && domain == "" {
|
||||
return nil
|
||||
@@ -42,46 +74,42 @@ func ClassifyURL(rawURL, domain string) *Classification {
|
||||
}
|
||||
}
|
||||
|
||||
// splitDomain returns (tld, sld) for a bare hostname.
|
||||
// Uses a simple heuristic: last label is TLD, second-to-last is SLD.
|
||||
// For compound TLDs like co.uk the full suffix is returned as TLD.
|
||||
// splitDomain returns (public suffix, registrable domain label).
|
||||
func splitDomain(domain string) (tld, sld string) {
|
||||
parts := strings.Split(domain, ".")
|
||||
if len(parts) < 2 {
|
||||
return domain, ""
|
||||
domain = normalizeDomain(domain)
|
||||
if domain == "" {
|
||||
return "", ""
|
||||
}
|
||||
// Known compound TLDs.
|
||||
compoundTLDs := map[string]bool{
|
||||
"co.uk": true, "co.jp": true, "co.in": true, "co.nz": true,
|
||||
"co.za": true, "com.au": true, "com.br": true, "com.mx": true,
|
||||
"gov.uk": true, "ac.uk": true, "edu.au": true, "gov.au": true,
|
||||
"or.jp": true, "ne.jp": true,
|
||||
}
|
||||
if len(parts) >= 3 {
|
||||
compound := parts[len(parts)-2] + "." + parts[len(parts)-1]
|
||||
if compoundTLDs[compound] {
|
||||
return compound, parts[len(parts)-3]
|
||||
|
||||
suffix, icann := publicsuffix.PublicSuffix(domain)
|
||||
if suffix == "" || !icann {
|
||||
parts := strings.Split(domain, ".")
|
||||
if len(parts) < 2 {
|
||||
return domain, ""
|
||||
}
|
||||
return parts[len(parts)-1], parts[len(parts)-2]
|
||||
}
|
||||
return parts[len(parts)-1], parts[len(parts)-2]
|
||||
|
||||
registrable, err := publicsuffix.EffectiveTLDPlusOne(domain)
|
||||
if err != nil {
|
||||
parts := strings.Split(domain, ".")
|
||||
if len(parts) < 2 {
|
||||
return suffix, ""
|
||||
}
|
||||
return suffix, parts[len(parts)-2]
|
||||
}
|
||||
|
||||
sld = strings.TrimSuffix(registrable, "."+suffix)
|
||||
return suffix, sld
|
||||
}
|
||||
|
||||
func isGovTLD(domain, tld string) bool {
|
||||
if tld == "gov" || tld == "gov.uk" || tld == "gov.au" {
|
||||
return true
|
||||
}
|
||||
return strings.HasSuffix(domain, ".gov") ||
|
||||
strings.HasSuffix(domain, ".gov.uk") ||
|
||||
strings.HasSuffix(domain, ".gov.au")
|
||||
return tld == "gov" || strings.HasSuffix(tld, ".gov") || strings.HasSuffix(domain, ".gov")
|
||||
}
|
||||
|
||||
func isEduTLD(domain, tld string) bool {
|
||||
if tld == "edu" || tld == "ac.uk" || tld == "edu.au" {
|
||||
return true
|
||||
}
|
||||
return strings.HasSuffix(domain, ".edu") ||
|
||||
strings.HasSuffix(domain, ".ac.uk") ||
|
||||
strings.HasSuffix(domain, ".edu.au")
|
||||
return tld == "edu" || strings.HasSuffix(tld, ".edu") || tld == "ac.uk" ||
|
||||
strings.HasSuffix(domain, ".edu") || strings.HasSuffix(domain, ".ac.uk")
|
||||
}
|
||||
|
||||
func isMilTLD(tld string) bool {
|
||||
@@ -111,81 +139,65 @@ func classifyContentType(rawURL string) string {
|
||||
}
|
||||
|
||||
func classifySourceHint(domain string) string {
|
||||
if hint, ok := domainSourceHints[domain]; ok {
|
||||
cfg := loadEnrichmentDomains()
|
||||
if hint, ok := cfg.DomainSourceHints[normalizeDomain(domain)]; ok {
|
||||
return hint
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// domainSourceHints maps known domains to a descriptive source hint.
|
||||
var domainSourceHints = map[string]string{
|
||||
"wikipedia.org": "encyclopedia",
|
||||
"en.wikipedia.org": "encyclopedia",
|
||||
"github.com": "code_repository",
|
||||
"gitlab.com": "code_repository",
|
||||
"stackoverflow.com": "qa_forum",
|
||||
"stackexchange.com": "qa_forum",
|
||||
"reddit.com": "social_forum",
|
||||
"nytimes.com": "news",
|
||||
"bbc.com": "news",
|
||||
"bbc.co.uk": "news",
|
||||
"reuters.com": "news",
|
||||
"theguardian.com": "news",
|
||||
"washingtonpost.com": "news",
|
||||
"forbes.com": "news",
|
||||
"techcrunch.com": "news",
|
||||
"medium.com": "blog_platform",
|
||||
"scholar.google.com": "academic",
|
||||
"arxiv.org": "academic",
|
||||
"pubmed.ncbi.nlm.nih.gov": "academic",
|
||||
"amazon.com": "marketplace",
|
||||
"ebay.com": "marketplace",
|
||||
"etsy.com": "marketplace",
|
||||
"docs.google.com": "document",
|
||||
"youtube.com": "video_platform",
|
||||
"vimeo.com": "video_platform",
|
||||
"twitter.com": "social_media",
|
||||
"x.com": "social_media",
|
||||
"facebook.com": "social_media",
|
||||
"linkedin.com": "professional_network",
|
||||
"instagram.com": "social_media",
|
||||
func loadEnrichmentDomains() enrichmentDomainsConfig {
|
||||
enrichmentDomainsOnce.Do(func() {
|
||||
enrichmentDomains = parseEnrichmentDomains(defaultEnrichmentDomainsYAML)
|
||||
if path := strings.TrimSpace(os.Getenv("OPENSERP_ENRICHMENT_DOMAINS_FILE")); path != "" {
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
enrichmentDomains = parseEnrichmentDomains(data)
|
||||
}
|
||||
}
|
||||
})
|
||||
return enrichmentDomains
|
||||
}
|
||||
|
||||
// newsDomains is the set of known news publisher domains.
|
||||
var newsDomains = map[string]bool{
|
||||
"nytimes.com": true, "bbc.com": true, "bbc.co.uk": true,
|
||||
"reuters.com": true, "apnews.com": true, "theguardian.com": true,
|
||||
"washingtonpost.com": true, "forbes.com": true, "techcrunch.com": true,
|
||||
"wired.com": true, "bloomberg.com": true, "cnn.com": true,
|
||||
"nbcnews.com": true, "cbsnews.com": true, "abcnews.go.com": true,
|
||||
"foxnews.com": true, "theverge.com": true, "engadget.com": true,
|
||||
"arstechnica.com": true, "zdnet.com": true, "venturebeat.com": true,
|
||||
"axios.com": true, "politico.com": true, "theatlantic.com": true,
|
||||
"economist.com": true, "ft.com": true, "wsj.com": true,
|
||||
"usatoday.com": true, "latimes.com": true, "nypost.com": true,
|
||||
func parseEnrichmentDomains(data []byte) enrichmentDomainsConfig {
|
||||
cfg := enrichmentDomainsConfig{
|
||||
DomainSourceHints: map[string]string{},
|
||||
NewsDomains: map[string]bool{},
|
||||
ForumDomains: map[string]bool{},
|
||||
MarketplaceDomains: map[string]bool{},
|
||||
SocialDomains: map[string]bool{},
|
||||
}
|
||||
|
||||
var file enrichmentDomainsFile
|
||||
if err := yaml.Unmarshal(data, &file); err != nil {
|
||||
return cfg
|
||||
}
|
||||
|
||||
for domain, hint := range file.DomainSourceHints {
|
||||
domain = normalizeDomain(domain)
|
||||
hint = strings.TrimSpace(hint)
|
||||
if domain != "" && hint != "" {
|
||||
cfg.DomainSourceHints[domain] = hint
|
||||
}
|
||||
}
|
||||
fillDomainSet(cfg.NewsDomains, file.NewsDomains)
|
||||
fillDomainSet(cfg.ForumDomains, file.ForumDomains)
|
||||
fillDomainSet(cfg.MarketplaceDomains, file.MarketplaceDomains)
|
||||
fillDomainSet(cfg.SocialDomains, file.SocialDomains)
|
||||
|
||||
return cfg
|
||||
}
|
||||
|
||||
// forumDomains is the set of known community/forum domains.
|
||||
var forumDomains = map[string]bool{
|
||||
"reddit.com": true, "news.ycombinator.com": true,
|
||||
"stackoverflow.com": true, "stackexchange.com": true,
|
||||
"superuser.com": true, "serverfault.com": true,
|
||||
"quora.com": true, "discourse.org": true,
|
||||
"boards.4chan.org": true, "hackernews.com": true,
|
||||
func fillDomainSet(dst map[string]bool, domains []string) {
|
||||
for _, domain := range domains {
|
||||
domain = normalizeDomain(domain)
|
||||
if domain != "" {
|
||||
dst[domain] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// marketplaceDomains is the set of known e-commerce/marketplace domains.
|
||||
var marketplaceDomains = map[string]bool{
|
||||
"amazon.com": true, "amazon.co.uk": true, "amazon.de": true,
|
||||
"ebay.com": true, "etsy.com": true, "walmart.com": true,
|
||||
"target.com": true, "bestbuy.com": true, "newegg.com": true,
|
||||
"aliexpress.com": true, "alibaba.com": true, "shopify.com": true,
|
||||
}
|
||||
|
||||
// socialDomains is the set of known social media platform domains.
|
||||
var socialDomains = map[string]bool{
|
||||
"twitter.com": true, "x.com": true, "facebook.com": true,
|
||||
"instagram.com": true, "tiktok.com": true, "snapchat.com": true,
|
||||
"pinterest.com": true, "tumblr.com": true, "linkedin.com": true,
|
||||
"youtube.com": true, "twitch.tv": true, "discord.com": true,
|
||||
func normalizeDomain(domain string) string {
|
||||
domain = strings.ToLower(strings.TrimSpace(domain))
|
||||
domain = strings.TrimPrefix(domain, "www.")
|
||||
return strings.TrimSuffix(domain, ".")
|
||||
}
|
||||
|
||||
103
core/enrichment_domains.yaml
Normal file
103
core/enrichment_domains.yaml
Normal file
@@ -0,0 +1,103 @@
|
||||
domain_source_hints:
|
||||
wikipedia.org: encyclopedia
|
||||
en.wikipedia.org: encyclopedia
|
||||
github.com: code_repository
|
||||
gitlab.com: code_repository
|
||||
stackoverflow.com: qa_forum
|
||||
stackexchange.com: qa_forum
|
||||
reddit.com: social_forum
|
||||
nytimes.com: news
|
||||
bbc.com: news
|
||||
bbc.co.uk: news
|
||||
reuters.com: news
|
||||
theguardian.com: news
|
||||
washingtonpost.com: news
|
||||
forbes.com: news
|
||||
techcrunch.com: news
|
||||
medium.com: blog_platform
|
||||
scholar.google.com: academic
|
||||
arxiv.org: academic
|
||||
pubmed.ncbi.nlm.nih.gov: academic
|
||||
amazon.com: marketplace
|
||||
ebay.com: marketplace
|
||||
etsy.com: marketplace
|
||||
docs.google.com: document
|
||||
youtube.com: video_platform
|
||||
vimeo.com: video_platform
|
||||
twitter.com: social_media
|
||||
x.com: social_media
|
||||
facebook.com: social_media
|
||||
linkedin.com: professional_network
|
||||
instagram.com: social_media
|
||||
|
||||
news_domains:
|
||||
- nytimes.com
|
||||
- bbc.com
|
||||
- bbc.co.uk
|
||||
- reuters.com
|
||||
- apnews.com
|
||||
- theguardian.com
|
||||
- washingtonpost.com
|
||||
- forbes.com
|
||||
- techcrunch.com
|
||||
- wired.com
|
||||
- bloomberg.com
|
||||
- cnn.com
|
||||
- nbcnews.com
|
||||
- cbsnews.com
|
||||
- abcnews.go.com
|
||||
- foxnews.com
|
||||
- theverge.com
|
||||
- engadget.com
|
||||
- arstechnica.com
|
||||
- zdnet.com
|
||||
- venturebeat.com
|
||||
- axios.com
|
||||
- politico.com
|
||||
- theatlantic.com
|
||||
- economist.com
|
||||
- ft.com
|
||||
- wsj.com
|
||||
- usatoday.com
|
||||
- latimes.com
|
||||
- nypost.com
|
||||
|
||||
forum_domains:
|
||||
- reddit.com
|
||||
- news.ycombinator.com
|
||||
- stackoverflow.com
|
||||
- stackexchange.com
|
||||
- superuser.com
|
||||
- serverfault.com
|
||||
- quora.com
|
||||
- discourse.org
|
||||
- boards.4chan.org
|
||||
- hackernews.com
|
||||
|
||||
marketplace_domains:
|
||||
- amazon.com
|
||||
- amazon.co.uk
|
||||
- amazon.de
|
||||
- ebay.com
|
||||
- etsy.com
|
||||
- walmart.com
|
||||
- target.com
|
||||
- bestbuy.com
|
||||
- newegg.com
|
||||
- aliexpress.com
|
||||
- alibaba.com
|
||||
- shopify.com
|
||||
|
||||
social_domains:
|
||||
- twitter.com
|
||||
- x.com
|
||||
- facebook.com
|
||||
- instagram.com
|
||||
- tiktok.com
|
||||
- snapchat.com
|
||||
- pinterest.com
|
||||
- tumblr.com
|
||||
- linkedin.com
|
||||
- youtube.com
|
||||
- twitch.tv
|
||||
- discord.com
|
||||
@@ -10,10 +10,7 @@ import (
|
||||
func RenderMarkdown(env *Envelope) []byte {
|
||||
var b strings.Builder
|
||||
|
||||
enginesStr := strings.Join(env.Meta.EnginesResponded, ", ")
|
||||
if enginesStr == "" {
|
||||
enginesStr = strings.Join(env.Query.EnginesRequested, ", ")
|
||||
}
|
||||
enginesStr := strings.Join(env.Query.EnginesRequested, ", ")
|
||||
fmt.Fprintf(&b, "# Search results for %q\n\n", env.Query.Text)
|
||||
fmt.Fprintf(&b, "**Query:** %s · **Engines:** %s · **Took:** %dms\n\n",
|
||||
env.Query.Text, enginesStr, env.Meta.TookMs)
|
||||
@@ -42,10 +39,7 @@ func RenderMarkdown(env *Envelope) []byte {
|
||||
func RenderMarkdownImage(env *ImageEnvelope) []byte {
|
||||
var b strings.Builder
|
||||
|
||||
enginesStr := strings.Join(env.Meta.EnginesResponded, ", ")
|
||||
if enginesStr == "" {
|
||||
enginesStr = strings.Join(env.Query.EnginesRequested, ", ")
|
||||
}
|
||||
enginesStr := strings.Join(env.Query.EnginesRequested, ", ")
|
||||
fmt.Fprintf(&b, "# Image results for %q\n\n", env.Query.Text)
|
||||
fmt.Fprintf(&b, "**Query:** %s · **Engines:** %s · **Took:** %dms\n\n",
|
||||
env.Query.Text, enginesStr, env.Meta.TookMs)
|
||||
|
||||
@@ -12,7 +12,7 @@ func RenderText(env *Envelope) []byte {
|
||||
var b strings.Builder
|
||||
|
||||
fmt.Fprintf(&b, "Search: %s\n", env.Query.Text)
|
||||
enginesStr := strings.Join(env.Meta.EnginesResponded, ", ")
|
||||
enginesStr := strings.Join(env.Query.EnginesRequested, ", ")
|
||||
if enginesStr != "" {
|
||||
fmt.Fprintf(&b, "Engines: %s\n", enginesStr)
|
||||
}
|
||||
|
||||
@@ -6,19 +6,16 @@ import "time"
|
||||
type QueryEcho struct {
|
||||
Text string `json:"text"`
|
||||
Lang string `json:"lang,omitempty"`
|
||||
Location *string `json:"location"`
|
||||
Device string `json:"device"`
|
||||
EnginesRequested []string `json:"engines_requested"`
|
||||
}
|
||||
|
||||
// ResponseMeta carries request-level metadata for observability and debugging.
|
||||
type ResponseMeta struct {
|
||||
RequestID string `json:"request_id"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
TookMs int64 `json:"took_ms"`
|
||||
EnginesResponded []string `json:"engines_responded"`
|
||||
EnginesFailed []string `json:"engines_failed"`
|
||||
Version string `json:"version"`
|
||||
RequestID string `json:"request_id"`
|
||||
RequestedAt string `json:"requested_at"`
|
||||
TookMs int64 `json:"took_ms"`
|
||||
EnginesFailed []string `json:"engines_failed"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// Pagination carries cursor information for client-side loop termination.
|
||||
@@ -60,10 +57,10 @@ type ClusterOccurrence struct {
|
||||
|
||||
// ImageEnvelope is the top-level v1 response wrapper for image search endpoints.
|
||||
type ImageEnvelope struct {
|
||||
Query QueryEcho `json:"query"`
|
||||
Meta ResponseMeta `json:"meta"`
|
||||
Query QueryEcho `json:"query"`
|
||||
Meta ResponseMeta `json:"meta"`
|
||||
Results []ImageResult `json:"results"`
|
||||
Pagination Pagination `json:"pagination"`
|
||||
Pagination Pagination `json:"pagination"`
|
||||
}
|
||||
|
||||
const apiVersion = "1.0"
|
||||
@@ -75,16 +72,13 @@ func NewEnvelope(q Query, requestID string, startedAt time.Time, engines []strin
|
||||
Query: QueryEcho{
|
||||
Text: q.Text,
|
||||
Lang: q.LangCode,
|
||||
Location: nil,
|
||||
Device: "desktop",
|
||||
EnginesRequested: engines,
|
||||
},
|
||||
Meta: ResponseMeta{
|
||||
RequestID: requestID,
|
||||
Timestamp: startedAt.UTC().Format(time.RFC3339),
|
||||
EnginesResponded: []string{},
|
||||
EnginesFailed: []string{},
|
||||
Version: apiVersion,
|
||||
RequestID: requestID,
|
||||
RequestedAt: startedAt.UTC().Format(time.RFC3339),
|
||||
EnginesFailed: []string{},
|
||||
Version: apiVersion,
|
||||
},
|
||||
Results: []Result{},
|
||||
Pagination: Pagination{},
|
||||
@@ -97,16 +91,13 @@ func NewImageEnvelope(q Query, requestID string, startedAt time.Time, engines []
|
||||
Query: QueryEcho{
|
||||
Text: q.Text,
|
||||
Lang: q.LangCode,
|
||||
Location: nil,
|
||||
Device: "desktop",
|
||||
EnginesRequested: engines,
|
||||
},
|
||||
Meta: ResponseMeta{
|
||||
RequestID: requestID,
|
||||
Timestamp: startedAt.UTC().Format(time.RFC3339),
|
||||
EnginesResponded: []string{},
|
||||
EnginesFailed: []string{},
|
||||
Version: apiVersion,
|
||||
RequestID: requestID,
|
||||
RequestedAt: startedAt.UTC().Format(time.RFC3339),
|
||||
EnginesFailed: []string{},
|
||||
Version: apiVersion,
|
||||
},
|
||||
Results: []ImageResult{},
|
||||
Pagination: Pagination{},
|
||||
|
||||
@@ -1,14 +1,23 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const responseIDBytes = 8
|
||||
|
||||
var imageDimensionPatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)height:\s*(\d+),\s*width:\s*(\d+)`),
|
||||
regexp.MustCompile(`(?i)\b(\d+)x(\d+)\b`),
|
||||
}
|
||||
|
||||
// EnrichContext carries request-scoped values needed to enrich a raw result.
|
||||
type EnrichContext struct {
|
||||
Engine string
|
||||
@@ -38,12 +47,8 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
|
||||
if limit <= 0 {
|
||||
limit = 25
|
||||
}
|
||||
onPage := raw.Rank
|
||||
if onPage < 0 {
|
||||
onPage = 0
|
||||
}
|
||||
absolute := ctx.Query.Start + onPage
|
||||
page := ctx.Query.Start/limit + 1
|
||||
absolute, onPage := computeResultPosition(raw.Rank, ctx.Query.Start)
|
||||
|
||||
result := Result{
|
||||
ID: buildResultID(ctx.Engine, normalizedURL),
|
||||
@@ -62,10 +67,6 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
|
||||
OnPage: onPage,
|
||||
},
|
||||
Engine: ctx.Engine,
|
||||
Rich: nil,
|
||||
EngineMeta: map[string]any{
|
||||
"raw_rank": raw.Rank,
|
||||
},
|
||||
}
|
||||
|
||||
result.DomainInfo = EnrichDomainInfo(domain)
|
||||
@@ -77,12 +78,15 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
|
||||
// EnrichImageResult converts a raw engine result into the v1 ImageResult shape.
|
||||
func EnrichImageResult(raw SearchResult, ctx EnrichContext) ImageResult {
|
||||
imageURL := normalizeURL(raw.URL)
|
||||
// raw.Description may hold the page URL for image results in some engines.
|
||||
pageURL := raw.Description
|
||||
meta := parseImageDescription(raw.Description)
|
||||
|
||||
pageURL := meta.PageURL
|
||||
if pageURL == "" {
|
||||
pageURL = imageURL
|
||||
}
|
||||
pageURL = normalizeURL(pageURL)
|
||||
sourceDomain := extractDomain(pageURL)
|
||||
imageWidth, imageHeight := meta.Width, meta.Height
|
||||
|
||||
return ImageResult{
|
||||
ID: buildImageID(ctx.Engine, imageURL),
|
||||
@@ -90,27 +94,42 @@ func EnrichImageResult(raw SearchResult, ctx EnrichContext) ImageResult {
|
||||
Type: ResultTypeImage,
|
||||
Title: raw.Title,
|
||||
Image: ImageData{
|
||||
URL: imageURL,
|
||||
URL: imageURL,
|
||||
Thumbnail: meta.ThumbnailURL,
|
||||
Width: imageWidth,
|
||||
Height: imageHeight,
|
||||
},
|
||||
Source: ImageSource{
|
||||
PageURL: pageURL,
|
||||
Domain: sourceDomain,
|
||||
},
|
||||
Engine: ctx.Engine,
|
||||
EngineMeta: map[string]any{"raw_rank": raw.Rank},
|
||||
Engine: ctx.Engine,
|
||||
}
|
||||
}
|
||||
|
||||
// buildResultID returns a stable "r_<hex>" ID for web results.
|
||||
// buildResultID returns a stable "s_<hex>" ID for web results.
|
||||
func buildResultID(engine, normalizedURL string) string {
|
||||
h := sha256.Sum256([]byte(engine + "|" + normalizedURL))
|
||||
return "r_" + hex.EncodeToString(h[:12])
|
||||
return "s_" + shortMD5(engine+"|"+normalizedURL)
|
||||
}
|
||||
|
||||
// buildImageID returns a stable "i_<hex>" ID for image results.
|
||||
func buildImageID(engine, imageURL string) string {
|
||||
h := sha256.Sum256([]byte(engine + "|" + imageURL))
|
||||
return "i_" + hex.EncodeToString(h[:12])
|
||||
return "i_" + shortMD5(engine+"|"+imageURL)
|
||||
}
|
||||
|
||||
func shortMD5(value string) string {
|
||||
h := md5.Sum([]byte(value))
|
||||
return hex.EncodeToString(h[:responseIDBytes])
|
||||
}
|
||||
|
||||
func computeResultPosition(rank, start int) (absolute, onPage int) {
|
||||
if rank <= 0 {
|
||||
return 0, 0
|
||||
}
|
||||
if start > 0 && rank > start {
|
||||
return rank, rank - start
|
||||
}
|
||||
return start + rank, rank
|
||||
}
|
||||
|
||||
// normalizeURL lowercases scheme+host, strips trailing slash, and removes
|
||||
@@ -235,6 +254,70 @@ func decodeBase64String(s string) ([]byte, error) {
|
||||
return base64.StdEncoding.DecodeString(s)
|
||||
}
|
||||
|
||||
type imageDescriptionMeta struct {
|
||||
PageURL string
|
||||
ThumbnailURL string
|
||||
Width int
|
||||
Height int
|
||||
}
|
||||
|
||||
func parseImageDescription(desc string) imageDescriptionMeta {
|
||||
meta := imageDescriptionMeta{}
|
||||
trimmed := strings.TrimSpace(desc)
|
||||
if trimmed == "" {
|
||||
return meta
|
||||
}
|
||||
|
||||
lower := strings.ToLower(trimmed)
|
||||
if idx := strings.Index(lower, "source page:"); idx >= 0 {
|
||||
meta.PageURL = strings.TrimSpace(trimmed[idx+len("source page:"):])
|
||||
if comma := strings.Index(meta.PageURL, ","); comma >= 0 {
|
||||
meta.PageURL = strings.TrimSpace(meta.PageURL[:comma])
|
||||
}
|
||||
} else if strings.HasPrefix(lower, "source:") {
|
||||
meta.PageURL = strings.TrimSpace(trimmed[len("source:"):])
|
||||
}
|
||||
|
||||
if idx := strings.Index(lower, "thumb_url:"); idx >= 0 {
|
||||
meta.ThumbnailURL = strings.TrimSpace(trimmed[idx+len("thumb_url:"):])
|
||||
if comma := strings.Index(meta.ThumbnailURL, ","); comma >= 0 {
|
||||
meta.ThumbnailURL = strings.TrimSpace(meta.ThumbnailURL[:comma])
|
||||
}
|
||||
}
|
||||
|
||||
for _, pattern := range imageDimensionPatterns {
|
||||
match := pattern.FindStringSubmatch(trimmed)
|
||||
if len(match) != 3 {
|
||||
continue
|
||||
}
|
||||
first, firstErr := strconv.Atoi(match[1])
|
||||
second, secondErr := strconv.Atoi(match[2])
|
||||
if firstErr != nil || secondErr != nil {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(strings.ToLower(match[0]), "height") {
|
||||
meta.Height = first
|
||||
meta.Width = second
|
||||
} else {
|
||||
meta.Width = first
|
||||
meta.Height = second
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if !isHTTPURL(meta.PageURL) {
|
||||
meta.PageURL = ""
|
||||
}
|
||||
if !isHTTPURL(meta.ThumbnailURL) {
|
||||
meta.ThumbnailURL = ""
|
||||
}
|
||||
return meta
|
||||
}
|
||||
|
||||
func isHTTPURL(value string) bool {
|
||||
return strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://")
|
||||
}
|
||||
|
||||
// NormalizeURLForClustering returns a URL suitable for cross-engine grouping
|
||||
// (same as normalizeURL but exported for use in cluster building).
|
||||
func NormalizeURLForClustering(rawURL string) string {
|
||||
|
||||
@@ -4,17 +4,17 @@ package core
|
||||
type ResultType string
|
||||
|
||||
const (
|
||||
ResultTypeOrganic ResultType = "organic"
|
||||
ResultTypeAd ResultType = "ad"
|
||||
ResultTypeOrganic ResultType = "organic"
|
||||
ResultTypeAd ResultType = "ad"
|
||||
ResultTypeFeaturedSnippet ResultType = "featured_snippet"
|
||||
ResultTypeKnowledgePanel ResultType = "knowledge_panel"
|
||||
ResultTypePeopleAlsoAsk ResultType = "people_also_ask"
|
||||
ResultTypeVideo ResultType = "video"
|
||||
ResultTypeImage ResultType = "image"
|
||||
ResultTypeNews ResultType = "news"
|
||||
ResultTypeShopping ResultType = "shopping"
|
||||
ResultTypeLocal ResultType = "local"
|
||||
ResultTypeAnswerBox ResultType = "answer_box"
|
||||
ResultTypeKnowledgePanel ResultType = "knowledge_panel"
|
||||
ResultTypePeopleAlsoAsk ResultType = "people_also_ask"
|
||||
ResultTypeVideo ResultType = "video"
|
||||
ResultTypeImage ResultType = "image"
|
||||
ResultTypeNews ResultType = "news"
|
||||
ResultTypeShopping ResultType = "shopping"
|
||||
ResultTypeLocal ResultType = "local"
|
||||
ResultTypeAnswerBox ResultType = "answer_box"
|
||||
)
|
||||
|
||||
// Position describes where a result sits in the overall result stream.
|
||||
@@ -27,21 +27,6 @@ type Position struct {
|
||||
OnPage int `json:"on_page"`
|
||||
}
|
||||
|
||||
// RichData is a placeholder for future structured SERP features such as star
|
||||
// ratings, prices, or sitelinks. It is null in v1.0.
|
||||
type RichData struct {
|
||||
Stars *float64 `json:"stars,omitempty"`
|
||||
Reviews *int `json:"reviews,omitempty"`
|
||||
Price *string `json:"price,omitempty"`
|
||||
Sitelinks []RichSitelink `json:"sitelinks,omitempty"`
|
||||
}
|
||||
|
||||
// RichSitelink is one navigational sub-link shown below a result.
|
||||
type RichSitelink struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// DomainInfo carries TLD-derived category signals for a result domain.
|
||||
type DomainInfo struct {
|
||||
TLD string `json:"tld"`
|
||||
@@ -63,21 +48,19 @@ type Classification struct {
|
||||
|
||||
// Result is the v1 normalized result returned in every search response.
|
||||
type Result struct {
|
||||
ID string `json:"id"`
|
||||
Rank int `json:"rank"`
|
||||
Type ResultType `json:"type"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
DisplayURL string `json:"display_url"`
|
||||
Snippet string `json:"snippet"`
|
||||
Domain string `json:"domain"`
|
||||
Favicon string `json:"favicon"`
|
||||
IsAd bool `json:"is_ad"`
|
||||
Position Position `json:"position"`
|
||||
Engine string `json:"engine"`
|
||||
Rich *RichData `json:"rich"`
|
||||
EngineMeta map[string]any `json:"engine_meta"`
|
||||
DomainInfo *DomainInfo `json:"domain_info,omitempty"`
|
||||
ID string `json:"id"`
|
||||
Rank int `json:"rank"`
|
||||
Type ResultType `json:"type"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
DisplayURL string `json:"display_url"`
|
||||
Snippet string `json:"snippet"`
|
||||
Domain string `json:"domain"`
|
||||
Favicon string `json:"favicon"`
|
||||
IsAd bool `json:"is_ad"`
|
||||
Position Position `json:"position"`
|
||||
Engine string `json:"engine"`
|
||||
DomainInfo *DomainInfo `json:"domain_info,omitempty"`
|
||||
Classification *Classification `json:"classification,omitempty"`
|
||||
}
|
||||
|
||||
@@ -97,12 +80,11 @@ type ImageSource struct {
|
||||
|
||||
// ImageResult is the v1 shape for image search results.
|
||||
type ImageResult struct {
|
||||
ID string `json:"id"`
|
||||
Rank int `json:"rank"`
|
||||
Type ResultType `json:"type"`
|
||||
Title string `json:"title"`
|
||||
Image ImageData `json:"image"`
|
||||
Source ImageSource `json:"source"`
|
||||
Engine string `json:"engine"`
|
||||
EngineMeta map[string]any `json:"engine_meta"`
|
||||
ID string `json:"id"`
|
||||
Rank int `json:"rank"`
|
||||
Type ResultType `json:"type"`
|
||||
Title string `json:"title"`
|
||||
Image ImageData `json:"image"`
|
||||
Source ImageSource `json:"source"`
|
||||
Engine string `json:"engine"`
|
||||
}
|
||||
|
||||
100
core/server.go
100
core/server.go
@@ -206,14 +206,17 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm
|
||||
WithField("action", action).
|
||||
Debugf("Starting %s request for query: %s", action, q.Text)
|
||||
|
||||
if hit, err := s.tryServeCacheHit(
|
||||
c,
|
||||
cacheHitCandidate{
|
||||
key: BuildCacheKey(engine.Name(), action, q),
|
||||
logMessage: fmt.Sprintf("Cache hit for %s %s: %s", engine.Name(), action, q.Text),
|
||||
},
|
||||
); hit || err != nil {
|
||||
return err
|
||||
if format == "json" {
|
||||
if hit, err := s.tryServeCacheHit(
|
||||
c,
|
||||
startedAt,
|
||||
cacheHitCandidate{
|
||||
key: BuildCacheKey(engine.Name(), action, q),
|
||||
logMessage: fmt.Sprintf("Cache hit for %s %s: %s", engine.Name(), action, q.Text),
|
||||
},
|
||||
); hit || err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
engineNames := []string{engine.Name()}
|
||||
@@ -237,7 +240,9 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm
|
||||
}
|
||||
|
||||
env := NewImageEnvelope(q, requestID, startedAt, engineNames)
|
||||
env.Meta.EnginesResponded = []string{usedEngine}
|
||||
if usedEngine != "" && usedEngine != engine.Name() {
|
||||
env.Meta.EnginesFailed = []string{engine.Name()}
|
||||
}
|
||||
ectx := EnrichContext{Engine: usedEngine, Query: q}
|
||||
for _, r := range res {
|
||||
env.Results = append(env.Results, EnrichImageResult(r, ectx))
|
||||
@@ -276,7 +281,9 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm
|
||||
}
|
||||
|
||||
env := NewEnvelope(q, requestID, startedAt, engineNames)
|
||||
env.Meta.EnginesResponded = []string{usedEngine}
|
||||
if usedEngine != "" && usedEngine != engine.Name() {
|
||||
env.Meta.EnginesFailed = []string{engine.Name()}
|
||||
}
|
||||
ectx := EnrichContext{Engine: usedEngine, Query: q}
|
||||
for _, r := range res {
|
||||
env.Results = append(env.Results, EnrichResult(r, ectx))
|
||||
@@ -687,15 +694,16 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string, run func(contex
|
||||
logMessage: fmt.Sprintf("Cache hit for mega %s partial set: engines=%s query=%s", action, engineNamesJoined, q.Text),
|
||||
})
|
||||
}
|
||||
if hit, err := s.tryServeCacheHit(c, cacheHitCandidates...); hit || err != nil {
|
||||
return err
|
||||
if format == "json" {
|
||||
if hit, err := s.tryServeCacheHit(c, startedAt, cacheHitCandidates...); hit || err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
rawResults, enginesResponded, enginesFailed := run(requestCtx, q, enginesToUse)
|
||||
rawResults, _, enginesFailed := run(requestCtx, q, enginesToUse)
|
||||
|
||||
if action == "image" {
|
||||
env := NewImageEnvelope(q, requestID, startedAt, engineNames)
|
||||
env.Meta.EnginesResponded = enginesResponded
|
||||
env.Meta.EnginesFailed = enginesFailed
|
||||
for _, r := range rawResults {
|
||||
ectx := EnrichContext{Engine: r.Engine, Query: q}
|
||||
@@ -724,7 +732,6 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string, run func(contex
|
||||
// Deduplicate the flat results list by normalized URL (keep best-ranked occurrence).
|
||||
dedupedRaw := s.deduplicateMegaResults(rawResults)
|
||||
env := NewEnvelope(q, requestID, startedAt, engineNames)
|
||||
env.Meta.EnginesResponded = enginesResponded
|
||||
env.Meta.EnginesFailed = enginesFailed
|
||||
for _, r := range dedupedRaw {
|
||||
ectx := EnrichContext{Engine: r.Engine, Query: q}
|
||||
@@ -800,33 +807,60 @@ func (s *Server) resolveEngines(enginesParam string) []SearchEngine {
|
||||
|
||||
func (s *Server) deduplicateMegaResults(results []MegaSearchResult) []MegaSearchResult {
|
||||
urlMap := make(map[string]MegaSearchResult)
|
||||
order := []string{}
|
||||
|
||||
for _, result := range results {
|
||||
if result.URL == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := urlMap[result.URL]; !exists {
|
||||
urlMap[result.URL] = result
|
||||
key := NormalizeURLForClustering(result.URL)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
existing, exists := urlMap[key]
|
||||
if !exists {
|
||||
urlMap[key] = result
|
||||
order = append(order, key)
|
||||
continue
|
||||
}
|
||||
if betterMegaResult(result, existing) {
|
||||
urlMap[key] = result
|
||||
}
|
||||
}
|
||||
|
||||
var deduped []MegaSearchResult
|
||||
for _, result := range urlMap {
|
||||
deduped = append(deduped, result)
|
||||
deduped := make([]MegaSearchResult, 0, len(urlMap))
|
||||
for _, key := range order {
|
||||
deduped = append(deduped, urlMap[key])
|
||||
}
|
||||
|
||||
sort.Slice(deduped, func(i, j int) bool {
|
||||
return deduped[i].Rank < deduped[j].Rank
|
||||
if deduped[i].Rank != deduped[j].Rank {
|
||||
return deduped[i].Rank < deduped[j].Rank
|
||||
}
|
||||
if deduped[i].Engine != deduped[j].Engine {
|
||||
return deduped[i].Engine < deduped[j].Engine
|
||||
}
|
||||
return NormalizeURLForClustering(deduped[i].URL) < NormalizeURLForClustering(deduped[j].URL)
|
||||
})
|
||||
return deduped
|
||||
}
|
||||
|
||||
func betterMegaResult(candidate, current MegaSearchResult) bool {
|
||||
if candidate.Rank > 0 && (current.Rank <= 0 || candidate.Rank < current.Rank) {
|
||||
return true
|
||||
}
|
||||
if candidate.Rank == current.Rank && candidate.Engine < current.Engine {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type cacheHitCandidate struct {
|
||||
key string
|
||||
logMessage string
|
||||
}
|
||||
|
||||
func (s *Server) tryServeCacheHit(c *fiber.Ctx, candidates ...cacheHitCandidate) (bool, error) {
|
||||
func (s *Server) tryServeCacheHit(c *fiber.Ctx, startedAt time.Time, candidates ...cacheHitCandidate) (bool, error) {
|
||||
if s.cache == nil {
|
||||
return false, nil
|
||||
}
|
||||
@@ -835,6 +869,7 @@ func (s *Server) tryServeCacheHit(c *fiber.Ctx, candidates ...cacheHitCandidate)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
cached = refreshCachedMeta(cached, RequestIDFromContext(c.UserContext()), startedAt)
|
||||
c.Set("Content-Type", "application/json")
|
||||
c.Set("X-Cache", "HIT")
|
||||
WithRequest(c.UserContext()).Debug(candidate.logMessage)
|
||||
@@ -843,6 +878,27 @@ func (s *Server) tryServeCacheHit(c *fiber.Ctx, candidates ...cacheHitCandidate)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func refreshCachedMeta(data []byte, requestID string, startedAt time.Time) []byte {
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(data, &payload); err != nil {
|
||||
return data
|
||||
}
|
||||
meta, ok := payload["meta"].(map[string]any)
|
||||
if !ok {
|
||||
return data
|
||||
}
|
||||
meta["request_id"] = requestID
|
||||
meta["requested_at"] = startedAt.UTC().Format(time.RFC3339)
|
||||
delete(meta, "timestamp")
|
||||
meta["took_ms"] = time.Since(startedAt).Milliseconds()
|
||||
|
||||
refreshed, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return data
|
||||
}
|
||||
return refreshed
|
||||
}
|
||||
|
||||
func (s *Server) cacheJSON(cacheKey string, payload interface{}) bool {
|
||||
if s.cache == nil {
|
||||
return false
|
||||
|
||||
@@ -645,6 +645,13 @@ func TestDedicatedEndpointFallbackBypassesCache(t *testing.T) {
|
||||
if got := first.Header.Get("X-Fallback-Engine"); got != "yandex" {
|
||||
t.Fatalf("expected fallback engine header, got %q", got)
|
||||
}
|
||||
var env Envelope
|
||||
if err := json.NewDecoder(first.Body).Decode(&env); err != nil {
|
||||
t.Fatalf("decode fallback envelope: %v", err)
|
||||
}
|
||||
if len(env.Meta.EnginesFailed) != 1 || env.Meta.EnginesFailed[0] != "google" {
|
||||
t.Fatalf("expected primary engine in engines_failed, got %v", env.Meta.EnginesFailed)
|
||||
}
|
||||
|
||||
second := request(t, srv, "/google/search?text=golang")
|
||||
if second.StatusCode != http.StatusOK {
|
||||
@@ -1653,6 +1660,172 @@ func TestFormatParamReturnsCorrectContentType(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatParamBypassesJSONCache(t *testing.T) {
|
||||
engine := &engineMock{name: "google", initialized: true}
|
||||
opts := DefaultServerOptions()
|
||||
opts.Resilience.Retry.MaxRetries = 0
|
||||
srv := NewServerWithOptions("127.0.0.1", 7205, opts, engine)
|
||||
|
||||
first := request(t, srv, "/google/search?text=cache-format&format=json")
|
||||
if first.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected first request 200, got %d", first.StatusCode)
|
||||
}
|
||||
|
||||
second := request(t, srv, "/google/search?text=cache-format&format=markdown")
|
||||
if second.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected second request 200, got %d", second.StatusCode)
|
||||
}
|
||||
if ct := second.Header.Get("Content-Type"); !strings.Contains(ct, "text/markdown") {
|
||||
t.Fatalf("expected markdown content type, got %q", ct)
|
||||
}
|
||||
body, _ := io.ReadAll(second.Body)
|
||||
if !strings.Contains(string(body), "# Search results") {
|
||||
t.Fatalf("expected markdown body, got %q", string(body))
|
||||
}
|
||||
|
||||
engine.mu.Lock()
|
||||
searchCalls := engine.searchCalls
|
||||
engine.mu.Unlock()
|
||||
if searchCalls != 2 {
|
||||
t.Fatalf("expected markdown request to bypass JSON cache, got %d search calls", searchCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachedEnvelopeRefreshesRequestID(t *testing.T) {
|
||||
engine := &engineMock{name: "google", initialized: true}
|
||||
opts := DefaultServerOptions()
|
||||
opts.Resilience.Retry.MaxRetries = 0
|
||||
srv := NewServerWithOptions("127.0.0.1", 7206, opts, engine)
|
||||
|
||||
first := requestWithHeader(t, srv, "/google/search?text=cache-id", "X-Request-ID", "req-first")
|
||||
if first.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected first request 200, got %d", first.StatusCode)
|
||||
}
|
||||
second := requestWithHeader(t, srv, "/google/search?text=cache-id", "X-Request-ID", "req-second")
|
||||
if second.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected second request 200, got %d", second.StatusCode)
|
||||
}
|
||||
if got := second.Header.Get("X-Cache"); got != "HIT" {
|
||||
t.Fatalf("expected cache hit, got %q", got)
|
||||
}
|
||||
|
||||
var env Envelope
|
||||
if err := json.NewDecoder(second.Body).Decode(&env); err != nil {
|
||||
t.Fatalf("decode envelope: %v", err)
|
||||
}
|
||||
if env.Meta.RequestID != "req-second" {
|
||||
t.Fatalf("expected refreshed meta.request_id, got %q", env.Meta.RequestID)
|
||||
}
|
||||
if got := second.Header.Get("X-Request-ID"); got != env.Meta.RequestID {
|
||||
t.Fatalf("expected header/body request IDs to match, header=%q body=%q", got, env.Meta.RequestID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPaginatedPositionUsesAbsoluteRank(t *testing.T) {
|
||||
engine := &engineMock{
|
||||
name: "google",
|
||||
initialized: true,
|
||||
searchFn: func(_ context.Context, q Query) ([]SearchResult, error) {
|
||||
return []SearchResult{{Rank: 11, URL: "https://example.com/page", Title: "Page"}}, nil
|
||||
},
|
||||
}
|
||||
opts := DefaultServerOptions()
|
||||
opts.Resilience.Retry.MaxRetries = 0
|
||||
srv := NewServerWithOptions("127.0.0.1", 7207, opts, engine)
|
||||
|
||||
resp := request(t, srv, "/google/search?text=page&start=10&limit=10")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var env Envelope
|
||||
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
|
||||
t.Fatalf("decode envelope: %v", err)
|
||||
}
|
||||
if len(env.Results) != 1 {
|
||||
t.Fatalf("expected one result, got %d", len(env.Results))
|
||||
}
|
||||
pos := env.Results[0].Position
|
||||
if pos.Absolute != 11 || pos.OnPage != 1 || pos.Page != 2 {
|
||||
t.Fatalf("unexpected position: %+v", pos)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageEnvelopeEnrichesMetadataFromDescription(t *testing.T) {
|
||||
engine := &engineMock{
|
||||
name: "google",
|
||||
initialized: true,
|
||||
imageFn: func(_ context.Context, q Query) ([]SearchResult, error) {
|
||||
return []SearchResult{{
|
||||
Rank: 1,
|
||||
URL: "https://cdn.example.com/image.jpg",
|
||||
Title: "Image",
|
||||
Description: "Height:800, Width:1200, Source Page: https://example.com/article",
|
||||
}}, nil
|
||||
},
|
||||
}
|
||||
opts := DefaultServerOptions()
|
||||
opts.Resilience.Retry.MaxRetries = 0
|
||||
srv := NewServerWithOptions("127.0.0.1", 7208, opts, engine)
|
||||
|
||||
resp := request(t, srv, "/google/image?text=image")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var env ImageEnvelope
|
||||
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
|
||||
t.Fatalf("decode envelope: %v", err)
|
||||
}
|
||||
if len(env.Results) != 1 {
|
||||
t.Fatalf("expected one image result, got %d", len(env.Results))
|
||||
}
|
||||
got := env.Results[0]
|
||||
if got.Image.Width != 1200 || got.Image.Height != 800 {
|
||||
t.Fatalf("expected image dimensions 1200x800, got %dx%d", got.Image.Width, got.Image.Height)
|
||||
}
|
||||
if got.Source.PageURL != "https://example.com/article" || got.Source.Domain != "example.com" {
|
||||
t.Fatalf("unexpected source: %+v", got.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMegaSearchDeduplicatesByNormalizedURLDeterministically(t *testing.T) {
|
||||
google := &engineMock{
|
||||
name: "google",
|
||||
initialized: true,
|
||||
searchFn: func(_ context.Context, q Query) ([]SearchResult, error) {
|
||||
return []SearchResult{{Rank: 2, URL: "https://example.com/page?utm_source=test", Title: "Google"}}, nil
|
||||
},
|
||||
}
|
||||
bing := &engineMock{
|
||||
name: "bing",
|
||||
initialized: true,
|
||||
searchFn: func(_ context.Context, q Query) ([]SearchResult, error) {
|
||||
return []SearchResult{{Rank: 1, URL: "https://example.com/page", Title: "Bing"}}, nil
|
||||
},
|
||||
}
|
||||
|
||||
opts := DefaultServerOptions()
|
||||
opts.Resilience.Retry.MaxRetries = 0
|
||||
srv := NewServerWithOptions("127.0.0.1", 7209, opts, google, bing)
|
||||
|
||||
resp := request(t, srv, "/mega/search?text=dedupe&engines=google,bing")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var env Envelope
|
||||
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
|
||||
t.Fatalf("decode envelope: %v", err)
|
||||
}
|
||||
if len(env.Results) != 1 {
|
||||
t.Fatalf("expected one deduped result, got %d", len(env.Results))
|
||||
}
|
||||
if env.Results[0].Engine != "bing" || env.Results[0].Rank != 1 {
|
||||
t.Fatalf("expected best-ranked bing result, got engine=%q rank=%d", env.Results[0].Engine, env.Results[0].Rank)
|
||||
}
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
|
||||
@@ -74,18 +74,15 @@ paths:
|
||||
query:
|
||||
text: golang
|
||||
lang: EN
|
||||
location: null
|
||||
device: desktop
|
||||
engines_requested: [google]
|
||||
meta:
|
||||
request_id: 01HXYZ...
|
||||
timestamp: "2026-04-24T12:00:00Z"
|
||||
requested_at: "2026-04-24T12:00:00Z"
|
||||
took_ms: 842
|
||||
engines_responded: [google]
|
||||
engines_failed: []
|
||||
version: "1.0"
|
||||
results:
|
||||
- id: r_a1b2c3d4e5f6a1b2c3d4e5f6
|
||||
- id: s_a1b2c3d4e5f6a1b2
|
||||
rank: 1
|
||||
type: organic
|
||||
title: The Go Programming Language
|
||||
@@ -100,9 +97,6 @@ paths:
|
||||
page: 1
|
||||
on_page: 1
|
||||
engine: google
|
||||
rich: null
|
||||
engine_meta:
|
||||
raw_rank: 1
|
||||
domain_info:
|
||||
tld: dev
|
||||
sld: go
|
||||
@@ -600,7 +594,7 @@ components:
|
||||
# ── v1 envelope ──────────────────────────────────────────────────
|
||||
QueryEcho:
|
||||
type: object
|
||||
required: [text, location, device, engines_requested]
|
||||
required: [text, engines_requested]
|
||||
properties:
|
||||
text:
|
||||
type: string
|
||||
@@ -608,12 +602,6 @@ components:
|
||||
lang:
|
||||
type: string
|
||||
example: EN
|
||||
location:
|
||||
type: string
|
||||
nullable: true
|
||||
device:
|
||||
type: string
|
||||
example: desktop
|
||||
engines_requested:
|
||||
type: array
|
||||
items:
|
||||
@@ -621,23 +609,18 @@ components:
|
||||
example: [google]
|
||||
ResponseMeta:
|
||||
type: object
|
||||
required: [request_id, timestamp, took_ms, engines_responded, engines_failed, version]
|
||||
required: [request_id, requested_at, took_ms, engines_failed, version]
|
||||
properties:
|
||||
request_id:
|
||||
type: string
|
||||
example: 01HXYZ...
|
||||
timestamp:
|
||||
requested_at:
|
||||
type: string
|
||||
format: date-time
|
||||
example: "2026-04-24T12:00:00Z"
|
||||
took_ms:
|
||||
type: integer
|
||||
example: 842
|
||||
engines_responded:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
example: [google, bing]
|
||||
engines_failed:
|
||||
type: array
|
||||
items:
|
||||
@@ -743,16 +726,14 @@ components:
|
||||
- is_ad
|
||||
- position
|
||||
- engine
|
||||
- rich
|
||||
- engine_meta
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
description: >
|
||||
Stable identifier: `r_` + hex(first 12 bytes of SHA-256(engine|normalized_url)).
|
||||
Stable identifier: `s_` + hex(first 8 bytes of MD5(engine|normalized_url)).
|
||||
Normalized URL: lowercase scheme+host, trailing slash stripped, utm_*/fbclid/gclid
|
||||
tracking params removed. Same URL → same ID across requests.
|
||||
example: r_a1b2c3d4e5f6a1b2c3d4e5f6
|
||||
example: s_a1b2c3d4e5f6a1b2
|
||||
rank:
|
||||
type: integer
|
||||
example: 1
|
||||
@@ -787,16 +768,6 @@ components:
|
||||
engine:
|
||||
type: string
|
||||
example: google
|
||||
rich:
|
||||
type: object
|
||||
nullable: true
|
||||
description: Reserved for future structured SERP data (stars, price, sitelinks).
|
||||
engine_meta:
|
||||
type: object
|
||||
additionalProperties: true
|
||||
description: Engine-specific data. Always contains `raw_rank`.
|
||||
example:
|
||||
raw_rank: 1
|
||||
domain_info:
|
||||
$ref: "#/components/schemas/DomainInfo"
|
||||
classification:
|
||||
@@ -830,12 +801,12 @@ components:
|
||||
example: example.com
|
||||
ImageResult:
|
||||
type: object
|
||||
required: [id, rank, type, title, image, source, engine, engine_meta]
|
||||
required: [id, rank, type, title, image, source, engine]
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
description: Stable identifier prefixed with `i_`.
|
||||
example: i_a1b2c3d4e5f6a1b2c3d4e5f6
|
||||
example: i_a1b2c3d4e5f6a1b2
|
||||
rank:
|
||||
type: integer
|
||||
example: 1
|
||||
@@ -850,9 +821,6 @@ components:
|
||||
$ref: "#/components/schemas/ImageSource"
|
||||
engine:
|
||||
type: string
|
||||
engine_meta:
|
||||
type: object
|
||||
additionalProperties: true
|
||||
# ── Clusters (mega only) ──────────────────────────────────────────
|
||||
ClusterOccurrence:
|
||||
type: object
|
||||
@@ -866,7 +834,7 @@ components:
|
||||
example: 1
|
||||
result_id:
|
||||
type: string
|
||||
example: r_a1b2c3d4e5f6a1b2c3d4e5f6
|
||||
example: s_a1b2c3d4e5f6a1b2
|
||||
Cluster:
|
||||
type: object
|
||||
required: [id, canonical_url, domain, title, occurrences, engines_count, best_rank, score]
|
||||
@@ -874,8 +842,8 @@ components:
|
||||
id:
|
||||
type: string
|
||||
description: >
|
||||
Stable identifier: `c_` + hex(first 12 bytes of SHA-256(normalized_url)).
|
||||
example: c_a1b2c3d4e5f6a1b2c3d4e5f6
|
||||
Stable identifier: `c_` + hex(first 8 bytes of MD5(normalized_url)).
|
||||
example: c_a1b2c3d4e5f6a1b2
|
||||
canonical_url:
|
||||
type: string
|
||||
example: https://go.dev/
|
||||
|
||||
2
go.mod
2
go.mod
@@ -20,6 +20,7 @@ require (
|
||||
github.com/ysmood/gson v0.7.3
|
||||
golang.org/x/net v0.43.0
|
||||
golang.org/x/time v0.12.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -50,5 +51,4 @@ require (
|
||||
golang.org/x/sys v0.35.0 // indirect
|
||||
golang.org/x/text v0.28.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -311,7 +311,7 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S
|
||||
Rank: img.Rank + 1,
|
||||
URL: img.OrigURL,
|
||||
Title: img.Title,
|
||||
Description: fmt.Sprintf("%dx%d, freshness:%s, thumb_url:%s", img.Height, img.Width, img.Freshness, img.ThumbURL),
|
||||
Description: fmt.Sprintf("%dx%d, freshness:%s, thumb_url:%s", img.Width, img.Height, img.Freshness, img.ThumbURL),
|
||||
}
|
||||
|
||||
searchResults = append(searchResults, res)
|
||||
|
||||
Reference in New Issue
Block a user