fix api response format issues

This commit is contained in:
Rustem Kamalov
2026-04-26 02:40:08 +03:00
parent 1bdb103b23
commit d47bf08523
16 changed files with 663 additions and 290 deletions

View File

@@ -287,10 +287,19 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
continue
}
res := core.SearchResult{
Rank: (searchPage * 30) + (i + 1),
URL: img.URL[0].Original,
Title: img.Title,
Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright),
Rank: (searchPage * 30) + (i + 1),
URL: img.URL[0].Original,
Title: img.Title,
Description: fmt.Sprintf(
"Source Page: %s, thumb_url:%s, %dx%d, date:%v, type:%v, copyright:%v",
img.URL[0].SourcePage,
img.ThumbURL,
img.Width,
img.Height,
img.PictureDate,
img.Type,
img.IsCopyright,
),
Ad: func() bool {
if img.AdType != "0" {
return true

View File

@@ -382,12 +382,14 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
// Extract information from the parsed data
srchRes.Title = imgData.T
srchRes.URL = imgData.IMGURL
srchRes.Description = imgData.Desc
// Add dimensions to description if available
if imgData.W > 0 && imgData.H > 0 {
srchRes.Description += fmt.Sprintf(" (%dx%d)", imgData.W, imgData.H)
}
srchRes.Description = fmt.Sprintf(
"%s Source Page: %s, thumb_url:%s, %dx%d",
imgData.Desc,
imgData.PURL,
imgData.TURL,
imgData.W,
imgData.H,
)
// Get the page URL
if imgData.MURL != "" {

View File

@@ -15,7 +15,7 @@ import (
)
const (
version = "0.6.7"
version = "0.7.0"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)

View File

@@ -1,7 +1,7 @@
package core
import (
"crypto/sha256"
"crypto/md5"
"encoding/hex"
"sort"
)
@@ -17,12 +17,12 @@ func BuildClusters(results []Result, enginesQueried int) []Cluster {
}
type clusterAccum struct {
occurrences []ClusterOccurrence
scoreSum float64
bestRank int
title string
occurrences []ClusterOccurrence
scoreSum float64
bestRank int
title string
canonicalURL string
domain string
domain string
}
// Group by result ID (which is derived from normalized URL + engine).
@@ -95,8 +95,8 @@ func BuildClusters(results []Result, enginesQueried int) []Cluster {
}
func buildClusterID(normalizedURL string) string {
h := sha256.Sum256([]byte(normalizedURL))
return "c_" + hex.EncodeToString(h[:12])
h := md5.Sum([]byte(normalizedURL))
return "c_" + hex.EncodeToString(h[:responseIDBytes])
}
func roundScore(s float64) float64 {

View File

@@ -1,7 +1,37 @@
package core
import (
_ "embed"
"os"
"strings"
"sync"
"golang.org/x/net/publicsuffix"
"gopkg.in/yaml.v3"
)
//go:embed enrichment_domains.yaml
var defaultEnrichmentDomainsYAML []byte
type enrichmentDomainsFile struct {
DomainSourceHints map[string]string `yaml:"domain_source_hints"`
NewsDomains []string `yaml:"news_domains"`
ForumDomains []string `yaml:"forum_domains"`
MarketplaceDomains []string `yaml:"marketplace_domains"`
SocialDomains []string `yaml:"social_domains"`
}
type enrichmentDomainsConfig struct {
DomainSourceHints map[string]string
NewsDomains map[string]bool
ForumDomains map[string]bool
MarketplaceDomains map[string]bool
SocialDomains map[string]bool
}
var (
enrichmentDomainsOnce sync.Once
enrichmentDomains enrichmentDomainsConfig
)
// EnrichDomainInfo derives TLD/category signals from a bare hostname.
@@ -10,7 +40,9 @@ func EnrichDomainInfo(domain string) *DomainInfo {
return nil
}
domain = normalizeDomain(domain)
tld, sld := splitDomain(domain)
cfg := loadEnrichmentDomains()
info := &DomainInfo{
TLD: tld,
@@ -18,16 +50,16 @@ func EnrichDomainInfo(domain string) *DomainInfo {
IsGov: isGovTLD(domain, tld),
IsEdu: isEduTLD(domain, tld),
IsMil: isMilTLD(tld),
IsNews: newsDomains[domain],
IsForum: forumDomains[domain],
IsMarketplace: marketplaceDomains[domain],
IsSocial: socialDomains[domain],
IsNews: cfg.NewsDomains[domain],
IsForum: cfg.ForumDomains[domain],
IsMarketplace: cfg.MarketplaceDomains[domain],
IsSocial: cfg.SocialDomains[domain],
}
return info
}
// ClassifyURL returns a rough content-type and source hint derived from the
// URL path alone no network calls.
// URL path alone; no network calls.
func ClassifyURL(rawURL, domain string) *Classification {
if rawURL == "" && domain == "" {
return nil
@@ -42,46 +74,42 @@ func ClassifyURL(rawURL, domain string) *Classification {
}
}
// splitDomain returns (tld, sld) for a bare hostname.
// Uses a simple heuristic: last label is TLD, second-to-last is SLD.
// For compound TLDs like co.uk the full suffix is returned as TLD.
// splitDomain returns (public suffix, registrable domain label).
func splitDomain(domain string) (tld, sld string) {
parts := strings.Split(domain, ".")
if len(parts) < 2 {
return domain, ""
domain = normalizeDomain(domain)
if domain == "" {
return "", ""
}
// Known compound TLDs.
compoundTLDs := map[string]bool{
"co.uk": true, "co.jp": true, "co.in": true, "co.nz": true,
"co.za": true, "com.au": true, "com.br": true, "com.mx": true,
"gov.uk": true, "ac.uk": true, "edu.au": true, "gov.au": true,
"or.jp": true, "ne.jp": true,
}
if len(parts) >= 3 {
compound := parts[len(parts)-2] + "." + parts[len(parts)-1]
if compoundTLDs[compound] {
return compound, parts[len(parts)-3]
suffix, icann := publicsuffix.PublicSuffix(domain)
if suffix == "" || !icann {
parts := strings.Split(domain, ".")
if len(parts) < 2 {
return domain, ""
}
return parts[len(parts)-1], parts[len(parts)-2]
}
return parts[len(parts)-1], parts[len(parts)-2]
registrable, err := publicsuffix.EffectiveTLDPlusOne(domain)
if err != nil {
parts := strings.Split(domain, ".")
if len(parts) < 2 {
return suffix, ""
}
return suffix, parts[len(parts)-2]
}
sld = strings.TrimSuffix(registrable, "."+suffix)
return suffix, sld
}
func isGovTLD(domain, tld string) bool {
if tld == "gov" || tld == "gov.uk" || tld == "gov.au" {
return true
}
return strings.HasSuffix(domain, ".gov") ||
strings.HasSuffix(domain, ".gov.uk") ||
strings.HasSuffix(domain, ".gov.au")
return tld == "gov" || strings.HasSuffix(tld, ".gov") || strings.HasSuffix(domain, ".gov")
}
func isEduTLD(domain, tld string) bool {
if tld == "edu" || tld == "ac.uk" || tld == "edu.au" {
return true
}
return strings.HasSuffix(domain, ".edu") ||
strings.HasSuffix(domain, ".ac.uk") ||
strings.HasSuffix(domain, ".edu.au")
return tld == "edu" || strings.HasSuffix(tld, ".edu") || tld == "ac.uk" ||
strings.HasSuffix(domain, ".edu") || strings.HasSuffix(domain, ".ac.uk")
}
func isMilTLD(tld string) bool {
@@ -111,81 +139,65 @@ func classifyContentType(rawURL string) string {
}
func classifySourceHint(domain string) string {
if hint, ok := domainSourceHints[domain]; ok {
cfg := loadEnrichmentDomains()
if hint, ok := cfg.DomainSourceHints[normalizeDomain(domain)]; ok {
return hint
}
return ""
}
// domainSourceHints maps known domains to a descriptive source hint.
var domainSourceHints = map[string]string{
"wikipedia.org": "encyclopedia",
"en.wikipedia.org": "encyclopedia",
"github.com": "code_repository",
"gitlab.com": "code_repository",
"stackoverflow.com": "qa_forum",
"stackexchange.com": "qa_forum",
"reddit.com": "social_forum",
"nytimes.com": "news",
"bbc.com": "news",
"bbc.co.uk": "news",
"reuters.com": "news",
"theguardian.com": "news",
"washingtonpost.com": "news",
"forbes.com": "news",
"techcrunch.com": "news",
"medium.com": "blog_platform",
"scholar.google.com": "academic",
"arxiv.org": "academic",
"pubmed.ncbi.nlm.nih.gov": "academic",
"amazon.com": "marketplace",
"ebay.com": "marketplace",
"etsy.com": "marketplace",
"docs.google.com": "document",
"youtube.com": "video_platform",
"vimeo.com": "video_platform",
"twitter.com": "social_media",
"x.com": "social_media",
"facebook.com": "social_media",
"linkedin.com": "professional_network",
"instagram.com": "social_media",
func loadEnrichmentDomains() enrichmentDomainsConfig {
enrichmentDomainsOnce.Do(func() {
enrichmentDomains = parseEnrichmentDomains(defaultEnrichmentDomainsYAML)
if path := strings.TrimSpace(os.Getenv("OPENSERP_ENRICHMENT_DOMAINS_FILE")); path != "" {
if data, err := os.ReadFile(path); err == nil {
enrichmentDomains = parseEnrichmentDomains(data)
}
}
})
return enrichmentDomains
}
// newsDomains is the set of known news publisher domains.
var newsDomains = map[string]bool{
"nytimes.com": true, "bbc.com": true, "bbc.co.uk": true,
"reuters.com": true, "apnews.com": true, "theguardian.com": true,
"washingtonpost.com": true, "forbes.com": true, "techcrunch.com": true,
"wired.com": true, "bloomberg.com": true, "cnn.com": true,
"nbcnews.com": true, "cbsnews.com": true, "abcnews.go.com": true,
"foxnews.com": true, "theverge.com": true, "engadget.com": true,
"arstechnica.com": true, "zdnet.com": true, "venturebeat.com": true,
"axios.com": true, "politico.com": true, "theatlantic.com": true,
"economist.com": true, "ft.com": true, "wsj.com": true,
"usatoday.com": true, "latimes.com": true, "nypost.com": true,
func parseEnrichmentDomains(data []byte) enrichmentDomainsConfig {
cfg := enrichmentDomainsConfig{
DomainSourceHints: map[string]string{},
NewsDomains: map[string]bool{},
ForumDomains: map[string]bool{},
MarketplaceDomains: map[string]bool{},
SocialDomains: map[string]bool{},
}
var file enrichmentDomainsFile
if err := yaml.Unmarshal(data, &file); err != nil {
return cfg
}
for domain, hint := range file.DomainSourceHints {
domain = normalizeDomain(domain)
hint = strings.TrimSpace(hint)
if domain != "" && hint != "" {
cfg.DomainSourceHints[domain] = hint
}
}
fillDomainSet(cfg.NewsDomains, file.NewsDomains)
fillDomainSet(cfg.ForumDomains, file.ForumDomains)
fillDomainSet(cfg.MarketplaceDomains, file.MarketplaceDomains)
fillDomainSet(cfg.SocialDomains, file.SocialDomains)
return cfg
}
// forumDomains is the set of known community/forum domains.
var forumDomains = map[string]bool{
"reddit.com": true, "news.ycombinator.com": true,
"stackoverflow.com": true, "stackexchange.com": true,
"superuser.com": true, "serverfault.com": true,
"quora.com": true, "discourse.org": true,
"boards.4chan.org": true, "hackernews.com": true,
func fillDomainSet(dst map[string]bool, domains []string) {
for _, domain := range domains {
domain = normalizeDomain(domain)
if domain != "" {
dst[domain] = true
}
}
}
// marketplaceDomains is the set of known e-commerce/marketplace domains.
var marketplaceDomains = map[string]bool{
"amazon.com": true, "amazon.co.uk": true, "amazon.de": true,
"ebay.com": true, "etsy.com": true, "walmart.com": true,
"target.com": true, "bestbuy.com": true, "newegg.com": true,
"aliexpress.com": true, "alibaba.com": true, "shopify.com": true,
}
// socialDomains is the set of known social media platform domains.
var socialDomains = map[string]bool{
"twitter.com": true, "x.com": true, "facebook.com": true,
"instagram.com": true, "tiktok.com": true, "snapchat.com": true,
"pinterest.com": true, "tumblr.com": true, "linkedin.com": true,
"youtube.com": true, "twitch.tv": true, "discord.com": true,
func normalizeDomain(domain string) string {
domain = strings.ToLower(strings.TrimSpace(domain))
domain = strings.TrimPrefix(domain, "www.")
return strings.TrimSuffix(domain, ".")
}

View File

@@ -0,0 +1,103 @@
domain_source_hints:
wikipedia.org: encyclopedia
en.wikipedia.org: encyclopedia
github.com: code_repository
gitlab.com: code_repository
stackoverflow.com: qa_forum
stackexchange.com: qa_forum
reddit.com: social_forum
nytimes.com: news
bbc.com: news
bbc.co.uk: news
reuters.com: news
theguardian.com: news
washingtonpost.com: news
forbes.com: news
techcrunch.com: news
medium.com: blog_platform
scholar.google.com: academic
arxiv.org: academic
pubmed.ncbi.nlm.nih.gov: academic
amazon.com: marketplace
ebay.com: marketplace
etsy.com: marketplace
docs.google.com: document
youtube.com: video_platform
vimeo.com: video_platform
twitter.com: social_media
x.com: social_media
facebook.com: social_media
linkedin.com: professional_network
instagram.com: social_media
news_domains:
- nytimes.com
- bbc.com
- bbc.co.uk
- reuters.com
- apnews.com
- theguardian.com
- washingtonpost.com
- forbes.com
- techcrunch.com
- wired.com
- bloomberg.com
- cnn.com
- nbcnews.com
- cbsnews.com
- abcnews.go.com
- foxnews.com
- theverge.com
- engadget.com
- arstechnica.com
- zdnet.com
- venturebeat.com
- axios.com
- politico.com
- theatlantic.com
- economist.com
- ft.com
- wsj.com
- usatoday.com
- latimes.com
- nypost.com
forum_domains:
- reddit.com
- news.ycombinator.com
- stackoverflow.com
- stackexchange.com
- superuser.com
- serverfault.com
- quora.com
- discourse.org
- boards.4chan.org
- hackernews.com
marketplace_domains:
- amazon.com
- amazon.co.uk
- amazon.de
- ebay.com
- etsy.com
- walmart.com
- target.com
- bestbuy.com
- newegg.com
- aliexpress.com
- alibaba.com
- shopify.com
social_domains:
- twitter.com
- x.com
- facebook.com
- instagram.com
- tiktok.com
- snapchat.com
- pinterest.com
- tumblr.com
- linkedin.com
- youtube.com
- twitch.tv
- discord.com

View File

@@ -10,10 +10,7 @@ import (
func RenderMarkdown(env *Envelope) []byte {
var b strings.Builder
enginesStr := strings.Join(env.Meta.EnginesResponded, ", ")
if enginesStr == "" {
enginesStr = strings.Join(env.Query.EnginesRequested, ", ")
}
enginesStr := strings.Join(env.Query.EnginesRequested, ", ")
fmt.Fprintf(&b, "# Search results for %q\n\n", env.Query.Text)
fmt.Fprintf(&b, "**Query:** %s · **Engines:** %s · **Took:** %dms\n\n",
env.Query.Text, enginesStr, env.Meta.TookMs)
@@ -42,10 +39,7 @@ func RenderMarkdown(env *Envelope) []byte {
func RenderMarkdownImage(env *ImageEnvelope) []byte {
var b strings.Builder
enginesStr := strings.Join(env.Meta.EnginesResponded, ", ")
if enginesStr == "" {
enginesStr = strings.Join(env.Query.EnginesRequested, ", ")
}
enginesStr := strings.Join(env.Query.EnginesRequested, ", ")
fmt.Fprintf(&b, "# Image results for %q\n\n", env.Query.Text)
fmt.Fprintf(&b, "**Query:** %s · **Engines:** %s · **Took:** %dms\n\n",
env.Query.Text, enginesStr, env.Meta.TookMs)

View File

@@ -12,7 +12,7 @@ func RenderText(env *Envelope) []byte {
var b strings.Builder
fmt.Fprintf(&b, "Search: %s\n", env.Query.Text)
enginesStr := strings.Join(env.Meta.EnginesResponded, ", ")
enginesStr := strings.Join(env.Query.EnginesRequested, ", ")
if enginesStr != "" {
fmt.Fprintf(&b, "Engines: %s\n", enginesStr)
}

View File

@@ -6,19 +6,16 @@ import "time"
type QueryEcho struct {
Text string `json:"text"`
Lang string `json:"lang,omitempty"`
Location *string `json:"location"`
Device string `json:"device"`
EnginesRequested []string `json:"engines_requested"`
}
// ResponseMeta carries request-level metadata for observability and debugging.
type ResponseMeta struct {
RequestID string `json:"request_id"`
Timestamp string `json:"timestamp"`
TookMs int64 `json:"took_ms"`
EnginesResponded []string `json:"engines_responded"`
EnginesFailed []string `json:"engines_failed"`
Version string `json:"version"`
RequestID string `json:"request_id"`
RequestedAt string `json:"requested_at"`
TookMs int64 `json:"took_ms"`
EnginesFailed []string `json:"engines_failed"`
Version string `json:"version"`
}
// Pagination carries cursor information for client-side loop termination.
@@ -60,10 +57,10 @@ type ClusterOccurrence struct {
// ImageEnvelope is the top-level v1 response wrapper for image search endpoints.
type ImageEnvelope struct {
Query QueryEcho `json:"query"`
Meta ResponseMeta `json:"meta"`
Query QueryEcho `json:"query"`
Meta ResponseMeta `json:"meta"`
Results []ImageResult `json:"results"`
Pagination Pagination `json:"pagination"`
Pagination Pagination `json:"pagination"`
}
const apiVersion = "1.0"
@@ -75,16 +72,13 @@ func NewEnvelope(q Query, requestID string, startedAt time.Time, engines []strin
Query: QueryEcho{
Text: q.Text,
Lang: q.LangCode,
Location: nil,
Device: "desktop",
EnginesRequested: engines,
},
Meta: ResponseMeta{
RequestID: requestID,
Timestamp: startedAt.UTC().Format(time.RFC3339),
EnginesResponded: []string{},
EnginesFailed: []string{},
Version: apiVersion,
RequestID: requestID,
RequestedAt: startedAt.UTC().Format(time.RFC3339),
EnginesFailed: []string{},
Version: apiVersion,
},
Results: []Result{},
Pagination: Pagination{},
@@ -97,16 +91,13 @@ func NewImageEnvelope(q Query, requestID string, startedAt time.Time, engines []
Query: QueryEcho{
Text: q.Text,
Lang: q.LangCode,
Location: nil,
Device: "desktop",
EnginesRequested: engines,
},
Meta: ResponseMeta{
RequestID: requestID,
Timestamp: startedAt.UTC().Format(time.RFC3339),
EnginesResponded: []string{},
EnginesFailed: []string{},
Version: apiVersion,
RequestID: requestID,
RequestedAt: startedAt.UTC().Format(time.RFC3339),
EnginesFailed: []string{},
Version: apiVersion,
},
Results: []ImageResult{},
Pagination: Pagination{},

View File

@@ -1,14 +1,23 @@
package core
import (
"crypto/sha256"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"fmt"
"net/url"
"regexp"
"strconv"
"strings"
)
const responseIDBytes = 8
var imageDimensionPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)height:\s*(\d+),\s*width:\s*(\d+)`),
regexp.MustCompile(`(?i)\b(\d+)x(\d+)\b`),
}
// EnrichContext carries request-scoped values needed to enrich a raw result.
type EnrichContext struct {
Engine string
@@ -38,12 +47,8 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
if limit <= 0 {
limit = 25
}
onPage := raw.Rank
if onPage < 0 {
onPage = 0
}
absolute := ctx.Query.Start + onPage
page := ctx.Query.Start/limit + 1
absolute, onPage := computeResultPosition(raw.Rank, ctx.Query.Start)
result := Result{
ID: buildResultID(ctx.Engine, normalizedURL),
@@ -62,10 +67,6 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
OnPage: onPage,
},
Engine: ctx.Engine,
Rich: nil,
EngineMeta: map[string]any{
"raw_rank": raw.Rank,
},
}
result.DomainInfo = EnrichDomainInfo(domain)
@@ -77,12 +78,15 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
// EnrichImageResult converts a raw engine result into the v1 ImageResult shape.
func EnrichImageResult(raw SearchResult, ctx EnrichContext) ImageResult {
imageURL := normalizeURL(raw.URL)
// raw.Description may hold the page URL for image results in some engines.
pageURL := raw.Description
meta := parseImageDescription(raw.Description)
pageURL := meta.PageURL
if pageURL == "" {
pageURL = imageURL
}
pageURL = normalizeURL(pageURL)
sourceDomain := extractDomain(pageURL)
imageWidth, imageHeight := meta.Width, meta.Height
return ImageResult{
ID: buildImageID(ctx.Engine, imageURL),
@@ -90,27 +94,42 @@ func EnrichImageResult(raw SearchResult, ctx EnrichContext) ImageResult {
Type: ResultTypeImage,
Title: raw.Title,
Image: ImageData{
URL: imageURL,
URL: imageURL,
Thumbnail: meta.ThumbnailURL,
Width: imageWidth,
Height: imageHeight,
},
Source: ImageSource{
PageURL: pageURL,
Domain: sourceDomain,
},
Engine: ctx.Engine,
EngineMeta: map[string]any{"raw_rank": raw.Rank},
Engine: ctx.Engine,
}
}
// buildResultID returns a stable "r_<hex>" ID for web results.
// buildResultID returns a stable "s_<hex>" ID for web results.
func buildResultID(engine, normalizedURL string) string {
h := sha256.Sum256([]byte(engine + "|" + normalizedURL))
return "r_" + hex.EncodeToString(h[:12])
return "s_" + shortMD5(engine+"|"+normalizedURL)
}
// buildImageID returns a stable "i_<hex>" ID for image results.
func buildImageID(engine, imageURL string) string {
h := sha256.Sum256([]byte(engine + "|" + imageURL))
return "i_" + hex.EncodeToString(h[:12])
return "i_" + shortMD5(engine+"|"+imageURL)
}
func shortMD5(value string) string {
h := md5.Sum([]byte(value))
return hex.EncodeToString(h[:responseIDBytes])
}
func computeResultPosition(rank, start int) (absolute, onPage int) {
if rank <= 0 {
return 0, 0
}
if start > 0 && rank > start {
return rank, rank - start
}
return start + rank, rank
}
// normalizeURL lowercases scheme+host, strips trailing slash, and removes
@@ -235,6 +254,70 @@ func decodeBase64String(s string) ([]byte, error) {
return base64.StdEncoding.DecodeString(s)
}
type imageDescriptionMeta struct {
PageURL string
ThumbnailURL string
Width int
Height int
}
func parseImageDescription(desc string) imageDescriptionMeta {
meta := imageDescriptionMeta{}
trimmed := strings.TrimSpace(desc)
if trimmed == "" {
return meta
}
lower := strings.ToLower(trimmed)
if idx := strings.Index(lower, "source page:"); idx >= 0 {
meta.PageURL = strings.TrimSpace(trimmed[idx+len("source page:"):])
if comma := strings.Index(meta.PageURL, ","); comma >= 0 {
meta.PageURL = strings.TrimSpace(meta.PageURL[:comma])
}
} else if strings.HasPrefix(lower, "source:") {
meta.PageURL = strings.TrimSpace(trimmed[len("source:"):])
}
if idx := strings.Index(lower, "thumb_url:"); idx >= 0 {
meta.ThumbnailURL = strings.TrimSpace(trimmed[idx+len("thumb_url:"):])
if comma := strings.Index(meta.ThumbnailURL, ","); comma >= 0 {
meta.ThumbnailURL = strings.TrimSpace(meta.ThumbnailURL[:comma])
}
}
for _, pattern := range imageDimensionPatterns {
match := pattern.FindStringSubmatch(trimmed)
if len(match) != 3 {
continue
}
first, firstErr := strconv.Atoi(match[1])
second, secondErr := strconv.Atoi(match[2])
if firstErr != nil || secondErr != nil {
continue
}
if strings.Contains(strings.ToLower(match[0]), "height") {
meta.Height = first
meta.Width = second
} else {
meta.Width = first
meta.Height = second
}
break
}
if !isHTTPURL(meta.PageURL) {
meta.PageURL = ""
}
if !isHTTPURL(meta.ThumbnailURL) {
meta.ThumbnailURL = ""
}
return meta
}
func isHTTPURL(value string) bool {
return strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://")
}
// NormalizeURLForClustering returns a URL suitable for cross-engine grouping
// (same as normalizeURL but exported for use in cluster building).
func NormalizeURLForClustering(rawURL string) string {

View File

@@ -4,17 +4,17 @@ package core
type ResultType string
const (
ResultTypeOrganic ResultType = "organic"
ResultTypeAd ResultType = "ad"
ResultTypeOrganic ResultType = "organic"
ResultTypeAd ResultType = "ad"
ResultTypeFeaturedSnippet ResultType = "featured_snippet"
ResultTypeKnowledgePanel ResultType = "knowledge_panel"
ResultTypePeopleAlsoAsk ResultType = "people_also_ask"
ResultTypeVideo ResultType = "video"
ResultTypeImage ResultType = "image"
ResultTypeNews ResultType = "news"
ResultTypeShopping ResultType = "shopping"
ResultTypeLocal ResultType = "local"
ResultTypeAnswerBox ResultType = "answer_box"
ResultTypeKnowledgePanel ResultType = "knowledge_panel"
ResultTypePeopleAlsoAsk ResultType = "people_also_ask"
ResultTypeVideo ResultType = "video"
ResultTypeImage ResultType = "image"
ResultTypeNews ResultType = "news"
ResultTypeShopping ResultType = "shopping"
ResultTypeLocal ResultType = "local"
ResultTypeAnswerBox ResultType = "answer_box"
)
// Position describes where a result sits in the overall result stream.
@@ -27,21 +27,6 @@ type Position struct {
OnPage int `json:"on_page"`
}
// RichData is a placeholder for future structured SERP features such as star
// ratings, prices, or sitelinks. It is null in v1.0.
type RichData struct {
Stars *float64 `json:"stars,omitempty"`
Reviews *int `json:"reviews,omitempty"`
Price *string `json:"price,omitempty"`
Sitelinks []RichSitelink `json:"sitelinks,omitempty"`
}
// RichSitelink is one navigational sub-link shown below a result.
type RichSitelink struct {
Title string `json:"title"`
URL string `json:"url"`
}
// DomainInfo carries TLD-derived category signals for a result domain.
type DomainInfo struct {
TLD string `json:"tld"`
@@ -63,21 +48,19 @@ type Classification struct {
// Result is the v1 normalized result returned in every search response.
type Result struct {
ID string `json:"id"`
Rank int `json:"rank"`
Type ResultType `json:"type"`
Title string `json:"title"`
URL string `json:"url"`
DisplayURL string `json:"display_url"`
Snippet string `json:"snippet"`
Domain string `json:"domain"`
Favicon string `json:"favicon"`
IsAd bool `json:"is_ad"`
Position Position `json:"position"`
Engine string `json:"engine"`
Rich *RichData `json:"rich"`
EngineMeta map[string]any `json:"engine_meta"`
DomainInfo *DomainInfo `json:"domain_info,omitempty"`
ID string `json:"id"`
Rank int `json:"rank"`
Type ResultType `json:"type"`
Title string `json:"title"`
URL string `json:"url"`
DisplayURL string `json:"display_url"`
Snippet string `json:"snippet"`
Domain string `json:"domain"`
Favicon string `json:"favicon"`
IsAd bool `json:"is_ad"`
Position Position `json:"position"`
Engine string `json:"engine"`
DomainInfo *DomainInfo `json:"domain_info,omitempty"`
Classification *Classification `json:"classification,omitempty"`
}
@@ -97,12 +80,11 @@ type ImageSource struct {
// ImageResult is the v1 shape for image search results.
type ImageResult struct {
ID string `json:"id"`
Rank int `json:"rank"`
Type ResultType `json:"type"`
Title string `json:"title"`
Image ImageData `json:"image"`
Source ImageSource `json:"source"`
Engine string `json:"engine"`
EngineMeta map[string]any `json:"engine_meta"`
ID string `json:"id"`
Rank int `json:"rank"`
Type ResultType `json:"type"`
Title string `json:"title"`
Image ImageData `json:"image"`
Source ImageSource `json:"source"`
Engine string `json:"engine"`
}

View File

@@ -206,14 +206,17 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm
WithField("action", action).
Debugf("Starting %s request for query: %s", action, q.Text)
if hit, err := s.tryServeCacheHit(
c,
cacheHitCandidate{
key: BuildCacheKey(engine.Name(), action, q),
logMessage: fmt.Sprintf("Cache hit for %s %s: %s", engine.Name(), action, q.Text),
},
); hit || err != nil {
return err
if format == "json" {
if hit, err := s.tryServeCacheHit(
c,
startedAt,
cacheHitCandidate{
key: BuildCacheKey(engine.Name(), action, q),
logMessage: fmt.Sprintf("Cache hit for %s %s: %s", engine.Name(), action, q.Text),
},
); hit || err != nil {
return err
}
}
engineNames := []string{engine.Name()}
@@ -237,7 +240,9 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm
}
env := NewImageEnvelope(q, requestID, startedAt, engineNames)
env.Meta.EnginesResponded = []string{usedEngine}
if usedEngine != "" && usedEngine != engine.Name() {
env.Meta.EnginesFailed = []string{engine.Name()}
}
ectx := EnrichContext{Engine: usedEngine, Query: q}
for _, r := range res {
env.Results = append(env.Results, EnrichImageResult(r, ectx))
@@ -276,7 +281,9 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm
}
env := NewEnvelope(q, requestID, startedAt, engineNames)
env.Meta.EnginesResponded = []string{usedEngine}
if usedEngine != "" && usedEngine != engine.Name() {
env.Meta.EnginesFailed = []string{engine.Name()}
}
ectx := EnrichContext{Engine: usedEngine, Query: q}
for _, r := range res {
env.Results = append(env.Results, EnrichResult(r, ectx))
@@ -687,15 +694,16 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string, run func(contex
logMessage: fmt.Sprintf("Cache hit for mega %s partial set: engines=%s query=%s", action, engineNamesJoined, q.Text),
})
}
if hit, err := s.tryServeCacheHit(c, cacheHitCandidates...); hit || err != nil {
return err
if format == "json" {
if hit, err := s.tryServeCacheHit(c, startedAt, cacheHitCandidates...); hit || err != nil {
return err
}
}
rawResults, enginesResponded, enginesFailed := run(requestCtx, q, enginesToUse)
rawResults, _, enginesFailed := run(requestCtx, q, enginesToUse)
if action == "image" {
env := NewImageEnvelope(q, requestID, startedAt, engineNames)
env.Meta.EnginesResponded = enginesResponded
env.Meta.EnginesFailed = enginesFailed
for _, r := range rawResults {
ectx := EnrichContext{Engine: r.Engine, Query: q}
@@ -724,7 +732,6 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string, run func(contex
// Deduplicate the flat results list by normalized URL (keep best-ranked occurrence).
dedupedRaw := s.deduplicateMegaResults(rawResults)
env := NewEnvelope(q, requestID, startedAt, engineNames)
env.Meta.EnginesResponded = enginesResponded
env.Meta.EnginesFailed = enginesFailed
for _, r := range dedupedRaw {
ectx := EnrichContext{Engine: r.Engine, Query: q}
@@ -800,33 +807,60 @@ func (s *Server) resolveEngines(enginesParam string) []SearchEngine {
func (s *Server) deduplicateMegaResults(results []MegaSearchResult) []MegaSearchResult {
urlMap := make(map[string]MegaSearchResult)
order := []string{}
for _, result := range results {
if result.URL == "" {
continue
}
if _, exists := urlMap[result.URL]; !exists {
urlMap[result.URL] = result
key := NormalizeURLForClustering(result.URL)
if key == "" {
continue
}
existing, exists := urlMap[key]
if !exists {
urlMap[key] = result
order = append(order, key)
continue
}
if betterMegaResult(result, existing) {
urlMap[key] = result
}
}
var deduped []MegaSearchResult
for _, result := range urlMap {
deduped = append(deduped, result)
deduped := make([]MegaSearchResult, 0, len(urlMap))
for _, key := range order {
deduped = append(deduped, urlMap[key])
}
sort.Slice(deduped, func(i, j int) bool {
return deduped[i].Rank < deduped[j].Rank
if deduped[i].Rank != deduped[j].Rank {
return deduped[i].Rank < deduped[j].Rank
}
if deduped[i].Engine != deduped[j].Engine {
return deduped[i].Engine < deduped[j].Engine
}
return NormalizeURLForClustering(deduped[i].URL) < NormalizeURLForClustering(deduped[j].URL)
})
return deduped
}
func betterMegaResult(candidate, current MegaSearchResult) bool {
if candidate.Rank > 0 && (current.Rank <= 0 || candidate.Rank < current.Rank) {
return true
}
if candidate.Rank == current.Rank && candidate.Engine < current.Engine {
return true
}
return false
}
type cacheHitCandidate struct {
key string
logMessage string
}
func (s *Server) tryServeCacheHit(c *fiber.Ctx, candidates ...cacheHitCandidate) (bool, error) {
func (s *Server) tryServeCacheHit(c *fiber.Ctx, startedAt time.Time, candidates ...cacheHitCandidate) (bool, error) {
if s.cache == nil {
return false, nil
}
@@ -835,6 +869,7 @@ func (s *Server) tryServeCacheHit(c *fiber.Ctx, candidates ...cacheHitCandidate)
if !ok {
continue
}
cached = refreshCachedMeta(cached, RequestIDFromContext(c.UserContext()), startedAt)
c.Set("Content-Type", "application/json")
c.Set("X-Cache", "HIT")
WithRequest(c.UserContext()).Debug(candidate.logMessage)
@@ -843,6 +878,27 @@ func (s *Server) tryServeCacheHit(c *fiber.Ctx, candidates ...cacheHitCandidate)
return false, nil
}
func refreshCachedMeta(data []byte, requestID string, startedAt time.Time) []byte {
var payload map[string]any
if err := json.Unmarshal(data, &payload); err != nil {
return data
}
meta, ok := payload["meta"].(map[string]any)
if !ok {
return data
}
meta["request_id"] = requestID
meta["requested_at"] = startedAt.UTC().Format(time.RFC3339)
delete(meta, "timestamp")
meta["took_ms"] = time.Since(startedAt).Milliseconds()
refreshed, err := json.Marshal(payload)
if err != nil {
return data
}
return refreshed
}
func (s *Server) cacheJSON(cacheKey string, payload interface{}) bool {
if s.cache == nil {
return false

View File

@@ -645,6 +645,13 @@ func TestDedicatedEndpointFallbackBypassesCache(t *testing.T) {
if got := first.Header.Get("X-Fallback-Engine"); got != "yandex" {
t.Fatalf("expected fallback engine header, got %q", got)
}
var env Envelope
if err := json.NewDecoder(first.Body).Decode(&env); err != nil {
t.Fatalf("decode fallback envelope: %v", err)
}
if len(env.Meta.EnginesFailed) != 1 || env.Meta.EnginesFailed[0] != "google" {
t.Fatalf("expected primary engine in engines_failed, got %v", env.Meta.EnginesFailed)
}
second := request(t, srv, "/google/search?text=golang")
if second.StatusCode != http.StatusOK {
@@ -1653,6 +1660,172 @@ func TestFormatParamReturnsCorrectContentType(t *testing.T) {
}
}
func TestFormatParamBypassesJSONCache(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()
opts.Resilience.Retry.MaxRetries = 0
srv := NewServerWithOptions("127.0.0.1", 7205, opts, engine)
first := request(t, srv, "/google/search?text=cache-format&format=json")
if first.StatusCode != http.StatusOK {
t.Fatalf("expected first request 200, got %d", first.StatusCode)
}
second := request(t, srv, "/google/search?text=cache-format&format=markdown")
if second.StatusCode != http.StatusOK {
t.Fatalf("expected second request 200, got %d", second.StatusCode)
}
if ct := second.Header.Get("Content-Type"); !strings.Contains(ct, "text/markdown") {
t.Fatalf("expected markdown content type, got %q", ct)
}
body, _ := io.ReadAll(second.Body)
if !strings.Contains(string(body), "# Search results") {
t.Fatalf("expected markdown body, got %q", string(body))
}
engine.mu.Lock()
searchCalls := engine.searchCalls
engine.mu.Unlock()
if searchCalls != 2 {
t.Fatalf("expected markdown request to bypass JSON cache, got %d search calls", searchCalls)
}
}
func TestCachedEnvelopeRefreshesRequestID(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()
opts.Resilience.Retry.MaxRetries = 0
srv := NewServerWithOptions("127.0.0.1", 7206, opts, engine)
first := requestWithHeader(t, srv, "/google/search?text=cache-id", "X-Request-ID", "req-first")
if first.StatusCode != http.StatusOK {
t.Fatalf("expected first request 200, got %d", first.StatusCode)
}
second := requestWithHeader(t, srv, "/google/search?text=cache-id", "X-Request-ID", "req-second")
if second.StatusCode != http.StatusOK {
t.Fatalf("expected second request 200, got %d", second.StatusCode)
}
if got := second.Header.Get("X-Cache"); got != "HIT" {
t.Fatalf("expected cache hit, got %q", got)
}
var env Envelope
if err := json.NewDecoder(second.Body).Decode(&env); err != nil {
t.Fatalf("decode envelope: %v", err)
}
if env.Meta.RequestID != "req-second" {
t.Fatalf("expected refreshed meta.request_id, got %q", env.Meta.RequestID)
}
if got := second.Header.Get("X-Request-ID"); got != env.Meta.RequestID {
t.Fatalf("expected header/body request IDs to match, header=%q body=%q", got, env.Meta.RequestID)
}
}
func TestPaginatedPositionUsesAbsoluteRank(t *testing.T) {
engine := &engineMock{
name: "google",
initialized: true,
searchFn: func(_ context.Context, q Query) ([]SearchResult, error) {
return []SearchResult{{Rank: 11, URL: "https://example.com/page", Title: "Page"}}, nil
},
}
opts := DefaultServerOptions()
opts.Resilience.Retry.MaxRetries = 0
srv := NewServerWithOptions("127.0.0.1", 7207, opts, engine)
resp := request(t, srv, "/google/search?text=page&start=10&limit=10")
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
var env Envelope
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
t.Fatalf("decode envelope: %v", err)
}
if len(env.Results) != 1 {
t.Fatalf("expected one result, got %d", len(env.Results))
}
pos := env.Results[0].Position
if pos.Absolute != 11 || pos.OnPage != 1 || pos.Page != 2 {
t.Fatalf("unexpected position: %+v", pos)
}
}
func TestImageEnvelopeEnrichesMetadataFromDescription(t *testing.T) {
engine := &engineMock{
name: "google",
initialized: true,
imageFn: func(_ context.Context, q Query) ([]SearchResult, error) {
return []SearchResult{{
Rank: 1,
URL: "https://cdn.example.com/image.jpg",
Title: "Image",
Description: "Height:800, Width:1200, Source Page: https://example.com/article",
}}, nil
},
}
opts := DefaultServerOptions()
opts.Resilience.Retry.MaxRetries = 0
srv := NewServerWithOptions("127.0.0.1", 7208, opts, engine)
resp := request(t, srv, "/google/image?text=image")
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
var env ImageEnvelope
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
t.Fatalf("decode envelope: %v", err)
}
if len(env.Results) != 1 {
t.Fatalf("expected one image result, got %d", len(env.Results))
}
got := env.Results[0]
if got.Image.Width != 1200 || got.Image.Height != 800 {
t.Fatalf("expected image dimensions 1200x800, got %dx%d", got.Image.Width, got.Image.Height)
}
if got.Source.PageURL != "https://example.com/article" || got.Source.Domain != "example.com" {
t.Fatalf("unexpected source: %+v", got.Source)
}
}
func TestMegaSearchDeduplicatesByNormalizedURLDeterministically(t *testing.T) {
google := &engineMock{
name: "google",
initialized: true,
searchFn: func(_ context.Context, q Query) ([]SearchResult, error) {
return []SearchResult{{Rank: 2, URL: "https://example.com/page?utm_source=test", Title: "Google"}}, nil
},
}
bing := &engineMock{
name: "bing",
initialized: true,
searchFn: func(_ context.Context, q Query) ([]SearchResult, error) {
return []SearchResult{{Rank: 1, URL: "https://example.com/page", Title: "Bing"}}, nil
},
}
opts := DefaultServerOptions()
opts.Resilience.Retry.MaxRetries = 0
srv := NewServerWithOptions("127.0.0.1", 7209, opts, google, bing)
resp := request(t, srv, "/mega/search?text=dedupe&engines=google,bing")
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
var env Envelope
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
t.Fatalf("decode envelope: %v", err)
}
if len(env.Results) != 1 {
t.Fatalf("expected one deduped result, got %d", len(env.Results))
}
if env.Results[0].Engine != "bing" || env.Results[0].Rank != 1 {
t.Fatalf("expected best-ranked bing result, got engine=%q rank=%d", env.Results[0].Engine, env.Results[0].Rank)
}
}
func min(a, b int) int {
if a < b {
return a

View File

@@ -74,18 +74,15 @@ paths:
query:
text: golang
lang: EN
location: null
device: desktop
engines_requested: [google]
meta:
request_id: 01HXYZ...
timestamp: "2026-04-24T12:00:00Z"
requested_at: "2026-04-24T12:00:00Z"
took_ms: 842
engines_responded: [google]
engines_failed: []
version: "1.0"
results:
- id: r_a1b2c3d4e5f6a1b2c3d4e5f6
- id: s_a1b2c3d4e5f6a1b2
rank: 1
type: organic
title: The Go Programming Language
@@ -100,9 +97,6 @@ paths:
page: 1
on_page: 1
engine: google
rich: null
engine_meta:
raw_rank: 1
domain_info:
tld: dev
sld: go
@@ -600,7 +594,7 @@ components:
# ── v1 envelope ──────────────────────────────────────────────────
QueryEcho:
type: object
required: [text, location, device, engines_requested]
required: [text, engines_requested]
properties:
text:
type: string
@@ -608,12 +602,6 @@ components:
lang:
type: string
example: EN
location:
type: string
nullable: true
device:
type: string
example: desktop
engines_requested:
type: array
items:
@@ -621,23 +609,18 @@ components:
example: [google]
ResponseMeta:
type: object
required: [request_id, timestamp, took_ms, engines_responded, engines_failed, version]
required: [request_id, requested_at, took_ms, engines_failed, version]
properties:
request_id:
type: string
example: 01HXYZ...
timestamp:
requested_at:
type: string
format: date-time
example: "2026-04-24T12:00:00Z"
took_ms:
type: integer
example: 842
engines_responded:
type: array
items:
type: string
example: [google, bing]
engines_failed:
type: array
items:
@@ -743,16 +726,14 @@ components:
- is_ad
- position
- engine
- rich
- engine_meta
properties:
id:
type: string
description: >
Stable identifier: `r_` + hex(first 12 bytes of SHA-256(engine|normalized_url)).
Stable identifier: `s_` + hex(first 8 bytes of MD5(engine|normalized_url)).
Normalized URL: lowercase scheme+host, trailing slash stripped, utm_*/fbclid/gclid
tracking params removed. Same URL → same ID across requests.
example: r_a1b2c3d4e5f6a1b2c3d4e5f6
example: s_a1b2c3d4e5f6a1b2
rank:
type: integer
example: 1
@@ -787,16 +768,6 @@ components:
engine:
type: string
example: google
rich:
type: object
nullable: true
description: Reserved for future structured SERP data (stars, price, sitelinks).
engine_meta:
type: object
additionalProperties: true
description: Engine-specific data. Always contains `raw_rank`.
example:
raw_rank: 1
domain_info:
$ref: "#/components/schemas/DomainInfo"
classification:
@@ -830,12 +801,12 @@ components:
example: example.com
ImageResult:
type: object
required: [id, rank, type, title, image, source, engine, engine_meta]
required: [id, rank, type, title, image, source, engine]
properties:
id:
type: string
description: Stable identifier prefixed with `i_`.
example: i_a1b2c3d4e5f6a1b2c3d4e5f6
example: i_a1b2c3d4e5f6a1b2
rank:
type: integer
example: 1
@@ -850,9 +821,6 @@ components:
$ref: "#/components/schemas/ImageSource"
engine:
type: string
engine_meta:
type: object
additionalProperties: true
# ── Clusters (mega only) ──────────────────────────────────────────
ClusterOccurrence:
type: object
@@ -866,7 +834,7 @@ components:
example: 1
result_id:
type: string
example: r_a1b2c3d4e5f6a1b2c3d4e5f6
example: s_a1b2c3d4e5f6a1b2
Cluster:
type: object
required: [id, canonical_url, domain, title, occurrences, engines_count, best_rank, score]
@@ -874,8 +842,8 @@ components:
id:
type: string
description: >
Stable identifier: `c_` + hex(first 12 bytes of SHA-256(normalized_url)).
example: c_a1b2c3d4e5f6a1b2c3d4e5f6
Stable identifier: `c_` + hex(first 8 bytes of MD5(normalized_url)).
example: c_a1b2c3d4e5f6a1b2
canonical_url:
type: string
example: https://go.dev/

2
go.mod
View File

@@ -20,6 +20,7 @@ require (
github.com/ysmood/gson v0.7.3
golang.org/x/net v0.43.0
golang.org/x/time v0.12.0
gopkg.in/yaml.v3 v3.0.1
)
require (
@@ -50,5 +51,4 @@ require (
golang.org/x/sys v0.35.0 // indirect
golang.org/x/text v0.28.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

View File

@@ -311,7 +311,7 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S
Rank: img.Rank + 1,
URL: img.OrigURL,
Title: img.Title,
Description: fmt.Sprintf("%dx%d, freshness:%s, thumb_url:%s", img.Height, img.Width, img.Freshness, img.ThumbURL),
Description: fmt.Sprintf("%dx%d, freshness:%s, thumb_url:%s", img.Width, img.Height, img.Freshness, img.ThumbURL),
}
searchResults = append(searchResults, res)