diff --git a/internal/agent/component/tool_component_test.go b/internal/agent/component/tool_component_test.go index c50736fd85..221607766f 100644 --- a/internal/agent/component/tool_component_test.go +++ b/internal/agent/component/tool_component_test.go @@ -472,11 +472,27 @@ func TestToolBackedComponentYahooFinanceIntegration(t *testing.T) { serverCalls := 0 server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { serverCalls++ - if symbols := request.URL.Query().Get("symbols"); symbols != "AAPL" { - t.Errorf("symbols = %q", symbols) - } writer.Header().Set("Content-Type", "application/json") - _, _ = writer.Write([]byte(`{"quoteResponse":{"result":[{"symbol":"AAPL","regularMarketPrice":189.5,"currency":"USD"}],"error":null}}`)) + switch request.URL.Path { + case "/v1/finance/search": + if q := request.URL.Query().Get("q"); q != "AAPL" { + t.Errorf("q = %q", q) + } + _, _ = writer.Write([]byte(`{"quotes":[{"symbol":"AAPL","currency":"USD"}],"news":[]}`)) + case "/v8/finance/chart/AAPL": + _, _ = writer.Write([]byte(`{ + "chart": { + "result": [{ + "meta": {"regularMarketPrice": 189.5}, + "timestamp": [], + "indicators": {"quote": [{}]} + }], + "error": null + } + }`)) + default: + t.Fatalf("unexpected path %s", request.URL.Path) + } })) defer server.Close() target, err := url.Parse(server.URL) @@ -508,7 +524,7 @@ func TestToolBackedComponentYahooFinanceIntegration(t *testing.T) { if !ok || !strings.Contains(report, "# Information:") || !strings.Contains(report, "| symbol | AAPL |") { t.Fatalf("report = %#v", out["report"]) } - if serverCalls != 1 { - t.Fatalf("server calls = %d, want 1", serverCalls) + if serverCalls != 2 { + t.Fatalf("server calls = %d, want 2", serverCalls) } } diff --git a/internal/agent/tool/registry.go b/internal/agent/tool/registry.go index e4751c1434..7388eeecef 100644 --- a/internal/agent/tool/registry.go +++ b/internal/agent/tool/registry.go @@ -582,12 +582,26 @@ func buildWikipediaTool(params map[string]any) (einotool.BaseTool, error) { func buildYahooFinanceTool(params map[string]any) (einotool.BaseTool, error) { defaults := defaultYahooFinanceParams() - if value, exists := params["info"]; exists { + boolFields := map[string]*bool{ + "info": &defaults.Info, + "history": &defaults.History, + "count": &defaults.Count, + "financials": &defaults.Financials, + "income_stmt": &defaults.IncomeStmt, + "balance_sheet": &defaults.BalanceSheet, + "cash_flow_statement": &defaults.CashFlowStatement, + "news": &defaults.News, + } + for key, target := range boolFields { + value, exists := params[key] + if !exists { + continue + } flag, valid := value.(bool) if !valid { - return nil, fmt.Errorf("agent tool: tool %q requires boolean node-level param info", "yahoo_finance") + return nil, fmt.Errorf("agent tool: tool %q requires boolean node-level param %s", "yahoo_finance", key) } - defaults.Info = flag + *target = flag } return NewYahooFinanceToolWithDefaults(nil, defaults), nil } diff --git a/internal/agent/tool/yahoo_finance.go b/internal/agent/tool/yahoo_finance.go index c6aa219701..c550bfa2a0 100644 --- a/internal/agent/tool/yahoo_finance.go +++ b/internal/agent/tool/yahoo_finance.go @@ -20,9 +20,11 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "net/url" "sort" + "strconv" "strings" "github.com/cloudwego/eino/components/tool" @@ -35,20 +37,66 @@ const yahooFinanceToolDescription = "The Yahoo Finance service provides access t const yahooFinanceStockCodeDescription = "The stock code or company name." -// yahooFinanceParams contains the model-emitted stock code and the supported -// Canvas-side information switch. Info exposes only StockCode. +const ( + yahooFinanceInfoDescription = "Fetch stock information." + yahooFinanceHistoryDescription = "Fetch historical market data." + yahooFinanceCountDescription = "Fetch share count data." + yahooFinanceFinancialsDescription = "Fetch financial calendar data." + yahooFinanceIncomeStatementDescription = "Fetch income statement data." + yahooFinanceBalanceSheetDescription = "Fetch balance sheet data." + yahooFinanceCashFlowStatementDescription = "Fetch cash flow statement data." + yahooFinanceNewsDescription = "Fetch related financial news." +) + +// yahooFinanceParams contains the resolved tool input and Canvas-side switches. type yahooFinanceParams struct { - StockCode string `json:"stock_code"` - Info bool `json:"info"` + StockCode string `json:"stock_code"` + Info bool `json:"info"` + History bool `json:"history"` + Count bool `json:"count"` + Financials bool `json:"financials"` + IncomeStmt bool `json:"income_stmt"` + BalanceSheet bool `json:"balance_sheet"` + CashFlowStatement bool `json:"cash_flow_statement"` + News bool `json:"news"` } -// yahooFinanceResponse is the upstream Yahoo Finance /v7/finance/quote -// envelope. -type yahooFinanceResponse struct { - QuoteResponse struct { +type yahooFinanceArgs struct { + StockCode string `json:"stock_code"` + Query string `json:"query"` + Info *bool `json:"info"` + History *bool `json:"history"` + Count *bool `json:"count"` + Financials *bool `json:"financials"` + IncomeStmt *bool `json:"income_stmt"` + BalanceSheet *bool `json:"balance_sheet"` + CashFlowStatement *bool `json:"cash_flow_statement"` + News *bool `json:"news"` +} + +type yahooFinanceSearchResponse struct { + Quotes []map[string]any `json:"quotes"` + News []map[string]any `json:"news"` +} + +type yahooFinanceChartResponse struct { + Chart struct { + Result []struct { + Meta map[string]any `json:"meta"` + Timestamp []int64 `json:"timestamp"` + Indicators struct { + Quote []map[string][]any `json:"quote"` + } `json:"indicators"` + } `json:"result"` + Error any `json:"error,omitempty"` + } `json:"chart"` +} + +type yahooFinanceSummaryResponse struct { + QuoteSummary struct { Result []map[string]any `json:"result"` Error any `json:"error,omitempty"` - } `json:"quoteResponse"` + } `json:"quoteSummary"` } // yahooFinanceEnvelope is the tool-to-component transport shape. @@ -57,14 +105,17 @@ type yahooFinanceEnvelope struct { Error string `json:"_ERROR,omitempty"` } -// yahooFinanceEndpoint is the Yahoo Finance quote URL. Exposed as a -// package var so tests can substitute a httptest.Server URL. -var yahooFinanceEndpoint = "https://query1.finance.yahoo.com/v7/finance/quote" +// yahooFinanceSearchEndpoint is the Yahoo Finance search URL -// YahooFinanceTool is the -// Yahoo Finance quote tool. -// It performs an unauthenticated GET against the public quote API -// via the shared HTTPHelper and returns the parsed quote records. +var ( + yahooFinanceSearchEndpoint = "https://query2.finance.yahoo.com/v1/finance/search" + yahooFinanceChartEndpoint = "https://query1.finance.yahoo.com/v8/finance/chart" + yahooFinanceSummaryEndpoint = "https://query2.finance.yahoo.com/v10/finance/quoteSummary" + yahooFinanceCookieEndpoint = "https://fc.yahoo.com" + yahooFinanceCrumbEndpoint = "https://query1.finance.yahoo.com/v1/test/getcrumb" +) + +// YahooFinanceTool fetches Yahoo Finance stock data via public Yahoo endpoints. type YahooFinanceTool struct { helper *HTTPHelper defaults yahooFinanceParams @@ -96,7 +147,7 @@ func NewYahooFinanceToolWithDefaults(h *HTTPHelper, defaults yahooFinanceParams) } func defaultYahooFinanceParams() yahooFinanceParams { - return yahooFinanceParams{Info: true} + return yahooFinanceParams{Info: true, News: true} } // Info returns the tool's metadata for the chat model. @@ -110,6 +161,46 @@ func (y *YahooFinanceTool) Info(_ context.Context) (*schema.ToolInfo, error) { Desc: yahooFinanceStockCodeDescription, Required: true, }, + "info": { + Type: schema.Boolean, + Desc: yahooFinanceInfoDescription, + Required: false, + }, + "history": { + Type: schema.Boolean, + Desc: yahooFinanceHistoryDescription, + Required: false, + }, + "count": { + Type: schema.Boolean, + Desc: yahooFinanceCountDescription, + Required: false, + }, + "financials": { + Type: schema.Boolean, + Desc: yahooFinanceFinancialsDescription, + Required: false, + }, + "income_stmt": { + Type: schema.Boolean, + Desc: yahooFinanceIncomeStatementDescription, + Required: false, + }, + "balance_sheet": { + Type: schema.Boolean, + Desc: yahooFinanceBalanceSheetDescription, + Required: false, + }, + "cash_flow_statement": { + Type: schema.Boolean, + Desc: yahooFinanceCashFlowStatementDescription, + Required: false, + }, + "news": { + Type: schema.Boolean, + Desc: yahooFinanceNewsDescription, + Required: false, + }, }), }, nil } @@ -117,7 +208,15 @@ func (y *YahooFinanceTool) Info(_ context.Context) (*schema.ToolInfo, error) { func (y *YahooFinanceTool) ComponentSpec() ComponentSpec { return ComponentSpec{ Inputs: map[string]string{ - "stock_code": yahooFinanceStockCodeDescription, + "stock_code": yahooFinanceStockCodeDescription, + "info": yahooFinanceInfoDescription, + "history": yahooFinanceHistoryDescription, + "count": yahooFinanceCountDescription, + "financials": yahooFinanceFinancialsDescription, + "income_stmt": yahooFinanceIncomeStatementDescription, + "balance_sheet": yahooFinanceBalanceSheetDescription, + "cash_flow_statement": yahooFinanceCashFlowStatementDescription, + "news": yahooFinanceNewsDescription, }, Outputs: map[string]string{"report": "Yahoo Finance data formatted as Markdown."}, InputForm: map[string]any{ @@ -126,57 +225,47 @@ func (y *YahooFinanceTool) ComponentSpec() ComponentSpec { } } -// buildYahooFinanceURL composes the quote URL for one Python-compatible -// stock_code input. Centralized for testability. +// buildYahooFinanceURL composes the search URL func buildYahooFinanceURL(stockCode string) string { q := url.Values{} - q.Set("symbols", stockCode) - return yahooFinanceEndpoint + "?" + q.Encode() + q.Set("q", stockCode) + q.Set("quotesCount", "1") + q.Set("newsCount", "10") + return yahooFinanceSearchEndpoint + "?" + q.Encode() } -// InvokableRun performs the Yahoo Finance quote lookup. +func buildYahooFinanceChartURL(stockCode string) string { + q := url.Values{} + q.Set("range", "1mo") + q.Set("interval", "1d") + return yahooFinanceChartEndpoint + "/" + url.PathEscape(stockCode) + "?" + q.Encode() +} + +func buildYahooFinanceSummaryURL(stockCode string, modules []string, crumb string) string { + q := url.Values{} + q.Set("modules", strings.Join(modules, ",")) + q.Set("crumb", crumb) + return yahooFinanceSummaryEndpoint + "/" + url.PathEscape(stockCode) + "?" + q.Encode() +} + +// InvokableRun performs the Yahoo Finance lookup. func (y *YahooFinanceTool) InvokableRun(ctx context.Context, argsJSON string, _ ...tool.Option) (string, error) { - var p yahooFinanceParams - if err := json.Unmarshal([]byte(argsJSON), &p); err != nil { + var args yahooFinanceArgs + if err := json.Unmarshal([]byte(argsJSON), &args); err != nil { return yahooFinanceErrJSON(fmt.Errorf("yahoo_finance: parse arguments: %w", err)), fmt.Errorf("yahoo_finance: parse arguments: %w", err) } - p = mergeYahooFinanceParams(y.defaults, p) + p := mergeYahooFinanceParams(y.defaults, args) p.StockCode = strings.TrimSpace(p.StockCode) - if p.StockCode == "" || !p.Info { + if p.StockCode == "" || !p.anySectionEnabled() { return yahooFinanceJSON(yahooFinanceEnvelope{Report: ""}), nil } - endpoint := buildYahooFinanceURL(p.StockCode) - // Yahoo Finance returns 401 unless we send a User-Agent that - // looks like a real browser. curl-style UA is the conventional - // workaround for the public (unauthenticated) endpoint. - headers := map[string]string{ - "User-Agent": "Mozilla/5.0 (compatible; ragflow/1.0)", - "Accept": "application/json", - } - - resp, err := y.helper.Do(ctx, http.MethodGet, endpoint, "", "", headers) + report, err := y.fetchYahooFinanceReport(ctx, p) if err != nil { return yahooFinanceErrJSON(err), err } - defer resp.Body.Close() - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return yahooFinanceErrJSON(fmt.Errorf("yahoo_finance: upstream returned %d", resp.StatusCode)), - fmt.Errorf("yahoo_finance: upstream returned %d", resp.StatusCode) - } - - var raw yahooFinanceResponse - if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { - return yahooFinanceErrJSON(fmt.Errorf("yahoo_finance: decode response: %w", err)), - fmt.Errorf("yahoo_finance: decode response: %w", err) - } - if raw.QuoteResponse.Error != nil { - err := fmt.Errorf("yahoo_finance: upstream error: %v", raw.QuoteResponse.Error) - return yahooFinanceErrJSON(err), err - } - return yahooFinanceJSON(yahooFinanceEnvelope{Report: renderYahooFinanceReport(raw.QuoteResponse.Result)}), nil + return yahooFinanceJSON(yahooFinanceEnvelope{Report: report}), nil } func (y *YahooFinanceTool) BuildComponentOutputs(envelope map[string]any) map[string]any { @@ -184,34 +273,404 @@ func (y *YahooFinanceTool) BuildComponentOutputs(envelope map[string]any) map[st return map[string]any{"report": report} } -func mergeYahooFinanceParams(defaults, params yahooFinanceParams) yahooFinanceParams { - params.Info = defaults.Info +func mergeYahooFinanceParams(defaults yahooFinanceParams, args yahooFinanceArgs) yahooFinanceParams { + params := defaults + params.StockCode = args.StockCode + if strings.TrimSpace(params.StockCode) == "" { + params.StockCode = args.Query + } + applyBool := func(value *bool, target *bool) { + if value != nil { + *target = *value + } + } + applyBool(args.Info, ¶ms.Info) + applyBool(args.History, ¶ms.History) + applyBool(args.Count, ¶ms.Count) + applyBool(args.Financials, ¶ms.Financials) + applyBool(args.IncomeStmt, ¶ms.IncomeStmt) + applyBool(args.BalanceSheet, ¶ms.BalanceSheet) + applyBool(args.CashFlowStatement, ¶ms.CashFlowStatement) + applyBool(args.News, ¶ms.News) return params } -// renderYahooFinanceReport keeps the existing public quote endpoint while -// returning the string report expected by the Canvas component. -func renderYahooFinanceReport(quotes []map[string]any) string { - if len(quotes) == 0 { +func (p yahooFinanceParams) anySectionEnabled() bool { + return p.Info || p.History || p.Count || p.Financials || p.IncomeStmt || + p.BalanceSheet || p.CashFlowStatement || p.News +} + +func (y *YahooFinanceTool) fetchYahooFinanceReport(ctx context.Context, p yahooFinanceParams) (string, error) { + var sections []string + var search *yahooFinanceSearchResponse + if p.Info || p.News || p.History || len(yahooFinanceSummaryModules(p)) > 0 { + raw, err := y.fetchYahooFinanceSearch(ctx, p.StockCode) + if err != nil { + return "", err + } + search = raw + } + symbol := yahooFinanceResolvedSymbol(search, p.StockCode) + + var chart *yahooFinanceChartResponse + if p.Info || p.History { + raw, err := y.fetchYahooFinanceChart(ctx, symbol) + if err != nil { + return "", err + } + chart = raw + if chart.Chart.Error != nil { + return "", fmt.Errorf("yahoo_finance: upstream chart error: %v", chart.Chart.Error) + } + } + + if p.Info { + sections = append(sections, "# Information:\n"+renderYahooFinanceMap(yahooFinanceInfo(search, chart))) + } + if p.History { + sections = append(sections, "# History:\n"+renderYahooFinanceHistory(chart)) + } + if p.News { + sections = append(sections, "# News:\n"+renderYahooFinanceRows(yahooFinanceNews(search))) + } + + modules := yahooFinanceSummaryModules(p) + if len(modules) > 0 { + summary, err := y.fetchYahooFinanceSummary(ctx, symbol, modules) + if err != nil { + return "", err + } + if summary.QuoteSummary.Error != nil { + return "", fmt.Errorf("yahoo_finance: upstream summary error: %v", summary.QuoteSummary.Error) + } + if len(summary.QuoteSummary.Result) > 0 { + result := summary.QuoteSummary.Result[0] + if p.Count { + sections = append(sections, "# Count:\n"+renderYahooFinanceMap(summaryMap(result, "defaultKeyStatistics"))) + } + if p.Financials { + sections = append(sections, "# Calendar:\n"+renderYahooFinanceMap(summaryMap(result, "calendarEvents"))) + } + if p.IncomeStmt { + sections = append(sections, "# Income statement:\n"+renderYahooFinanceMap(summaryMap(result, "incomeStatementHistory"))) + sections = append(sections, "# Quarterly income statement:\n"+renderYahooFinanceMap(summaryMap(result, "incomeStatementHistoryQuarterly"))) + } + if p.BalanceSheet { + sections = append(sections, "# Balance sheet:\n"+renderYahooFinanceMap(summaryMap(result, "balanceSheetHistory"))) + sections = append(sections, "# Quarterly balance sheet:\n"+renderYahooFinanceMap(summaryMap(result, "balanceSheetHistoryQuarterly"))) + } + if p.CashFlowStatement { + sections = append(sections, "# Cash flow statement:\n"+renderYahooFinanceMap(summaryMap(result, "cashflowStatementHistory"))) + sections = append(sections, "# Quarterly cash flow statement:\n"+renderYahooFinanceMap(summaryMap(result, "cashflowStatementHistoryQuarterly"))) + } + } + } + + return strings.Join(sections, "\n\n"), nil +} + +func yahooFinanceResolvedSymbol(search *yahooFinanceSearchResponse, fallback string) string { + fallback = strings.TrimSpace(fallback) + if search == nil || len(search.Quotes) == 0 { + return fallback + } + symbol, _ := search.Quotes[0]["symbol"].(string) + symbol = strings.TrimSpace(symbol) + if symbol == "" { + return fallback + } + return symbol +} + +func (y *YahooFinanceTool) fetchYahooFinanceSearch(ctx context.Context, stockCode string) (*yahooFinanceSearchResponse, error) { + var raw yahooFinanceSearchResponse + if err := y.fetchYahooFinanceJSON(ctx, buildYahooFinanceURL(stockCode), &raw); err != nil { + return nil, err + } + return &raw, nil +} + +func (y *YahooFinanceTool) fetchYahooFinanceChart(ctx context.Context, stockCode string) (*yahooFinanceChartResponse, error) { + var raw yahooFinanceChartResponse + if err := y.fetchYahooFinanceJSON(ctx, buildYahooFinanceChartURL(stockCode), &raw); err != nil { + return nil, err + } + return &raw, nil +} + +func (y *YahooFinanceTool) fetchYahooFinanceSummary(ctx context.Context, stockCode string, modules []string) (*yahooFinanceSummaryResponse, error) { + cookieHeader, crumb, err := y.fetchYahooFinanceCrumb(ctx) + if err != nil { + return nil, err + } + var raw yahooFinanceSummaryResponse + if err := y.fetchYahooFinanceJSONWithHeaders(ctx, buildYahooFinanceSummaryURL(stockCode, modules, crumb), &raw, map[string]string{ + "Cookie": cookieHeader, + }); err != nil { + return nil, err + } + return &raw, nil +} + +func (y *YahooFinanceTool) fetchYahooFinanceJSON(ctx context.Context, endpoint string, target any) error { + return y.fetchYahooFinanceJSONWithHeaders(ctx, endpoint, target, nil) +} + +func (y *YahooFinanceTool) fetchYahooFinanceJSONWithHeaders(ctx context.Context, endpoint string, target any, extraHeaders map[string]string) error { + headers := map[string]string{ + "User-Agent": "Mozilla/5.0 (compatible; ragflow/1.0)", + "Accept": "application/json", + } + for key, value := range extraHeaders { + if value != "" { + headers[key] = value + } + } + resp, err := y.helper.Do(ctx, http.MethodGet, endpoint, "", "", headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("yahoo_finance: upstream returned %d", resp.StatusCode) + } + if err := json.NewDecoder(resp.Body).Decode(target); err != nil { + return fmt.Errorf("yahoo_finance: decode response: %w", err) + } + return nil +} + +func (y *YahooFinanceTool) fetchYahooFinanceCrumb(ctx context.Context) (string, string, error) { + headers := map[string]string{ + "User-Agent": "Mozilla/5.0 (compatible; ragflow/1.0)", + "Accept": "text/plain,*/*", + } + resp, err := y.helper.Do(ctx, http.MethodGet, yahooFinanceCookieEndpoint, "", "", headers) + if err != nil { + return "", "", err + } + cookies := resp.Cookies() + _ = resp.Body.Close() + cookieHeader := yahooFinanceCookieHeader(cookies) + if cookieHeader == "" { + return "", "", fmt.Errorf("yahoo_finance: missing Yahoo cookie") + } + + headers["Cookie"] = cookieHeader + resp, err = y.helper.Do(ctx, http.MethodGet, yahooFinanceCrumbEndpoint, "", "", headers) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", "", fmt.Errorf("yahoo_finance: crumb endpoint returned %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", "", fmt.Errorf("yahoo_finance: read crumb: %w", err) + } + crumb := strings.TrimSpace(string(body)) + if crumb == "" { + return "", "", fmt.Errorf("yahoo_finance: empty crumb") + } + return cookieHeader, crumb, nil +} + +func yahooFinanceCookieHeader(cookies []*http.Cookie) string { + values := make([]string, 0, len(cookies)) + for _, cookie := range cookies { + if cookie == nil || cookie.Name == "" { + continue + } + values = append(values, cookie.Name+"="+cookie.Value) + } + return strings.Join(values, "; ") +} + +func yahooFinanceSummaryModules(p yahooFinanceParams) []string { + var modules []string + if p.Count { + modules = append(modules, "defaultKeyStatistics") + } + if p.Financials { + modules = append(modules, "calendarEvents") + } + if p.IncomeStmt { + modules = append(modules, "incomeStatementHistory", "incomeStatementHistoryQuarterly") + } + if p.BalanceSheet { + modules = append(modules, "balanceSheetHistory", "balanceSheetHistoryQuarterly") + } + if p.CashFlowStatement { + modules = append(modules, "cashflowStatementHistory", "cashflowStatementHistoryQuarterly") + } + return modules +} + +func yahooFinanceInfo(search *yahooFinanceSearchResponse, chart *yahooFinanceChartResponse) map[string]any { + info := map[string]any{} + if search != nil && len(search.Quotes) > 0 { + for key, value := range search.Quotes[0] { + info[key] = value + } + } + if chart != nil && len(chart.Chart.Result) > 0 { + for key, value := range chart.Chart.Result[0].Meta { + info[key] = value + } + } + return info +} + +func yahooFinanceNews(search *yahooFinanceSearchResponse) []map[string]any { + if search == nil { + return nil + } + return search.News +} + +func summaryMap(result map[string]any, key string) map[string]any { + value, ok := result[key].(map[string]any) + if !ok { + return nil + } + return value +} + +func renderYahooFinanceMap(values map[string]any) string { + if len(values) == 0 { return "" } - sections := make([]string, 0, len(quotes)) - for _, quote := range quotes { - keys := make([]string, 0, len(quote)) - for key := range quote { - keys = append(keys, key) - } - sort.Strings(keys) + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) - rows := []string{"# Information:", "| | 0 |", "|:---|:---|"} - for _, key := range keys { - rows = append(rows, fmt.Sprintf("| %s | %s |", markdownCell(key), markdownCell(yahooFinanceValue(quote[key])))) + scalars := make([]string, 0, len(keys)) + tables := make([]string, 0) + for _, key := range keys { + if rows := yahooFinanceRows(values[key]); len(rows) > 0 { + tables = append(tables, "### "+markdownCell(key)+"\n"+renderYahooFinanceTable(rows)) + continue } + scalars = append(scalars, fmt.Sprintf("| %s | %s |", markdownCell(key), markdownCell(yahooFinanceValue(values[key])))) + } + + sections := make([]string, 0, 1+len(tables)) + if len(scalars) > 0 { + rows := []string{"| | 0 |", "|:---|:---|"} + rows = append(rows, scalars...) sections = append(sections, strings.Join(rows, "\n")) } + sections = append(sections, tables...) return strings.Join(sections, "\n\n") } +func renderYahooFinanceRows(values []map[string]any) string { + if len(values) == 0 { + return "" + } + return renderYahooFinanceTable(values) +} + +func yahooFinanceRows(value any) []map[string]any { + switch rows := value.(type) { + case []map[string]any: + return rows + case []any: + result := make([]map[string]any, 0, len(rows)) + for _, row := range rows { + asMap, ok := row.(map[string]any) + if !ok { + return nil + } + result = append(result, asMap) + } + return result + default: + return nil + } +} + +func renderYahooFinanceTable(values []map[string]any) string { + if len(values) == 0 { + return "" + } + keys := yahooFinanceTableKeys(values) + if len(keys) == 0 { + return "" + } + + rows := []string{"| " + strings.Join(keys, " | ") + " |"} + align := make([]string, len(keys)) + for i := range align { + align[i] = ":---" + } + rows = append(rows, "|"+strings.Join(align, "|")+"|") + for _, value := range values { + cells := make([]string, 0, len(keys)) + for _, key := range keys { + cells = append(cells, markdownCell(yahooFinanceValue(value[key]))) + } + rows = append(rows, "| "+strings.Join(cells, " | ")+" |") + } + return strings.Join(rows, "\n") +} + +func yahooFinanceTableKeys(values []map[string]any) []string { + seen := map[string]bool{} + keys := make([]string, 0) + for _, value := range values { + rowKeys := make([]string, 0, len(value)) + for key := range value { + if !seen[key] { + rowKeys = append(rowKeys, key) + } + } + sort.Strings(rowKeys) + for _, key := range rowKeys { + seen[key] = true + keys = append(keys, key) + } + } + return keys +} + +func renderYahooFinanceHistory(chart *yahooFinanceChartResponse) string { + if chart == nil || len(chart.Chart.Result) == 0 || len(chart.Chart.Result[0].Indicators.Quote) == 0 { + return "" + } + result := chart.Chart.Result[0] + quotes := result.Indicators.Quote[0] + keys := make([]string, 0, len(quotes)) + for key := range quotes { + keys = append(keys, key) + } + sort.Strings(keys) + + rows := []string{"| timestamp | " + strings.Join(keys, " | ") + " |"} + align := []string{":---"} + for range keys { + align = append(align, ":---") + } + rows = append(rows, "|"+strings.Join(align, "|")+"|") + for i, ts := range result.Timestamp { + cells := []string{fmt.Sprint(ts)} + for _, key := range keys { + values := quotes[key] + if i >= len(values) { + cells = append(cells, "") + continue + } + cells = append(cells, yahooFinanceValue(values[i])) + } + rows = append(rows, "| "+strings.Join(cells, " | ")+" |") + } + return strings.Join(rows, "\n") +} + func yahooFinanceValue(value any) string { if value == nil { return "None" @@ -219,12 +678,70 @@ func yahooFinanceValue(value any) string { if text, ok := value.(string); ok { return text } - if encoded, err := json.Marshal(value); err == nil { - return string(encoded) + if formatted, ok := yahooFinanceFormattedValue(value); ok { + return formatted + } + switch typed := value.(type) { + case float64: + return strconv.FormatFloat(typed, 'f', -1, 64) + case float32: + return strconv.FormatFloat(float64(typed), 'f', -1, 32) + case map[string]any: + return yahooFinanceMapValue(typed) + case []any: + return yahooFinanceArrayValue(typed) + case []string: + return strings.Join(typed, ", ") } return fmt.Sprint(value) } +func yahooFinanceFormattedValue(value any) (string, bool) { + fields, ok := value.(map[string]any) + if !ok { + return "", false + } + if text, ok := fields["fmt"].(string); ok { + return text, true + } + if text, ok := fields["longFmt"].(string); ok { + return text, true + } + raw, ok := fields["raw"] + if !ok || len(fields) > 3 { + return "", false + } + return yahooFinanceValue(raw), true +} + +func yahooFinanceMapValue(fields map[string]any) string { + if len(fields) == 0 { + return "None" + } + keys := make([]string, 0, len(fields)) + for key := range fields { + keys = append(keys, key) + } + sort.Strings(keys) + + parts := make([]string, 0, len(keys)) + for _, key := range keys { + parts = append(parts, key+": "+yahooFinanceValue(fields[key])) + } + return strings.Join(parts, "; ") +} + +func yahooFinanceArrayValue(values []any) string { + if len(values) == 0 { + return "None" + } + parts := make([]string, 0, len(values)) + for _, value := range values { + parts = append(parts, yahooFinanceValue(value)) + } + return strings.Join(parts, ", ") +} + func markdownCell(value string) string { value = strings.ReplaceAll(value, "|", `\|`) return strings.ReplaceAll(value, "\n", "
") diff --git a/internal/agent/tool/yahoo_finance_test.go b/internal/agent/tool/yahoo_finance_test.go index b24dd64726..6196be870d 100644 --- a/internal/agent/tool/yahoo_finance_test.go +++ b/internal/agent/tool/yahoo_finance_test.go @@ -34,40 +34,60 @@ func TestYahooFinanceBuildURL(t *testing.T) { if err != nil { t.Fatalf("url.Parse(%q): %v", got, err) } - if parsed.Host != "query1.finance.yahoo.com" { + if parsed.Host != "query2.finance.yahoo.com" { t.Fatalf("host = %q", parsed.Host) } - if parsed.Path != "/v7/finance/quote" { + if parsed.Path != "/v1/finance/search" { t.Fatalf("path = %q", parsed.Path) } - if symbols := parsed.Query().Get("symbols"); symbols != "0005.HK" { - t.Fatalf("symbols = %q", symbols) + if q := parsed.Query().Get("q"); q != "0005.HK" { + t.Fatalf("q = %q", q) } - if _, exists := parsed.Query()["fields"]; exists { - t.Fatalf("unexpected legacy fields query: %s", parsed.RawQuery) + if quotesCount := parsed.Query().Get("quotesCount"); quotesCount != "1" { + t.Fatalf("quotesCount = %q", quotesCount) } } func TestYahooFinanceInvokableRunBuildsMarkdownReport(t *testing.T) { t.Parallel() - var gotSymbols, gotUserAgent string + var gotQuery, gotUserAgent string + var gotPaths []string server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - gotSymbols = request.URL.Query().Get("symbols") + gotPaths = append(gotPaths, request.URL.Path) gotUserAgent = request.Header.Get("User-Agent") writer.Header().Set("Content-Type", "application/json") - _, _ = writer.Write([]byte(`{ - "quoteResponse": { - "result": [{ + switch request.URL.Path { + case "/v1/finance/search": + gotQuery = request.URL.Query().Get("q") + _, _ = writer.Write([]byte(`{ + "quotes": [{ "symbol":"AAPL", - "regularMarketPrice":189.5, "currency":"USD", - "marketState":"REGULAR", "note":"left|right" }], - "error": null - } - }`)) + "news": [{ + "title":"Apple market update", + "publisher":"Example" + }] + }`)) + case "/v8/finance/chart/AAPL": + _, _ = writer.Write([]byte(`{ + "chart": { + "result": [{ + "meta": { + "regularMarketPrice":189.5, + "marketState":"REGULAR" + }, + "timestamp": [1710000000], + "indicators": {"quote": [{"close": [189.5]}]} + }], + "error": null + } + }`)) + default: + t.Fatalf("unexpected path %s", request.URL.Path) + } })) defer server.Close() @@ -76,8 +96,11 @@ func TestYahooFinanceInvokableRunBuildsMarkdownReport(t *testing.T) { if err != nil { t.Fatalf("InvokableRun: %v", err) } - if gotSymbols != "AAPL" { - t.Fatalf("symbols = %q", gotSymbols) + if gotQuery != "AAPL" { + t.Fatalf("q = %q", gotQuery) + } + if strings.Join(gotPaths, ",") != "/v1/finance/search,/v8/finance/chart/AAPL" { + t.Fatalf("paths = %#v", gotPaths) } if !strings.Contains(gotUserAgent, "ragflow") { t.Fatalf("User-Agent = %q", gotUserAgent) @@ -94,6 +117,9 @@ func TestYahooFinanceInvokableRunBuildsMarkdownReport(t *testing.T) { `| note | left\|right |`, "| regularMarketPrice | 189.5 |", "| symbol | AAPL |", + "# News:", + "| publisher | title |", + "| Example | Apple market update |", } { if !strings.Contains(envelope.Report, expected) { t.Fatalf("report missing %q:\n%s", expected, envelope.Report) @@ -104,6 +130,134 @@ func TestYahooFinanceInvokableRunBuildsMarkdownReport(t *testing.T) { } } +func TestYahooFinanceInvokableRunAcceptsQueryAlias(t *testing.T) { + t.Parallel() + + var gotQuery string + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.Header().Set("Content-Type", "application/json") + switch request.URL.Path { + case "/v1/finance/search": + gotQuery = request.URL.Query().Get("q") + _, _ = writer.Write([]byte(`{"quotes":[{"symbol":"3800.HK"}],"news":[]}`)) + case "/v8/finance/chart/3800.HK": + _, _ = writer.Write([]byte(`{ + "chart": { + "result": [{ + "meta": {"regularMarketPrice": 0.6}, + "timestamp": [], + "indicators": {"quote": [{}]} + }], + "error": null + } + }`)) + default: + t.Fatalf("unexpected path %s", request.URL.Path) + } + })) + defer server.Close() + + helper := NewHTTPHelper().WithClient(&http.Client{Transport: rewriteHostTransport(server.URL)}) + raw, err := NewYahooFinanceToolWith(helper).InvokableRun(context.Background(), `{"query":"3800.HK"}`) + if err != nil { + t.Fatalf("InvokableRun: %v", err) + } + if gotQuery != "3800.HK" { + t.Fatalf("q = %q", gotQuery) + } + var envelope yahooFinanceEnvelope + if err := json.Unmarshal([]byte(raw), &envelope); err != nil { + t.Fatalf("unmarshal output %q: %v", raw, err) + } + if !strings.Contains(envelope.Report, "| symbol | 3800.HK |") || + !strings.Contains(envelope.Report, "| regularMarketPrice | 0.6 |") { + t.Fatalf("report = %s", envelope.Report) + } +} + +func TestYahooFinanceUsesSearchResolvedSymbolForDownstreamRequests(t *testing.T) { + t.Parallel() + + var paths []string + var gotSearchQuery string + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + paths = append(paths, request.URL.Path) + writer.Header().Set("Content-Type", "application/json") + switch request.URL.Path { + case "/v1/finance/search": + gotSearchQuery = request.URL.Query().Get("q") + _, _ = writer.Write([]byte(`{"quotes":[{"symbol":"AAPL","longname":"Apple Inc."}],"news":[]}`)) + case "/v8/finance/chart/AAPL": + _, _ = writer.Write([]byte(`{ + "chart": { + "result": [{ + "meta": {"regularMarketPrice": 189.5}, + "timestamp": [], + "indicators": {"quote": [{}]} + }], + "error": null + } + }`)) + case "/": + http.SetCookie(writer, &http.Cookie{Name: "A1", Value: "cookie-value"}) + writer.WriteHeader(http.StatusNotFound) + case "/v1/test/getcrumb": + if cookie := request.Header.Get("Cookie"); cookie != "A1=cookie-value" { + t.Fatalf("crumb Cookie = %q", cookie) + } + writer.Header().Set("Content-Type", "text/plain") + _, _ = writer.Write([]byte("crumb-value")) + case "/v10/finance/quoteSummary/AAPL": + if cookie := request.Header.Get("Cookie"); cookie != "A1=cookie-value" { + t.Fatalf("summary Cookie = %q", cookie) + } + if crumb := request.URL.Query().Get("crumb"); crumb != "crumb-value" { + t.Fatalf("crumb = %q", crumb) + } + _, _ = writer.Write([]byte(`{ + "quoteSummary": { + "result": [{ + "defaultKeyStatistics": { + "sharesOutstanding": {"raw": 14687356000, "fmt": "14.69B"} + } + }], + "error": null + } + }`)) + default: + t.Fatalf("unexpected path %s", request.URL.Path) + } + })) + defer server.Close() + + helper := NewHTTPHelper().WithClient(&http.Client{Transport: rewriteHostTransport(server.URL)}) + yahoo := NewYahooFinanceToolWithDefaults(helper, yahooFinanceParams{Info: true, Count: true}) + raw, err := yahoo.InvokableRun(context.Background(), `{"stock_code":"Apple"}`) + if err != nil { + t.Fatalf("InvokableRun: %v", err) + } + if gotSearchQuery != "Apple" { + t.Fatalf("search q = %q", gotSearchQuery) + } + gotPaths := strings.Join(paths, ",") + if strings.Contains(gotPaths, "/Apple") { + t.Fatalf("downstream paths used unresolved company name: %v", paths) + } + for _, expected := range []string{"/v1/finance/search", "/v8/finance/chart/AAPL", "/v10/finance/quoteSummary/AAPL"} { + if !strings.Contains(gotPaths, expected) { + t.Fatalf("paths missing %s: %v", expected, paths) + } + } + var envelope yahooFinanceEnvelope + if err := json.Unmarshal([]byte(raw), &envelope); err != nil { + t.Fatalf("unmarshal output %q: %v", raw, err) + } + if !strings.Contains(envelope.Report, "| longname | Apple Inc. |") || + !strings.Contains(envelope.Report, "| sharesOutstanding | 14.69B |") { + t.Fatalf("report = %s", envelope.Report) + } +} + func TestYahooFinanceEmptyStockCodeSkipsRequest(t *testing.T) { t.Parallel() @@ -156,18 +310,148 @@ func TestYahooFinanceAllSectionsDisabledSkipsRequest(t *testing.T) { } } +func TestYahooFinanceSummaryUsesCrumbAndCookie(t *testing.T) { + t.Parallel() + + var sawCookieOnCrumb, sawCookieOnSummary bool + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.Header().Set("Content-Type", "application/json") + switch request.URL.Path { + case "/v1/finance/search": + _, _ = writer.Write([]byte(`{"quotes":[{"symbol":"AAPL"}],"news":[]}`)) + case "/": + http.SetCookie(writer, &http.Cookie{Name: "A1", Value: "cookie-value"}) + writer.WriteHeader(http.StatusNotFound) + case "/v1/test/getcrumb": + sawCookieOnCrumb = request.Header.Get("Cookie") == "A1=cookie-value" + writer.Header().Set("Content-Type", "text/plain") + _, _ = writer.Write([]byte("crumb-value")) + case "/v10/finance/quoteSummary/AAPL": + sawCookieOnSummary = request.Header.Get("Cookie") == "A1=cookie-value" + if crumb := request.URL.Query().Get("crumb"); crumb != "crumb-value" { + t.Fatalf("crumb = %q", crumb) + } + if modules := request.URL.Query().Get("modules"); modules != "defaultKeyStatistics" { + t.Fatalf("modules = %q", modules) + } + _, _ = writer.Write([]byte(`{ + "quoteSummary": { + "result": [{ + "defaultKeyStatistics": { + "sharesOutstanding": {"raw": 14687356000, "fmt": "14.69B"} + } + }], + "error": null + } + }`)) + default: + t.Fatalf("unexpected path %s", request.URL.Path) + } + })) + defer server.Close() + + helper := NewHTTPHelper().WithClient(&http.Client{Transport: rewriteHostTransport(server.URL)}) + yahoo := NewYahooFinanceToolWithDefaults(helper, yahooFinanceParams{Count: true}) + raw, err := yahoo.InvokableRun(context.Background(), `{"stock_code":"AAPL"}`) + if err != nil { + t.Fatalf("InvokableRun: %v", err) + } + if !sawCookieOnCrumb || !sawCookieOnSummary { + t.Fatalf("cookie propagation crumb=%v summary=%v", sawCookieOnCrumb, sawCookieOnSummary) + } + var envelope yahooFinanceEnvelope + if err := json.Unmarshal([]byte(raw), &envelope); err != nil { + t.Fatalf("unmarshal output: %v", err) + } + if !strings.Contains(envelope.Report, "# Count:") || + !strings.Contains(envelope.Report, "| sharesOutstanding | 14.69B |") { + t.Fatalf("report = %s", envelope.Report) + } +} + +func TestRenderYahooFinanceMapFormatsNestedRows(t *testing.T) { + t.Parallel() + + report := renderYahooFinanceMap(map[string]any{ + "maxAge": float64(86400), + "cashflowStatements": []any{ + map[string]any{ + "endDate": map[string]any{"fmt": "2025-12-31", "raw": float64(1767139200)}, + "maxAge": float64(1), + "netIncome": map[string]any{"fmt": "-2.87B", "longFmt": "-2,867,891,000", "raw": float64(-2867891000)}, + }, + map[string]any{ + "endDate": map[string]any{"fmt": "2024-12-31", "raw": float64(1735603200)}, + "maxAge": float64(1), + "netIncome": map[string]any{"fmt": "-4.75B", "longFmt": "-4,750,396,000", "raw": float64(-4750396000)}, + }, + }, + }) + + for _, expected := range []string{ + "| maxAge | 86400 |", + "### cashflowStatements", + "| endDate | maxAge | netIncome |", + "| 2025-12-31 | 1 | -2.87B |", + "| 2024-12-31 | 1 | -4.75B |", + } { + if !strings.Contains(report, expected) { + t.Fatalf("report missing %q:\n%s", expected, report) + } + } + if strings.Contains(report, `"raw"`) || strings.Contains(report, `"fmt"`) { + t.Fatalf("report leaked JSON internals:\n%s", report) + } +} + +func TestYahooFinanceValueFormatsNestedObjectsWithoutJSON(t *testing.T) { + t.Parallel() + + report := renderYahooFinanceMap(map[string]any{ + "currentTradingPeriod": map[string]any{ + "regular": map[string]any{"start": float64(1784251800), "end": float64(1784275800), "timezone": "HKT"}, + }, + "validRanges": []any{"1d", "5d", "1mo"}, + "thumbnail": map[string]any{ + "resolutions": []any{ + map[string]any{"tag": "original", "url": "https://example.test/original.png", "width": float64(768)}, + }, + }, + }) + + for _, expected := range []string{ + "| currentTradingPeriod | regular: end: 1784275800; start: 1784251800; timezone: HKT |", + "| validRanges | 1d, 5d, 1mo |", + "| thumbnail | resolutions: tag: original; url: https://example.test/original.png; width: 768 |", + } { + if !strings.Contains(report, expected) { + t.Fatalf("report missing %q:\n%s", expected, report) + } + } + for _, leaked := range []string{`{"`, `":`, `["`, `"]`} { + if strings.Contains(report, leaked) { + t.Fatalf("report leaked JSON token %q:\n%s", leaked, report) + } + } +} + func TestMergeYahooFinanceParamsKeepsStockCodeAndUsesNodeConfig(t *testing.T) { t.Parallel() - defaults := yahooFinanceParams{Info: true} - params := yahooFinanceParams{ + defaults := yahooFinanceParams{Info: true, News: true} + info := false + history := true + args := yahooFinanceArgs{ StockCode: "AAPL", - Info: false, + Info: &info, + History: &history, } - got := mergeYahooFinanceParams(defaults, params) + got := mergeYahooFinanceParams(defaults, args) want := defaults want.StockCode = "AAPL" + want.Info = false + want.History = true if got != want { t.Fatalf("merged params = %#v, want %#v", got, want) } @@ -181,15 +465,20 @@ func TestYahooFinanceErrorsReturnEnvelope(t *testing.T) { statusCode int body string wantError string + chartError bool }{ {name: "http status", statusCode: http.StatusUnauthorized, body: `denied`, wantError: "upstream returned 401"}, {name: "invalid json", statusCode: http.StatusOK, body: `{`, wantError: "decode response"}, - {name: "upstream envelope", statusCode: http.StatusOK, body: `{"quoteResponse":{"result":[],"error":{"code":"Not Found"}}}`, wantError: "upstream error"}, + {name: "upstream envelope", statusCode: http.StatusOK, body: `{"chart":{"result":[],"error":{"code":"Not Found"}}}`, wantError: "upstream chart error", chartError: true}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { writer.WriteHeader(test.statusCode) + if test.chartError && request.URL.Path == "/v1/finance/search" { + _, _ = writer.Write([]byte(`{"quotes":[{"symbol":"AAPL"}],"news":[]}`)) + return + } _, _ = writer.Write([]byte(test.body)) })) defer server.Close() @@ -244,7 +533,7 @@ func TestYahooFinanceComponentContract(t *testing.T) { } } -func TestYahooFinanceInfoOnlyExposesStockCode(t *testing.T) { +func TestYahooFinanceInfoExposesPythonCompatibleParams(t *testing.T) { t.Parallel() info, err := NewYahooFinanceTool().Info(context.Background()) @@ -263,14 +552,25 @@ func TestYahooFinanceInfoOnlyExposesStockCode(t *testing.T) { t.Fatalf("marshal schema: %v", err) } schemaText := string(raw) - for _, expected := range []string{`"stock_code"`, `"required":["stock_code"]`} { + for _, expected := range []string{ + `"stock_code"`, + `"required":["stock_code"]`, + `"info"`, + `"history"`, + `"count"`, + `"financials"`, + `"income_stmt"`, + `"balance_sheet"`, + `"cash_flow_statement"`, + `"news"`, + } { if !strings.Contains(schemaText, expected) { t.Fatalf("schema missing %s: %s", expected, schemaText) } } - for _, leaked := range []string{`"symbols"`, `"fields"`, `"info"`, `"news"`} { + for _, leaked := range []string{`"symbols"`, `"fields"`} { if strings.Contains(schemaText, leaked) { - t.Fatalf("schema leaked node config %s: %s", leaked, schemaText) + t.Fatalf("schema leaked upstream field %s: %s", leaked, schemaText) } } } @@ -292,6 +592,9 @@ func TestBuildYahooFinanceToolUsesNodeDefaults(t *testing.T) { if yahoo.defaults.Info { t.Fatalf("defaults = %#v", yahoo.defaults) } + if !yahoo.defaults.History || !yahoo.defaults.BalanceSheet || !yahoo.defaults.News { + t.Fatalf("defaults = %#v", yahoo.defaults) + } if yahoo.defaults.StockCode != "" { t.Fatalf("runtime stock_code leaked into defaults: %#v", yahoo.defaults) }