Implement builtin chunk method as ingestion pipeline in GO (#16822)

### Summary Implement builtin chunk mehtod as ingestion pipeline in GO
2026-08-04 23:00:30 +08:00 · 2026-07-13 13:51:40 +08:00
parent ed27b5f9f8
commit d549194562
97 changed files with 11427 additions and 3285 deletions
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -18,32 +18,84 @@ package parser

 import (
 	"bytes"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"regexp"
 	"strings"
+	"sync"
+	"time"

 	"github.com/gomarkdown/markdown/ast"
 	"github.com/gomarkdown/markdown/parser"
 )

-type MarkdownParser struct{}
+// mdImagePattern matches markdown inline image syntax: ![alt](url).
+var mdImagePattern = regexp.MustCompile(`!\[[^\]]*\]\(([^)\s]+)\)`)

-func NewMarkdownParser() *MarkdownParser {
-	return &MarkdownParser{}
+// dataURIPrefix is the MIME prefix for data URI images.
+const dataURIPrefix = "data:image/"
+
+// GoMarkdown is the lib_type identifier for the pure-Go markdown backend.
+const GoMarkdown = "go_markdown"
+
+// ssrfAllowLoopback lets tests exercise the HTTP image fetch path against a
+// loopback httptest server. It stays false in production so loopback
+// addresses are rejected (SSRF protection).
+var ssrfAllowLoopback bool
+
+type MarkdownParser struct {
+	libType      string
+	ParseMethod  string
+	OutputFormat string
+	VLM          map[string]any
+}
+
+func NewMarkdownParser(libType string) (*MarkdownParser, error) {
+	switch libType {
+	case GoMarkdown:
+		return &MarkdownParser{
+			libType: GoMarkdown,
+		}, nil
+	default:
+		return nil, fmt.Errorf("unsupported Markdown library type: %s", libType)
+	}
+}
+
+func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {
+	if p == nil || setup == nil {
+		return
+	}
+	if v, ok := setup["parse_method"].(string); ok && v != "" {
+		p.ParseMethod = v
+	}
+	if v, ok := setup["output_format"].(string); ok && v != "" {
+		p.OutputFormat = v
+	}
+	if v, ok := setup["vlm"].(map[string]any); ok {
+		p.VLM = v
+	}
 }

 // ParseWithResult implements ParseResultProducer (plan §6.5) and
 // returns a structured markdown payload that mirrors the Python
 // parser's `output_format == "json"` shape. Each top-level block
-// emits one item with `text` + `doc_type_kwd: "text"`. The legacy
-// debug-print path has been removed; callers consume ParseResult directly.
+// emits one item with `text` + `doc_type_kwd: "text"`. When the
+// block contains a markdown image reference (![alt](src)), the image
+// data is resolved and the item carries `doc_type_kwd: "image"` with
+// the base64-encoded image payload. The legacy debug-print path has
+// been removed; callers consume ParseResult directly.
 func (p *MarkdownParser) ParseWithResult(filename string, data []byte) ParseResult {
 	doc := markdownNew().Parse(data)
+	rawText := string(data)

 	var items []map[string]any
-	walkMarkdownBlocks(doc, &items)
+	walkMarkdownBlocksWithImages(doc, rawText, &items)
 	if items == nil {
-		// No blocks emitted — surface a single empty item so the
-		// downstream chunker sees a non-nil JSON slice (mirrors the
-		// Python contract of always producing at least one item).
 		items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
 	}
 	return ParseResult{
@@ -66,55 +118,222 @@ func markdownNew() *parser.Parser {
 	return parser.NewWithExtensions(extensions)
 }

-// walkMarkdownBlocks emits one normalized item per top-level block.
-// Headings (LeafBlock / H*) are emitted with the heading text so a
-// downstream title chunker can find them; paragraphs (Paragraph)
-// emit the leaf-node text. The walker is intentionally a
-// best-effort pass — full TOC / outline handling lands with the
-// deepdoc/parser port — but it satisfies the per-block "emit
-// text+doc_type_kwd" contract enough for a Phase-1a migration
-// test to verify wire-shape parity.
-func walkMarkdownBlocks(doc ast.Node, out *[]map[string]any) {
+// walkMarkdownBlocksWithImages emits one normalized item per
+// top-level block. Headings, paragraphs, lists, and code blocks are
+// emitted with their text. When a block contains a markdown image
+// reference (![alt](src)), the image data is resolved via
+// resolveMarkdownImage and the item carries `doc_type_kwd: "image"`
+// together with the base64-encoded image payload.
+func walkMarkdownBlocksWithImages(doc ast.Node, rawText string, out *[]map[string]any) {
 	for _, child := range doc.GetChildren() {
+		var ckType string
+		var docTypeKwd string
+		var txt string
+
 		switch n := child.(type) {
 		case *ast.Heading:
-			*out = append(*out, map[string]any{
-				"text":         headingText(n),
-				"doc_type_kwd": "text",
-				"ck_type":      "heading",
-			})
+			txt = headingText(n)
+			ckType = "heading"
+			docTypeKwd = "text"
 		case *ast.Paragraph:
-			*out = append(*out, map[string]any{
-				"text":         leafText(n),
-				"doc_type_kwd": "text",
-				"ck_type":      "text",
-			})
+			txt = leafText(n)
+			ckType = "text"
+			docTypeKwd = "text"
 		case *ast.List:
-			*out = append(*out, map[string]any{
-				"text":         leafText(n),
-				"doc_type_kwd": "text",
-				"ck_type":      "list",
-			})
+			txt = leafText(n)
+			ckType = "list"
+			docTypeKwd = "text"
 		case *ast.CodeBlock:
-			*out = append(*out, map[string]any{
-				"text":         leafText(n),
-				"doc_type_kwd": "text",
-				"ck_type":      "code",
-			})
+			txt = leafText(n)
+			ckType = "code"
+			docTypeKwd = "text"
 		default:
-			// Block types we don't yet normalize (HTML, tables,
-			// images, definitions) are best-effort: emit the leaf
-			// text without a ck_type so downstream components can
-			// still treat them as text chunks.
-			txt := leafText(n)
-			if strings.TrimSpace(txt) != "" {
-				*out = append(*out, map[string]any{
-					"text":         txt,
-					"doc_type_kwd": "text",
-				})
+			txt = leafText(n)
+			if strings.TrimSpace(txt) == "" {
+				continue
 			}
+			docTypeKwd = "text"
+		}
+
+		item := map[string]any{
+			"text":         txt,
+			"doc_type_kwd": docTypeKwd,
+		}
+		if ckType != "" {
+			item["ck_type"] = ckType
+		}
+
+		// Detect markdown image references in the raw source text
+		// that corresponds to this block. When found, resolve the
+		// image data so downstream vision enhancement can describe it.
+		if imgData, imgFound := resolveMarkdownImage(txt, rawText); imgFound && imgData != "" {
+			item["doc_type_kwd"] = "image"
+			item["image"] = imgData
+		}
+
+		*out = append(*out, item)
+	}
+}
+
+// resolveMarkdownImage extracts the first markdown image reference
+// from the given text and returns its base64-encoded data. Supports:
+//   - data:image/... URIs → decoded directly
+//   - http:// / https:// URLs → fetched (with basic SSRF filtering)
+//
+// Returns (base64String, true) on success, ("", false) when no image
+// is found or resolution fails.
+func resolveMarkdownImage(leafText, rawFullText string) (string, bool) {
+	// Prefer matching against the raw full text to catch images
+	// whose alt-text was split across markdown rendering.
+	searchIn := rawFullText
+	if strings.TrimSpace(searchIn) == "" {
+		searchIn = leafText
+	}
+	matches := mdImagePattern.FindStringSubmatch(searchIn)
+	if len(matches) < 2 {
+		return "", false
+	}
+	url := matches[1]
+
+	if strings.HasPrefix(url, dataURIPrefix) {
+		// data:image/png;base64,xxxx
+		idx := strings.Index(url, "base64,")
+		if idx < 0 {
+			return "", false
+		}
+		return url[idx+len("base64,"):], true
+	}
+	if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
+		b64, err := fetchImageAsBase64(url)
+		if err != nil {
+			return "", false
+		}
+		return b64, true
+	}
+	// Local / relative paths — not fetched for security.
+	return "", false
+}
+
+// fetchImageAsBase64 fetches an HTTP(S) image URL and returns its
+// content as a base64-encoded string. Local/private addresses and
+// redirects to them are rejected (SSRF guard). Hostnames are resolved
+// once and the validated IP is pinned in a custom DialContext to
+// prevent DNS-rebinding TOCTOU attacks.
+func fetchImageAsBase64(rawURL string) (string, error) {
+	rawURL = strings.TrimSpace(rawURL)
+
+	parsed, err := url.Parse(rawURL)
+	if err != nil {
+		return "", fmt.Errorf("markdown: invalid image URL: %w", err)
+	}
+
+	// pinned maps hostname (without port) → validated IP. The hostname
+	// is resolved once per host, and the transport dials the pinned IP
+	// directly instead of re-resolving DNS.
+	var pinnedMu sync.Mutex
+	pinned := make(map[string]net.IP)
+
+	pinHost := func(host string) error {
+		ip, err := resolveAndValidateHost(host)
+		if err != nil {
+			return err
+		}
+		h, _, _ := net.SplitHostPort(host)
+		if h == "" {
+			h = host
+		}
+		pinnedMu.Lock()
+		pinned[h] = ip
+		pinnedMu.Unlock()
+		return nil
+	}
+
+	if err := pinHost(parsed.Host); err != nil {
+		return "", err
+	}
+
+	transport := &http.Transport{
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			host, port, err := net.SplitHostPort(addr)
+			if err != nil {
+				return nil, err
+			}
+			pinnedMu.Lock()
+			ip, ok := pinned[host]
+			pinnedMu.Unlock()
+			if ok {
+				return (&net.Dialer{}).DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
+			}
+			return (&net.Dialer{}).DialContext(ctx, network, addr)
+		},
+	}
+
+	client := &http.Client{
+		Transport: transport,
+		Timeout:   30 * time.Second,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) >= 5 {
+				return fmt.Errorf("markdown: too many redirects")
+			}
+			return pinHost(req.URL.Host)
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return "", fmt.Errorf("markdown: create image request: %w", err)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("markdown: fetch image %s: %w", rawURL, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", fmt.Errorf("markdown: fetch image %s: HTTP %d", rawURL, resp.StatusCode)
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024)) // 32 MiB cap
+	if err != nil {
+		return "", fmt.Errorf("markdown: read image %s: %w", rawURL, err)
+	}
+	return base64.StdEncoding.EncodeToString(body), nil
+}
+
+// resolveAndValidateHost resolves a host (which may include a port),
+// validates none of its IPs are internal/private, and returns the
+// first public IP for connection pinning.
+func resolveAndValidateHost(host string) (net.IP, error) {
+	hostname := host
+	if h, _, err := net.SplitHostPort(host); err == nil {
+		hostname = h
+	}
+
+	ip := net.ParseIP(hostname)
+	if ip != nil {
+		if (ip.IsLoopback() && !ssrfAllowLoopback) || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
+			ip.IsPrivate() || ip.IsUnspecified() {
+			return nil, fmt.Errorf("markdown: rejected image URL to internal address: %s", host)
+		}
+		return ip, nil
+	}
+
+	addrs, err := net.DefaultResolver.LookupIPAddr(context.Background(), hostname)
+	if err != nil {
+		return nil, fmt.Errorf("markdown: cannot resolve image host: %s", hostname)
+	}
+	for _, addr := range addrs {
+		ip := addr.IP
+		if (ip.IsLoopback() && !ssrfAllowLoopback) || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
+			ip.IsPrivate() || ip.IsUnspecified() {
+			return nil, fmt.Errorf("markdown: rejected image URL resolving to internal address: %s (%s)", host, ip)
 		}
 	}
+	if len(addrs) == 0 {
+		return nil, fmt.Errorf("markdown: no addresses resolved for host: %s", hostname)
+	}
+	return addrs[0].IP, nil
 }

 // headingText returns the inline-text of a heading node by