Align Go parser backends and PDF pipeline with Python (#16676)

Ports remaining Go parser wiring and PDF backends, adds tenant-aware VLM dispatch, aligns post-processing with Python, and adds end-to-end pipeline coverage with a generated six-page PDF.
2026-07-30 04:29:24 +08:00 · 2026-07-06 19:50:54 +08:00
parent 3044283442
commit 55af1d70f3
36 changed files with 4918 additions and 28 deletions
--- a/internal/parser/parser/pdf_parser_somark_test.go
+++ b/internal/parser/parser/pdf_parser_somark_test.go
@@ -0,0 +1,140 @@
+package parser
+
+import (
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestPDFParser_ParseWithResult_SoMarkJSONIntegration(t *testing.T) {
+	var submitSeen bool
+	var pollSeen bool
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/parse/async":
+			submitSeen = true
+			reader, err := r.MultipartReader()
+			if err != nil {
+				t.Errorf("MultipartReader: %v", err)
+				return
+			}
+			fields := map[string]string{}
+			for {
+				part, err := reader.NextPart()
+				if err == io.EOF {
+					break
+				}
+				if err != nil {
+					t.Errorf("NextPart: %v", err)
+					return
+				}
+				body, _ := io.ReadAll(part)
+				fields[part.FormName()] = string(body)
+			}
+			if fields["api_key"] != "somark-secret" {
+				t.Errorf("api_key = %q, want somark-secret", fields["api_key"])
+				return
+			}
+			if !strings.Contains(fields["element_formats"], "image") {
+				t.Errorf("element_formats = %q", fields["element_formats"])
+				return
+			}
+			_, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task-1"}}`))
+		case "/parse/async_check":
+			pollSeen = true
+			body, _ := io.ReadAll(r.Body)
+			if !strings.Contains(string(body), "task_id=task-1") {
+				t.Errorf("poll body = %q, want task_id", string(body))
+				return
+			}
+			_, _ = w.Write([]byte(`{"code":0,"data":{"status":"SUCCESS","result":{"outputs":{"json":{"pages":[{"blocks":[{"type":"title","content":"SoMark Title","title_level":2},{"type":"figure","content":"Figure caption"},{"type":"table","content":"<table><tr><td>x</td></tr></table>"}]}]}}}}}`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+
+	pdf := NewPDFParser()
+	pdf.ConfigureFromSetup(map[string]any{
+		"parse_method":    "SoMark",
+		"output_format":   "json",
+		"somark_base_url": server.URL,
+		"somark_api_key":  "somark-secret",
+	})
+	res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+	if !submitSeen || !pollSeen {
+		t.Fatalf("submit/poll seen = %v/%v, want true/true", submitSeen, pollSeen)
+	}
+	if len(res.JSON) < 3 {
+		t.Fatalf("JSON len = %d, want >=3", len(res.JSON))
+	}
+}
+
+func TestPDFParser_ParseWithResult_SoMarkMarkdownIntegration(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/parse/async":
+			_, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task-2"}}`))
+		case "/parse/async_check":
+			_, _ = w.Write([]byte(`{"code":0,"data":{"status":"SUCCESS","result":{"outputs":{"json":{"pages":[{"blocks":[{"type":"title","content":"SoMark Title","title_level":1},{"type":"text","content":"Body"}]}]}}}}}`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+	pdf := NewPDFParser()
+	pdf.ConfigureFromSetup(map[string]any{
+		"parse_method":    "SoMark",
+		"output_format":   "markdown",
+		"somark_base_url": server.URL,
+	})
+	res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+	if !strings.Contains(res.Markdown, "SoMark Title") {
+		t.Fatalf("Markdown = %q, want title", res.Markdown)
+	}
+}
+
+func TestPDFParser_ParseWithResult_SoMarkRequiresBaseURL(t *testing.T) {
+	pdf := NewPDFParser()
+	pdf.ConfigureFromSetup(map[string]any{"parse_method": "SoMark"})
+	pdf.SoMarkBaseURL = ""
+	res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
+	if res.Err == nil || !strings.Contains(res.Err.Error(), "somark_base_url") {
+		t.Fatalf("error = %v, want somark_base_url context", res.Err)
+	}
+}
+
+func TestSoMarkBlockToItem_DropsHeaderByDefault(t *testing.T) {
+	if item := soMarkBlockToItem(map[string]any{"type": "header", "content": "x"}, false); item != nil {
+		t.Fatalf("item = %#v, want nil", item)
+	}
+}
+
+func TestSoMarkSubmitMultipartShape(t *testing.T) {
+	var form multipart.Form
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseMultipartForm(1 << 20)
+		form = *r.MultipartForm
+		_, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task-3"}}`))
+	}))
+	defer server.Close()
+	taskID, err := soMarkSubmit(server.URL, "sample.pdf", []byte("%PDF"), NewPDFParser(), "key")
+	if err != nil {
+		t.Fatalf("soMarkSubmit: %v", err)
+	}
+	if taskID != "task-3" {
+		t.Fatalf("taskID = %q, want task-3", taskID)
+	}
+	if got := form.Value["api_key"][0]; got != "key" {
+		t.Fatalf("api_key = %q, want key", got)
+	}
+}