From c77741e7a8097310f3112486263e5062992a2d09 Mon Sep 17 00:00:00 2001 From: Wang Qi Date: Wed, 29 Jul 2026 09:39:47 +0800 Subject: [PATCH] Porting change from #17477 python to go, parse table into chunk for markdown file (#17488) Porting change from #17477 python to go, parse table into chunk for markdown file Before: image After: image --- internal/parser/parser/markdown_parser.go | 120 +++++++++++++++++- .../parser/parser/markdown_parser_test.go | 27 ++++ 2 files changed, 142 insertions(+), 5 deletions(-) diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go index 1457f5de45..54f9f838f0 100644 --- a/internal/parser/parser/markdown_parser.go +++ b/internal/parser/parser/markdown_parser.go @@ -30,8 +30,9 @@ import ( "sync" "time" + markdownlib "github.com/gomarkdown/markdown" "github.com/gomarkdown/markdown/ast" - "github.com/gomarkdown/markdown/parser" + mdparser "github.com/gomarkdown/markdown/parser" ) // mdImagePattern matches markdown inline image syntax: ![alt](url). @@ -94,8 +95,18 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) { // the base64-encoded image payload. The legacy debug-print path has // been removed; callers consume ParseResult directly. func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult { - doc := markdownNew().Parse(data) rawText := string(data) + if rendered, ok := renderMarkdownTablesInline(rawText); ok { + return ParseResult{ + OutputFormat: "json", + File: map[string]any{ + "name": filename, + }, + JSON: []map[string]any{{"text": rendered, "doc_type_kwd": "text"}}, + } + } + + doc := markdownNew().Parse(data) var items []map[string]any walkMarkdownBlocksWithImages(doc, rawText, &items, p.FlattenMediaToText) @@ -117,9 +128,108 @@ func (p *MarkdownParser) String() string { // markdownNew is a thin constructor so the extension set is owned // in one place (both Parse and ParseWithResult consume it). -func markdownNew() *parser.Parser { - extensions := parser.CommonExtensions | parser.AutoHeadingIDs | parser.NoEmptyLineBeforeBlock - return parser.NewWithExtensions(extensions) +func markdownNew() *mdparser.Parser { + extensions := mdparser.CommonExtensions | mdparser.AutoHeadingIDs | mdparser.NoEmptyLineBeforeBlock + return mdparser.NewWithExtensions(extensions) +} + +func renderMarkdownTablesInline(text string) (string, bool) { + lines := strings.SplitAfter(strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n"), "\n") + var buf strings.Builder + changed := false + inFence := false + var fenceChar byte + var fenceLen int + + for i := 0; i < len(lines); { + line := strings.TrimRight(lines[i], "\n") + if ch, n, ok := markdownFenceMarker(line); ok { + if inFence && ch == fenceChar && n >= fenceLen { + inFence = false + } else if !inFence { + inFence = true + fenceChar = ch + fenceLen = n + } + buf.WriteString(lines[i]) + i++ + continue + } + if !inFence && i+1 < len(lines) && isMarkdownTableRow(line) && isMarkdownTableSeparator(strings.TrimRight(lines[i+1], "\n")) { + start := i + i += 2 + for i < len(lines) && isMarkdownTableRow(strings.TrimRight(lines[i], "\n")) { + i++ + } + for i < len(lines) && strings.TrimSpace(lines[i]) == "" { + i++ + } + tableHTML := markdownlib.ToHTML([]byte(strings.Join(lines[start:i], "")), markdownNew(), nil) + buf.WriteString(strings.TrimRight(string(tableHTML), "\r\n")) + buf.WriteByte('\n') + changed = true + continue + } + buf.WriteString(lines[i]) + i++ + } + return buf.String(), changed +} + +func markdownFenceMarker(line string) (byte, int, bool) { + trimmed := strings.TrimLeft(line, " \t") + if len(line)-len(trimmed) > 3 || len(trimmed) < 3 { + return 0, 0, false + } + ch := trimmed[0] + if ch != '`' && ch != '~' { + return 0, 0, false + } + n := 0 + for n < len(trimmed) && trimmed[n] == ch { + n++ + } + return ch, n, n >= 3 +} + +func isMarkdownTableRow(line string) bool { + cells := markdownTableCells(line) + if len(cells) < 2 { + return false + } + for _, cell := range cells { + if strings.TrimSpace(cell) != "" { + return true + } + } + return false +} + +func isMarkdownTableSeparator(line string) bool { + cells := markdownTableCells(line) + if len(cells) < 2 { + return false + } + for _, cell := range cells { + cell = strings.ReplaceAll(strings.TrimSpace(cell), " ", "") + if cell == "" { + return false + } + cell = strings.TrimPrefix(strings.TrimSuffix(cell, ":"), ":") + if strings.Trim(cell, "-") != "" || !strings.Contains(cell, "-") { + return false + } + } + return true +} + +func markdownTableCells(line string) []string { + line = strings.TrimSpace(line) + if !strings.Contains(line, "|") { + return nil + } + line = strings.TrimPrefix(strings.TrimSuffix(line, "|"), "|") + return strings.Split(line, "|") } // walkMarkdownBlocksWithImages emits one normalized item per diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go index 8247710ddd..64afa19b56 100644 --- a/internal/parser/parser/markdown_parser_test.go +++ b/internal/parser/parser/markdown_parser_test.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "net/http" "net/http/httptest" + "strings" "testing" ) @@ -88,6 +89,32 @@ func TestMarkdownParser_ParseWithResult_NoImage(t *testing.T) { } } +func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) { + ctx := t.Context() + p, _ := NewMarkdownParser(GoMarkdown) + md := "[M03] Health check package comparison:\n\n| Check item | Basic 699 CNY | Advanced 1299 CNY |\n| --- | --- | --- |\n| Blood routine / Urine routine | Yes | Yes |\n\nNote: All packages require fasting.\n" + res := p.ParseWithResult(ctx, "test.md", []byte(md)) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if len(res.JSON) != 1 { + t.Fatalf("len(JSON) = %d, want 1", len(res.JSON)) + } + text, _ := res.JSON[0]["text"].(string) + if got, _ := res.JSON[0]["doc_type_kwd"].(string); got != "text" { + t.Fatalf("doc_type_kwd = %q, want text", got) + } + if !strings.Contains(text, "") || !strings.Contains(text, "") { + t.Fatalf("table was not rendered inline: %q", text) + } + if strings.Contains(text, "| Check item |") { + t.Fatalf("raw markdown table leaked into text: %q", text) + } + if gap := text[strings.Index(text, "
Check item
"):strings.Index(text, "Note:")]; gap != "\n" { + t.Fatalf("gap after table = %q, want %q", gap, "\n") + } +} + func TestMarkdownParser_ConfigureFromSetup(t *testing.T) { p, _ := NewMarkdownParser(GoMarkdown) p.ConfigureFromSetup(map[string]any{