mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-02 13:57:30 +08:00
0403d19b5a
## Summary Fix the Go ingestion pipeline so that several parser setup switches and the image VLM prompt are actually honored end-to-end (previously the DSL fields existed but the Go code never read them). - **DOCX** (`docx_parser.go`, `docx_postprocess.go`): read `remove_toc` and `remove_header_footer`; apply to both JSON and markdown output paths (outline-based TOC removal with a text-heuristic fallback, plus header/footer section filtering). - **HTML** (`html_parser.go`, `html_postprocess.go`, `text_toc.go`): read `remove_header_footer` (pre-parse strip of `<header>`/`<footer>` and ARIA `banner`/`contentinfo`) and `remove_toc` (post-parse `remove_contents_table` heuristic). - **Markdown** (`markdown_parser.go`): read `flatten_media_to_text` and force media blocks to text when enabled. - **Image VLM** (`media_dispatch.go`): read `system_prompt` instead of `prompt` so the user-configured image VLM prompt is no longer silently dropped (`prompt` remains the video family key). All flags are wired through `ConfigureFromSetup`, which the dispatch layer already invokes for every family, so the behavior is live rather than dead code. ## Test plan - New unit tests: `docx_postprocess_test.go`, `html_parser_test.go`, `text_toc_test.go`, `markdown_parser_test.go`, `media_dispatch_test.go`. - `bash build.sh --test ./internal/parser/parser/... ./internal/ingestion/component/...` ## Notes - The `File` component is excluded from this migration scope. - Relates to the Python→Go parity diff (Parser 1.8–1.11, 1.15).
139 lines
3.8 KiB
Go
139 lines
3.8 KiB
Go
package parser
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestHTMLParser_ConfigureFromSetup(t *testing.T) {
|
|
p := NewHTMLParser()
|
|
p.ConfigureFromSetup(map[string]any{
|
|
"remove_header_footer": true,
|
|
"remove_toc": true,
|
|
})
|
|
if !p.RemoveHeaderFooter {
|
|
t.Error("RemoveHeaderFooter = false, want true")
|
|
}
|
|
if !p.RemoveTOC {
|
|
t.Error("RemoveTOC = false, want true")
|
|
}
|
|
}
|
|
|
|
func TestHTMLParser_ConfigureFromSetup_NilSafe(t *testing.T) {
|
|
p := NewHTMLParser()
|
|
p.ConfigureFromSetup(nil) // should not panic
|
|
if p.RemoveHeaderFooter || p.RemoveTOC {
|
|
t.Error("flags should stay false after nil setup")
|
|
}
|
|
}
|
|
|
|
// TestHTMLParser_RemoveHeaderFooter verifies that <header>/<footer>
|
|
// tags and role="banner"/role="contentinfo" elements are stripped
|
|
// before parsing (mirrors Python parser.py:1083-1084 pre-parse).
|
|
func TestHTMLParser_RemoveHeaderFooter(t *testing.T) {
|
|
|
|
p := NewHTMLParser()
|
|
p.ConfigureFromSetup(map[string]any{"remove_header_footer": true})
|
|
|
|
html := `<html><body>
|
|
<header>Site Navigation</header>
|
|
<main>
|
|
<p>Main content here</p>
|
|
<div role="banner">Ad banner</div>
|
|
<footer>Copyright 2024</footer>
|
|
<div role="contentinfo">Legal info</div>
|
|
</main>
|
|
</body></html>`
|
|
|
|
res := p.ParseWithResult(context.Background(), "test.html", []byte(html))
|
|
if res.Err != nil {
|
|
t.Fatalf("ParseWithResult: %v", res.Err)
|
|
}
|
|
for _, item := range res.JSON {
|
|
text, _ := item["text"].(string)
|
|
for _, banned := range []string{"Site Navigation", "Copyright 2024", "Ad banner", "Legal info"} {
|
|
if strings.Contains(text, banned) {
|
|
t.Errorf("header/footer text %q leaked into output item: %q", banned, text)
|
|
}
|
|
}
|
|
}
|
|
// Main content must survive.
|
|
found := false
|
|
for _, item := range res.JSON {
|
|
if text, _ := item["text"].(string); strings.Contains(text, "Main content here") {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("main content was stripped along with header/footer")
|
|
}
|
|
}
|
|
|
|
// TestHTMLParser_RemoveHeaderFooter_DisabledByDefault verifies that
|
|
// without the flag, <header>/<footer> content is preserved.
|
|
func TestHTMLParser_RemoveHeaderFooter_DisabledByDefault(t *testing.T) {
|
|
|
|
p := NewHTMLParser()
|
|
|
|
html := `<html><body><header>Nav</header><p>Body</p></body></html>`
|
|
res := p.ParseWithResult(context.Background(), "test.html", []byte(html))
|
|
if res.Err != nil {
|
|
t.Fatalf("ParseWithResult: %v", res.Err)
|
|
}
|
|
found := false
|
|
for _, item := range res.JSON {
|
|
if text, _ := item["text"].(string); strings.Contains(text, "Nav") {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("header text should be preserved when flag is off")
|
|
}
|
|
}
|
|
|
|
// TestHTMLParser_RemoveTOC verifies that TOC sections are filtered
|
|
// after parsing (mirrors Python parser.py:1087-1088 post-parse).
|
|
func TestHTMLParser_RemoveTOC(t *testing.T) {
|
|
|
|
p := NewHTMLParser()
|
|
p.ConfigureFromSetup(map[string]any{"remove_toc": true})
|
|
|
|
html := `<html><body>
|
|
<h1>前言</h1>
|
|
<h1>目录</h1>
|
|
<p>第一章 概述</p>
|
|
<p>第二章 方法</p>
|
|
<p>正文开始</p>
|
|
</body></html>`
|
|
|
|
res := p.ParseWithResult(context.Background(), "test.html", []byte(html))
|
|
if res.Err != nil {
|
|
t.Fatalf("ParseWithResult: %v", res.Err)
|
|
}
|
|
// "目录" heading and "第一章 概述" (first entry after heading)
|
|
// should be removed by removeContentsTable.
|
|
for _, item := range res.JSON {
|
|
text, _ := item["text"].(string)
|
|
if text == "目录" {
|
|
t.Errorf("TOC heading %q was not removed", text)
|
|
}
|
|
if strings.HasPrefix(text, "第一章") {
|
|
t.Errorf("first TOC entry %q was not removed", text)
|
|
}
|
|
}
|
|
// "前言" and "正文开始" should survive.
|
|
wantTexts := map[string]bool{"前言": false, "正文开始": false}
|
|
for _, item := range res.JSON {
|
|
text, _ := item["text"].(string)
|
|
if _, ok := wantTexts[text]; ok {
|
|
wantTexts[text] = true
|
|
}
|
|
}
|
|
for text, found := range wantTexts {
|
|
if !found {
|
|
t.Errorf("expected %q to survive, but it was removed", text)
|
|
}
|
|
}
|
|
}
|