mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-23 17:06:42 +08:00
## Summary Three groups of changes across the Go ingestion pipeline: ### 1. DOCX parsing improvements - **docx_parser.go**: Enhanced DOCX parsing with better structure extraction and media handling - **docx_parser_cgo_test.go**, **docx_parser_test.go**: Companion tests - **office_parsers_no_cgo.go**: Stub sync for non-CGO builds ### 2. Email (.eml) parsing: base64 Content-Transfer-Encoding decoding - **email_parser.go** (`decodeCTE`): Added Content-Transfer-Encoding decoding for base64 and quoted-printable. Go's `mime/multipart.Reader` does not decode Content-Transfer-Encoding automatically, so attachments with `Content-Transfer-Encoding: base64` remained base64-encoded in the output. The new `decodeCTE` helper is called after reading each multipart part's raw bytes in `readMailBody`, mirroring Python's `part.get_payload(decode=True)`. - **email_parser_test.go**: Two new tests — simple base64 attachment and nested multipart/alternative with base64 attachment. ### 3. Extractor LLM driver fix + ModelDriver consolidation - **extractor.go**: Fixed a bug where the Extractor component used `ModelFactory.CreateModelDriver()`, which creates bare model instances without API keys or provider configuration. Switched to `models.GetPreconfiguredDriver()` which resolves the actual pre-configured driver from `ProviderManager`, matching the codepath used by `llm.go`. This fixes auto keyword/question extraction in DSL pipelines that require LLM calls. - **get_driver.go** (new): Extracted shared `GetPreconfiguredDriver()` from `llm.go:newChatModelDriver()` so both `llm.go` and `extractor.go` use the same codepath. - **get_driver_test.go** (new): Tests for the shared driver resolution. - **llm.go**: Replaced inline driver resolution with `models.GetPreconfiguredDriver()`. ### 4. Chunker fixes and observability - **group.go** (`extractLineRecords`): Fixed to also read `markdown` and `html` payload keys — previously it only read `text`/`content`, causing GroupTitleChunker to silently return empty results for markdown-format parser output. - **common.go** (`compileDelimPattern`): Aligned with Python's `_compile_delimiter_pattern` — only backtick-wrapped delimiters produce an active regex pattern; plain delimiters are not compiled into the split regex. - **token.go** (`applyChildrenDelim`): Set `DocType` and `CKType` to `"text"` on created ChunkDocs so the token-size merge path correctly identifies and merges text segments. - **parser.go**, **extractor.go**, **tokenizer.go**, **group.go**, **hierarchy.go**: Added debug-level logging for pipeline diagnostics. - **parser_dispatch_test.go**, **group_test.go**: New tests. ## Verification - All Go tests pass: `bash build.sh --test ./internal/parser/parser/...` and `bash build.sh --test ./internal/ingestion/component/...` - Build succeeds: `bash build.sh --go`
128 lines
4.0 KiB
Go
128 lines
4.0 KiB
Go
//go:build cgo
|
|
|
|
package parser
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"testing"
|
|
)
|
|
|
|
func TestDOCXParser_ParseWithResult_JSON(t *testing.T) {
|
|
p := NewDOCXParser()
|
|
p.outputFormat = "json"
|
|
data := minimalDOCX(t, "Hello from JSON path")
|
|
res := p.ParseWithResult("sample.docx", data)
|
|
if res.Err != nil {
|
|
t.Fatalf("ParseWithResult: %v", res.Err)
|
|
}
|
|
if got, want := res.OutputFormat, "json"; got != want {
|
|
t.Fatalf("OutputFormat = %q, want %q", got, want)
|
|
}
|
|
if len(res.JSON) == 0 {
|
|
t.Fatal("JSON items is empty; expected parsed content")
|
|
}
|
|
for i, item := range res.JSON {
|
|
if _, ok := item["text"]; !ok {
|
|
t.Errorf("item[%d] missing 'text' field", i)
|
|
}
|
|
if _, ok := item["doc_type_kwd"]; !ok {
|
|
t.Errorf("item[%d] missing 'doc_type_kwd' field", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDOCXParser_ConfigureFromSetup_JSON(t *testing.T) {
|
|
p := NewDOCXParser()
|
|
p.ConfigureFromSetup(map[string]any{"output_format": "json"})
|
|
if p.outputFormat != "json" {
|
|
t.Fatalf("After ConfigureFromSetup, outputFormat = %q, want %q", p.outputFormat, "json")
|
|
}
|
|
if p.libType != "" {
|
|
t.Errorf("libType unexpectedly = %q", p.libType)
|
|
}
|
|
// Full round-trip: json config → json output
|
|
data := minimalDOCX(t, "Config test")
|
|
res := p.ParseWithResult("sample.docx", data)
|
|
if res.Err != nil {
|
|
t.Fatalf("ParseWithResult: %v", res.Err)
|
|
}
|
|
if res.OutputFormat != "json" {
|
|
t.Errorf("OutputFormat = %q, want %q", res.OutputFormat, "json")
|
|
}
|
|
if len(res.JSON) == 0 {
|
|
t.Error("JSON items is empty")
|
|
}
|
|
}
|
|
|
|
func TestDOCXParser_ConfigureFromSetup_Markdown(t *testing.T) {
|
|
p := NewDOCXParser()
|
|
p.ConfigureFromSetup(map[string]any{"output_format": "markdown"})
|
|
if p.outputFormat != "markdown" {
|
|
t.Fatalf("After ConfigureFromSetup, outputFormat = %q, want %q", p.outputFormat, "markdown")
|
|
}
|
|
data := minimalDOCX(t, "Config md test")
|
|
res := p.ParseWithResult("sample.docx", data)
|
|
if res.Err != nil {
|
|
t.Fatalf("ParseWithResult: %v", res.Err)
|
|
}
|
|
if res.OutputFormat != "markdown" {
|
|
t.Errorf("OutputFormat = %q, want %q", res.OutputFormat, "markdown")
|
|
}
|
|
if res.Markdown == "" {
|
|
t.Error("Markdown is empty")
|
|
}
|
|
}
|
|
|
|
func TestDOCXParser_ParseWithResult_CGOMinimalDocument(t *testing.T) {
|
|
p := NewDOCXParser()
|
|
data := minimalDOCX(t, "Hello from DOCX parser")
|
|
res := p.ParseWithResult("sample.docx", data)
|
|
if res.Err != nil {
|
|
t.Fatalf("ParseWithResult: %v", res.Err)
|
|
}
|
|
if got, want := res.OutputFormat, "markdown"; got != want {
|
|
t.Fatalf("OutputFormat = %q, want %q", got, want)
|
|
}
|
|
if res.Markdown == "" {
|
|
t.Fatal("Markdown is empty; want parsed content")
|
|
}
|
|
}
|
|
|
|
func minimalDOCX(t *testing.T, text string) []byte {
|
|
t.Helper()
|
|
var buf bytes.Buffer
|
|
zw := zip.NewWriter(&buf)
|
|
writeZipFile(t, zw, "[Content_Types].xml", `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
|
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
|
|
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
|
|
<Default Extension="xml" ContentType="application/xml"/>
|
|
<Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
|
|
</Types>`)
|
|
writeZipFile(t, zw, "_rels/.rels", `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
|
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
|
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
|
|
</Relationships>`)
|
|
writeZipFile(t, zw, "word/document.xml", `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
|
<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
|
|
<w:body>
|
|
<w:p><w:r><w:t>`+text+`</w:t></w:r></w:p>
|
|
</w:body>
|
|
</w:document>`)
|
|
if err := zw.Close(); err != nil {
|
|
t.Fatalf("zip close: %v", err)
|
|
}
|
|
return buf.Bytes()
|
|
}
|
|
|
|
func writeZipFile(t *testing.T, zw *zip.Writer, name, body string) {
|
|
t.Helper()
|
|
w, err := zw.Create(name)
|
|
if err != nil {
|
|
t.Fatalf("create zip entry %s: %v", name, err)
|
|
}
|
|
if _, err := w.Write([]byte(body)); err != nil {
|
|
t.Fatalf("write zip entry %s: %v", name, err)
|
|
}
|
|
}
|