fix: docx/email parsing, extractor LLM driver, and chunker alignment (#17144)

## Summary

Three groups of changes across the Go ingestion pipeline:

### 1. DOCX parsing improvements
- **docx_parser.go**: Enhanced DOCX parsing with better structure
extraction and media handling
- **docx_parser_cgo_test.go**, **docx_parser_test.go**: Companion tests
- **office_parsers_no_cgo.go**: Stub sync for non-CGO builds

### 2. Email (.eml) parsing: base64 Content-Transfer-Encoding decoding
- **email_parser.go** (`decodeCTE`): Added Content-Transfer-Encoding
decoding for base64 and quoted-printable. Go's `mime/multipart.Reader`
does not decode Content-Transfer-Encoding automatically, so attachments
with `Content-Transfer-Encoding: base64` remained base64-encoded in the
output. The new `decodeCTE` helper is called after reading each
multipart part's raw bytes in `readMailBody`, mirroring Python's
`part.get_payload(decode=True)`.
- **email_parser_test.go**: Two new tests — simple base64 attachment and
nested multipart/alternative with base64 attachment.

### 3. Extractor LLM driver fix + ModelDriver consolidation
- **extractor.go**: Fixed a bug where the Extractor component used
`ModelFactory.CreateModelDriver()`, which creates bare model instances
without API keys or provider configuration. Switched to
`models.GetPreconfiguredDriver()` which resolves the actual
pre-configured driver from `ProviderManager`, matching the codepath used
by `llm.go`. This fixes auto keyword/question extraction in DSL
pipelines that require LLM calls.
- **get_driver.go** (new): Extracted shared `GetPreconfiguredDriver()`
from `llm.go:newChatModelDriver()` so both `llm.go` and `extractor.go`
use the same codepath.
- **get_driver_test.go** (new): Tests for the shared driver resolution.
- **llm.go**: Replaced inline driver resolution with
`models.GetPreconfiguredDriver()`.

### 4. Chunker fixes and observability
- **group.go** (`extractLineRecords`): Fixed to also read `markdown` and
`html` payload keys — previously it only read `text`/`content`, causing
GroupTitleChunker to silently return empty results for markdown-format
parser output.
- **common.go** (`compileDelimPattern`): Aligned with Python's
`_compile_delimiter_pattern` — only backtick-wrapped delimiters produce
an active regex pattern; plain delimiters are not compiled into the
split regex.
- **token.go** (`applyChildrenDelim`): Set `DocType` and `CKType` to
`"text"` on created ChunkDocs so the token-size merge path correctly
identifies and merges text segments.
- **parser.go**, **extractor.go**, **tokenizer.go**, **group.go**,
**hierarchy.go**: Added debug-level logging for pipeline diagnostics.
- **parser_dispatch_test.go**, **group_test.go**: New tests.

## Verification
- All Go tests pass: `bash build.sh --test ./internal/parser/parser/...`
and `bash build.sh --test ./internal/ingestion/component/...`
- Build succeeds: `bash build.sh --go`
This commit is contained in:
Jack
2026-07-21 13:51:17 +08:00
committed by GitHub
parent a730ce7aa6
commit bb6b43b5c9
18 changed files with 997 additions and 90 deletions

View File

@@ -22,6 +22,7 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
"html"
"strings"
officeOxide "github.com/yfedoseev/office_oxide/go"
@@ -38,20 +39,33 @@ type DOCXFigure struct {
}
type DOCXParser struct {
libType string
libType string
outputFormat string // from DSL config; "json" or "markdown"
}
func NewDOCXParser() *DOCXParser {
return &DOCXParser{}
}
// ParseWithResult captures the office_oxide ToMarkdown output
// and additionally extracts embedded images with their surrounding
// text context so the downstream vision-figure dispatch can enrich
// the markdown with LLM-generated image descriptions.
// ConfigureFromSetup implements parserSetupConfigurer, receiving the
// DSL "docx" family setup map. The output_format key drives whether
// ParseWithResult produces JSON items (structured) or markdown.
func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {
if p == nil || setup == nil {
return
}
if v, ok := setup["output_format"].(string); ok && v != "" {
p.outputFormat = v
}
}
// ParseWithResult produces structured JSON items (when
// p.outputFormat == "json") or markdown (default) from a
// docx document. Embedded images are extracted in both paths
// for downstream vision-figure dispatch.
//
// Mirrors python naive.py: Docx() → naive_merge_docx() →
// vision_figure_parser_docx_wrapper_naive().
// JSON path mirrors python parser.py:_docx() output_format == "json".
// Markdown path mirrors python naive.py: Docx() → naive_merge_docx().
func (p *DOCXParser) ParseWithResult(filename string, data []byte) ParseResult {
doc, err := officeOxide.OpenFromBytes(data, "docx")
if err != nil {
@@ -59,38 +73,43 @@ func (p *DOCXParser) ParseWithResult(filename string, data []byte) ParseResult {
}
defer doc.Close()
md, err := doc.ToMarkdown()
if err != nil {
return ParseResult{Err: fmt.Errorf("docx to-markdown: %w", err)}
}
fileMeta := map[string]any{
"name": filename,
"format": "docx",
}
// Extract embedded images with their text context from the
// office_oxide IR so the downstream vision dispatch can
// enrich them. The already-opened doc handle is reused
// (no second OpenFromBytes).
// Extract IR JSON for section building (JSON path) and
// embedded-image extraction (both paths).
irJSON, irErr := doc.ToIRJSON()
var figures []DOCXFigure
if irErr == nil {
figures = extractDOCXFiguresFromIR(irJSON)
}
if len(figures) > 0 {
figs := make([]map[string]any, 0, len(figures))
for _, f := range figures {
figs = append(figs, map[string]any{
"image": f.Image,
"context_above": f.ContextAbove,
"context_below": f.ContextBelow,
"marker": f.Marker,
})
}
fileMeta["figures"] = figs
fileMeta["figures"] = buildFiguresMap(figures)
}
if p.outputFormat == "json" {
if irErr != nil {
return ParseResult{Err: fmt.Errorf("docx to-ir-json: %w", irErr)}
}
var sections []map[string]any
sections = buildDOCXJSONSections(irJSON)
if len(sections) == 0 {
sections = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
return ParseResult{
OutputFormat: "json",
File: fileMeta,
JSON: sections,
}
}
// Default / markdown path.
md, err := doc.ToMarkdown()
if err != nil {
return ParseResult{Err: fmt.Errorf("docx to-markdown: %w", err)}
}
return ParseResult{
OutputFormat: "markdown",
File: fileMeta,
@@ -218,6 +237,101 @@ func joinDOCXIRRuns(runs []docxIRRun) string {
return b.String()
}
// buildFiguresMap converts the internal DOCXFigure slice to the
// map form attached to fileMeta["figures"].
func buildFiguresMap(figures []DOCXFigure) []map[string]any {
figs := make([]map[string]any, 0, len(figures))
for _, f := range figures {
figs = append(figs, map[string]any{
"image": f.Image,
"context_above": f.ContextAbove,
"context_below": f.ContextBelow,
"marker": f.Marker,
})
}
return figs
}
// joinCellText concatenates all paragraph texts inside a table cell,
// joined by newlines.
func joinCellText(cell docxIRCell) string {
var parts []string
for _, el := range cell.Content {
if text := joinDOCXIRRuns(el.Content); text != "" {
parts = append(parts, text)
}
}
return strings.Join(parts, "\n")
}
// docxIRTableToHTML converts a table IR element to an HTML table string.
func docxIRTableToHTML(el docxIRElement) string {
var sb strings.Builder
sb.WriteString("<table>")
for _, row := range el.Rows {
sb.WriteString("<tr>")
for _, cell := range row.Cells {
sb.WriteString("<td>")
sb.WriteString(html.EscapeString(joinCellText(cell)))
sb.WriteString("</td>")
}
sb.WriteString("</tr>")
}
sb.WriteString("</table>")
return sb.String()
}
// buildDOCXJSONSections converts an office_oxide IR JSON string into a
// slice of structured items compatible with the chunker's JSON input
// contract. Each item carries at least text and doc_type_kwd.
func buildDOCXJSONSections(irJSON string) []map[string]any {
var ir docxIRDocument
if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {
return nil
}
var sections []map[string]any
for _, sec := range ir.Sections {
for _, el := range sec.Elements {
switch el.Type {
case "paragraph", "heading":
text := joinDOCXIRRuns(el.Content)
if strings.TrimSpace(text) == "" {
continue
}
item := map[string]any{
"text": text,
"image": nil,
"doc_type_kwd": "text",
}
if el.Type == "heading" {
item["ck_type"] = "heading"
}
sections = append(sections, item)
case "image":
b64 := base64.StdEncoding.EncodeToString(el.Data)
sections = append(sections, map[string]any{
"text": "",
"image": b64,
"doc_type_kwd": "image",
})
case "table":
html := docxIRTableToHTML(el)
if html == "<table></table>" {
continue
}
sections = append(sections, map[string]any{
"text": html,
"image": nil,
"doc_type_kwd": "table",
})
}
}
}
return sections
}
// --- office_oxide IR types (local copy, independent of deepdoc) ---
type docxIRDocument struct {
@@ -230,14 +344,26 @@ type docxIRSection struct {
}
type docxIRElement struct {
Type string `json:"type"`
Content []docxIRRun `json:"content"`
Data []byte `json:"data"`
Type string `json:"type"` // "paragraph", "heading", "table", "image"
Level int `json:"level"` // heading level (1-6)
Style string `json:"style"` // Word style name (e.g. "Normal", "Heading 1")
Content []docxIRRun `json:"content"` // rich text runs
Data []byte `json:"data"` // raw image bytes (for "image" type)
Rows []docxIRRow `json:"rows"` // table rows
}
type docxIRRun struct {
Type string `json:"type"`
Text string `json:"text"`
Type string `json:"type"` // "text", "image"
Text string `json:"text"`
Content []docxIRElement `json:"content"` // nested elements (used in table cells)
}
type docxIRRow struct {
Cells []docxIRCell `json:"cells"`
}
type docxIRCell struct {
Content []docxIRElement `json:"content"` // nested paragraphs inside table cell
}
func (p *DOCXParser) String() string {

View File

@@ -8,6 +8,72 @@ import (
"testing"
)
func TestDOCXParser_ParseWithResult_JSON(t *testing.T) {
p := NewDOCXParser()
p.outputFormat = "json"
data := minimalDOCX(t, "Hello from JSON path")
res := p.ParseWithResult("sample.docx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if got, want := res.OutputFormat, "json"; got != want {
t.Fatalf("OutputFormat = %q, want %q", got, want)
}
if len(res.JSON) == 0 {
t.Fatal("JSON items is empty; expected parsed content")
}
for i, item := range res.JSON {
if _, ok := item["text"]; !ok {
t.Errorf("item[%d] missing 'text' field", i)
}
if _, ok := item["doc_type_kwd"]; !ok {
t.Errorf("item[%d] missing 'doc_type_kwd' field", i)
}
}
}
func TestDOCXParser_ConfigureFromSetup_JSON(t *testing.T) {
p := NewDOCXParser()
p.ConfigureFromSetup(map[string]any{"output_format": "json"})
if p.outputFormat != "json" {
t.Fatalf("After ConfigureFromSetup, outputFormat = %q, want %q", p.outputFormat, "json")
}
if p.libType != "" {
t.Errorf("libType unexpectedly = %q", p.libType)
}
// Full round-trip: json config → json output
data := minimalDOCX(t, "Config test")
res := p.ParseWithResult("sample.docx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if res.OutputFormat != "json" {
t.Errorf("OutputFormat = %q, want %q", res.OutputFormat, "json")
}
if len(res.JSON) == 0 {
t.Error("JSON items is empty")
}
}
func TestDOCXParser_ConfigureFromSetup_Markdown(t *testing.T) {
p := NewDOCXParser()
p.ConfigureFromSetup(map[string]any{"output_format": "markdown"})
if p.outputFormat != "markdown" {
t.Fatalf("After ConfigureFromSetup, outputFormat = %q, want %q", p.outputFormat, "markdown")
}
data := minimalDOCX(t, "Config md test")
res := p.ParseWithResult("sample.docx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if res.OutputFormat != "markdown" {
t.Errorf("OutputFormat = %q, want %q", res.OutputFormat, "markdown")
}
if res.Markdown == "" {
t.Error("Markdown is empty")
}
}
func TestDOCXParser_ParseWithResult_CGOMinimalDocument(t *testing.T) {
p := NewDOCXParser()
data := minimalDOCX(t, "Hello from DOCX parser")

View File

@@ -0,0 +1,210 @@
//go:build cgo
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package parser
import (
"encoding/base64"
"strings"
"testing"
)
func TestBuildDOCXJSONSections_Paragraphs(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"paragraph","content":[{"type":"text","text":"Hello world"}],"style":"Normal"}
]}]}`
sections := buildDOCXJSONSections(ir)
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1", len(sections))
}
item := sections[0]
if got, ok := item["text"].(string); !ok || got != "Hello world" {
t.Errorf("text = %q, want %q", got, "Hello world")
}
if got, ok := item["doc_type_kwd"].(string); !ok || got != "text" {
t.Errorf("doc_type_kwd = %q, want %q", got, "text")
}
}
func TestBuildDOCXJSONSections_Headings(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"heading","level":1,"content":[{"type":"text","text":"Chapter 1"}]}
]}]}`
sections := buildDOCXJSONSections(ir)
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1", len(sections))
}
item := sections[0]
if got, ok := item["text"].(string); !ok || got != "Chapter 1" {
t.Errorf("text = %q, want %q", got, "Chapter 1")
}
if got, ok := item["doc_type_kwd"].(string); !ok || got != "text" {
t.Errorf("doc_type_kwd = %q, want %q", got, "text")
}
if got, ok := item["ck_type"].(string); !ok || got != "heading" {
t.Errorf("ck_type = %q, want %q", got, "heading")
}
}
func TestBuildDOCXJSONSections_Images(t *testing.T) {
b64 := base64.StdEncoding.EncodeToString([]byte("fake-image-data"))
ir := `{"sections":[{"title":"","elements":[
{"type":"image","data":"` + b64 + `"}
]}]}`
sections := buildDOCXJSONSections(ir)
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1", len(sections))
}
item := sections[0]
if got, ok := item["text"].(string); !ok || got != "" {
t.Errorf("text = %q, want empty", got)
}
if got, ok := item["image"].(string); !ok || got != b64 {
t.Errorf("image = %q, want %q", got, b64)
}
if got, ok := item["doc_type_kwd"].(string); !ok || got != "image" {
t.Errorf("doc_type_kwd = %q, want %q", got, "image")
}
}
func TestBuildDOCXJSONSections_Tables(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"table","rows":[
{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"A1"}]}]},{"content":[{"type":"paragraph","content":[{"type":"text","text":"B1"}]}]}]},
{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"A2"}]}]},{"content":[{"type":"paragraph","content":[{"type":"text","text":"B2"}]}]}]}
]}
]}]}`
sections := buildDOCXJSONSections(ir)
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1", len(sections))
}
item := sections[0]
html, ok := item["text"].(string)
if !ok {
t.Fatal("text field missing or not string")
}
if !strings.Contains(html, "<table>") || !strings.Contains(html, "</table>") {
t.Errorf("html = %q, missing <table> tags", html)
}
if !strings.Contains(html, "<tr>") || !strings.Contains(html, "</tr>") {
t.Errorf("html = %q, missing <tr> tags", html)
}
if !strings.Contains(html, "<td>A1</td>") || !strings.Contains(html, "<td>B2</td>") {
t.Errorf("html = %q, missing cell content", html)
}
if got, ok := item["doc_type_kwd"].(string); !ok || got != "table" {
t.Errorf("doc_type_kwd = %q, want %q", got, "table")
}
}
func TestBuildDOCXJSONSections_MixedContent(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"paragraph","content":[{"type":"text","text":"First para"}]},
{"type":"image","data":"aW1hZ2U="},
{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"}]}]}]}]},
{"type":"heading","level":2,"content":[{"type":"text","text":"Sub title"}]}
]}]}`
sections := buildDOCXJSONSections(ir)
if len(sections) != 4 {
t.Fatalf("got %d sections, want 4", len(sections))
}
// paragraph first
if got, _ := sections[0]["doc_type_kwd"].(string); got != "text" {
t.Errorf("item[0].doc_type_kwd = %q, want %q", got, "text")
}
// image second
if got, _ := sections[1]["doc_type_kwd"].(string); got != "image" {
t.Errorf("item[1].doc_type_kwd = %q, want %q", got, "image")
}
// table third
if got, _ := sections[2]["doc_type_kwd"].(string); got != "table" {
t.Errorf("item[2].doc_type_kwd = %q, want %q", got, "table")
}
// heading fourth
if got, _ := sections[3]["doc_type_kwd"].(string); got != "text" {
t.Errorf("item[3].doc_type_kwd = %q, want %q", got, "text")
}
if got, _ := sections[3]["ck_type"].(string); got != "heading" {
t.Errorf("item[3].ck_type = %q, want %q", got, "heading")
}
}
func TestBuildDOCXJSONSections_EmptySkipped(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"paragraph","content":[{"type":"text","text":""}]},
{"type":"paragraph","content":[{"type":"text","text":"Only this matters"}]},
{"type":"table","rows":[]}
]}]}`
sections := buildDOCXJSONSections(ir)
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1 (only non-empty paragraph)", len(sections))
}
if got, _ := sections[0]["text"].(string); got != "Only this matters" {
t.Errorf("text = %q, want %q", got, "Only this matters")
}
}
func TestDocxIRTableToHTML_Empty(t *testing.T) {
el := docxIRElement{Type: "table", Rows: []docxIRRow{}}
html := docxIRTableToHTML(el)
if html != "<table></table>" {
t.Errorf("empty table html = %q, want %q", html, "<table></table>")
}
}
func TestDocxIRTableToHTML_Single(t *testing.T) {
el := docxIRElement{
Type: "table",
Rows: []docxIRRow{{
Cells: []docxIRCell{{
Content: []docxIRElement{{
Type: "paragraph",
Content: []docxIRRun{{Type: "text", Text: "hello"}},
}},
}},
}},
}
html := docxIRTableToHTML(el)
want := "<table><tr><td>hello</td></tr></table>"
if html != want {
t.Errorf("single cell html = %q, want %q", html, want)
}
}
func TestDocxIRTableToHTML_MultiRowCol(t *testing.T) {
cell := func(text string) docxIRCell {
return docxIRCell{
Content: []docxIRElement{{
Type: "paragraph",
Content: []docxIRRun{{Type: "text", Text: text}},
}},
}
}
el := docxIRElement{
Type: "table",
Rows: []docxIRRow{
{Cells: []docxIRCell{cell("A1"), cell("B1")}},
{Cells: []docxIRCell{cell("A2"), cell("B2")}},
},
}
html := docxIRTableToHTML(el)
want := "<table><tr><td>A1</td><td>B1</td></tr><tr><td>A2</td><td>B2</td></tr></table>"
if html != want {
t.Errorf("multi cell html = %q, want %q", html, want)
}
}

View File

@@ -18,10 +18,12 @@ package parser
import (
"bytes"
"encoding/base64"
"fmt"
"io"
"mime"
"mime/multipart"
"mime/quotedprintable"
"net/mail"
"path/filepath"
"strings"
@@ -255,6 +257,8 @@ func readMailBody(body io.Reader, contentType string, collectAttachments bool) (
// Check if this part is an attachment.
if collectAttachments && isAttachmentPart(part) {
raw, _ := io.ReadAll(part)
raw = decodeCTE(raw, part.Header.Get("Content-Transfer-Encoding"))
attachments = append(attachments, map[string]any{
"filename": attachmentFilename(part),
"payload": decodeMailPayload(raw, partParams["charset"]),
@@ -263,6 +267,7 @@ func readMailBody(body io.Reader, contentType string, collectAttachments bool) (
}
raw, _ := io.ReadAll(part)
raw = decodeCTE(raw, part.Header.Get("Content-Transfer-Encoding"))
decoded := decodeMailPayload(raw, partParams["charset"])
switch partMedia {
@@ -278,6 +283,33 @@ func readMailBody(body io.Reader, contentType string, collectAttachments bool) (
// isAttachmentPart checks whether a multipart part should be treated as
// an attachment (Content-Disposition starts with "attachment"). Mirrors
// Python's check in _email().
// decodeCTE decodes Content-Transfer-Encoding (base64, quoted-printable, etc.).
// Mirrors Python part.get_payload(decode=True).
func decodeCTE(raw []byte, cte string) []byte {
switch strings.ToLower(strings.TrimSpace(cte)) {
case "base64":
// Real-world MIME base64 is often line-wrapped (~76 chars per line).
// Remove all whitespace before decoding; base64.StdEncoding.DecodeString
// strictly rejects interior whitespace.
cleanRaw := bytes.ReplaceAll(raw, []byte("\n"), nil)
cleanRaw = bytes.ReplaceAll(cleanRaw, []byte("\r"), nil)
d, err := base64.StdEncoding.DecodeString(string(cleanRaw))
if err != nil {
return raw
}
return d
case "quoted-printable":
r := quotedprintable.NewReader(bytes.NewReader(raw))
d, err := io.ReadAll(r)
if err != nil {
return raw
}
return d
default:
return raw
}
}
func isAttachmentPart(part *multipart.Part) bool {
disp := part.Header.Get("Content-Disposition")
if disp == "" {

View File

@@ -17,6 +17,7 @@
package parser
import (
"encoding/base64"
"strings"
"testing"
)
@@ -118,6 +119,135 @@ func TestEmailParser_MsgNotSupported(t *testing.T) {
}
}
func TestEmailParser_Base64Attachment(t *testing.T) {
attachmentContent := "Hello! This is the decoded content of the attachment."
encoded := base64.StdEncoding.EncodeToString([]byte(attachmentContent))
// Simulate MIME line-wrapping (typically 76 chars per line).
if len(encoded) > 20 {
encoded = encoded[:20] + "\r\n" + encoded[20:]
}
boundary := "attachboundary"
raw := strings.Join([]string{
"From: sender@test.com",
"To: receiver@test.com",
"Subject: Base64 Attachment Test",
"Content-Type: multipart/mixed; boundary=" + boundary,
"",
"--" + boundary,
"Content-Type: text/plain; charset=utf-8",
"",
"Body text here.",
"--" + boundary,
"Content-Type: text/plain; charset=utf-8",
"Content-Disposition: attachment; filename=\"test.txt\"",
"Content-Transfer-Encoding: base64",
"",
encoded,
"--" + boundary + "--",
}, "\r\n")
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
"output_format": "json",
"fields": []string{"from", "body", "attachments"},
})
result := p.ParseWithResult("test.eml", []byte(raw))
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
item := result.JSON[0]
atts, ok := item["attachments"].([]map[string]any)
if !ok {
t.Fatalf("attachments missing or wrong type: %T", item["attachments"])
}
if len(atts) != 1 {
t.Fatalf("expected 1 attachment, got %d", len(atts))
}
payload, ok := atts[0]["payload"].(string)
if !ok {
t.Fatalf("payload missing or wrong type: %T", atts[0]["payload"])
}
if payload != attachmentContent {
t.Errorf("attachment payload = %q, want %q (should be decoded from base64, not raw base64)", payload, attachmentContent)
}
if fn, _ := atts[0]["filename"].(string); fn != "test.txt" {
t.Errorf("filename = %q, want test.txt", fn)
}
}
func TestEmailParser_Base64AttachmentInMixedMultipart(t *testing.T) {
// Simulates the original test email structure:
// multipart/mixed → multipart/alternative (text/plain + text/html) + base64 attachment
innerBoundary := "inneralt"
outerBoundary := "outermixed"
attachmentContent := "<html><body><h1>Bookmarks</h1><p>Test data</p></body></html>"
encoded := base64.StdEncoding.EncodeToString([]byte(attachmentContent))
// Simulate MIME line-wrapping (typically 76 chars per line).
if len(encoded) > 20 {
encoded = encoded[:20] + "\r\n" + encoded[20:]
}
raw := strings.Join([]string{
"From: sender@test.com",
"To: receiver@test.com",
"Subject: Mixed Multipart Test",
"Content-Type: multipart/mixed; boundary=" + outerBoundary,
"",
"--" + outerBoundary,
"Content-Type: multipart/alternative; boundary=" + innerBoundary,
"",
"--" + innerBoundary,
"Content-Type: text/plain; charset=utf-8",
"",
"Plain text body.",
"--" + innerBoundary,
"Content-Type: text/html; charset=utf-8",
"",
"<p>HTML body.</p>",
"--" + innerBoundary + "--",
"--" + outerBoundary,
"Content-Type: text/html; charset=utf-8",
"Content-Disposition: attachment; filename=\"bookmarks.html\"",
"Content-Transfer-Encoding: base64",
"",
encoded,
"--" + outerBoundary + "--",
}, "\r\n")
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
"output_format": "json",
"fields": []string{"from", "body", "attachments"},
})
result := p.ParseWithResult("test.eml", []byte(raw))
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
item := result.JSON[0]
// Verify body text was extracted from nested multipart/alternative
if v, ok := item["text"].(string); !ok || !strings.Contains(v, "Plain text body") {
t.Errorf("text: got %q, want to contain 'Plain text body'", v)
}
if v, ok := item["text_html"].(string); !ok || !strings.Contains(v, "HTML body") {
t.Errorf("text_html: got %q, want to contain 'HTML body'", v)
}
// Verify attachment is decoded from base64
atts, ok := item["attachments"].([]map[string]any)
if !ok || len(atts) != 1 {
t.Fatalf("expected 1 attachment, got %d", len(atts))
}
payload, _ := atts[0]["payload"].(string)
if payload != attachmentContent {
t.Errorf("attachment payload = %q, want %q (should be decoded from base64)", payload, attachmentContent)
}
}
func TestEmailParser_Multipart(t *testing.T) {
boundary := "boundary123"
raw := strings.Join([]string{

View File

@@ -61,6 +61,11 @@ func NewDOCXParser() *DOCXParser {
return &DOCXParser{}
}
func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {
// No-op in the no-CGO stub: the real implementation in
// docx_parser.go reads output_format from setup.
}
func (p *DOCXParser) String() string {
return "DOCXParser(no-cgo)"
}