diff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.go index 414b2db37d..2a0535ed06 100644 --- a/internal/agent/component/llm.go +++ b/internal/agent/component/llm.go @@ -300,34 +300,7 @@ func toEinoMessages(msgs []schema.Message) []*schema.Message { // chat. Provider-specific endpoint suffixes remain owned by conf/models/*.json; // a tenant base_url override replaces only the endpoint root. func newChatModelDriver(driver, override string) (models.ModelDriver, error) { - pm := models.GetProviderManager() - if pm != nil { - provider := pm.FindProvider(driver) - if provider != nil && provider.ModelDriver != nil { - modelDriver := provider.ModelDriver - if strings.TrimSpace(override) != "" { - modelDriver = modelDriver.NewInstance( - map[string]string{ - "default": strings.TrimRight(override, "/"), - }, - ) - if modelDriver == nil { - return nil, fmt.Errorf("provider does not support a custom base_url") - } - } - return modelDriver, nil - } - } - - // Dummy is an explicit test/development driver and has no provider config. - if strings.EqualFold(driver, "dummy") { - baseURL := map[string]string(nil) - if strings.TrimSpace(override) != "" { - baseURL = map[string]string{"default": strings.TrimRight(override, "/")} - } - return models.NewDummyModel(baseURL, models.URLSuffix{Chat: "chat/completions"}), nil - } - return nil, fmt.Errorf("provider is not configured") + return models.GetPreconfiguredDriver(driver, override) } // NewLLMComponent builds an LLMComponent from raw params. diff --git a/internal/entity/models/get_driver.go b/internal/entity/models/get_driver.go new file mode 100644 index 0000000000..e6d22da31b --- /dev/null +++ b/internal/entity/models/get_driver.go @@ -0,0 +1,62 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package models + +import ( + "fmt" + "strings" +) + +// GetPreconfiguredDriver returns the provider's pre-built ModelDriver from +// ProviderManager, with an optional base URL override applied. +// +// The caller MUST NOT mutate the returned driver — use NewInstance to +// create a derived copy with custom base URLs. +// +// Returns: +// - driver, nil — found and (optionally) overridden +// - nil, error — provider manager uninitialized, provider unknown, +// or provider does not support custom base URLs +func GetPreconfiguredDriver(driver string, baseURLOverride string) (ModelDriver, error) { + baseURLOverride = strings.TrimSpace(baseURLOverride) + if strings.EqualFold(driver, "dummy") { + baseURL := map[string]string(nil) + if baseURLOverride != "" { + baseURL = map[string]string{"default": strings.TrimRight(baseURLOverride, "/")} + } + return NewDummyModel(baseURL, URLSuffix{Chat: "chat/completions"}), nil + } + + pm := GetProviderManager() + if pm == nil { + return nil, fmt.Errorf("provider manager not initialized") + } + provider := pm.FindProvider(driver) + if provider == nil || provider.ModelDriver == nil { + return nil, fmt.Errorf("provider %q is not configured", driver) + } + md := provider.ModelDriver + if baseURLOverride != "" { + md = md.NewInstance(map[string]string{ + "default": strings.TrimRight(baseURLOverride, "/"), + }) + if md == nil { + return nil, fmt.Errorf("provider %q does not support a custom base_url", driver) + } + } + return md, nil +} diff --git a/internal/entity/models/get_driver_test.go b/internal/entity/models/get_driver_test.go new file mode 100644 index 0000000000..29c1d8a262 --- /dev/null +++ b/internal/entity/models/get_driver_test.go @@ -0,0 +1,154 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package models + +import ( + "strings" + "testing" +) + +func TestGetPreconfiguredDriverReturnsPreBuiltDriver(t *testing.T) { + dir, restore := setupProviderTestDir(t, "aliyun.json") + defer restore() + + if err := InitProviderManager(dir); err != nil { + t.Fatalf("InitProviderManager: %v", err) + } + + driver, err := GetPreconfiguredDriver("Tongyi-Qianwen", "") + if err != nil { + t.Fatalf("GetPreconfiguredDriver: %v", err) + } + if driver == nil { + t.Fatal("GetPreconfiguredDriver returned nil, want non-nil") + } + if _, ok := driver.(*AliyunModel); !ok { + t.Fatalf("GetPreconfiguredDriver returned %T, want *AliyunModel", driver) + } +} + +func TestGetPreconfiguredDriverWithBaseURLOverride(t *testing.T) { + dir, restore := setupProviderTestDir(t, "aliyun.json") + defer restore() + + if err := InitProviderManager(dir); err != nil { + t.Fatalf("InitProviderManager: %v", err) + } + + customURL := "https://custom-endpoint.example.com/v1" + driver, err := GetPreconfiguredDriver("Tongyi-Qianwen", customURL) + if err != nil { + t.Fatalf("GetPreconfiguredDriver: %v", err) + } + if driver == nil { + t.Fatal("GetPreconfiguredDriver returned nil, want non-nil") + } + + // Verify the override took effect by checking GetBaseURL. + aliModel, ok := driver.(*AliyunModel) + if !ok { + t.Fatalf("GetPreconfiguredDriver returned %T, want *AliyunModel", driver) + } + gotURL, err := aliModel.baseModel.GetBaseURL(&APIConfig{}) + if err != nil { + t.Fatalf("GetBaseURL: %v", err) + } + if expected := strings.TrimSuffix(customURL, "/"); gotURL != expected { + t.Errorf("GetBaseURL = %q, want %q", gotURL, expected) + } +} + +func TestGetPreconfiguredDriverProviderNotFound(t *testing.T) { + dir, restore := setupProviderTestDir(t, "aliyun.json") + defer restore() + + if err := InitProviderManager(dir); err != nil { + t.Fatalf("InitProviderManager: %v", err) + } + + _, err := GetPreconfiguredDriver("NonExistentProvider", "") + if err == nil { + t.Fatal("GetPreconfiguredDriver should error for unknown provider, got nil") + } +} + +func TestGetPreconfiguredDriverManagerNotInitialized(t *testing.T) { + // Save and reset the global so we can test the nil-manager path. + saved := providerManager + providerManager = nil + defer func() { providerManager = saved }() + + _, err := GetPreconfiguredDriver("Tongyi-Qianwen", "") + if err == nil { + t.Fatal("GetPreconfiguredDriver should error when manager is nil, got nil") + } +} + +func TestGetPreconfiguredDriverSuffixTrimmed(t *testing.T) { + dir, restore := setupProviderTestDir(t, "aliyun.json") + defer restore() + + if err := InitProviderManager(dir); err != nil { + t.Fatalf("InitProviderManager: %v", err) + } + + // Trailing slash should be stripped. + driver, err := GetPreconfiguredDriver("Tongyi-Qianwen", "https://example.com/") + if err != nil { + t.Fatalf("GetPreconfiguredDriver: %v", err) + } + aliModel, ok := driver.(*AliyunModel) + if !ok { + t.Fatalf("expected *AliyunModel, got %T", driver) + } + gotURL, err := aliModel.baseModel.GetBaseURL(&APIConfig{}) + if err != nil { + t.Fatalf("GetBaseURL: %v", err) + } + if gotURL != "https://example.com" { + t.Errorf("GetBaseURL = %q, want %q", gotURL, "https://example.com") + } +} + +func TestGetPreconfiguredDriverDummyNoOverride(t *testing.T) { + driver, err := GetPreconfiguredDriver("dummy", "") + if err != nil { + t.Fatalf("GetPreconfiguredDriver(dummy): %v", err) + } + if _, ok := driver.(*DummyModel); !ok { + t.Fatalf("expected *DummyModel, got %T", driver) + } +} + +func TestGetPreconfiguredDriverDummyWithOverride(t *testing.T) { + driver, err := GetPreconfiguredDriver("Dummy", "https://override.example.com") + if err != nil { + t.Fatalf("GetPreconfiguredDriver(Dummy): %v", err) + } + dummy, ok := driver.(*DummyModel) + if !ok { + t.Fatalf("expected *DummyModel, got %T", driver) + } + // The override should be stored in the driver's BaseURL. + gotURL, err := dummy.baseModel.GetBaseURL(&APIConfig{}) + if err != nil { + t.Fatalf("GetBaseURL: %v", err) + } + if gotURL != "https://override.example.com" { + t.Errorf("GetBaseURL = %q, want %q", gotURL, "https://override.example.com") + } +} diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go index e12a6a93f2..7633d03425 100644 --- a/internal/ingestion/component/chunker/common.go +++ b/internal/ingestion/component/chunker/common.go @@ -105,25 +105,25 @@ func stringListFromAny(in []any) []string { // alternation. Entries wrapped in backticks are treated as regex // literals and regex-escaped; plain strings are simply regex-escaped. // Longer patterns win (matches python `sorted(set, key=len, reverse=True)`). +// Mirrors Python _compile_delimiter_pattern: only backtick-wrapped delimiters +// produce an active pattern. Plain delimiters are not compiled — they are only +// used by naive_merge / mergeByTokenSize for sentence-level splitting when no +// active pattern exists. func compileDelimPattern(delims []string) *regexp.Regexp { var custom []string - var plain []string for _, d := range delims { if d == "" { continue } if strings.HasPrefix(d, "`") && strings.HasSuffix(d, "`") && len(d) >= 2 { custom = append(custom, regexp.QuoteMeta(d[1:len(d)-1])) - } else { - plain = append(plain, regexp.QuoteMeta(d)) } } - all := append(plain, custom...) - if len(all) == 0 { + if len(custom) == 0 { return nil } - sort.SliceStable(all, func(i, j int) bool { return len(all[i]) > len(all[j]) }) - return regexp.MustCompile(strings.Join(all, "|")) + sort.SliceStable(custom, func(i, j int) bool { return len(custom[i]) > len(custom[j]) }) + return regexp.MustCompile(strings.Join(custom, "|")) } // splitKeepingDelim is the Go equivalent of python's diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.go index e3a652de31..3ddf351ec7 100644 --- a/internal/ingestion/component/chunker/group.go +++ b/internal/ingestion/component/chunker/group.go @@ -38,7 +38,10 @@ import ( "fmt" "strings" + "go.uber.org/zap" + "ragflow/internal/agent/runtime" + "ragflow/internal/common" "ragflow/internal/ingestion/component/globals" "ragflow/internal/ingestion/component/schema" "ragflow/internal/tokenizer" @@ -104,11 +107,34 @@ func buildSectionIDs(levels []int, targetLevel int) []int { // record-buckets for the merge pass). func invokeGroup(_ context.Context, inputs map[string]any, p *titleChunkerParam) (map[string]any, error) { records := extractLineRecords(inputs) + common.Debug("chunker stage", + zap.String("component", "Chunker"), + zap.String("variant", "group"), + zap.Int("records", len(records)), + ) if len(records) == 0 { return emptyOutputs(), nil } ctx := newLevelContext(records, p) levels := ctx.Levels() + // Count heading level distribution for debugging. + headingCounts := make(map[int]int) + for _, lvl := range levels { + if lvl < bodyLevel { + headingCounts[lvl]++ + } + } + bodyCount := len(levels) + for _, c := range headingCounts { + bodyCount -= c + } + common.Debug("chunker stage", + zap.String("component", "Chunker"), + zap.String("variant", "group"), + zap.Int("records", len(records)), + zap.Int("body_level", bodyCount), + zap.Any("heading_levels", headingCounts), + ) // Mirror python group_chunker._resolve_group_target_level: when // `hierarchy` is unset the target level is `most_level` directly @@ -120,7 +146,18 @@ func invokeGroup(_ context.Context, inputs map[string]any, p *titleChunkerParam) secIDs := buildSectionIDs(levels, targetLevel) groups := groupRecords(records, secIDs, p) + common.Debug("chunker stage", + zap.String("component", "Chunker"), + zap.String("variant", "group"), + zap.Int("groups", len(groups)), + ) chunks := buildChunksFromRecordGroups(groups, p, isPlainTextFormat(inputs)) + common.Debug("chunker stage", + zap.String("component", "Chunker"), + zap.String("variant", "group"), + zap.Int("chunks", len(chunks)), + zap.Bool("plain_text", isPlainTextFormat(inputs)), + ) if len(chunks) == 0 { return emptyOutputs(), nil } @@ -253,7 +290,7 @@ func extractLineRecords(inputs map[string]any) []lineRecord { if docs := chunksFromInputs(inputs); docs != nil { return recordsFromStructured(docs) } - text, _ := stringFromInputs(inputs, "text", "content") + text, _ := stringFromInputs(inputs, "text", "content", "markdown", "html") if text == "" { return nil } diff --git a/internal/ingestion/component/chunker/group_test.go b/internal/ingestion/component/chunker/group_test.go index b1206e71d8..2cb1ebae6e 100644 --- a/internal/ingestion/component/chunker/group_test.go +++ b/internal/ingestion/component/chunker/group_test.go @@ -64,6 +64,48 @@ func TestGroupTitleChunker_NewRejectsHierarchyWithoutHierarchyParam(t *testing.T } } +// TestExtractLineRecords_MarkdownKey verifies extractLineRecords reads +// the "markdown" payload key. Before this fix extractLineRecords only +// looked at "text"/"content" and silently returned nil for parser +// output that carried output_format="markdown". +func TestExtractLineRecords_MarkdownKey(t *testing.T) { + records := extractLineRecords(map[string]any{ + "output_format": "markdown", + "markdown": "line1\nline2\nline3", + }) + if len(records) != 3 { + t.Fatalf("got %d records, want 3", len(records)) + } + for i, r := range records { + if r.textOrEmpty() == "" { + t.Errorf("record[%d]: empty text", i) + } + } +} + +// TestExtractLineRecords_HTMLKey verifies extractLineRecords reads the +// "html" payload key, same safety-net rationale as TestMarkdownKey. +func TestExtractLineRecords_HTMLKey(t *testing.T) { + records := extractLineRecords(map[string]any{ + "output_format": "html", + "html": "

first

\n

second

", + }) + if len(records) != 2 { + t.Fatalf("got %d records, want 2", len(records)) + } +} + +// TestExtractLineRecords_TextKeyStillWorks ensures the existing "text" +// key path is not broken by the addition of "markdown"/"html". +func TestExtractLineRecords_TextKeyStillWorks(t *testing.T) { + records := extractLineRecords(map[string]any{ + "text": "hello\nworld", + }) + if len(records) != 2 { + t.Fatalf("got %d records, want 2", len(records)) + } +} + func TestGroupTitleChunker_InvokeEmptyInput(t *testing.T) { c, err := NewGroupTitleChunker(map[string]any{ "levels": [][]string{{`^# `}}, diff --git a/internal/ingestion/component/chunker/hierarchy.go b/internal/ingestion/component/chunker/hierarchy.go index 6f761cbc86..79f81e3dce 100644 --- a/internal/ingestion/component/chunker/hierarchy.go +++ b/internal/ingestion/component/chunker/hierarchy.go @@ -42,7 +42,10 @@ import ( "context" "fmt" + "go.uber.org/zap" + "ragflow/internal/agent/runtime" + "ragflow/internal/common" "ragflow/internal/ingestion/component/globals" ) @@ -137,11 +140,34 @@ func (n *chunkNode) getPaths(paths *[][]int, titles []int, depth int, includeHea // invokeHierarchy runs the HierarchyTitleChunker strategy. func invokeHierarchy(_ context.Context, inputs map[string]any, p *titleChunkerParam) (map[string]any, error) { records := extractLineRecords(inputs) + common.Debug("chunker stage", + zap.String("component", "Chunker"), + zap.String("variant", "hierarchy"), + zap.Int("records", len(records)), + ) if len(records) == 0 { return emptyOutputs(), nil } ctx := newLevelContext(records, p) levels := ctx.Levels() + // Count heading level distribution for debugging. + headingCounts := make(map[int]int) + for _, lvl := range levels { + if lvl < bodyLevel { + headingCounts[lvl]++ + } + } + bodyCount := len(levels) + for _, c := range headingCounts { + bodyCount -= c + } + common.Debug("chunker stage", + zap.String("component", "Chunker"), + zap.String("variant", "hierarchy"), + zap.Int("records", len(records)), + zap.Int("body_level", bodyCount), + zap.Any("heading_levels", headingCounts), + ) // Mirror python HierarchyTitleChunker.build_chunks: accumulate // contiguous text records into a run; on each non-text record flush @@ -206,8 +232,19 @@ func invokeHierarchy(_ context.Context, inputs map[string]any, p *titleChunkerPa recordGroups = append(recordGroups, []lineRecord{rec}) } flush() + common.Debug("chunker stage", + zap.String("component", "Chunker"), + zap.String("variant", "hierarchy"), + zap.Int("record_groups", len(recordGroups)), + ) chunks := buildChunksFromRecordGroups(recordGroups, p, isPlainTextFormat(inputs)) + common.Debug("chunker stage", + zap.String("component", "Chunker"), + zap.String("variant", "hierarchy"), + zap.Int("chunks", len(chunks)), + zap.Bool("plain_text", isPlainTextFormat(inputs)), + ) if len(chunks) == 0 { return emptyOutputs(), nil } diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go index 0975e9415b..975f36df16 100644 --- a/internal/ingestion/component/chunker/token.go +++ b/internal/ingestion/component/chunker/token.go @@ -813,7 +813,11 @@ func applyChildrenDelim(segs []string, pattern *regexp.Regexp) []schema.ChunkDoc if pattern == nil { out := make([]schema.ChunkDoc, 0, len(segs)) for _, s := range segs { - out = append(out, schema.ChunkDoc{Text: s}) + out = append(out, schema.ChunkDoc{ + Text: s, + DocType: "text", + CKType: "text", + }) } return out } diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go index 7a620553dd..d8759f6e62 100644 --- a/internal/ingestion/component/extractor.go +++ b/internal/ingestion/component/extractor.go @@ -326,18 +326,10 @@ func (e *einoExtractorChatInvoker) Chat(ctx context.Context, req extractorChatRe if driver == "" { driver = "dummy" } - var baseURL map[string]string - if req.BaseURL != "" { - baseURL = map[string]string{"default": req.BaseURL} - } - urlSuffix := extractorChatURLSuffixFor(driver) - d, err := models.NewModelFactory().CreateModelDriver(driver, baseURL, urlSuffix) + d, err := models.GetPreconfiguredDriver(driver, req.BaseURL) if err != nil { return nil, fmt.Errorf("extractor: resolve driver %q: %w", driver, err) } - if d == nil { - return nil, fmt.Errorf("extractor: no driver for %q", driver) - } apiKey := req.APIKey cfg := &models.APIConfig{ApiKey: &apiKey} cm := models.NewChatModel(d, &modelName, cfg) @@ -379,19 +371,6 @@ func splitExtractorLLIDPair(s string) (modelName, provider string, ok bool) { } } -// extractorChatURLSuffixFor matches -// internal/agent/component/llm.go:chatURLSuffixFor — anthropic -// uses v1/messages, everything else falls through to the openai- -// compatible chat/completions default. -func extractorChatURLSuffixFor(driver string) models.URLSuffix { - switch strings.ToLower(driver) { - case "anthropic": - return models.URLSuffix{Chat: "v1/messages"} - default: - return models.URLSuffix{Chat: "chat/completions"} - } -} - // toExtractorEinoMessages converts eschema.Message → *eschema.Message // for the eino bridge. The user / system / assistant roles pass // through; multi-modal content is intentionally not propagated — @@ -520,6 +499,10 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, inputs map[string]any) return nil, fmt.Errorf("extractor: %w", err) } in := c.resolveInputs(inputs) + common.Debug("extractor stage", + zap.String("component", "Extractor"), + zap.Int("input_chunks", len(in.chunks)), + ) if in.fieldName == "toc" { return nil, fmt.Errorf("extractor: field_name %q requires the TOC prompt generator which is not yet ported to Go", "toc") } @@ -564,6 +547,10 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, inputs map[string]any) }); err != nil { return nil, fmt.Errorf("extractor: %w", err) } + common.Debug("extractor stage", + zap.String("component", "Extractor"), + zap.Int("output_chunks", len(in.chunks)), + ) return map[string]any{ "chunks": in.chunks, "output_format": "chunks", diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go index fb0c10d6d8..589e3b5a90 100644 --- a/internal/ingestion/component/parser.go +++ b/internal/ingestion/component/parser.go @@ -81,7 +81,10 @@ import ( "strings" "unicode/utf8" + "go.uber.org/zap" + "ragflow/internal/agent/runtime" + "ragflow/internal/common" "ragflow/internal/ingestion/component/globals" "ragflow/internal/ingestion/component/schema" "ragflow/internal/utility" @@ -486,6 +489,19 @@ func (c *ParserComponent) Invoke(ctx context.Context, inputs map[string]any) (ma // forwards only this explicit output to the next node, so shared // fields must live in Globals. globals.PublishGlobals(ctx, out) + // Debug log: summarize parser output for pipeline debugging. + if dispatched.OutputFormat == "json" { + common.Debug("parser stage output", + zap.String("component", "Parser"), + zap.String("output_format", "json"), + zap.Int("json_items", len(dispatched.JSON)), + ) + } else if dispatched.OutputFormat != "" { + common.Debug("parser stage output", + zap.String("component", "Parser"), + zap.String("output_format", dispatched.OutputFormat), + ) + } // Progress (_created_time / _elapsed_time stamping, start/done // callbacks) is owned by the canvas framework (realComponentBody), // not by this component, so we return the work result directly. diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go index 117273096c..350383e09d 100644 --- a/internal/ingestion/component/parser_dispatch_test.go +++ b/internal/ingestion/component/parser_dispatch_test.go @@ -288,6 +288,21 @@ func TestResolveOutputFormat_DefaultsAndWhitelist(t *testing.T) { } } +func TestDefaultSetups_DOCX_OutputFormatMarkdown(t *testing.T) { + setups := defaultSetups() + docx, ok := setups["docx"] + if !ok { + t.Fatal("defaultSetups: docx key missing") + } + got, ok := docx["output_format"].(string) + if !ok { + t.Fatal("defaultSetups: docx.output_format missing or not a string") + } + if got != "json" { + t.Errorf("docx.output_format = %q, want %q", got, "json") + } +} + func TestConfigureParserFromSetups_UsesPythonFamilySetup(t *testing.T) { setups := defaultSetups() got := &captureSetupConfigurer{} diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go index 61959fb26d..a66285cb0b 100644 --- a/internal/ingestion/component/tokenizer.go +++ b/internal/ingestion/component/tokenizer.go @@ -101,7 +101,10 @@ import ( "strings" "time" + "go.uber.org/zap" + "ragflow/internal/agent/runtime" + "ragflow/internal/common" "ragflow/internal/ingestion/component/globals" "ragflow/internal/ingestion/component/schema" "ragflow/internal/tokenizer" @@ -345,6 +348,10 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, inputs map[string]any) return nil, err } chunks := chunksFromTokenizerUpstream(upstream) + common.Debug("tokenizer stage", + zap.String("component", "Tokenizer"), + zap.Int("input_chunks", len(chunks)), + ) titleStem := titleExtRE.ReplaceAllString(name, "") normalizeChunkTextFallback(chunks) @@ -372,6 +379,10 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, inputs map[string]any) return nil, err } + common.Debug("tokenizer stage", + zap.String("component", "Tokenizer"), + zap.Int("output_chunks", len(chunks)), + ) return out, nil } diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.go index 3d8d2e68df..04525d169c 100644 --- a/internal/parser/parser/docx_parser.go +++ b/internal/parser/parser/docx_parser.go @@ -22,6 +22,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "html" "strings" officeOxide "github.com/yfedoseev/office_oxide/go" @@ -38,20 +39,33 @@ type DOCXFigure struct { } type DOCXParser struct { - libType string + libType string + outputFormat string // from DSL config; "json" or "markdown" } func NewDOCXParser() *DOCXParser { return &DOCXParser{} } -// ParseWithResult captures the office_oxide ToMarkdown output -// and additionally extracts embedded images with their surrounding -// text context so the downstream vision-figure dispatch can enrich -// the markdown with LLM-generated image descriptions. +// ConfigureFromSetup implements parserSetupConfigurer, receiving the +// DSL "docx" family setup map. The output_format key drives whether +// ParseWithResult produces JSON items (structured) or markdown. +func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) { + if p == nil || setup == nil { + return + } + if v, ok := setup["output_format"].(string); ok && v != "" { + p.outputFormat = v + } +} + +// ParseWithResult produces structured JSON items (when +// p.outputFormat == "json") or markdown (default) from a +// docx document. Embedded images are extracted in both paths +// for downstream vision-figure dispatch. // -// Mirrors python naive.py: Docx() → naive_merge_docx() → -// vision_figure_parser_docx_wrapper_naive(). +// JSON path mirrors python parser.py:_docx() output_format == "json". +// Markdown path mirrors python naive.py: Docx() → naive_merge_docx(). func (p *DOCXParser) ParseWithResult(filename string, data []byte) ParseResult { doc, err := officeOxide.OpenFromBytes(data, "docx") if err != nil { @@ -59,38 +73,43 @@ func (p *DOCXParser) ParseWithResult(filename string, data []byte) ParseResult { } defer doc.Close() - md, err := doc.ToMarkdown() - if err != nil { - return ParseResult{Err: fmt.Errorf("docx to-markdown: %w", err)} - } - fileMeta := map[string]any{ "name": filename, "format": "docx", } - // Extract embedded images with their text context from the - // office_oxide IR so the downstream vision dispatch can - // enrich them. The already-opened doc handle is reused - // (no second OpenFromBytes). + // Extract IR JSON for section building (JSON path) and + // embedded-image extraction (both paths). irJSON, irErr := doc.ToIRJSON() var figures []DOCXFigure if irErr == nil { figures = extractDOCXFiguresFromIR(irJSON) } if len(figures) > 0 { - figs := make([]map[string]any, 0, len(figures)) - for _, f := range figures { - figs = append(figs, map[string]any{ - "image": f.Image, - "context_above": f.ContextAbove, - "context_below": f.ContextBelow, - "marker": f.Marker, - }) - } - fileMeta["figures"] = figs + fileMeta["figures"] = buildFiguresMap(figures) } + if p.outputFormat == "json" { + if irErr != nil { + return ParseResult{Err: fmt.Errorf("docx to-ir-json: %w", irErr)} + } + var sections []map[string]any + sections = buildDOCXJSONSections(irJSON) + if len(sections) == 0 { + sections = []map[string]any{{"text": "", "doc_type_kwd": "text"}} + } + return ParseResult{ + OutputFormat: "json", + File: fileMeta, + JSON: sections, + } + } + + // Default / markdown path. + md, err := doc.ToMarkdown() + if err != nil { + return ParseResult{Err: fmt.Errorf("docx to-markdown: %w", err)} + } return ParseResult{ OutputFormat: "markdown", File: fileMeta, @@ -218,6 +237,101 @@ func joinDOCXIRRuns(runs []docxIRRun) string { return b.String() } +// buildFiguresMap converts the internal DOCXFigure slice to the +// map form attached to fileMeta["figures"]. +func buildFiguresMap(figures []DOCXFigure) []map[string]any { + figs := make([]map[string]any, 0, len(figures)) + for _, f := range figures { + figs = append(figs, map[string]any{ + "image": f.Image, + "context_above": f.ContextAbove, + "context_below": f.ContextBelow, + "marker": f.Marker, + }) + } + return figs +} + +// joinCellText concatenates all paragraph texts inside a table cell, +// joined by newlines. +func joinCellText(cell docxIRCell) string { + var parts []string + for _, el := range cell.Content { + if text := joinDOCXIRRuns(el.Content); text != "" { + parts = append(parts, text) + } + } + return strings.Join(parts, "\n") +} + +// docxIRTableToHTML converts a table IR element to an HTML table string. +func docxIRTableToHTML(el docxIRElement) string { + var sb strings.Builder + sb.WriteString("") + for _, row := range el.Rows { + sb.WriteString("") + for _, cell := range row.Cells { + sb.WriteString("") + } + sb.WriteString("") + } + sb.WriteString("
") + sb.WriteString(html.EscapeString(joinCellText(cell))) + sb.WriteString("
") + return sb.String() +} + +// buildDOCXJSONSections converts an office_oxide IR JSON string into a +// slice of structured items compatible with the chunker's JSON input +// contract. Each item carries at least text and doc_type_kwd. +func buildDOCXJSONSections(irJSON string) []map[string]any { + var ir docxIRDocument + if err := json.Unmarshal([]byte(irJSON), &ir); err != nil { + return nil + } + var sections []map[string]any + for _, sec := range ir.Sections { + for _, el := range sec.Elements { + switch el.Type { + case "paragraph", "heading": + text := joinDOCXIRRuns(el.Content) + if strings.TrimSpace(text) == "" { + continue + } + item := map[string]any{ + "text": text, + "image": nil, + "doc_type_kwd": "text", + } + if el.Type == "heading" { + item["ck_type"] = "heading" + } + sections = append(sections, item) + + case "image": + b64 := base64.StdEncoding.EncodeToString(el.Data) + sections = append(sections, map[string]any{ + "text": "", + "image": b64, + "doc_type_kwd": "image", + }) + + case "table": + html := docxIRTableToHTML(el) + if html == "
" { + continue + } + sections = append(sections, map[string]any{ + "text": html, + "image": nil, + "doc_type_kwd": "table", + }) + } + } + } + return sections +} + // --- office_oxide IR types (local copy, independent of deepdoc) --- type docxIRDocument struct { @@ -230,14 +344,26 @@ type docxIRSection struct { } type docxIRElement struct { - Type string `json:"type"` - Content []docxIRRun `json:"content"` - Data []byte `json:"data"` + Type string `json:"type"` // "paragraph", "heading", "table", "image" + Level int `json:"level"` // heading level (1-6) + Style string `json:"style"` // Word style name (e.g. "Normal", "Heading 1") + Content []docxIRRun `json:"content"` // rich text runs + Data []byte `json:"data"` // raw image bytes (for "image" type) + Rows []docxIRRow `json:"rows"` // table rows } type docxIRRun struct { - Type string `json:"type"` - Text string `json:"text"` + Type string `json:"type"` // "text", "image" + Text string `json:"text"` + Content []docxIRElement `json:"content"` // nested elements (used in table cells) +} + +type docxIRRow struct { + Cells []docxIRCell `json:"cells"` +} + +type docxIRCell struct { + Content []docxIRElement `json:"content"` // nested paragraphs inside table cell } func (p *DOCXParser) String() string { diff --git a/internal/parser/parser/docx_parser_cgo_test.go b/internal/parser/parser/docx_parser_cgo_test.go index 859483cf2b..8eeb0cfca6 100644 --- a/internal/parser/parser/docx_parser_cgo_test.go +++ b/internal/parser/parser/docx_parser_cgo_test.go @@ -8,6 +8,72 @@ import ( "testing" ) +func TestDOCXParser_ParseWithResult_JSON(t *testing.T) { + p := NewDOCXParser() + p.outputFormat = "json" + data := minimalDOCX(t, "Hello from JSON path") + res := p.ParseWithResult("sample.docx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if got, want := res.OutputFormat, "json"; got != want { + t.Fatalf("OutputFormat = %q, want %q", got, want) + } + if len(res.JSON) == 0 { + t.Fatal("JSON items is empty; expected parsed content") + } + for i, item := range res.JSON { + if _, ok := item["text"]; !ok { + t.Errorf("item[%d] missing 'text' field", i) + } + if _, ok := item["doc_type_kwd"]; !ok { + t.Errorf("item[%d] missing 'doc_type_kwd' field", i) + } + } +} + +func TestDOCXParser_ConfigureFromSetup_JSON(t *testing.T) { + p := NewDOCXParser() + p.ConfigureFromSetup(map[string]any{"output_format": "json"}) + if p.outputFormat != "json" { + t.Fatalf("After ConfigureFromSetup, outputFormat = %q, want %q", p.outputFormat, "json") + } + if p.libType != "" { + t.Errorf("libType unexpectedly = %q", p.libType) + } + // Full round-trip: json config → json output + data := minimalDOCX(t, "Config test") + res := p.ParseWithResult("sample.docx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if res.OutputFormat != "json" { + t.Errorf("OutputFormat = %q, want %q", res.OutputFormat, "json") + } + if len(res.JSON) == 0 { + t.Error("JSON items is empty") + } +} + +func TestDOCXParser_ConfigureFromSetup_Markdown(t *testing.T) { + p := NewDOCXParser() + p.ConfigureFromSetup(map[string]any{"output_format": "markdown"}) + if p.outputFormat != "markdown" { + t.Fatalf("After ConfigureFromSetup, outputFormat = %q, want %q", p.outputFormat, "markdown") + } + data := minimalDOCX(t, "Config md test") + res := p.ParseWithResult("sample.docx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if res.OutputFormat != "markdown" { + t.Errorf("OutputFormat = %q, want %q", res.OutputFormat, "markdown") + } + if res.Markdown == "" { + t.Error("Markdown is empty") + } +} + func TestDOCXParser_ParseWithResult_CGOMinimalDocument(t *testing.T) { p := NewDOCXParser() data := minimalDOCX(t, "Hello from DOCX parser") diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.go new file mode 100644 index 0000000000..6548a876d7 --- /dev/null +++ b/internal/parser/parser/docx_parser_test.go @@ -0,0 +1,210 @@ +//go:build cgo + +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package parser + +import ( + "encoding/base64" + "strings" + "testing" +) + +func TestBuildDOCXJSONSections_Paragraphs(t *testing.T) { + ir := `{"sections":[{"title":"","elements":[ + {"type":"paragraph","content":[{"type":"text","text":"Hello world"}],"style":"Normal"} + ]}]}` + sections := buildDOCXJSONSections(ir) + if len(sections) != 1 { + t.Fatalf("got %d sections, want 1", len(sections)) + } + item := sections[0] + if got, ok := item["text"].(string); !ok || got != "Hello world" { + t.Errorf("text = %q, want %q", got, "Hello world") + } + if got, ok := item["doc_type_kwd"].(string); !ok || got != "text" { + t.Errorf("doc_type_kwd = %q, want %q", got, "text") + } +} + +func TestBuildDOCXJSONSections_Headings(t *testing.T) { + ir := `{"sections":[{"title":"","elements":[ + {"type":"heading","level":1,"content":[{"type":"text","text":"Chapter 1"}]} + ]}]}` + sections := buildDOCXJSONSections(ir) + if len(sections) != 1 { + t.Fatalf("got %d sections, want 1", len(sections)) + } + item := sections[0] + if got, ok := item["text"].(string); !ok || got != "Chapter 1" { + t.Errorf("text = %q, want %q", got, "Chapter 1") + } + if got, ok := item["doc_type_kwd"].(string); !ok || got != "text" { + t.Errorf("doc_type_kwd = %q, want %q", got, "text") + } + if got, ok := item["ck_type"].(string); !ok || got != "heading" { + t.Errorf("ck_type = %q, want %q", got, "heading") + } +} + +func TestBuildDOCXJSONSections_Images(t *testing.T) { + b64 := base64.StdEncoding.EncodeToString([]byte("fake-image-data")) + ir := `{"sections":[{"title":"","elements":[ + {"type":"image","data":"` + b64 + `"} + ]}]}` + sections := buildDOCXJSONSections(ir) + if len(sections) != 1 { + t.Fatalf("got %d sections, want 1", len(sections)) + } + item := sections[0] + if got, ok := item["text"].(string); !ok || got != "" { + t.Errorf("text = %q, want empty", got) + } + if got, ok := item["image"].(string); !ok || got != b64 { + t.Errorf("image = %q, want %q", got, b64) + } + if got, ok := item["doc_type_kwd"].(string); !ok || got != "image" { + t.Errorf("doc_type_kwd = %q, want %q", got, "image") + } +} + +func TestBuildDOCXJSONSections_Tables(t *testing.T) { + ir := `{"sections":[{"title":"","elements":[ + {"type":"table","rows":[ + {"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"A1"}]}]},{"content":[{"type":"paragraph","content":[{"type":"text","text":"B1"}]}]}]}, + {"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"A2"}]}]},{"content":[{"type":"paragraph","content":[{"type":"text","text":"B2"}]}]}]} + ]} + ]}]}` + sections := buildDOCXJSONSections(ir) + if len(sections) != 1 { + t.Fatalf("got %d sections, want 1", len(sections)) + } + item := sections[0] + html, ok := item["text"].(string) + if !ok { + t.Fatal("text field missing or not string") + } + if !strings.Contains(html, "") || !strings.Contains(html, "
") { + t.Errorf("html = %q, missing tags", html) + } + if !strings.Contains(html, "") || !strings.Contains(html, "") { + t.Errorf("html = %q, missing tags", html) + } + if !strings.Contains(html, "") || !strings.Contains(html, "") { + t.Errorf("html = %q, missing cell content", html) + } + if got, ok := item["doc_type_kwd"].(string); !ok || got != "table" { + t.Errorf("doc_type_kwd = %q, want %q", got, "table") + } +} + +func TestBuildDOCXJSONSections_MixedContent(t *testing.T) { + ir := `{"sections":[{"title":"","elements":[ + {"type":"paragraph","content":[{"type":"text","text":"First para"}]}, + {"type":"image","data":"aW1hZ2U="}, + {"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"}]}]}]}]}, + {"type":"heading","level":2,"content":[{"type":"text","text":"Sub title"}]} + ]}]}` + sections := buildDOCXJSONSections(ir) + if len(sections) != 4 { + t.Fatalf("got %d sections, want 4", len(sections)) + } + // paragraph first + if got, _ := sections[0]["doc_type_kwd"].(string); got != "text" { + t.Errorf("item[0].doc_type_kwd = %q, want %q", got, "text") + } + // image second + if got, _ := sections[1]["doc_type_kwd"].(string); got != "image" { + t.Errorf("item[1].doc_type_kwd = %q, want %q", got, "image") + } + // table third + if got, _ := sections[2]["doc_type_kwd"].(string); got != "table" { + t.Errorf("item[2].doc_type_kwd = %q, want %q", got, "table") + } + // heading fourth + if got, _ := sections[3]["doc_type_kwd"].(string); got != "text" { + t.Errorf("item[3].doc_type_kwd = %q, want %q", got, "text") + } + if got, _ := sections[3]["ck_type"].(string); got != "heading" { + t.Errorf("item[3].ck_type = %q, want %q", got, "heading") + } +} + +func TestBuildDOCXJSONSections_EmptySkipped(t *testing.T) { + ir := `{"sections":[{"title":"","elements":[ + {"type":"paragraph","content":[{"type":"text","text":""}]}, + {"type":"paragraph","content":[{"type":"text","text":"Only this matters"}]}, + {"type":"table","rows":[]} + ]}]}` + sections := buildDOCXJSONSections(ir) + if len(sections) != 1 { + t.Fatalf("got %d sections, want 1 (only non-empty paragraph)", len(sections)) + } + if got, _ := sections[0]["text"].(string); got != "Only this matters" { + t.Errorf("text = %q, want %q", got, "Only this matters") + } +} + +func TestDocxIRTableToHTML_Empty(t *testing.T) { + el := docxIRElement{Type: "table", Rows: []docxIRRow{}} + html := docxIRTableToHTML(el) + if html != "
A1B2
" { + t.Errorf("empty table html = %q, want %q", html, "
") + } +} + +func TestDocxIRTableToHTML_Single(t *testing.T) { + el := docxIRElement{ + Type: "table", + Rows: []docxIRRow{{ + Cells: []docxIRCell{{ + Content: []docxIRElement{{ + Type: "paragraph", + Content: []docxIRRun{{Type: "text", Text: "hello"}}, + }}, + }}, + }}, + } + html := docxIRTableToHTML(el) + want := "
hello
" + if html != want { + t.Errorf("single cell html = %q, want %q", html, want) + } +} + +func TestDocxIRTableToHTML_MultiRowCol(t *testing.T) { + cell := func(text string) docxIRCell { + return docxIRCell{ + Content: []docxIRElement{{ + Type: "paragraph", + Content: []docxIRRun{{Type: "text", Text: text}}, + }}, + } + } + el := docxIRElement{ + Type: "table", + Rows: []docxIRRow{ + {Cells: []docxIRCell{cell("A1"), cell("B1")}}, + {Cells: []docxIRCell{cell("A2"), cell("B2")}}, + }, + } + html := docxIRTableToHTML(el) + want := "
A1B1
A2B2
" + if html != want { + t.Errorf("multi cell html = %q, want %q", html, want) + } +} diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go index b7a42ce68b..2aea280337 100644 --- a/internal/parser/parser/email_parser.go +++ b/internal/parser/parser/email_parser.go @@ -18,10 +18,12 @@ package parser import ( "bytes" + "encoding/base64" "fmt" "io" "mime" "mime/multipart" + "mime/quotedprintable" "net/mail" "path/filepath" "strings" @@ -255,6 +257,8 @@ func readMailBody(body io.Reader, contentType string, collectAttachments bool) ( // Check if this part is an attachment. if collectAttachments && isAttachmentPart(part) { raw, _ := io.ReadAll(part) + raw = decodeCTE(raw, part.Header.Get("Content-Transfer-Encoding")) + attachments = append(attachments, map[string]any{ "filename": attachmentFilename(part), "payload": decodeMailPayload(raw, partParams["charset"]), @@ -263,6 +267,7 @@ func readMailBody(body io.Reader, contentType string, collectAttachments bool) ( } raw, _ := io.ReadAll(part) + raw = decodeCTE(raw, part.Header.Get("Content-Transfer-Encoding")) decoded := decodeMailPayload(raw, partParams["charset"]) switch partMedia { @@ -278,6 +283,33 @@ func readMailBody(body io.Reader, contentType string, collectAttachments bool) ( // isAttachmentPart checks whether a multipart part should be treated as // an attachment (Content-Disposition starts with "attachment"). Mirrors // Python's check in _email(). +// decodeCTE decodes Content-Transfer-Encoding (base64, quoted-printable, etc.). +// Mirrors Python part.get_payload(decode=True). +func decodeCTE(raw []byte, cte string) []byte { + switch strings.ToLower(strings.TrimSpace(cte)) { + case "base64": + // Real-world MIME base64 is often line-wrapped (~76 chars per line). + // Remove all whitespace before decoding; base64.StdEncoding.DecodeString + // strictly rejects interior whitespace. + cleanRaw := bytes.ReplaceAll(raw, []byte("\n"), nil) + cleanRaw = bytes.ReplaceAll(cleanRaw, []byte("\r"), nil) + d, err := base64.StdEncoding.DecodeString(string(cleanRaw)) + if err != nil { + return raw + } + return d + case "quoted-printable": + r := quotedprintable.NewReader(bytes.NewReader(raw)) + d, err := io.ReadAll(r) + if err != nil { + return raw + } + return d + default: + return raw + } +} + func isAttachmentPart(part *multipart.Part) bool { disp := part.Header.Get("Content-Disposition") if disp == "" { diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.go index 9cc60afc90..7ad32196f9 100644 --- a/internal/parser/parser/email_parser_test.go +++ b/internal/parser/parser/email_parser_test.go @@ -17,6 +17,7 @@ package parser import ( + "encoding/base64" "strings" "testing" ) @@ -118,6 +119,135 @@ func TestEmailParser_MsgNotSupported(t *testing.T) { } } +func TestEmailParser_Base64Attachment(t *testing.T) { + attachmentContent := "Hello! This is the decoded content of the attachment." + encoded := base64.StdEncoding.EncodeToString([]byte(attachmentContent)) + // Simulate MIME line-wrapping (typically 76 chars per line). + if len(encoded) > 20 { + encoded = encoded[:20] + "\r\n" + encoded[20:] + } + + boundary := "attachboundary" + raw := strings.Join([]string{ + "From: sender@test.com", + "To: receiver@test.com", + "Subject: Base64 Attachment Test", + "Content-Type: multipart/mixed; boundary=" + boundary, + "", + "--" + boundary, + "Content-Type: text/plain; charset=utf-8", + "", + "Body text here.", + "--" + boundary, + "Content-Type: text/plain; charset=utf-8", + "Content-Disposition: attachment; filename=\"test.txt\"", + "Content-Transfer-Encoding: base64", + "", + encoded, + "--" + boundary + "--", + }, "\r\n") + + p := NewEmailParser() + p.ConfigureFromSetup(map[string]any{ + "output_format": "json", + "fields": []string{"from", "body", "attachments"}, + }) + + result := p.ParseWithResult("test.eml", []byte(raw)) + if result.Err != nil { + t.Fatalf("unexpected error: %v", result.Err) + } + item := result.JSON[0] + + atts, ok := item["attachments"].([]map[string]any) + if !ok { + t.Fatalf("attachments missing or wrong type: %T", item["attachments"]) + } + if len(atts) != 1 { + t.Fatalf("expected 1 attachment, got %d", len(atts)) + } + payload, ok := atts[0]["payload"].(string) + if !ok { + t.Fatalf("payload missing or wrong type: %T", atts[0]["payload"]) + } + if payload != attachmentContent { + t.Errorf("attachment payload = %q, want %q (should be decoded from base64, not raw base64)", payload, attachmentContent) + } + if fn, _ := atts[0]["filename"].(string); fn != "test.txt" { + t.Errorf("filename = %q, want test.txt", fn) + } +} + +func TestEmailParser_Base64AttachmentInMixedMultipart(t *testing.T) { + // Simulates the original test email structure: + // multipart/mixed → multipart/alternative (text/plain + text/html) + base64 attachment + innerBoundary := "inneralt" + outerBoundary := "outermixed" + attachmentContent := "

Bookmarks

Test data

" + encoded := base64.StdEncoding.EncodeToString([]byte(attachmentContent)) + // Simulate MIME line-wrapping (typically 76 chars per line). + if len(encoded) > 20 { + encoded = encoded[:20] + "\r\n" + encoded[20:] + } + + raw := strings.Join([]string{ + "From: sender@test.com", + "To: receiver@test.com", + "Subject: Mixed Multipart Test", + "Content-Type: multipart/mixed; boundary=" + outerBoundary, + "", + "--" + outerBoundary, + "Content-Type: multipart/alternative; boundary=" + innerBoundary, + "", + "--" + innerBoundary, + "Content-Type: text/plain; charset=utf-8", + "", + "Plain text body.", + "--" + innerBoundary, + "Content-Type: text/html; charset=utf-8", + "", + "

HTML body.

", + "--" + innerBoundary + "--", + "--" + outerBoundary, + "Content-Type: text/html; charset=utf-8", + "Content-Disposition: attachment; filename=\"bookmarks.html\"", + "Content-Transfer-Encoding: base64", + "", + encoded, + "--" + outerBoundary + "--", + }, "\r\n") + + p := NewEmailParser() + p.ConfigureFromSetup(map[string]any{ + "output_format": "json", + "fields": []string{"from", "body", "attachments"}, + }) + + result := p.ParseWithResult("test.eml", []byte(raw)) + if result.Err != nil { + t.Fatalf("unexpected error: %v", result.Err) + } + item := result.JSON[0] + + // Verify body text was extracted from nested multipart/alternative + if v, ok := item["text"].(string); !ok || !strings.Contains(v, "Plain text body") { + t.Errorf("text: got %q, want to contain 'Plain text body'", v) + } + if v, ok := item["text_html"].(string); !ok || !strings.Contains(v, "HTML body") { + t.Errorf("text_html: got %q, want to contain 'HTML body'", v) + } + + // Verify attachment is decoded from base64 + atts, ok := item["attachments"].([]map[string]any) + if !ok || len(atts) != 1 { + t.Fatalf("expected 1 attachment, got %d", len(atts)) + } + payload, _ := atts[0]["payload"].(string) + if payload != attachmentContent { + t.Errorf("attachment payload = %q, want %q (should be decoded from base64)", payload, attachmentContent) + } +} + func TestEmailParser_Multipart(t *testing.T) { boundary := "boundary123" raw := strings.Join([]string{ diff --git a/internal/parser/parser/office_parsers_no_cgo.go b/internal/parser/parser/office_parsers_no_cgo.go index 25a1f2ab6c..ce1e3a1604 100644 --- a/internal/parser/parser/office_parsers_no_cgo.go +++ b/internal/parser/parser/office_parsers_no_cgo.go @@ -61,6 +61,11 @@ func NewDOCXParser() *DOCXParser { return &DOCXParser{} } +func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) { + // No-op in the no-CGO stub: the real implementation in + // docx_parser.go reads output_format from setup. +} + func (p *DOCXParser) String() string { return "DOCXParser(no-cgo)" }