Files
ragflow/internal/entity/models/get_driver_test.go
Jack bb6b43b5c9 fix: docx/email parsing, extractor LLM driver, and chunker alignment (#17144)
## Summary

Three groups of changes across the Go ingestion pipeline:

### 1. DOCX parsing improvements
- **docx_parser.go**: Enhanced DOCX parsing with better structure
extraction and media handling
- **docx_parser_cgo_test.go**, **docx_parser_test.go**: Companion tests
- **office_parsers_no_cgo.go**: Stub sync for non-CGO builds

### 2. Email (.eml) parsing: base64 Content-Transfer-Encoding decoding
- **email_parser.go** (`decodeCTE`): Added Content-Transfer-Encoding
decoding for base64 and quoted-printable. Go's `mime/multipart.Reader`
does not decode Content-Transfer-Encoding automatically, so attachments
with `Content-Transfer-Encoding: base64` remained base64-encoded in the
output. The new `decodeCTE` helper is called after reading each
multipart part's raw bytes in `readMailBody`, mirroring Python's
`part.get_payload(decode=True)`.
- **email_parser_test.go**: Two new tests — simple base64 attachment and
nested multipart/alternative with base64 attachment.

### 3. Extractor LLM driver fix + ModelDriver consolidation
- **extractor.go**: Fixed a bug where the Extractor component used
`ModelFactory.CreateModelDriver()`, which creates bare model instances
without API keys or provider configuration. Switched to
`models.GetPreconfiguredDriver()` which resolves the actual
pre-configured driver from `ProviderManager`, matching the codepath used
by `llm.go`. This fixes auto keyword/question extraction in DSL
pipelines that require LLM calls.
- **get_driver.go** (new): Extracted shared `GetPreconfiguredDriver()`
from `llm.go:newChatModelDriver()` so both `llm.go` and `extractor.go`
use the same codepath.
- **get_driver_test.go** (new): Tests for the shared driver resolution.
- **llm.go**: Replaced inline driver resolution with
`models.GetPreconfiguredDriver()`.

### 4. Chunker fixes and observability
- **group.go** (`extractLineRecords`): Fixed to also read `markdown` and
`html` payload keys — previously it only read `text`/`content`, causing
GroupTitleChunker to silently return empty results for markdown-format
parser output.
- **common.go** (`compileDelimPattern`): Aligned with Python's
`_compile_delimiter_pattern` — only backtick-wrapped delimiters produce
an active regex pattern; plain delimiters are not compiled into the
split regex.
- **token.go** (`applyChildrenDelim`): Set `DocType` and `CKType` to
`"text"` on created ChunkDocs so the token-size merge path correctly
identifies and merges text segments.
- **parser.go**, **extractor.go**, **tokenizer.go**, **group.go**,
**hierarchy.go**: Added debug-level logging for pipeline diagnostics.
- **parser_dispatch_test.go**, **group_test.go**: New tests.

## Verification
- All Go tests pass: `bash build.sh --test ./internal/parser/parser/...`
and `bash build.sh --test ./internal/ingestion/component/...`
- Build succeeds: `bash build.sh --go`
2026-07-21 13:51:17 +08:00

155 lines
4.5 KiB
Go

//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package models
import (
"strings"
"testing"
)
func TestGetPreconfiguredDriverReturnsPreBuiltDriver(t *testing.T) {
dir, restore := setupProviderTestDir(t, "aliyun.json")
defer restore()
if err := InitProviderManager(dir); err != nil {
t.Fatalf("InitProviderManager: %v", err)
}
driver, err := GetPreconfiguredDriver("Tongyi-Qianwen", "")
if err != nil {
t.Fatalf("GetPreconfiguredDriver: %v", err)
}
if driver == nil {
t.Fatal("GetPreconfiguredDriver returned nil, want non-nil")
}
if _, ok := driver.(*AliyunModel); !ok {
t.Fatalf("GetPreconfiguredDriver returned %T, want *AliyunModel", driver)
}
}
func TestGetPreconfiguredDriverWithBaseURLOverride(t *testing.T) {
dir, restore := setupProviderTestDir(t, "aliyun.json")
defer restore()
if err := InitProviderManager(dir); err != nil {
t.Fatalf("InitProviderManager: %v", err)
}
customURL := "https://custom-endpoint.example.com/v1"
driver, err := GetPreconfiguredDriver("Tongyi-Qianwen", customURL)
if err != nil {
t.Fatalf("GetPreconfiguredDriver: %v", err)
}
if driver == nil {
t.Fatal("GetPreconfiguredDriver returned nil, want non-nil")
}
// Verify the override took effect by checking GetBaseURL.
aliModel, ok := driver.(*AliyunModel)
if !ok {
t.Fatalf("GetPreconfiguredDriver returned %T, want *AliyunModel", driver)
}
gotURL, err := aliModel.baseModel.GetBaseURL(&APIConfig{})
if err != nil {
t.Fatalf("GetBaseURL: %v", err)
}
if expected := strings.TrimSuffix(customURL, "/"); gotURL != expected {
t.Errorf("GetBaseURL = %q, want %q", gotURL, expected)
}
}
func TestGetPreconfiguredDriverProviderNotFound(t *testing.T) {
dir, restore := setupProviderTestDir(t, "aliyun.json")
defer restore()
if err := InitProviderManager(dir); err != nil {
t.Fatalf("InitProviderManager: %v", err)
}
_, err := GetPreconfiguredDriver("NonExistentProvider", "")
if err == nil {
t.Fatal("GetPreconfiguredDriver should error for unknown provider, got nil")
}
}
func TestGetPreconfiguredDriverManagerNotInitialized(t *testing.T) {
// Save and reset the global so we can test the nil-manager path.
saved := providerManager
providerManager = nil
defer func() { providerManager = saved }()
_, err := GetPreconfiguredDriver("Tongyi-Qianwen", "")
if err == nil {
t.Fatal("GetPreconfiguredDriver should error when manager is nil, got nil")
}
}
func TestGetPreconfiguredDriverSuffixTrimmed(t *testing.T) {
dir, restore := setupProviderTestDir(t, "aliyun.json")
defer restore()
if err := InitProviderManager(dir); err != nil {
t.Fatalf("InitProviderManager: %v", err)
}
// Trailing slash should be stripped.
driver, err := GetPreconfiguredDriver("Tongyi-Qianwen", "https://example.com/")
if err != nil {
t.Fatalf("GetPreconfiguredDriver: %v", err)
}
aliModel, ok := driver.(*AliyunModel)
if !ok {
t.Fatalf("expected *AliyunModel, got %T", driver)
}
gotURL, err := aliModel.baseModel.GetBaseURL(&APIConfig{})
if err != nil {
t.Fatalf("GetBaseURL: %v", err)
}
if gotURL != "https://example.com" {
t.Errorf("GetBaseURL = %q, want %q", gotURL, "https://example.com")
}
}
func TestGetPreconfiguredDriverDummyNoOverride(t *testing.T) {
driver, err := GetPreconfiguredDriver("dummy", "")
if err != nil {
t.Fatalf("GetPreconfiguredDriver(dummy): %v", err)
}
if _, ok := driver.(*DummyModel); !ok {
t.Fatalf("expected *DummyModel, got %T", driver)
}
}
func TestGetPreconfiguredDriverDummyWithOverride(t *testing.T) {
driver, err := GetPreconfiguredDriver("Dummy", "https://override.example.com")
if err != nil {
t.Fatalf("GetPreconfiguredDriver(Dummy): %v", err)
}
dummy, ok := driver.(*DummyModel)
if !ok {
t.Fatalf("expected *DummyModel, got %T", driver)
}
// The override should be stored in the driver's BaseURL.
gotURL, err := dummy.baseModel.GetBaseURL(&APIConfig{})
if err != nil {
t.Fatalf("GetBaseURL: %v", err)
}
if gotURL != "https://override.example.com" {
t.Errorf("GetBaseURL = %q, want %q", gotURL, "https://override.example.com")
}
}