zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-31 04:59:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Go: add context to lots of interface (#17253)

Browse Source 
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-07-22 22:30:57 +08:00
committed by GitHub
parent b3d394954d
commit d19a036cda
221 changed files with 3215 additions and 2337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
internal/parser/parser/audio_parser.go
View File

@@ -22,6 +22,7 @@
package parser
import (
"context"
"fmt"
"path/filepath"
"strings"
@@ -70,7 +71,7 @@ func (p *AudioParser) ConfigureFromSetup(setup map[string]any) {
// file extension against the audio extension whitelist. The actual
// speech-to-text transcription happens via maybeDispatchAudio at the
// component layer (mirrors Python's LLMBundle.transcription call).
func (p *AudioParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *AudioParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
ext := strings.ToLower(filepath.Ext(filename))
if len(ext) > 1 && ext[0] == '.' {
ext = ext[1:]

15
internal/parser/parser/audio_parser_test.go
View File

@@ -22,6 +22,7 @@ import (
)
func TestAudioParser_ValidExtension(t *testing.T) {
ctx := t.Context()
p := NewAudioParser()
p.ConfigureFromSetup(map[string]any{
"output_format": "text",
@@ -30,7 +31,7 @@ func TestAudioParser_ValidExtension(t *testing.T) {
},
})
result := p.ParseWithResult("test.mp3", []byte{1, 2, 3, 4})
result := p.ParseWithResult(ctx, "test.mp3", []byte{1, 2, 3, 4})
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
@@ -43,9 +44,10 @@ func TestAudioParser_ValidExtension(t *testing.T) {
}
func TestAudioParser_AllExtensions(t *testing.T) {
ctx := t.Context()
for _, ext := range []string{"da", "wave", "wav", "mp3", "aac", "flac", "ogg", "aiff", "au", "midi", "wma", "realaudio", "vqf", "oggvorbis", "ape"} {
p := NewAudioParser()
result := p.ParseWithResult("audio."+ext, []byte{})
result := p.ParseWithResult(ctx, "audio."+ext, []byte{})
if result.Err != nil {
t.Errorf("extension .%s rejected: %v", ext, result.Err)
}
@@ -53,8 +55,9 @@ func TestAudioParser_AllExtensions(t *testing.T) {
}
func TestAudioParser_InvalidExtension(t *testing.T) {
ctx := t.Context()
p := NewAudioParser()
result := p.ParseWithResult("notaudio.txt", []byte{})
result := p.ParseWithResult(ctx, "notaudio.txt", []byte{})
if result.Err == nil {
t.Fatal("expected error for .txt extension")
}
@@ -76,19 +79,21 @@ func TestAudioParser_ConfigureVLM(t *testing.T) {
}
func TestAudioParser_ConfigureNilSafe(t *testing.T) {
ctx := t.Context()
p := NewAudioParser()
p.ConfigureFromSetup(nil)
p.ConfigureFromSetup(map[string]any{})
// Should not panic.
result := p.ParseWithResult("test.wav", []byte{})
result := p.ParseWithResult(ctx, "test.wav", []byte{})
if result.Err != nil {
t.Errorf("unexpected error: %v", result.Err)
}
}
func TestAudioParser_DefaultOutputFormat(t *testing.T) {
ctx := t.Context()
p := NewAudioParser()
result := p.ParseWithResult("test.flac", []byte{})
result := p.ParseWithResult(ctx, "test.flac", []byte{})
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}

3
internal/parser/parser/csv_parser.go
View File

@@ -31,6 +31,7 @@
package parser
import (
"context"
"encoding/csv"
"fmt"
"html"
@@ -113,7 +114,7 @@ func (p *CSVParser) ConfigureFromSetup(setup map[string]any) {
// Python's RAGFlowExcelParser.html().
// When TCADP parse_method is configured, the file is dispatched to
// the Tencent Cloud Document Parsing API.
func (p *CSVParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
method := normalizeXLSXParseMethod(p.ParseMethod)
switch method {
case "tcadp":

3
internal/parser/parser/doc_parser.go
View File

@@ -19,6 +19,7 @@
package parser
import (
"context"
"fmt"
officeOxide "github.com/yfedoseev/office_oxide/go"
@@ -39,7 +40,7 @@ func (p *DOCParser) String() string {
// the Go side uses office_oxide which supports DOC via PlainText.
// OutputFormat="text" — the python side falls back to text for
// legacy DOC files since structured extraction is unreliable.
func (p *DOCParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
doc, err := officeOxide.OpenFromBytes(data, "doc")
if err != nil {
return ParseResult{Err: fmt.Errorf("doc open: %w", err)}

3
internal/parser/parser/docx_parser.go
View File

@@ -19,6 +19,7 @@
package parser
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
@@ -66,7 +67,7 @@ func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {
//
// JSON path mirrors python parser.py:_docx() output_format == "json".
// Markdown path mirrors python naive.py: Docx() → naive_merge_docx().
func (p *DOCXParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
doc, err := officeOxide.OpenFromBytes(data, "docx")
if err != nil {
return ParseResult{Err: fmt.Errorf("docx open: %w", err)}

12
internal/parser/parser/docx_parser_cgo_test.go
View File

@@ -9,10 +9,11 @@ import (
)
func TestDOCXParser_ParseWithResult_JSON(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
p.outputFormat = "json"
data := minimalDOCX(t, "Hello from JSON path")
res := p.ParseWithResult("sample.docx", data)
res := p.ParseWithResult(ctx, "sample.docx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -33,6 +34,7 @@ func TestDOCXParser_ParseWithResult_JSON(t *testing.T) {
}
func TestDOCXParser_ConfigureFromSetup_JSON(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
p.ConfigureFromSetup(map[string]any{"output_format": "json"})
if p.outputFormat != "json" {
@@ -43,7 +45,7 @@ func TestDOCXParser_ConfigureFromSetup_JSON(t *testing.T) {
}
// Full round-trip: json config → json output
data := minimalDOCX(t, "Config test")
res := p.ParseWithResult("sample.docx", data)
res := p.ParseWithResult(ctx, "sample.docx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -56,13 +58,14 @@ func TestDOCXParser_ConfigureFromSetup_JSON(t *testing.T) {
}
func TestDOCXParser_ConfigureFromSetup_Markdown(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
p.ConfigureFromSetup(map[string]any{"output_format": "markdown"})
if p.outputFormat != "markdown" {
t.Fatalf("After ConfigureFromSetup, outputFormat = %q, want %q", p.outputFormat, "markdown")
}
data := minimalDOCX(t, "Config md test")
res := p.ParseWithResult("sample.docx", data)
res := p.ParseWithResult(ctx, "sample.docx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -75,9 +78,10 @@ func TestDOCXParser_ConfigureFromSetup_Markdown(t *testing.T) {
}
func TestDOCXParser_ParseWithResult_CGOMinimalDocument(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
data := minimalDOCX(t, "Hello from DOCX parser")
res := p.ParseWithResult("sample.docx", data)
res := p.ParseWithResult(ctx, "sample.docx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}

3
internal/parser/parser/email_parser.go
View File

@@ -18,6 +18,7 @@ package parser
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"io"
@@ -73,7 +74,7 @@ func (p *EmailParser) ConfigureFromSetup(setup map[string]any) {
}
}
func (p *EmailParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *EmailParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
ext := strings.ToLower(filepath.Ext(filename))
if ext == ".msg" {
return ParseResult{

18
internal/parser/parser/email_parser_test.go
View File

@@ -23,6 +23,7 @@ import (
)
func TestEmailParser_EmlJSON(t *testing.T) {
ctx := t.Context()
raw := strings.Join([]string{
"From: sender@example.com",
"To: recipient@example.com",
@@ -41,7 +42,7 @@ func TestEmailParser_EmlJSON(t *testing.T) {
"fields": []string{"from", "to", "cc", "date", "subject", "body", "metadata"},
})
result := p.ParseWithResult("test.eml", []byte(raw))
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
@@ -78,6 +79,7 @@ func TestEmailParser_EmlJSON(t *testing.T) {
}
func TestEmailParser_EmlText(t *testing.T) {
ctx := t.Context()
raw := strings.Join([]string{
"From: sender@test.com",
"To: recipient@test.com",
@@ -93,7 +95,7 @@ func TestEmailParser_EmlText(t *testing.T) {
"fields": []string{"from", "to", "subject", "body"},
})
result := p.ParseWithResult("test.eml", []byte(raw))
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
@@ -109,8 +111,9 @@ func TestEmailParser_EmlText(t *testing.T) {
}
func TestEmailParser_MsgNotSupported(t *testing.T) {
ctx := t.Context()
p := NewEmailParser()
result := p.ParseWithResult("test.msg", []byte{})
result := p.ParseWithResult(ctx, "test.msg", []byte{})
if result.Err == nil {
t.Fatal("expected error for .msg file")
}
@@ -120,6 +123,7 @@ func TestEmailParser_MsgNotSupported(t *testing.T) {
}
func TestEmailParser_Base64Attachment(t *testing.T) {
ctx := t.Context()
attachmentContent := "Hello! This is the decoded content of the attachment."
encoded := base64.StdEncoding.EncodeToString([]byte(attachmentContent))
// Simulate MIME line-wrapping (typically 76 chars per line).
@@ -153,7 +157,7 @@ func TestEmailParser_Base64Attachment(t *testing.T) {
"fields": []string{"from", "body", "attachments"},
})
result := p.ParseWithResult("test.eml", []byte(raw))
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
@@ -179,6 +183,7 @@ func TestEmailParser_Base64Attachment(t *testing.T) {
}
func TestEmailParser_Base64AttachmentInMixedMultipart(t *testing.T) {
ctx := t.Context()
// Simulates the original test email structure:
// multipart/mixed → multipart/alternative (text/plain + text/html) + base64 attachment
innerBoundary := "inneralt"
@@ -223,7 +228,7 @@ func TestEmailParser_Base64AttachmentInMixedMultipart(t *testing.T) {
"fields": []string{"from", "body", "attachments"},
})
result := p.ParseWithResult("test.eml", []byte(raw))
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
@@ -249,6 +254,7 @@ func TestEmailParser_Base64AttachmentInMixedMultipart(t *testing.T) {
}
func TestEmailParser_Multipart(t *testing.T) {
ctx := t.Context()
boundary := "boundary123"
raw := strings.Join([]string{
"From: multipart@test.com",
@@ -273,7 +279,7 @@ func TestEmailParser_Multipart(t *testing.T) {
"fields": []string{"from", "body"},
})
result := p.ParseWithResult("test.eml", []byte(raw))
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}

3
internal/parser/parser/epub_parser.go
View File

@@ -31,6 +31,7 @@ package parser
import (
"archive/zip"
"bytes"
"context"
"encoding/xml"
"fmt"
"io"
@@ -53,7 +54,7 @@ func (p *EPUBParser) String() string {
// ParseWithResult implements ParseResultProducer. It extracts XHTML
// content from the EPUB spine and emits one JSON item per spine entry
// with {text, doc_type_kwd:"text"}.
func (p *EPUBParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
if len(data) == 0 {
return ParseResult{
OutputFormat: "json",

3
internal/parser/parser/html_parser.go
View File

@@ -18,6 +18,7 @@ package parser
import (
"bytes"
"context"
"fmt"
"strings"
@@ -47,7 +48,7 @@ func (p *HTMLParser) String() string {
// (bold / links / images) is intentionally NOT surfaced as a
// separate ck_type — the python HtmlParser collapses inline
// formatting into the parent block's text.
func (p *HTMLParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
doc, err := html.Parse(bytes.NewReader(data))
if err != nil {
return ParseResult{Err: fmt.Errorf("html parse: %w", err)}

3
internal/parser/parser/json_parser.go
View File

@@ -32,6 +32,7 @@ package parser
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"strings"
@@ -51,7 +52,7 @@ func (p *JSONParser) String() string {
// ParseWithResult implements ParseResultProducer. It detects the JSON
// shape (array, single object, or line-delimited) and emits one JSON
// item per logical record, with {text, doc_type_kwd:"text"}.
func (p *JSONParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *JSONParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
text := string(bytes.TrimSpace(data))
if text == "" {
return ParseResult{

2
internal/parser/parser/markdown_parser.go
View File

@@ -89,7 +89,7 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {
// data is resolved and the item carries `doc_type_kwd: "image"` with
// the base64-encoded image payload. The legacy debug-print path has
// been removed; callers consume ParseResult directly.
func (p *MarkdownParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
doc := markdownNew().Parse(data)
rawText := string(data)

12
internal/parser/parser/markdown_parser_test.go
View File

@@ -8,12 +8,13 @@ import (
)
func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {
ctx := t.Context()
p, err := NewMarkdownParser(GoMarkdown)
if err != nil {
t.Fatalf("NewMarkdownParser: %v", err)
}
md := "# Hello\n\nThis is a paragraph.\n\n* List item 1\n* List item 2\n\n```go\nfunc main() {}\n```\n"
res := p.ParseWithResult("test.md", []byte(md))
res := p.ParseWithResult(ctx, "test.md", []byte(md))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -33,8 +34,9 @@ func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {
}
func TestMarkdownParser_ParseWithResult_EmptyInput(t *testing.T) {
ctx := t.Context()
p, _ := NewMarkdownParser(GoMarkdown)
res := p.ParseWithResult("empty.md", []byte(""))
res := p.ParseWithResult(ctx, "empty.md", []byte(""))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -44,13 +46,14 @@ func TestMarkdownParser_ParseWithResult_EmptyInput(t *testing.T) {
}
func TestMarkdownParser_ParseWithResult_ImageDataURI(t *testing.T) {
ctx := t.Context()
p, _ := NewMarkdownParser(GoMarkdown)
// 1×1 pixel transparent PNG encoded as data URI
pixelPNG := make([]byte, 68) // minimal 1x1 PNG header
pixelB64 := base64.StdEncoding.EncodeToString([]byte("fake-png-data"))
md := "Some text with an image\n![test](data:image/png;base64," + pixelB64 + ")\n"
_ = pixelPNG
res := p.ParseWithResult("test.md", []byte(md))
res := p.ParseWithResult(ctx, "test.md", []byte(md))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -71,9 +74,10 @@ func TestMarkdownParser_ParseWithResult_ImageDataURI(t *testing.T) {
}
func TestMarkdownParser_ParseWithResult_NoImage(t *testing.T) {
ctx := t.Context()
p, _ := NewMarkdownParser(GoMarkdown)
md := "# Title\n\nJust some text, no images here.\n\nMore text."
res := p.ParseWithResult("test.md", []byte(md))
res := p.ParseWithResult(ctx, "test.md", []byte(md))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}

4
internal/parser/parser/parse_result.go
View File

@@ -31,6 +31,8 @@
package parser
import "context"
// ParseResult is the structured return value of a successful parse.
// Exactly one of the payload fields (JSON / Markdown / Text / HTML)
// is populated on success, matching the Python contract — see
@@ -84,5 +86,5 @@ type ParseResult struct {
// ParseResultProducer is the parser package's single structured-output
// contract. Every parser returned by GetParser must implement it.
type ParseResultProducer interface {
ParseWithResult(filename string, data []byte) ParseResult
ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult
}

3
internal/parser/parser/parse_result_test.go
View File

@@ -130,9 +130,10 @@ func (s sentinelErr) Error() string { return string(s) }
// tiny: 1 heading, 1 paragraph, 1 unordered list item, no nested
// formatting.
func TestMarkdownParser_ParseWithResult(t *testing.T) {
ctx := t.Context()
p, _ := NewMarkdownParser(GoMarkdown)
src := []byte("# Title\n\nFirst paragraph.\n\n- Item one\n")
res := p.ParseWithResult("doc.md", src)
res := p.ParseWithResult(ctx, "doc.md", src)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}

18
internal/parser/parser/parse_with_result_test.go
View File

@@ -45,7 +45,8 @@ import (
func TestTextParser_ParseWithResult_ParaSplit(t *testing.T) {
p := NewTextParser()
src := []byte("First paragraph.\n\nSecond paragraph.\n\nThird.")
res := p.ParseWithResult("doc.txt", src)
ctx := t.Context()
res := p.ParseWithResult(ctx, "doc.txt", src)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -71,8 +72,9 @@ func TestTextParser_ParseWithResult_ParaSplit(t *testing.T) {
// sees a non-nil JSON slice. Mirrors the MarkdownParser convention
// at markdown_parser.go:71-76.
func TestTextParser_ParseWithResult_Empty(t *testing.T) {
ctx := t.Context()
p := NewTextParser()
res := p.ParseWithResult("empty.txt", []byte{})
res := p.ParseWithResult(ctx, "empty.txt", []byte{})
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -85,9 +87,10 @@ func TestTextParser_ParseWithResult_Empty(t *testing.T) {
// maxItemBytes boundary behaviour. A single paragraph longer
// than 8192 bytes is sliced at the nearest line boundary.
func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) {
ctx := t.Context()
p := NewTextParser()
long := strings.Repeat("a", 9000)
res := p.ParseWithResult("long.txt", []byte(long))
res := p.ParseWithResult(ctx, "long.txt", []byte(long))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -105,9 +108,10 @@ func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) {
// validation rule. Invalid bytes produce an error in the result
// (matching the python TxtParser's behaviour).
func TestTextParser_ParseWithResult_InvalidUTF8(t *testing.T) {
ctx := t.Context()
p := NewTextParser()
bad := []byte{0xff, 0xfe, 0xfd}
res := p.ParseWithResult("bad.txt", bad)
res := p.ParseWithResult(ctx, "bad.txt", bad)
if res.Err == nil {
t.Fatal("want error for invalid UTF-8, got nil")
}
@@ -117,13 +121,14 @@ func TestTextParser_ParseWithResult_InvalidUTF8(t *testing.T) {
// Three block elements (heading, paragraph, list) yield three
// items with the python-compatible ck_type vocabulary.
func TestHTMLParser_ParseWithResult_BlockSplit(t *testing.T) {
ctx := t.Context()
p := NewHTMLParser()
src := []byte(`<!DOCTYPE html><html><body>
<h1>Title</h1>
<p>First paragraph.</p>
<ul><li>Item one</li></ul>
</body></html>`)
res := p.ParseWithResult("doc.html", src)
res := p.ParseWithResult(ctx, "doc.html", src)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -157,6 +162,7 @@ func TestHTMLParser_ParseWithResult_BlockSplit(t *testing.T) {
// rule that <script> / <style> subtrees are skipped entirely so
// they don't pollute the downstream chunker input.
func TestHTMLParser_ParseWithResult_SkipsScriptAndStyle(t *testing.T) {
ctx := t.Context()
p := NewHTMLParser()
src := []byte(`<html><body>
<p>Visible.</p>
@@ -164,7 +170,7 @@ func TestHTMLParser_ParseWithResult_SkipsScriptAndStyle(t *testing.T) {
<style>body { color: red; }</style>
<p>Also visible.</p>
</body></html>`)
res := p.ParseWithResult("doc.html", src)
res := p.ParseWithResult(ctx, "doc.html", src)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}

12
internal/parser/parser/pdf_parser_cgo.go
View File

@@ -12,7 +12,7 @@ import (
deepdoctype "ragflow/internal/deepdoc/parser/type"
)
func (p *PDFParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *PDFParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
if err := p.validateParseMethod(); err != nil {
return ParseResult{Err: err}
}
@@ -21,13 +21,13 @@ func (p *PDFParser) ParseWithResult(filename string, data []byte) ParseResult {
case "plain_text":
return parsePDFWithPlainText(filename, data, p)
case "mineru":
return parsePDFWithMinerU(filename, data, p)
return parsePDFWithMinerU(ctx, filename, data, p)
case "paddleocr":
return parsePDFWithPaddleOCR(filename, data, p)
return parsePDFWithPaddleOCR(ctx, filename, data, p)
case "docling":
return parsePDFWithDocling(filename, data, p)
return parsePDFWithDocling(ctx, filename, data, p)
case "opendataloader":
return parsePDFWithOpenDataLoader(filename, data, p)
return parsePDFWithOpenDataLoader(ctx, filename, data, p)
case "somark":
return parsePDFWithSoMark(filename, data, p)
case "tcadp":
@@ -36,7 +36,7 @@ func (p *PDFParser) ParseWithResult(filename string, data []byte) ParseResult {
cfg := deepdoctype.DefaultParserConfig()
cfg.SkipOCR = false
parser := deepdocpdf.NewParser(cfg)
res := parsePDFWithDeepDocOptions(context.Background(), filename, data, pdfPostProcessOptions{
res := parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{
outputFormat: p.OutputFormat,
zoom: cfg.Zoom,
enableMultiColumn: p.EnableMultiColumn,

6
internal/parser/parser/pdf_parser_cgo_test.go
View File

@@ -8,19 +8,21 @@ import (
)
func TestPDFParser_ParseWithResult_CGOInvalidPDF(t *testing.T) {
ctx := t.Context()
t.Setenv("DEEPDOC_URL", "")
t.Setenv("OSSDEEPDOC_URL", "")
pdf := NewPDFParser()
res := pdf.ParseWithResult("bad.pdf", []byte("not a valid pdf"))
res := pdf.ParseWithResult(ctx, "bad.pdf", []byte("not a valid pdf"))
if res.Err == nil {
t.Fatal("want parse error for invalid PDF bytes, got nil")
}
}
func TestPDFParser_ParseWithResult_CGOEmpty(t *testing.T) {
ctx := t.Context()
pdf := NewPDFParser()
res := pdf.ParseWithResult("empty.pdf", nil)
res := pdf.ParseWithResult(ctx, "empty.pdf", nil)
if res.Err != nil {
t.Fatalf("empty input: want nil err, got %v", res.Err)
}

10
internal/parser/parser/pdf_parser_docling.go
View File

@@ -36,7 +36,7 @@ type doclingResponse struct {
Results []doclingResult `json:"results"`
}
func parsePDFWithDocling(filename string, data []byte, parser *PDFParser) ParseResult {
func parsePDFWithDocling(ctx context.Context, filename string, data []byte, parser *PDFParser) ParseResult {
if len(data) == 0 {
return emptyPDFResult(filename)
}
@@ -116,7 +116,7 @@ func parsePDFWithDocling(filename string, data []byte, parser *PDFParser) ParseR
lastErr = fmt.Errorf("%s: chunked response contained no usable text", candidate.endpoint)
continue
}
if res, ok := parseDoclingStandardResult(filename, body, parser.OutputFormat); ok {
if res, ok := parseDoclingStandardResult(ctx, filename, body, parser.OutputFormat); ok {
return res
}
lastErr = fmt.Errorf("%s: standard response contained no parsed document", candidate.endpoint)
@@ -189,7 +189,7 @@ func parseDoclingChunkedResult(filename string, body []byte, outputFormat string
return doclingTextsToResult(filename, texts, outputFormat, pageCount), true
}
func parseDoclingStandardResult(filename string, body []byte, outputFormat string) (ParseResult, bool) {
func parseDoclingStandardResult(ctx context.Context, filename string, body []byte, outputFormat string) (ParseResult, bool) {
var payload doclingResponse
if err := json.Unmarshal(body, &payload); err != nil {
return ParseResult{}, false
@@ -209,13 +209,13 @@ func parseDoclingStandardResult(filename string, body []byte, outputFormat strin
}
for _, doc := range docs {
if md := strings.TrimSpace(doc.MDContent); md != "" {
return parseMinerUMarkdownResult(filename, md, outputFormat, len(docs)), true
return parseMinerUMarkdownResult(ctx, filename, md, outputFormat, len(docs)), true
}
if txt := strings.TrimSpace(doc.TextContent); txt != "" {
return doclingTextsToResult(filename, []string{txt}, outputFormat, len(docs)), true
}
if md, _ := doc.JSONContent["md_content"].(string); strings.TrimSpace(md) != "" {
return parseMinerUMarkdownResult(filename, md, outputFormat, len(docs)), true
return parseMinerUMarkdownResult(ctx, filename, md, outputFormat, len(docs)), true
}
}
return ParseResult{}, false

9
internal/parser/parser/pdf_parser_docling_test.go
View File

@@ -70,7 +70,8 @@ func TestPDFParser_ParseWithResult_DoclingChunkedMarkdownIntegration(t *testing.
"docling_api_key": "doc-secret",
})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -132,7 +133,8 @@ func TestPDFParser_ParseWithResult_DoclingFallbackToStandardJSONIntegration(t *t
"docling_server_url": server.URL,
})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -151,7 +153,8 @@ func TestPDFParser_ParseWithResult_DoclingRequiresServerURL(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{"parse_method": "Docling"})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err == nil {
t.Fatal("ParseWithResult: want error when docling_server_url is missing, got nil")
}

9
internal/parser/parser/pdf_parser_fixture_cgo_test.go
View File

@@ -19,7 +19,8 @@ func TestPDFParser_ParseWithResult_CGOFixture(t *testing.T) {
t.Fatalf("ReadFile(%s): %v", path, err)
}
pdf := NewPDFParser()
res := pdf.ParseWithResult("Doc1.pdf", data)
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "Doc1.pdf", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -49,7 +50,8 @@ func TestPDFParser_ParseWithResult_CGOFixtureMarkdown(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{"output_format": "markdown"})
res := pdf.ParseWithResult("Doc1.pdf", data)
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "Doc1.pdf", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -72,7 +74,8 @@ func TestPDFParser_ParseWithResult_CGOFixturePlainText(t *testing.T) {
}
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{"parse_method": "plain_text"})
res := pdf.ParseWithResult("Doc1.pdf", data)
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "Doc1.pdf", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}

17
internal/parser/parser/pdf_parser_mineru.go
View File

@@ -1,6 +1,7 @@
package parser
import (
"context"
"fmt"
"ragflow/internal/common"
"strings"
@@ -12,7 +13,7 @@ import (
const minerUPollTimeout = 30 * time.Second
const minerUPollInterval = 200 * time.Millisecond
func parsePDFWithMinerU(filename string, data []byte, parser *PDFParser) ParseResult {
func parsePDFWithMinerU(ctx context.Context, filename string, data []byte, parser *PDFParser) ParseResult {
if len(data) == 0 {
return emptyPDFResult(filename)
}
@@ -50,11 +51,11 @@ func parsePDFWithMinerU(filename string, data []byte, parser *PDFParser) ParseRe
apiConfig.ApiKey = &apiKey
}
task, err := driver.ParseFile(&backend, data, nil, apiConfig, &models.ParseFileConfig{}, nil)
task, err := driver.ParseFile(ctx, &backend, data, nil, apiConfig, &models.ParseFileConfig{}, nil)
if err != nil {
return ParseResult{Err: fmt.Errorf("parser: MinerU submit: %w", err)}
}
content, err := pollMinerUTask(driver, task.TaskID, apiConfig, timeout)
content, err := pollMinerUTask(ctx, driver, task.TaskID, apiConfig, timeout)
if err != nil {
return ParseResult{Err: fmt.Errorf("parser: MinerU result: %w", err)}
}
@@ -62,17 +63,17 @@ func parsePDFWithMinerU(filename string, data []byte, parser *PDFParser) ParseRe
if strings.TrimSpace(content) != "" {
pageCount = 1
}
return parseMinerUMarkdownResult(filename, content, parser.OutputFormat, pageCount)
return parseMinerUMarkdownResult(ctx, filename, content, parser.OutputFormat, pageCount)
}
func pollMinerUTask(driver *models.MinerULocalModel, taskID string, apiConfig *models.APIConfig, timeout time.Duration) (string, error) {
func pollMinerUTask(ctx context.Context, driver *models.MinerULocalModel, taskID string, apiConfig *models.APIConfig, timeout time.Duration) (string, error) {
if timeout <= 0 {
timeout = minerUPollTimeout
}
deadline := time.Now().Add(timeout)
var lastErr error
for {
task, err := driver.ShowTask(taskID, apiConfig)
task, err := driver.ShowTask(ctx, taskID, apiConfig)
if err == nil {
for _, segment := range task.Segments {
if strings.TrimSpace(segment.Content) != "" {
@@ -93,12 +94,12 @@ func pollMinerUTask(driver *models.MinerULocalModel, taskID string, apiConfig *m
}
}
func parseMinerUMarkdownResult(filename, markdown, outputFormat string, pageCount int) ParseResult {
func parseMinerUMarkdownResult(ctx context.Context, filename, markdown, outputFormat string, pageCount int) ParseResult {
fileMeta := pdfFileMeta(filename, pageCount)
switch strings.ToLower(strings.TrimSpace(outputFormat)) {
case "", "json":
mp, _ := NewMarkdownParser(GoMarkdown)
res := mp.ParseWithResult(filename, []byte(markdown))
res := mp.ParseWithResult(ctx, filename, []byte(markdown))
if res.Err != nil {
return res
}

9
internal/parser/parser/pdf_parser_mineru_test.go
View File

@@ -80,7 +80,8 @@ func TestPDFParser_ParseWithResult_MinerUMarkdownIntegration(t *testing.T) {
"mineru_backend": "pipeline",
})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -120,7 +121,8 @@ func TestPDFParser_ParseWithResult_MinerUJSONIntegration(t *testing.T) {
"mineru_apiserver": server.URL,
})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -139,7 +141,8 @@ func TestPDFParser_ParseWithResult_MinerURequiresAPIServer(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{"parse_method": "MinerU"})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err == nil {
t.Fatal("ParseWithResult: want error when mineru_apiserver is missing, got nil")
}

4
internal/parser/parser/pdf_parser_opendataloader.go
View File

@@ -13,7 +13,7 @@ import (
models "ragflow/internal/entity/models"
)
func parsePDFWithOpenDataLoader(filename string, data []byte, parser *PDFParser) ParseResult {
func parsePDFWithOpenDataLoader(ctx context.Context, filename string, data []byte, parser *PDFParser) ParseResult {
if len(data) == 0 {
return emptyPDFResult(filename)
}
@@ -67,7 +67,7 @@ func parsePDFWithOpenDataLoader(filename string, data []byte, parser *PDFParser)
}
}
if strings.TrimSpace(payload.MDText) != "" {
return parseMinerUMarkdownResult(filename, payload.MDText, parser.OutputFormat, 1)
return parseMinerUMarkdownResult(ctx, filename, payload.MDText, parser.OutputFormat, 1)
}
return ParseResult{Err: fmt.Errorf("parser: OpenDataLoader returned no parsed content")}
}

9
internal/parser/parser/pdf_parser_opendataloader_test.go
View File

@@ -70,7 +70,8 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderJSONIntegration(t *testing.T) {
"sanitize": sanitize,
})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -99,7 +100,8 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderMarkdownFallback(t *testing.T)
"output_format": "markdown",
"opendataloader_apiserver": server.URL,
})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -111,7 +113,8 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderMarkdownFallback(t *testing.T)
func TestPDFParser_ParseWithResult_OpenDataLoaderRequiresAPIServer(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{"parse_method": "OpenDataLoader"})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err == nil || !strings.Contains(res.Err.Error(), "opendataloader_apiserver") {
t.Fatalf("error = %v, want opendataloader_apiserver context", res.Err)
}

7
internal/parser/parser/pdf_parser_paddleocr.go
View File

@@ -1,6 +1,7 @@
package parser
import (
"context"
"fmt"
"ragflow/internal/common"
"strings"
@@ -8,7 +9,7 @@ import (
models "ragflow/internal/entity/models"
)
func parsePDFWithPaddleOCR(filename string, data []byte, parser *PDFParser) ParseResult {
func parsePDFWithPaddleOCR(ctx context.Context, filename string, data []byte, parser *PDFParser) ParseResult {
if len(data) == 0 {
return emptyPDFResult(filename)
}
@@ -45,7 +46,7 @@ func parsePDFWithPaddleOCR(filename string, data []byte, parser *PDFParser) Pars
apiConfig.ApiKey = &apiKey
}
resp, err := driver.OCRFile(&algorithm, data, &filename, apiConfig, &models.OCRConfig{
resp, err := driver.OCRFile(ctx, &algorithm, data, &filename, apiConfig, &models.OCRConfig{
Algorithm: algorithm,
}, nil)
if err != nil {
@@ -58,5 +59,5 @@ func parsePDFWithPaddleOCR(filename string, data []byte, parser *PDFParser) Pars
if resp.Text != nil && strings.TrimSpace(*resp.Text) == "" {
pageCount = 0
}
return parseMinerUMarkdownResult(filename, *resp.Text, parser.OutputFormat, pageCount)
return parseMinerUMarkdownResult(ctx, filename, *resp.Text, parser.OutputFormat, pageCount)
}

9
internal/parser/parser/pdf_parser_paddleocr_test.go
View File

@@ -62,7 +62,8 @@ func TestPDFParser_ParseWithResult_PaddleOCRMarkdownIntegration(t *testing.T) {
"paddleocr_api_key": "paddle-secret",
})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -98,7 +99,8 @@ func TestPDFParser_ParseWithResult_PaddleOCRJSONIntegration(t *testing.T) {
"paddleocr_base_url": server.URL,
})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -117,7 +119,8 @@ func TestPDFParser_ParseWithResult_PaddleOCRRequiresBaseURL(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{"parse_method": "PaddleOCR"})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err == nil {
t.Fatal("ParseWithResult: want error when paddleocr_base_url is missing, got nil")
}

9
internal/parser/parser/pdf_parser_somark_test.go
View File

@@ -64,7 +64,8 @@ func TestPDFParser_ParseWithResult_SoMarkJSONIntegration(t *testing.T) {
"somark_base_url": server.URL,
"somark_api_key": "somark-secret",
})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -94,7 +95,8 @@ func TestPDFParser_ParseWithResult_SoMarkMarkdownIntegration(t *testing.T) {
"output_format": "markdown",
"somark_base_url": server.URL,
})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -107,7 +109,8 @@ func TestPDFParser_ParseWithResult_SoMarkRequiresBaseURL(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{"parse_method": "SoMark"})
pdf.SoMarkBaseURL = ""
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err == nil || !strings.Contains(res.Err.Error(), "somark_base_url") {
t.Fatalf("error = %v, want somark_base_url context", res.Err)
}

9
internal/parser/parser/pdf_parser_tcadp_test.go
View File

@@ -36,7 +36,8 @@ func TestPDFParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
"tcadp_apiserver": server.URL,
"tcadp_api_key": "tcadp-secret",
})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -65,7 +66,8 @@ func TestPDFParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {
"output_format": "markdown",
"tcadp_apiserver": server.URL,
})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -77,7 +79,8 @@ func TestPDFParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {
func TestPDFParser_ParseWithResult_TCADPRequiresAPIServer(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser"})
res := pdf.ParseWithResult("sample.pdf", []byte("%PDF-1.4\nmock"))
ctx := t.Context()
res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
if res.Err == nil || !strings.Contains(res.Err.Error(), "tcadp_apiserver") {
t.Fatalf("error = %v, want tcadp_apiserver context", res.Err)
}

3
internal/parser/parser/picture_parser.go
View File

@@ -30,6 +30,7 @@
package parser
import (
"context"
"fmt"
"path/filepath"
"strings"
@@ -98,7 +99,7 @@ func (p *PictureParser) ConfigureFromSetup(setup map[string]any) {
// file extension against the image extension whitelist. The actual
// OCR and VLM description happens via maybeDispatchImage at the
// component layer (mirrors Python's picture.py:chunk()).
func (p *PictureParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
ext := strings.ToLower(filepath.Ext(filename))
if len(ext) > 1 && ext[0] == '.' {
ext = ext[1:]

15
internal/parser/parser/picture_parser_test.go
View File

@@ -66,9 +66,10 @@ func TestPictureParser_ConfigureFromSetup(t *testing.T) {
}
func TestPictureParser_ParseWithResult_NilSetup(t *testing.T) {
ctx := t.Context()
p := NewPictureParser()
p.ConfigureFromSetup(nil)
res := p.ParseWithResult("photo.png", []byte("\x89PNG"))
res := p.ParseWithResult(ctx, "photo.png", []byte("\x89PNG"))
if res.Err != nil {
t.Errorf("unexpected error for nil setup: %v", res.Err)
}
@@ -82,11 +83,12 @@ func TestPictureParser_ParseWithResult_NilSetup(t *testing.T) {
}
func TestPictureParser_ParseWithResult_ValidExtensions(t *testing.T) {
ctx := t.Context()
exts := []string{"png", "jpg", "jpeg", "gif", "bmp", "tiff", "tif", "webp", "svg", "ico", "avif", "heic", "apng"}
p := NewPictureParser()
for _, ext := range exts {
fn := "img." + ext
res := p.ParseWithResult(fn, []byte{1, 2, 3})
res := p.ParseWithResult(ctx, fn, []byte{1, 2, 3})
if res.Err != nil {
t.Errorf("unexpected error for .%s: %v", ext, res.Err)
}
@@ -97,8 +99,9 @@ func TestPictureParser_ParseWithResult_ValidExtensions(t *testing.T) {
}
func TestPictureParser_ParseWithResult_InvalidExtension(t *testing.T) {
ctx := t.Context()
p := NewPictureParser()
res := p.ParseWithResult("sound.mp3", []byte{1})
res := p.ParseWithResult(ctx, "sound.mp3", []byte{1})
if res.Err == nil {
t.Error("expected error for .mp3, got nil")
}
@@ -108,8 +111,9 @@ func TestPictureParser_ParseWithResult_InvalidExtension(t *testing.T) {
}
func TestPictureParser_ParseWithResult_VideoExtension(t *testing.T) {
ctx := t.Context()
p := NewPictureParser()
res := p.ParseWithResult("video.mp4", []byte{1})
res := p.ParseWithResult(ctx, "video.mp4", []byte{1})
if res.Err == nil {
t.Error("expected error for video .mp4, got nil")
}
@@ -119,11 +123,12 @@ func TestPictureParser_ParseWithResult_VideoExtension(t *testing.T) {
}
func TestPictureParser_ParseWithResult_OutputFormat(t *testing.T) {
ctx := t.Context()
p := NewPictureParser()
p.ConfigureFromSetup(map[string]any{
"output_format": "json",
})
res := p.ParseWithResult("photo.jpg", []byte{1, 2, 3})
res := p.ParseWithResult(ctx, "photo.jpg", []byte{1, 2, 3})
if res.Err != nil {
t.Fatalf("unexpected error: %v", res.Err)
}

6
internal/parser/parser/ppt_parser.go
View File

@@ -18,6 +18,8 @@
package parser
import "context"
type PPTParser struct{}
func NewPPTParser() *PPTParser {
@@ -33,6 +35,6 @@ func (p *PPTParser) String() string {
// hint (OLE binary). The two file families differ only in the
// binary container; the python parser.py:slides branch treats
// them uniformly.
func (p *PPTParser) ParseWithResult(filename string, data []byte) ParseResult {
return (&PPTXParser{format: "ppt"}).ParseWithResult(filename, data)
func (p *PPTParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
return (&PPTXParser{format: "ppt"}).ParseWithResult(ctx, filename, data)
}

6
internal/parser/parser/ppt_parser_cgo_test.go
View File

@@ -26,9 +26,10 @@ func TestPPTXParser_FormatField(t *testing.T) {
// JSON items. Uses office_oxide's own PptxWriter to produce the
// test data so no external file is needed.
func TestPPTXParser_ParseWithResult_CGO(t *testing.T) {
ctx := t.Context()
p := NewPPTXParser()
data := buildPPTX(t, "Hello World")
res := p.ParseWithResult("test.pptx", data)
res := p.ParseWithResult(ctx, "test.pptx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -47,12 +48,13 @@ func TestPPTXParser_ParseWithResult_CGO(t *testing.T) {
// delegates correctly to PPTXParser{format:"ppt"} and produces
// output with File["format"] = "ppt".
func TestPPTParser_ParseWithResult_CGO(t *testing.T) {
ctx := t.Context()
p := NewPPTParser()
// Use PPTX content — office_oxide may reject it with format="ppt"
// hint (expects OLE binary). When it does, skip gracefully; when
// it succeeds, verify the metadata contract.
data := buildPPTX(t, "Hello")
res := p.ParseWithResult("test.ppt", data)
res := p.ParseWithResult(ctx, "test.ppt", data)
if res.Err != nil {
t.Skip("PPTParser with PPTX data (expected maybe to fail):", res.Err)
}

3
internal/parser/parser/pptx_parser.go
View File

@@ -19,6 +19,7 @@
package parser
import (
"context"
"fmt"
"strings"
@@ -45,7 +46,7 @@ func (p *PPTXParser) String() string {
// ParseWithResult emits one JSON item per slide with the slide's
// plain text. Mirrors the python parser.py:slides branch which
// forces output_format="json" for the slide family.
func (p *PPTXParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
doc, err := officeOxide.OpenFromBytes(data, p.format)
if err != nil {
return ParseResult{Err: fmt.Errorf("presentation open: %w", err)}

3
internal/parser/parser/text_parser.go
View File

@@ -34,6 +34,7 @@ package parser
import (
"bytes"
"context"
"strings"
)
@@ -60,7 +61,7 @@ func NewTextParser() *TextParser {
// The items slice is always non-nil so downstream chunkers see a
// non-empty JSON payload even for an empty input (mirrors the
// MarkdownParser convention at markdown_parser.go:71-76).
func (p *TextParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
if !utf8Valid(data) {
return ParseResult{Err: errInvalidUTF8}
}

3
internal/parser/parser/xls_parser.go
View File

@@ -18,6 +18,7 @@ package parser
import (
"bytes"
"context"
"fmt"
"strings"
@@ -73,7 +74,7 @@ func (p *XLSParser) ConfigureFromSetup(setup map[string]any) {
}
}
func (p *XLSParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
method := normalizeXLSXParseMethod(p.ParseMethod)
switch method {
case "tcadp":

30
internal/parser/parser/xls_tcadp_test.go
View File

@@ -7,6 +7,7 @@ import (
)
func TestXLSXParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
ctx := t.Context()
zipPayload := tcadpZipFixture(t)
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -34,7 +35,7 @@ func TestXLSXParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
"table_result_type": "1",
"markdown_image_response_type": "1",
})
res := p.ParseWithResult("sample.xlsx", []byte("mock xlsx content"))
res := p.ParseWithResult(ctx, "sample.xlsx", []byte("mock xlsx content"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -44,6 +45,7 @@ func TestXLSXParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
}
func TestXLSParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
ctx := t.Context()
zipPayload := tcadpZipFixture(t)
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -68,7 +70,7 @@ func TestXLSParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
"tcadp_apiserver": server.URL,
"tcadp_api_key": "tcadp-secret",
})
res := p.ParseWithResult("sample.xls", []byte("mock xls content"))
res := p.ParseWithResult(ctx, "sample.xls", []byte("mock xls content"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -78,6 +80,7 @@ func TestXLSParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
}
func TestCSVParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
ctx := t.Context()
zipPayload := tcadpZipFixture(t)
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -99,7 +102,7 @@ func TestCSVParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
"tcadp_apiserver": server.URL,
"tcadp_api_key": "tcadp-secret",
})
res := p.ParseWithResult("sample.csv", []byte("a,b,c\n1,2,3"))
res := p.ParseWithResult(ctx, "sample.csv", []byte("a,b,c\n1,2,3"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -109,6 +112,7 @@ func TestCSVParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
}
func TestXLSXParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {
ctx := t.Context()
zipPayload := tcadpZipFixture(t)
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -132,7 +136,7 @@ func TestXLSXParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {
"output_format": "markdown",
"tcadp_apiserver": server.URL,
})
res := p.ParseWithResult("sample.xlsx", []byte("mock xlsx content"))
res := p.ParseWithResult(ctx, "sample.xlsx", []byte("mock xlsx content"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
@@ -142,63 +146,69 @@ func TestXLSXParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {
}
func TestXLSXParser_ParseWithResult_TCADPRequiresAPIServer(t *testing.T) {
ctx := t.Context()
p, err := NewXLSXParser("")
if err != nil {
t.Fatalf("NewXLSXParser: %v", err)
}
p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser"})
res := p.ParseWithResult("sample.xlsx", []byte("mock xlsx content"))
res := p.ParseWithResult(ctx, "sample.xlsx", []byte("mock xlsx content"))
if res.Err == nil {
t.Fatal("expected error about tcadp_apiserver, got nil")
}
}
func TestXLSParser_ParseWithResult_TCADPRequiresAPIServer(t *testing.T) {
ctx := t.Context()
p, err := NewXLSParser("")
if err != nil {
t.Fatalf("NewXLSParser: %v", err)
}
p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser"})
res := p.ParseWithResult("sample.xls", []byte("mock xls content"))
res := p.ParseWithResult(ctx, "sample.xls", []byte("mock xls content"))
if res.Err == nil {
t.Fatal("expected error about tcadp_apiserver, got nil")
}
}
func TestCSVParser_ParseWithResult_TCADPRequiresAPIServer(t *testing.T) {
ctx := t.Context()
p := NewCSVParser()
p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser"})
res := p.ParseWithResult("sample.csv", []byte("mock csv content"))
res := p.ParseWithResult(ctx, "sample.csv", []byte("mock csv content"))
if res.Err == nil {
t.Fatal("expected error about tcadp_apiserver, got nil")
}
}
func TestXLSXParser_ParseWithResult_InvalidXLSXHandled(t *testing.T) {
ctx := t.Context()
p, err := NewXLSXParser("")
if err != nil {
t.Fatalf("NewXLSXParser: %v", err)
}
res := p.ParseWithResult("sample.xlsx", []byte("not a valid xlsx"))
res := p.ParseWithResult(ctx, "sample.xlsx", []byte("not a valid xlsx"))
if res.Err == nil {
t.Fatal("expected error for invalid xlsx, got nil")
}
}
func TestXLSParser_ParseWithResult_InvalidXLSHandled(t *testing.T) {
ctx := t.Context()
p, err := NewXLSParser("")
if err != nil {
t.Fatalf("NewXLSParser: %v", err)
}
res := p.ParseWithResult("sample.xls", []byte("not a valid xls"))
res := p.ParseWithResult(ctx, "sample.xls", []byte("not a valid xls"))
if res.Err == nil {
t.Fatal("expected error for invalid xls, got nil")
}
}
func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) {
ctx := t.Context()
p := NewCSVParser()
res := p.ParseWithResult("sample.csv", []byte("a,b\n1,2"))
res := p.ParseWithResult(ctx, "sample.csv", []byte("a,b\n1,2"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}

9
internal/parser/parser/xlsx_parse_method_test.go
View File

@@ -69,7 +69,8 @@ func TestXLSXParser_DeepDocParseMethod(t *testing.T) {
}
p.ConfigureFromSetup(map[string]any{"parse_method": tc.method})
res := p.ParseWithResult("test.xlsx", buf.Bytes())
ctx := t.Context()
res := p.ParseWithResult(ctx, "test.xlsx", buf.Bytes())
if res.Err != nil {
t.Fatalf("ParseWithResult(%s): %v", tc.method, res.Err)
}
@@ -89,7 +90,8 @@ func TestCSVParser_DeepDocParseMethod(t *testing.T) {
p := NewCSVParser()
p.ConfigureFromSetup(map[string]any{"parse_method": "deepdoc"})
res := p.ParseWithResult("test.csv", []byte("a,b\n1,2"))
ctx := t.Context()
res := p.ParseWithResult(ctx, "test.csv", []byte("a,b\n1,2"))
if res.Err != nil {
t.Fatalf("ParseWithResult(deepdoc): %v", res.Err)
}
@@ -114,7 +116,8 @@ func TestXLSParser_DeepDocParseMethod_NoUnsupportedError(t *testing.T) {
}
p.ConfigureFromSetup(map[string]any{"parse_method": "deepdoc"})
res := p.ParseWithResult("test.xls", []byte("not a real xls"))
ctx := t.Context()
res := p.ParseWithResult(ctx, "test.xls", []byte("not a real xls"))
if res.Err != nil && strings.Contains(res.Err.Error(), "unsupported XLS parse method") {
t.Fatalf("deepdoc must not be rejected as unsupported: %v", res.Err)
}

3
internal/parser/parser/xlsx_parser.go
View File

@@ -18,6 +18,7 @@ package parser
import (
"bytes"
"context"
"fmt"
"strings"
@@ -90,7 +91,7 @@ func normalizeXLSXParseMethod(raw string) string {
return method
}
func (p *XLSXParser) ParseWithResult(filename string, data []byte) ParseResult {
func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
method := normalizeXLSXParseMethod(p.ParseMethod)
switch method {
case "tcadp":