zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-08 16:38:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: parser component param check and family mapping fixes (#17312)

Browse Source 
## Summary

Adds construction-time parameter validation to the ingestion
`ParserComponent` (mirroring the applicable subset of Python
`ParserParam.check()`), fixes a family-mapping mismatch that silently
skipped `output_format` validation and setup configuration for
image/audio files, and aligns the no-CGO parser stubs with the CGO
variants by threading `context.Context` through `ParseWithResult`.
This commit is contained in:
Jack
2026-07-23 22:06:22 +08:00
committed by GitHub
parent 61201e437d
commit 902d05b22b
8 changed files with 349 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/parser/parser/office_parsers_no_cgo.go
View File

@@ -3,6 +3,7 @@
package parser
import (
"context"
"errors"
"fmt"
)
@@ -17,28 +18,28 @@ import (
// shape used by the rest of the package.
var ErrOfficeCGORequired = errors.New("parser: office family requires CGO (office_oxide)")
func (p *DOCXParser) ParseWithResult(filename string, _ []byte) ParseResult {
func (p *DOCXParser) ParseWithResult(_ context.Context, filename string, _ []byte) ParseResult {
return ParseResult{
File: map[string]any{"name": filename},
Err: fmt.Errorf("%w: docx", ErrOfficeCGORequired),
}
}
func (p *DOCParser) ParseWithResult(filename string, _ []byte) ParseResult {
func (p *DOCParser) ParseWithResult(_ context.Context, filename string, _ []byte) ParseResult {
return ParseResult{
File: map[string]any{"name": filename},
Err: fmt.Errorf("%w: doc", ErrOfficeCGORequired),
}
}
func (p *PPTParser) ParseWithResult(filename string, _ []byte) ParseResult {
func (p *PPTParser) ParseWithResult(_ context.Context, filename string, _ []byte) ParseResult {
return ParseResult{
File: map[string]any{"name": filename},
Err: fmt.Errorf("%w: ppt", ErrOfficeCGORequired),
}
}
func (p *PPTXParser) ParseWithResult(filename string, _ []byte) ParseResult {
func (p *PPTXParser) ParseWithResult(_ context.Context, filename string, _ []byte) ParseResult {
return ParseResult{
File: map[string]any{"name": filename},
Err: fmt.Errorf("%w: pptx", ErrOfficeCGORequired),

10
internal/parser/parser/office_parsers_nocgo_test.go
View File

@@ -3,19 +3,21 @@
package parser
import (
"context"
"errors"
"testing"
)
func TestOfficeParsers_ParseWithResult_NoCGO(t *testing.T) {
ctx := context.Background()
cases := []struct {
name string
res ParseResult
}{
{name: "docx", res: (&DOCXParser{}).ParseWithResult("a.docx", nil)},
{name: "doc", res: (&DOCParser{}).ParseWithResult("a.doc", nil)},
{name: "pptx", res: (&PPTXParser{}).ParseWithResult("a.pptx", nil)},
{name: "ppt", res: (&PPTParser{}).ParseWithResult("a.ppt", nil)},
{name: "docx", res: (&DOCXParser{}).ParseWithResult(ctx, "a.docx", nil)},
{name: "doc", res: (&DOCParser{}).ParseWithResult(ctx, "a.doc", nil)},
{name: "pptx", res: (&PPTXParser{}).ParseWithResult(ctx, "a.pptx", nil)},
{name: "ppt", res: (&PPTParser{}).ParseWithResult(ctx, "a.ppt", nil)},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {

9
internal/parser/parser/pdf_parser_nocgo.go
View File

@@ -3,6 +3,7 @@
package parser
import (
"context"
"fmt"
)
@@ -14,13 +15,13 @@ func (p *PDFParser) ParseWithResult(ctx context.Context, filename string, data [
case "plain_text":
return parsePDFWithPlainText(filename, data, p)
case "mineru":
return parsePDFWithMinerU(filename, data, p)
return parsePDFWithMinerU(ctx, filename, data, p)
case "paddleocr":
return parsePDFWithPaddleOCR(filename, data, p)
return parsePDFWithPaddleOCR(ctx, filename, data, p)
case "docling":
return parsePDFWithDocling(filename, data, p)
return parsePDFWithDocling(ctx, filename, data, p)
case "opendataloader":
return parsePDFWithOpenDataLoader(filename, data, p)
return parsePDFWithOpenDataLoader(ctx, filename, data, p)
case "somark":
return parsePDFWithSoMark(filename, data, p)
case "tcadp":

6
internal/parser/parser/pdf_parser_nocgo_test.go
View File

@@ -3,14 +3,16 @@
package parser
import (
"context"
"errors"
"testing"
)
func TestPDFParser_ParseWithResult_NoCGO(t *testing.T) {
ctx := context.Background()
pdf := NewPDFParser()
empty := pdf.ParseWithResult("empty.pdf", nil)
empty := pdf.ParseWithResult(ctx, "empty.pdf", nil)
if empty.Err != nil {
t.Fatalf("empty input: want nil err, got %v", empty.Err)
}
@@ -21,7 +23,7 @@ func TestPDFParser_ParseWithResult_NoCGO(t *testing.T) {
t.Fatalf("empty input JSON len = %d, want 1", len(empty.JSON))
}
res := pdf.ParseWithResult("a.pdf", []byte("%PDF-1.4"))
res := pdf.ParseWithResult(ctx, "a.pdf", []byte("%PDF-1.4"))
if res.Err == nil {
t.Fatal("want ErrPDFEngineUnavailable, got nil")
}