zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-14 08:58:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Align Go ingestion boundaries with Python (#16647)

Browse Source 
Moves doc_id blob resolution into Parser, tightens chunker/tokenizer to
Python output_format semantics, updates extractor list handling, and
fixes real-template integration tests.
This commit is contained in:
Zhichang Yu
2026-07-05 20:43:52 +08:00
committed by GitHub
parent 0fcfb38365
commit 014c3f634f
119 changed files with 18083 additions and 4712 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
internal/cli/user_command.go
View File

@@ -29,8 +29,8 @@ import (
"os/exec"
"path/filepath"
"ragflow/internal/common"
"ragflow/internal/ingestion"
"ragflow/internal/ingestion/parser"
"ragflow/internal/parser/chunk"
"ragflow/internal/parser/parser"
"ragflow/internal/utility"
"strings"
"time"
@@ -3561,8 +3561,9 @@ func (c *CLI) APIParseLocalFileCommand(cmd *Command) (ResponseIf, error) {
return nil, fmt.Errorf("failed to read dsl file: %w", err)
}
if err = fileParser.Parse(filename, fileContent); err != nil {
return nil, formatRequestError("parse local file", err)
parseResult := fileParser.ParseWithResult(filename, fileContent)
if parseResult.Err != nil {
return nil, formatRequestError("parse local file", parseResult.Err)
}
var result SimpleResponse
@@ -3773,13 +3774,17 @@ func (c *CLI) DevChunkCommand(cmd *Command) (ResponseIf, error) {
if !ok {
return nil, fmt.Errorf("filename not provided")
}
dslFilename, ok := cmd.Params["dsl"].(string)
optionsFilename, ok := cmd.Params["dsl"].(string)
if !ok {
return nil, fmt.Errorf("dsl not provided")
return nil, fmt.Errorf("chunk options file not provided")
}
dsl, err := os.ReadFile(dslFilename)
optionsRaw, err := os.ReadFile(optionsFilename)
if err != nil {
return nil, fmt.Errorf("failed to read dsl file: %w", err)
return nil, fmt.Errorf("failed to read chunk options file: %w", err)
}
options, err := parseChunkOptions(optionsRaw)
if err != nil {
return nil, err
}
explain, ok := cmd.Params["explain"].(bool)
@@ -3787,19 +3792,11 @@ func (c *CLI) DevChunkCommand(cmd *Command) (ResponseIf, error) {
explain = false
}
engine := ingestion.NewChunkEngine()
plan, err := engine.Compile(string(dsl))
if err != nil {
return nil, fmt.Errorf("compile failed: %w", err)
}
if explain {
explanation, err := engine.Explain(plan)
explanation, err := explainChunkOptions(options)
if err != nil {
return nil, fmt.Errorf("explain error: %w", err)
}
result.Message = explanation
} else {
fileToChunking, err := os.ReadFile(filename)
@@ -3807,7 +3804,7 @@ func (c *CLI) DevChunkCommand(cmd *Command) (ResponseIf, error) {
return nil, fmt.Errorf("failed to read file: %w", err)
}
chunkContext, err := engine.Execute(plan, string(fileToChunking))
chunkContext, err := chunk.Run(string(fileToChunking), options)
if err != nil {
return nil, fmt.Errorf("chunking error: %w", err)
}
@@ -3825,6 +3822,22 @@ func (c *CLI) DevChunkCommand(cmd *Command) (ResponseIf, error) {
return &result, nil
}
func parseChunkOptions(raw []byte) (chunk.ChunkOptions, error) {
var options chunk.ChunkOptions
if err := json.Unmarshal(raw, &options); err != nil {
return options, fmt.Errorf("failed to parse chunk options file: %w", err)
}
return options, nil
}
func explainChunkOptions(options chunk.ChunkOptions) (string, error) {
formatted, err := json.MarshalIndent(options, "", " ")
if err != nil {
return "", err
}
return string(formatted), nil
}
// APIOpenaiChatCommand dispatches the parsed OPENAI_CHAT command to either a
// non-streaming oneshot call or a streaming SSE call, depending on the
// `stream` option.