Go: parse ingestion DSL (#15938)

PR #15938

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>

internal/cli/cli.go

@@ -1209,6 +1209,12 @@ func (c *CLI) handleMetaCommand(cmd *Command) error {
 		c.running = false
 	case "?", "h", "help":
 		c.printHelp()
+	case "pwd":
+		dir, err := os.Getwd()
+		if err != nil {
+			return fmt.Errorf("get working directory: %w", err)
+		}
+		fmt.Println(dir)
 	default:
 		return fmt.Errorf("unknown meta command: \\%s", command)
 	}

internal/cli/response.go

@@ -529,6 +529,33 @@ func (r *TaskResponse) PrintOut() {
 	}
 }
 
+type ExplainResponse struct {
+	Code         int    `json:"code"`
+	Message      string `json:"message"`
+	Duration     float64
+	OutputFormat OutputFormat
+}
+
+func (r *ExplainResponse) Type() string {
+	return "explain"
+}
+
+func (r *ExplainResponse) TimeCost() float64 {
+	return r.Duration
+}
+
+func (r *ExplainResponse) SetOutputFormat(format OutputFormat) {
+	r.OutputFormat = format
+}
+
+func (r *ExplainResponse) PrintOut() {
+	if r.Code == 0 {
+		fmt.Printf("\n%s\n", r.Message)
+	} else {
+		fmt.Printf("ERROR %d\n", r.Code)
+	}
+}
+
 // FileSystemResponse wraps the raw text output from executeFilesystem().
 type FileSystemResponse struct {
 	Output       string

internal/cli/user_command.go

@@ -28,6 +28,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"ragflow/internal/ingestion"
 	"strings"
 	"time"
 )
@@ -3445,27 +3446,60 @@ func (c *CLI) ChunkCommand(cmd *Command) (ResponseIf, error) {
 		return nil, fmt.Errorf("this command is only allowed in USER mode")
 	}
 
+	var result ExplainResponse
+	start := time.Now()
+
 	filename, ok := cmd.Params["filename"].(string)
 	if !ok {
 		return nil, fmt.Errorf("filename not provided")
 	}
-	dsl, ok := cmd.Params["dsl"].(string)
+	dslFilename, ok := cmd.Params["dsl"].(string)
 	if !ok {
 		return nil, fmt.Errorf("dsl not provided")
 	}
+	dsl, err := os.ReadFile(dslFilename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read dsl file: %w", err)
+	}
+
 	explain, ok := cmd.Params["explain"].(bool)
 	if !ok {
 		explain = false
 	}
 
-	if explain {
-		fmt.Printf("Explain chunk file: %s, DSL: %s\n", filename, dsl)
-	} else {
-		// TODO: not implemented
-		fmt.Printf("Chunk file: %s, DSL: %s\n", filename, dsl)
+	engine := ingestion.NewChunkEngine()
+	plan, err := engine.Compile(string(dsl))
+	if err != nil {
+		return nil, fmt.Errorf("compile failed: %w", err)
 	}
 
-	var result SimpleResponse
+	if explain {
+
+		explanation, err := engine.Explain(plan)
+		if err != nil {
+			return nil, fmt.Errorf("explain error: %w", err)
+		}
+
+		result.Message = explanation
+	} else {
+		fileToChunking, err := os.ReadFile(filename)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read file: %w", err)
+		}
+
+		chunkContext, err := engine.Execute(plan, string(fileToChunking))
+		if err != nil {
+			return nil, fmt.Errorf("chunking error: %w", err)
+		}
+
+		for _, resultChunk := range chunkContext.ResultChunks {
+			fmt.Printf("Chunk index: %d\n", resultChunk.Index)
+			fmt.Printf("Chunk size: %d\n", resultChunk.Size)
+			fmt.Printf("Chunk content: \n%s\n", resultChunk.Content)
+		}
+	}
+
+	result.Duration = time.Since(start).Seconds()
 	result.Code = 0
 	result.Message = fmt.Sprintf("Success to chunk %s", filename)
 	return &result, nil