mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-01 21:37:33 +08:00
## Summary
Continuation of the Python→Go ingestion pipeline migration (File →
Parser → Chunker → Extractor → Tokenizer). Fixes cover Parser, Chunker,
and Tokenizer gaps identified. Fix page number (0-indexed and 1-index
mixed before fix; use 1-indexed after fix) and chunk order issues.
### Parser
- **Slides TCADP (1.7):** `pptx_tcadp.go` + TCADP branch in
`pptx_parser.go`/`ppt_parser.go` — PowerPoint files now support
`parse_method="tcadp"` via the TCADP cloud service, matching the
spreadsheet-family TCADP pattern. PPT containers pass `"PPT"` as
fileType (not hardcoded `"PPTX"`).
- **Audio default output_format (2.11):** `defaultSetups()` audio
default changed from `"text"` to `"json"`, aligning with Python
`parser.py:232` and `AllowedOutputFormat["audio"]={"json"}`.
- **PDF VLM enhancement (1.1):** `maybeDispatchPDFVisionEnhancement` in
`pdf_vision_dispatch.go` enriches image/table items with IMAGE2TEXT
model descriptions after PDF parsing, mirroring Python
`enhance_media_sections_with_vision`. Semaphore fix: acquire before
goroutine start to prevent unbounded goroutine creation.
- **json family (2.3):** reclassified as Keep Go — `json_parser.go` is a
functional enhancement, not a parity gap.
- **page number:** changed from "mixed use of 1-indexed & 0-indexed" to
"1-indexed"
### Chunker
- **BULLET_PATTERN fallback (1.7):** 4th-level fallback in
`resolveTitleLevels` (`title.go`) detects bullet/numbered-list patterns
(Chinese legal, numbering, English) when outline + regex levels produce
only bodyLevel. Guarded by `allBodyLevel` to never override existing
structure.
- **Tag/One chunker fields (1.8):** `tag.go` sets `TopInt` from source
row index; `one.go` preserves `Positions`/`PDFPositions` from source
items. TSV multi-line RowNum fix: tracks `contentStart` for correct row
attribution.
- **Overlapped_percent normalization (2.6):**
`NormalizeOverlappedPercent` in `schema/chunker.go` mirrors Python
`common/float_utils.py:50-58` — accepts `[0,1)` fraction or `[0,90]`
percent, normalizes to canonical `[0,90]`.
- **Paragraph splitting (2.7):** aligned to Python flow `naive_merge` —
`CRLF` normalization, `splitKeepingDelimiter` preserves sentence
delimiters, single-section merge with token-budget-governed chunking.
- **chunk order:** sort by reading order
### Tokenizer
- **Phantom chunk filtering (Omission 2):** `isPhantomChunk` + filter
loop in `chunksFromTokenizerUpstream` skips zero-value ChunkDocs.
- **Batch size env var (Omission 3):** `embeddingBatchSize()` reads
`TOKENIZER_EMBEDDING_BATCH_SIZE`, defaults to 16.
- **Summary empty check (Diff 5):** `TrimSpace(s) != ""` → `s != ""`,
matching Python truthy check.
- **chunk_order_int all paths (Diff 8):** set unconditionally before
full_text/embedding branching.
- **Timeout default (Diff 10):** `600s` → `60s`, matching Python
`@timeout(60)`.
- **Small maxTokens truncation (Diff 14):** `truncateForEmbedding`
returns `""` when `maxTokens <= 10`, matching Python.
### Code review fixes
- Semaphore acquire moved before goroutine in `pdf_vision_dispatch.go`
(concurrency control)
- Context propagation in `pptx_tcadp.go` (cancellation support)
- Test resolver leak fix in `media_dispatch_test.go` (defer restore)
- Migration history comments removed per AGENTS.md
## Test plan
```
bash build.sh --test ./internal/parser/parser/... ./internal/ingestion/component/...
```
## Notes
- Migration diff tracking: `docs/migration_python_go_diff.md`
- Remaining gaps: Extractor component only (21 items)
145 lines
4.3 KiB
Go
145 lines
4.3 KiB
Go
//go:build cgo
|
|
|
|
//
|
|
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
//
|
|
|
|
package parser
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
officeOxide "github.com/yfedoseev/office_oxide/go"
|
|
)
|
|
|
|
// PPTXParser parses both .pptx (OOXML) and .ppt (OLE binary)
|
|
// files via the office_oxide backend. The format field controls
|
|
// the container format passed to OpenFromBytes — "pptx" for
|
|
// ZIP-based OOXML presentations and "ppt" for the legacy binary
|
|
// OLE format.
|
|
type PPTXParser struct {
|
|
format string
|
|
|
|
// TCADP cloud-parsing configuration
|
|
ParseMethod string
|
|
TCADPAPIServer string
|
|
TCADPAPIKey string
|
|
TCADPTableResultType string
|
|
TCADPMarkdownImageResponseType string
|
|
OutputFormat string
|
|
}
|
|
|
|
func NewPPTXParser() *PPTXParser {
|
|
return &PPTXParser{format: "pptx"}
|
|
}
|
|
|
|
func (p *PPTXParser) String() string {
|
|
return "PPTXParser"
|
|
}
|
|
|
|
// ConfigureFromSetup reads the slides-family setup map. Mirrors the
|
|
// XLSXParser ConfigureFromSetup pattern
|
|
func (p *PPTXParser) ConfigureFromSetup(setup map[string]any) {
|
|
if p == nil || setup == nil {
|
|
return
|
|
}
|
|
if v, ok := setup["parse_method"].(string); ok {
|
|
p.ParseMethod = v
|
|
}
|
|
if v, ok := setup["tcadp_apiserver"].(string); ok {
|
|
p.TCADPAPIServer = v
|
|
}
|
|
if v, ok := setup["tcadp_api_key"].(string); ok {
|
|
p.TCADPAPIKey = v
|
|
}
|
|
if v, ok := setup["table_result_type"].(string); ok {
|
|
p.TCADPTableResultType = v
|
|
}
|
|
if v, ok := setup["markdown_image_response_type"].(string); ok {
|
|
p.TCADPMarkdownImageResponseType = v
|
|
}
|
|
if v, ok := setup["output_format"].(string); ok {
|
|
p.OutputFormat = v
|
|
}
|
|
if p.OutputFormat == "" {
|
|
p.OutputFormat = "json"
|
|
}
|
|
}
|
|
|
|
// ParseWithResult emits one JSON item per slide with the slide's
|
|
// plain text. Mirrors the python parser.py:slides branch which
|
|
// forces output_format="json" for the slide family.
|
|
func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
|
|
// p == nil guard: the struct is embedded by value in PPTParser and
|
|
// always created via NewPPTXParser or the "ppt"-format constructor in
|
|
// PPTParser, so this branch is unreachable from normal call paths.
|
|
// Kept as defensive guard — a nil dereference here would obscure the
|
|
// root cause behind a nil-pointer panic.
|
|
if p == nil {
|
|
return ParseResult{Err: fmt.Errorf("PPTXParser is nil")}
|
|
}
|
|
method := strings.ToLower(strings.TrimSpace(p.ParseMethod))
|
|
switch method {
|
|
case "tcadp":
|
|
return parsePresentationWithTCADP(ctx,
|
|
filename, data, strings.ToUpper(p.format),
|
|
p.TCADPAPIServer, p.TCADPAPIKey,
|
|
p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,
|
|
p.OutputFormat,
|
|
)
|
|
case "", "deepdoc":
|
|
// Continue with the local office_oxide parser.
|
|
default:
|
|
// PDF-specific methods like "paddleocr" / "mineru" are
|
|
// meaningless for PPTX; treat as default path.
|
|
}
|
|
doc, err := officeOxide.OpenFromBytes(data, p.format)
|
|
if err != nil {
|
|
return ParseResult{Err: fmt.Errorf("presentation open: %w", err)}
|
|
}
|
|
defer doc.Close()
|
|
|
|
text, err := doc.PlainText()
|
|
if err != nil {
|
|
return ParseResult{Err: fmt.Errorf("pptx plain-text: %w", err)}
|
|
}
|
|
|
|
// Split on form-feed (the python TxtParser convention used by
|
|
// RAGFlow's slide parser) — each block becomes a JSON item.
|
|
var items []map[string]any
|
|
for i, raw := range strings.Split(text, "\f") {
|
|
trimmed := strings.TrimSpace(raw)
|
|
if trimmed == "" {
|
|
continue
|
|
}
|
|
items = append(items, map[string]any{
|
|
"text": trimmed,
|
|
"doc_type_kwd": "text",
|
|
"slide_number": i + 1,
|
|
})
|
|
}
|
|
if items == nil {
|
|
items = []map[string]any{{"text": strings.TrimSpace(text), "doc_type_kwd": "text"}}
|
|
}
|
|
|
|
return ParseResult{
|
|
OutputFormat: "json",
|
|
File: map[string]any{"name": filename, "format": p.format},
|
|
JSON: items,
|
|
}
|
|
}
|