mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-29 20:19:24 +08:00
9b0719fa94
## Summary
Continuation of the Python→Go ingestion pipeline migration (File →
Parser → Chunker → Extractor → Tokenizer). Fixes cover Parser, Chunker,
and Tokenizer gaps identified. Fix page number (0-indexed and 1-index
mixed before fix; use 1-indexed after fix) and chunk order issues.
### Parser
- **Slides TCADP (1.7):** `pptx_tcadp.go` + TCADP branch in
`pptx_parser.go`/`ppt_parser.go` — PowerPoint files now support
`parse_method="tcadp"` via the TCADP cloud service, matching the
spreadsheet-family TCADP pattern. PPT containers pass `"PPT"` as
fileType (not hardcoded `"PPTX"`).
- **Audio default output_format (2.11):** `defaultSetups()` audio
default changed from `"text"` to `"json"`, aligning with Python
`parser.py:232` and `AllowedOutputFormat["audio"]={"json"}`.
- **PDF VLM enhancement (1.1):** `maybeDispatchPDFVisionEnhancement` in
`pdf_vision_dispatch.go` enriches image/table items with IMAGE2TEXT
model descriptions after PDF parsing, mirroring Python
`enhance_media_sections_with_vision`. Semaphore fix: acquire before
goroutine start to prevent unbounded goroutine creation.
- **json family (2.3):** reclassified as Keep Go — `json_parser.go` is a
functional enhancement, not a parity gap.
- **page number:** changed from "mixed use of 1-indexed & 0-indexed" to
"1-indexed"
### Chunker
- **BULLET_PATTERN fallback (1.7):** 4th-level fallback in
`resolveTitleLevels` (`title.go`) detects bullet/numbered-list patterns
(Chinese legal, numbering, English) when outline + regex levels produce
only bodyLevel. Guarded by `allBodyLevel` to never override existing
structure.
- **Tag/One chunker fields (1.8):** `tag.go` sets `TopInt` from source
row index; `one.go` preserves `Positions`/`PDFPositions` from source
items. TSV multi-line RowNum fix: tracks `contentStart` for correct row
attribution.
- **Overlapped_percent normalization (2.6):**
`NormalizeOverlappedPercent` in `schema/chunker.go` mirrors Python
`common/float_utils.py:50-58` — accepts `[0,1)` fraction or `[0,90]`
percent, normalizes to canonical `[0,90]`.
- **Paragraph splitting (2.7):** aligned to Python flow `naive_merge` —
`CRLF` normalization, `splitKeepingDelimiter` preserves sentence
delimiters, single-section merge with token-budget-governed chunking.
- **chunk order:** sort by reading order
### Tokenizer
- **Phantom chunk filtering (Omission 2):** `isPhantomChunk` + filter
loop in `chunksFromTokenizerUpstream` skips zero-value ChunkDocs.
- **Batch size env var (Omission 3):** `embeddingBatchSize()` reads
`TOKENIZER_EMBEDDING_BATCH_SIZE`, defaults to 16.
- **Summary empty check (Diff 5):** `TrimSpace(s) != ""` → `s != ""`,
matching Python truthy check.
- **chunk_order_int all paths (Diff 8):** set unconditionally before
full_text/embedding branching.
- **Timeout default (Diff 10):** `600s` → `60s`, matching Python
`@timeout(60)`.
- **Small maxTokens truncation (Diff 14):** `truncateForEmbedding`
returns `""` when `maxTokens <= 10`, matching Python.
### Code review fixes
- Semaphore acquire moved before goroutine in `pdf_vision_dispatch.go`
(concurrency control)
- Context propagation in `pptx_tcadp.go` (cancellation support)
- Test resolver leak fix in `media_dispatch_test.go` (defer restore)
- Migration history comments removed per AGENTS.md
## Test plan
```
bash build.sh --test ./internal/parser/parser/... ./internal/ingestion/component/...
```
## Notes
- Migration diff tracking: `docs/migration_python_go_diff.md`
- Remaining gaps: Extractor component only (21 items)
102 lines
3.4 KiB
Go
102 lines
3.4 KiB
Go
package task
|
||
|
||
import (
|
||
"testing"
|
||
)
|
||
|
||
// =============================================================================
|
||
// AddPositions
|
||
// Canonical format: [pageNum, left, right, top, bottom] × N
|
||
// Contract: input page numbers are ALREADY 1-indexed (the 0→1 conversion
|
||
// happens once, at the parser boundary in normalizePDFPageNumber). This
|
||
// function is a passthrough — it must NOT add +1, otherwise callers that
|
||
// already feed 1-indexed values (the PDF path after normalization) get a
|
||
// double-incremented page number.
|
||
// =============================================================================
|
||
|
||
func TestAddPositions_Basic(t *testing.T) {
|
||
chunk := map[string]any{}
|
||
// [pn=1 (first page, 1-indexed), left=100, right=50, top=200, bottom=150]
|
||
positions := []float64{1, 100, 50, 200, 150}
|
||
AddPositions(chunk, positions)
|
||
|
||
pageNum, ok := chunk["page_num_int"].([]int)
|
||
if !ok || len(pageNum) != 1 || pageNum[0] != 1 {
|
||
t.Errorf("page_num_int = %v, want [1]", pageNum)
|
||
}
|
||
top, ok := chunk["top_int"].([]int)
|
||
if !ok || len(top) != 1 || top[0] != 200 {
|
||
t.Errorf("top_int = %v, want [200]", top)
|
||
}
|
||
position, ok := chunk["position_int"].([][]int)
|
||
if !ok || len(position) != 1 {
|
||
t.Fatalf("position_int = %v, want [[1 100 50 200 150]]", position)
|
||
}
|
||
if position[0][0] != 1 || position[0][1] != 100 || position[0][2] != 50 || position[0][3] != 200 || position[0][4] != 150 {
|
||
t.Errorf("position_int[0] = %v, want [1 100 50 200 150]", position[0])
|
||
}
|
||
}
|
||
|
||
func TestAddPositions_MultiplePositions(t *testing.T) {
|
||
chunk := map[string]any{}
|
||
positions := []float64{
|
||
1, 100, 50, 200, 150, // pn=1, left=100, right=50, top=200, bottom=150
|
||
2, 200, 60, 300, 250, // pn=2, left=200, right=60, top=300, bottom=250
|
||
}
|
||
AddPositions(chunk, positions)
|
||
|
||
pageNum := chunk["page_num_int"].([]int)
|
||
if len(pageNum) != 2 || pageNum[0] != 1 || pageNum[1] != 2 {
|
||
t.Errorf("page_num_int = %v, want [1 2]", pageNum)
|
||
}
|
||
top := chunk["top_int"].([]int)
|
||
if len(top) != 2 || top[0] != 200 || top[1] != 300 {
|
||
t.Errorf("top_int = %v, want [200 300]", top)
|
||
}
|
||
position := chunk["position_int"].([][]int)
|
||
if len(position) != 2 {
|
||
t.Fatalf("position_int len = %d, want 2", len(position))
|
||
}
|
||
}
|
||
|
||
func TestAddPositions_NilPositions(t *testing.T) {
|
||
chunk := map[string]any{}
|
||
AddPositions(chunk, nil)
|
||
if _, exists := chunk["page_num_int"]; exists {
|
||
t.Error("page_num_int should not be set for nil positions")
|
||
}
|
||
}
|
||
|
||
func TestAddPositions_EmptyPositions(t *testing.T) {
|
||
chunk := map[string]any{}
|
||
AddPositions(chunk, []float64{})
|
||
if _, exists := chunk["page_num_int"]; exists {
|
||
t.Error("page_num_int should not be set for empty positions")
|
||
}
|
||
}
|
||
|
||
func TestAddPositions_PartialPositions(t *testing.T) {
|
||
chunk := map[string]any{}
|
||
positions := []float64{1, 100} // only 2 elements, not a complete position
|
||
AddPositions(chunk, positions)
|
||
if _, exists := chunk["page_num_int"]; exists {
|
||
t.Error("page_num_int should not be set for partial positions")
|
||
}
|
||
}
|
||
|
||
func TestAddPositions_PassthroughNoOffset(t *testing.T) {
|
||
// page numbers are 1-indexed on entry; AddPositions must not add +1.
|
||
chunk := map[string]any{}
|
||
positions := []float64{6, 100, 50, 200, 150} // pn=6, 1-indexed
|
||
AddPositions(chunk, positions)
|
||
|
||
pageNum := chunk["page_num_int"].([]int)
|
||
if pageNum[0] != 6 {
|
||
t.Errorf("page_num_int = %d, want 6 (passthrough, no +1)", pageNum[0])
|
||
}
|
||
position := chunk["position_int"].([][]int)
|
||
if position[0][0] != 6 {
|
||
t.Errorf("position_int[0][0] = %d, want 6 (passthrough, no +1)", position[0][0])
|
||
}
|
||
}
|