mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-15 13:14:28 +08:00
feat(chunker): add ManualChunker for manual-layout PDFs (#18272)
Add `ManualChunker`, the Go port of Python's `manual` doc-type chunk method (`rag/app/manual.py`). Like `GroupTitleChunker` it merges adjacent text records into heading-bounded groups, but it first re-sorts the records into physical reading order before grouping.
This commit is contained in:
Jack
@@ -146,7 +146,7 @@ func TestComponentsHandler_FilterIngestion(t *testing.T) {
|
||||
_, _, data := decodeEnvelope(t, w.Body.Bytes())
|
||||
|
||||
wantNames := []string{
|
||||
"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
|
||||
"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
|
||||
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
|
||||
"titlechunker", "tokenchunker", "tokenizer",
|
||||
}
|
||||
@@ -172,7 +172,7 @@ func TestComponentsHandler_FilterMultiple(t *testing.T) {
|
||||
_, _, data := decodeEnvelope(t, w.Body.Bytes())
|
||||
|
||||
wantNames := []string{
|
||||
"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
|
||||
"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
|
||||
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
|
||||
"titlechunker", "tokenchunker", "tokenizer",
|
||||
}
|
||||
@@ -273,7 +273,7 @@ func TestComponentsHandler_CaseInsensitive(t *testing.T) {
|
||||
}
|
||||
_, _, data := decodeEnvelope(t, w.Body.Bytes())
|
||||
wantNames := []string{
|
||||
"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
|
||||
"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
|
||||
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
|
||||
"titlechunker", "tokenchunker", "tokenizer",
|
||||
}
|
||||
|
||||
@@ -41,6 +41,8 @@ func newChunkerByName(name string, params map[string]any) (runtime.Component, er
|
||||
return NewTitleChunker(params)
|
||||
case ComponentNameGroupTitleChunker:
|
||||
return NewGroupTitleChunker(params)
|
||||
case ComponentNameManualChunker:
|
||||
return NewManualChunker(params)
|
||||
case ComponentNameHierarchyTitleChunker:
|
||||
return NewHierarchyTitleChunker(params)
|
||||
case ComponentNameQAChunker:
|
||||
|
||||
@@ -107,9 +107,9 @@ func buildSectionIDs(levels []int, targetLevel int) []int {
|
||||
}
|
||||
|
||||
// invokeGroup runs the GroupTitleChunker strategy against the
|
||||
// supplied inputs. Detected headings + adjacent merges happen in two
|
||||
// goroutines (heading detection sequential, then a fan-out over
|
||||
// record-buckets for the merge pass).
|
||||
// supplied inputs. It extracts the line records and defers to the
|
||||
// shared chunkFromRecords pipeline (which ManualChunker also uses,
|
||||
// after a physical-position resort).
|
||||
func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam) (map[string]any, error) {
|
||||
records := extractLineRecords(inputs)
|
||||
common.Debug("chunker stage",
|
||||
@@ -120,6 +120,18 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any,
|
||||
if len(records) == 0 {
|
||||
return emptyOutputs(), nil
|
||||
}
|
||||
return chunkFromRecords(parentCtx, db, inputs, p, records)
|
||||
}
|
||||
|
||||
// chunkFromRecords runs the shared GroupTitle / Manual grouping pipeline over
|
||||
// an already-extracted record list: resolve heading levels, split into
|
||||
// sections, merge adjacent text records, build chunks, and perform on-demand
|
||||
// PDF cropping. GroupTitleChunker feeds records in input order; ManualChunker
|
||||
// feeds them after a (page, top, left) resort. Sharing this body guarantees
|
||||
// both strategies emit byte-identical output for coordinate-free input (the
|
||||
// docx manual branch) — the no-regression contract locked by
|
||||
// TestManualChunker_NoPositionsEqualsGroupChunker.
|
||||
func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam, records []lineRecord) (map[string]any, error) {
|
||||
ctx := newLevelContext(records, outlineFromInputs(inputs), p)
|
||||
levels := ctx.Levels()
|
||||
// Count heading level distribution for debugging.
|
||||
@@ -170,7 +182,7 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any,
|
||||
if upstream, uErr := decodeChunkerFromUpstream(inputs); uErr == nil {
|
||||
engine, eErr := newPDFEngineFromUpstream(parentCtx, db, upstream)
|
||||
if eErr != nil {
|
||||
slog.Warn("GroupTitleChunker: could not open PDF for on-demand cropping", "err", eErr)
|
||||
slog.Warn("chunker: could not open PDF for on-demand cropping", "err", eErr)
|
||||
}
|
||||
if engine != nil {
|
||||
defer engine.Close()
|
||||
@@ -330,6 +342,21 @@ func removeTag(text string) string {
|
||||
return posTagRemove.ReplaceAllString(text, "")
|
||||
}
|
||||
|
||||
// pdfPosRowLess orders two PDF coordinate 5-tuples [page,left,right,top,bottom]
|
||||
// by (page, top, left) — i.e. by indices (0, 3, 1). It is the single shared
|
||||
// (page, top, left) comparator used both when re-sorting chunker records
|
||||
// (ManualChunker) and when de-duplicating/merging position matrices
|
||||
// (mergePositionMatrix). Both rows must have length >= 5.
|
||||
func pdfPosRowLess(a, b []float64) bool {
|
||||
if a[0] != b[0] {
|
||||
return a[0] < b[0]
|
||||
}
|
||||
if a[3] != b[3] {
|
||||
return a[3] < b[3]
|
||||
}
|
||||
return a[1] < b[1]
|
||||
}
|
||||
|
||||
// mergePositionMatrix aggregates multiple PDF coordinate matrices into a
|
||||
// single de-duplicated, sorted matrix. Mirrors Python
|
||||
// pdf_chunk_metadata.py:127 merge_pdf_positions: rows are 5-tuples
|
||||
@@ -363,13 +390,7 @@ func mergePositionMatrix(sources ...json.RawMessage) [][]float64 {
|
||||
return nil
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i][0] != out[j][0] {
|
||||
return out[i][0] < out[j][0]
|
||||
}
|
||||
if out[i][3] != out[j][3] {
|
||||
return out[i][3] < out[j][3]
|
||||
}
|
||||
return out[i][1] < out[j][1]
|
||||
return pdfPosRowLess(out[i], out[j])
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
205
internal/ingestion/component/chunker/manual.go
Normal file
205
internal/ingestion/component/chunker/manual.go
Normal file
@@ -0,0 +1,205 @@
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// ManualChunker is the Go port of Python's `manual` doc-type chunk method
|
||||
// (rag/app/manual.py). Like GroupTitleChunker it merges adjacent text records
|
||||
// into heading-bounded groups, but it first re-sorts the records into physical
|
||||
// reading order before grouping.
|
||||
//
|
||||
// Why the resort matters: for multi-column / manually-laid-out PDFs the parser
|
||||
// emits records in READING order (column-by-column), so a naive grouping
|
||||
// interleaves the left column's later content with the right column's earlier
|
||||
// content. ManualChunker re-orders records by (page, top, left) — mirroring
|
||||
// Python's
|
||||
//
|
||||
// sorted(sections, key=lambda x: (x[-1][0][0], x[-1][0][3], x[-1][0][1]))
|
||||
//
|
||||
// so grouping then follows the true top-down, left-to-right layout.
|
||||
//
|
||||
// Crucially, docx (and any coordinate-free payload) emits records in
|
||||
// document-logical order with NO positions — exactly as Python's manual docx
|
||||
// branch, which performs no resort. When no record carries coordinates,
|
||||
// ManualChunker skips the sort entirely and reuses the identical grouping
|
||||
// pipeline as GroupTitleChunker, so its output is bit-for-bit equal (locked by
|
||||
// TestManualChunker_NoPositionsEqualsGroupChunker). No parser change is needed.
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"ragflow/internal/agent/runtime"
|
||||
"ragflow/internal/ingestion/component/globals"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const ComponentNameManualChunker = "ManualChunker"
|
||||
|
||||
// ManualChunkerComponent is the standalone manual-layout chunker. It shares
|
||||
// the GroupTitleChunker grouping body (chunkFromRecords) and only differs by
|
||||
// the optional physical-position pre-sort.
|
||||
type ManualChunkerComponent struct {
|
||||
name string
|
||||
param titleChunkerParam
|
||||
}
|
||||
|
||||
// NewManualChunker constructs the component. It accepts the same heading-level
|
||||
// params as TitleChunker (method is pinned to "group"); the position resort is
|
||||
// automatic based on whether the upstream payload carries coordinates.
|
||||
func NewManualChunker(params map[string]any) (runtime.Component, error) {
|
||||
// method is pinned to "group": ManualChunker's entire value-add is the
|
||||
// physical-position resort, which only fires inside the group path.
|
||||
// A caller must not downgrade it to "naive"/"title" (that would skip the
|
||||
// resort and silently diverge from Python's manual.py), so method is
|
||||
// deliberately ignored here even if passed in params.
|
||||
conf := map[string]any{"method": "group"}
|
||||
for k, v := range params {
|
||||
if k == "method" {
|
||||
continue
|
||||
}
|
||||
conf[k] = v
|
||||
}
|
||||
p := defaultsTitle()
|
||||
p.Update(conf)
|
||||
if err := p.TitleChunkerParam.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("ManualChunker: %w", err)
|
||||
}
|
||||
return &ManualChunkerComponent{
|
||||
name: ComponentNameManualChunker,
|
||||
param: p,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *ManualChunkerComponent) Inputs() map[string]string { return ChunkerInputs }
|
||||
func (c *ManualChunkerComponent) Outputs() map[string]string { return ChunkerOutputs }
|
||||
|
||||
func (c *ManualChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
|
||||
if inputs == nil {
|
||||
inputs = map[string]any{}
|
||||
}
|
||||
// `name` is read from the workflow-wide Globals bag (seeded at
|
||||
// pipeline start, published by the File component), not from the
|
||||
// upstream output map.
|
||||
name := globals.GlobalOrInput(ctx, inputs, "name", "")
|
||||
if name == "" {
|
||||
return map[string]any{
|
||||
"output_format": "chunks",
|
||||
"chunks": []map[string]any{},
|
||||
"_ERROR": "ManualChunker: missing required upstream field \"name\"",
|
||||
}, nil
|
||||
}
|
||||
return c.invoke(ctx, db, withName(inputs, name))
|
||||
}
|
||||
|
||||
func (c *ManualChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
|
||||
records := extractLineRecords(inputs)
|
||||
if len(records) == 0 {
|
||||
return emptyOutputs(), nil
|
||||
}
|
||||
// Coordinate-free payloads (docx, plain text) need no resort — and
|
||||
// skipping it keeps the output identical to GroupTitleChunker.
|
||||
if hasPdfPositions(records) {
|
||||
sortRecordsByPosition(records)
|
||||
}
|
||||
return chunkFromRecords(ctx, db, inputs, &c.param, records)
|
||||
}
|
||||
|
||||
// hasPdfPositions reports whether any record carries a PDF coordinate matrix
|
||||
// (either the structured `_pdf_positions` or the legacy `positions` form).
|
||||
func hasPdfPositions(records []lineRecord) bool {
|
||||
for _, r := range records {
|
||||
if len(r.pdfPositions) > 0 || len(r.positions) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sortRecordsByPosition re-orders records into physical reading order
|
||||
// (page, then top, then left) using a stable sort, so records that share a
|
||||
// key keep their original relative order. Records without coordinates act as
|
||||
// immovable barriers: they keep their original relative order and are never
|
||||
// compared against positioned records.
|
||||
//
|
||||
// The slice is split into maximal contiguous runs of positioned records and
|
||||
// each run is sorted independently. This is required because a single
|
||||
// sort.SliceStable over a MIXED slice (some records with coordinates, some
|
||||
// without) is not a strict weak ordering: a coordinate-free record B sits
|
||||
// incomparable between two positioned records A and C, yet A and C remain
|
||||
// comparable — breaking equivalence-class transitivity and making the sort
|
||||
// undefined behaviour. Segmenting on coordinate-free records keeps each sort
|
||||
// a valid strict weak ordering while preserving the documented behaviour that
|
||||
// coordinate-free records keep their input order (see hasPdfPositions gate).
|
||||
func sortRecordsByPosition(records []lineRecord) {
|
||||
runStart := -1
|
||||
flush := func(end int) {
|
||||
if runStart < 0 {
|
||||
return
|
||||
}
|
||||
sort.SliceStable(records[runStart:end], func(i, j int) bool {
|
||||
ar, _ := firstPositionRow(records[runStart+i])
|
||||
br, _ := firstPositionRow(records[runStart+j])
|
||||
return pdfPosRowLess(ar, br)
|
||||
})
|
||||
runStart = -1
|
||||
}
|
||||
for i := 0; i <= len(records); i++ {
|
||||
positioned := i < len(records)
|
||||
if positioned {
|
||||
_, positioned = firstPositionRow(records[i])
|
||||
}
|
||||
if positioned {
|
||||
if runStart < 0 {
|
||||
runStart = i
|
||||
}
|
||||
} else {
|
||||
flush(i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// firstPositionRow returns the first PDF coordinate 5-tuple
|
||||
// [page,left,right,top,bottom] of a record's coordinate matrix, or
|
||||
// (nil, false) when the record carries no usable coordinates. It reads the
|
||||
// structured `_pdf_positions` key first and falls back to the legacy
|
||||
// `positions` key. Callers (pdfPosRowLess) index offsets 0/1/3, so we enforce
|
||||
// len(row) >= 5 here.
|
||||
func firstPositionRow(r lineRecord) ([]float64, bool) {
|
||||
var raw json.RawMessage
|
||||
switch {
|
||||
case len(r.pdfPositions) > 0:
|
||||
raw = r.pdfPositions
|
||||
case len(r.positions) > 0:
|
||||
raw = r.positions
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
var mat [][]float64
|
||||
if err := json.Unmarshal(raw, &mat); err != nil || len(mat) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
row := mat[0]
|
||||
if len(row) < 5 {
|
||||
return nil, false
|
||||
}
|
||||
return row, true
|
||||
}
|
||||
|
||||
// init registers ManualChunker under CategoryIngestion.
|
||||
func init() {
|
||||
MustRegisterChunker(ComponentNameManualChunker)
|
||||
}
|
||||
97
internal/ingestion/component/chunker/manual_pdf_cgo_test.go
Normal file
97
internal/ingestion/component/chunker/manual_pdf_cgo_test.go
Normal file
@@ -0,0 +1,97 @@
|
||||
//go:build cgo && integration
|
||||
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/parser/parser"
|
||||
)
|
||||
|
||||
// TestManualChunker_RealPDFResort proves the (page, top, left) resort operates
|
||||
// on the ACTUAL records the Go PDF parser produces. This closes gap 1 from the
|
||||
// review: the template integration test only feeds a .txt fixture with no
|
||||
// coordinates, so nothing exercised the resort against real parser output.
|
||||
//
|
||||
// It parses a real PDF, normalizes its records through the SAME extractLineRecords
|
||||
// path the chunker uses in production, then asserts the resort yields a
|
||||
// non-decreasing physical order — i.e. the shared pdfPosRowLess comparator
|
||||
// genuinely consumes the parser-emitted _pdf_positions rather than silently
|
||||
// no-op'ing. The assertion is layout-independent: whether or not the reading
|
||||
// order differed from the physical order, the sorted result must be monotonic.
|
||||
func TestManualChunker_RealPDFResort(t *testing.T) {
|
||||
t.Setenv("DEEPDOC_URL", "")
|
||||
t.Setenv("OSSDEEPDOC_URL", "")
|
||||
|
||||
path := filepath.Join("..", "..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf")
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%s): %v", path, err)
|
||||
}
|
||||
pdf := parser.NewPDFParser()
|
||||
res := pdf.ParseWithResult(t.Context(), "Doc1.pdf", data)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
if len(res.JSON) == 0 {
|
||||
t.Fatal("parser produced no JSON items")
|
||||
}
|
||||
|
||||
// Same shape the chunker receives in production.
|
||||
input := map[string]any{
|
||||
"name": "Doc1.pdf",
|
||||
"output_format": "json",
|
||||
"chunks": res.JSON,
|
||||
}
|
||||
records := extractLineRecords(input)
|
||||
if len(records) == 0 {
|
||||
t.Fatal("extractLineRecords produced no records from real parser output")
|
||||
}
|
||||
|
||||
positioned := 0
|
||||
for _, r := range records {
|
||||
if _, ok := firstPositionRow(r); ok {
|
||||
positioned++
|
||||
}
|
||||
}
|
||||
if positioned == 0 {
|
||||
t.Fatal("no real parser records carried _pdf_positions; resort would be a no-op")
|
||||
}
|
||||
|
||||
sortRecordsByPosition(records)
|
||||
|
||||
// After the resort, the physical (page, top, left) order must be
|
||||
// non-decreasing. pdfPosRowLess(cur, prev) being true would mean a record
|
||||
// physically precedes its predecessor — a resort violation. A coordinate-free
|
||||
// record (no firstPositionRow) is skipped and never becomes prev, so prev may
|
||||
// be nil until the first positioned record; guard against that.
|
||||
var prev []float64
|
||||
for i := 0; i < len(records); i++ {
|
||||
cur, ok := firstPositionRow(records[i])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if prev != nil && pdfPosRowLess(cur, prev) {
|
||||
t.Fatalf("resort produced out-of-order records at %d: row %v precedes %v", i, cur, prev)
|
||||
}
|
||||
prev = cur
|
||||
}
|
||||
}
|
||||
431
internal/ingestion/component/chunker/manual_test.go
Normal file
431
internal/ingestion/component/chunker/manual_test.go
Normal file
@@ -0,0 +1,431 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/agent/runtime"
|
||||
)
|
||||
|
||||
// manualChunkerInput is a small builder for a structured (output_format=chunks)
|
||||
// upstream payload, mirroring the shape the Parser emits for PDF/JSON output.
|
||||
func manualChunkerInput(name string, items ...map[string]any) map[string]any {
|
||||
return map[string]any{
|
||||
"name": name,
|
||||
"output_format": "chunks",
|
||||
"chunks": items,
|
||||
}
|
||||
}
|
||||
|
||||
// posItem builds a chunk item carrying a PDF position matrix
|
||||
// [page,left,right,top,bottom] under the `_pdf_positions` key (matching
|
||||
// schema.ChunkDoc.PDFPositions).
|
||||
func posItem(text string, page, left, right, top, bottom float64) map[string]any {
|
||||
return map[string]any{
|
||||
"text": text,
|
||||
"doc_type_kwd": "text",
|
||||
"_pdf_positions": [][]float64{{page, left, right, top, bottom}},
|
||||
}
|
||||
}
|
||||
|
||||
// mustPosJSON marshals a single 5-tuple position matrix into the
|
||||
// json.RawMessage form used by lineRecord.pdfPositions / positions.
|
||||
func mustPosJSON(t *testing.T, page, left, right, top, bottom float64) json.RawMessage {
|
||||
t.Helper()
|
||||
b, err := json.Marshal([][]float64{{page, left, right, top, bottom}})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal positions: %v", err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func plainItem(text string) map[string]any {
|
||||
return map[string]any{"text": text, "doc_type_kwd": "text"}
|
||||
}
|
||||
|
||||
func mustManual(t *testing.T, params map[string]any) *ManualChunkerComponent {
|
||||
t.Helper()
|
||||
c, err := NewManualChunker(params)
|
||||
if err != nil {
|
||||
t.Fatalf("NewManualChunker: %v", err)
|
||||
}
|
||||
mc, ok := c.(*ManualChunkerComponent)
|
||||
if !ok {
|
||||
t.Fatalf("NewManualChunker returned %T, want *ManualChunkerComponent", c)
|
||||
}
|
||||
return mc
|
||||
}
|
||||
|
||||
// manualChunkTexts pulls the ordered chunk texts out of a chunker output map.
|
||||
func manualChunkTexts(out map[string]any) []string {
|
||||
chunks, _ := out["chunks"].([]map[string]any)
|
||||
out2 := make([]string, 0, len(chunks))
|
||||
for _, ck := range chunks {
|
||||
if s, _ := ck["text"].(string); s != "" {
|
||||
out2 = append(out2, s)
|
||||
}
|
||||
}
|
||||
return out2
|
||||
}
|
||||
|
||||
// TestManualChunker_Registered pins that the component is discoverable via the
|
||||
// runtime registry under the CategoryIngestion category with non-empty
|
||||
// input/output metadata.
|
||||
func TestManualChunker_Registered(t *testing.T) {
|
||||
factory, cat, meta, ok := runtime.DefaultRegistry.Lookup(ComponentNameManualChunker)
|
||||
if !ok {
|
||||
t.Fatal("ManualChunker: registry miss")
|
||||
}
|
||||
if cat != runtime.CategoryIngestion {
|
||||
t.Errorf("category = %q, want %q", cat, runtime.CategoryIngestion)
|
||||
}
|
||||
if factory == nil {
|
||||
t.Error("factory is nil")
|
||||
}
|
||||
if len(meta.Inputs) == 0 {
|
||||
t.Error("inputs metadata empty")
|
||||
}
|
||||
if len(meta.Outputs) == 0 {
|
||||
t.Error("outputs metadata empty")
|
||||
}
|
||||
}
|
||||
|
||||
// TestManualChunker_Empty is the trivial boundary: no records => empty outputs.
|
||||
func TestManualChunker_Empty(t *testing.T) {
|
||||
c := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
|
||||
out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "doc"})
|
||||
if err != nil {
|
||||
t.Fatalf("Invoke: %v", err)
|
||||
}
|
||||
if got, want := out["output_format"], "chunks"; got != want {
|
||||
t.Errorf("output_format = %v, want %v", got, want)
|
||||
}
|
||||
chunks, _ := out["chunks"].([]map[string]any)
|
||||
if len(chunks) != 0 {
|
||||
t.Errorf("chunks = %d, want 0", len(chunks))
|
||||
}
|
||||
}
|
||||
|
||||
// TestManualChunker_NoPositionsEqualsGroupChunker is the regression lock: when
|
||||
// the upstream payload carries NO physical coordinates (e.g. docx, which Go
|
||||
// emits in document-logical order — exactly as Python's manual docx branch,
|
||||
// which performs no coordinate resort), ManualChunker MUST behave bit-for-bit
|
||||
// like GroupTitleChunker. Any divergence here is an unintended regression.
|
||||
func TestManualChunker_NoPositionsEqualsGroupChunker(t *testing.T) {
|
||||
var mkLevels = [][]string{{`^# `}, {`^## `}}
|
||||
input := manualChunkerInput("doc",
|
||||
plainItem("# H1"),
|
||||
plainItem("body under h1 a"),
|
||||
plainItem("body under h1 b"),
|
||||
plainItem("## H2"),
|
||||
plainItem("body under h2"),
|
||||
plainItem("more body under h2"),
|
||||
)
|
||||
|
||||
gc, err := NewGroupTitleChunker(map[string]any{"levels": mkLevels})
|
||||
if err != nil {
|
||||
t.Fatalf("NewGroupTitleChunker: %v", err)
|
||||
}
|
||||
gOut, err := gc.Invoke(t.Context(), nil, input)
|
||||
if err != nil {
|
||||
t.Fatalf("group Invoke: %v", err)
|
||||
}
|
||||
|
||||
mc := mustManual(t, map[string]any{"levels": mkLevels})
|
||||
mOut, err := mc.Invoke(t.Context(), nil, input)
|
||||
if err != nil {
|
||||
t.Fatalf("manual Invoke: %v", err)
|
||||
}
|
||||
|
||||
gChunks, _ := gOut["chunks"].([]map[string]any)
|
||||
mChunks, _ := mOut["chunks"].([]map[string]any)
|
||||
if !reflect.DeepEqual(gChunks, mChunks) {
|
||||
t.Fatalf("ManualChunker output diverges from GroupTitleChunker on no-coordinate input\n group: %#v\n manual: %#v", gChunks, mChunks)
|
||||
}
|
||||
}
|
||||
|
||||
// TestManualChunker_SortsByPageTopLeft is the core behavioural difference
|
||||
// vs. GroupTitleChunker. For a multi-column / manual PDF, the parser emits
|
||||
// records in READING order (column-by-column), not in PHYSICAL/top-down
|
||||
// order. ManualChunker must resort records by (page, top, left) before
|
||||
// grouping so that, e.g., the left column's content precedes the right
|
||||
// column's when they share a page band.
|
||||
func TestManualChunker_SortsByPageTopLeft(t *testing.T) {
|
||||
// page 1, all body (no heading markers). Physical layout:
|
||||
// top=100, left=10 -> "left col top"
|
||||
// top=100, left=400 -> "right col"
|
||||
// top=300, left=10 -> "left col bottom"
|
||||
// top=400, left=200 -> "footer title"
|
||||
// Parser reading order is scrambled relative to physical order.
|
||||
input := manualChunkerInput("doc",
|
||||
posItem("footer title", 1, 200, 400, 400, 500),
|
||||
posItem("left col bottom", 1, 10, 200, 300, 400),
|
||||
posItem("right col", 1, 400, 600, 100, 400),
|
||||
posItem("left col top", 1, 10, 200, 100, 200),
|
||||
)
|
||||
|
||||
mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
|
||||
out, err := mc.Invoke(t.Context(), nil, input)
|
||||
if err != nil {
|
||||
t.Fatalf("Invoke: %v", err)
|
||||
}
|
||||
|
||||
// Whether merged into one chunk or several, the concatenated text MUST
|
||||
// follow physical order: left col top, right col, left col bottom,
|
||||
// footer title.
|
||||
var full strings.Builder
|
||||
for _, s := range manualChunkTexts(out) {
|
||||
full.WriteString(s)
|
||||
full.WriteString("\n")
|
||||
}
|
||||
joined := full.String()
|
||||
wantOrder := []string{"left col top", "right col", "left col bottom", "footer title"}
|
||||
last := -1
|
||||
for _, w := range wantOrder {
|
||||
idx := strings.Index(joined, w)
|
||||
if idx < 0 {
|
||||
t.Fatalf("expected %q to appear in manual output %q", w, joined)
|
||||
}
|
||||
if idx <= last {
|
||||
t.Fatalf("physical order violated: %q appears after a later element in %q", w, joined)
|
||||
}
|
||||
last = idx
|
||||
}
|
||||
}
|
||||
|
||||
// TestManualChunker_SortDiffersFromGroupWhenPositionsPresent proves the resort
|
||||
// actually fires (and is not a no-op): the same scrambled input must produce a
|
||||
// DIFFERENT text order from GroupTitleChunker, which groups in input order.
|
||||
func TestManualChunker_SortDiffersFromGroupWhenPositionsPresent(t *testing.T) {
|
||||
input := manualChunkerInput("doc",
|
||||
posItem("parser-reads-2nd-col-first", 1, 400, 600, 100, 400),
|
||||
posItem("parser-reads-1st-col-first", 1, 10, 200, 100, 200),
|
||||
)
|
||||
|
||||
gc, _ := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
|
||||
gOut, err := gc.Invoke(t.Context(), nil, input)
|
||||
if err != nil {
|
||||
t.Fatalf("group Invoke: %v", err)
|
||||
}
|
||||
mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
|
||||
mOut, err := mc.Invoke(t.Context(), nil, input)
|
||||
if err != nil {
|
||||
t.Fatalf("manual Invoke: %v", err)
|
||||
}
|
||||
if reflect.DeepEqual(gOut["chunks"], mOut["chunks"]) {
|
||||
t.Fatalf("ManualChunker did not reorder relative to GroupTitleChunker; both = %#v", mOut["chunks"])
|
||||
}
|
||||
// ManualChunker must place the 1st-column item first.
|
||||
if texts := manualChunkTexts(mOut); len(texts) == 0 || !strings.Contains(texts[0], "1st-col") {
|
||||
t.Fatalf("expected 1st-column text first in manual output, got %#v", texts)
|
||||
}
|
||||
}
|
||||
|
||||
// TestManualChunker_MergesPositions pins parity with GroupTitleChunker: when
|
||||
// adjacent text records are merged into one chunk, the PDF position matrices
|
||||
// are MERGED across the merged records (not dropped), and parser position tags
|
||||
// are stripped from the text.
|
||||
func TestManualChunker_MergesPositions(t *testing.T) {
|
||||
input := manualChunkerInput("doc",
|
||||
posItem("body one", 1, 10, 20, 30, 40),
|
||||
posItem("body two", 2, 15, 25, 35, 45),
|
||||
)
|
||||
mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
|
||||
out, err := mc.Invoke(t.Context(), nil, input)
|
||||
if err != nil {
|
||||
t.Fatalf("Invoke: %v", err)
|
||||
}
|
||||
chunks, _ := out["chunks"].([]map[string]any)
|
||||
found := false
|
||||
for _, ck := range chunks {
|
||||
text, _ := ck["text"].(string)
|
||||
if !strings.Contains(text, "body one") || !strings.Contains(text, "body two") {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if strings.Contains(text, "@@") {
|
||||
t.Errorf("parser position tags leaked into chunk text: %q", text)
|
||||
}
|
||||
pos, ok := ck["_pdf_positions"].([][]float64)
|
||||
if !ok {
|
||||
t.Fatalf("_pdf_positions missing or wrong type %T on merged group chunk", ck["_pdf_positions"])
|
||||
}
|
||||
if len(pos) != 2 {
|
||||
// Hard fail: the page/top/left checks below would index out of
|
||||
// range if the merged matrix does not have exactly two rows.
|
||||
t.Fatalf("merged _pdf_positions = %d rows, want 2 (both records)", len(pos))
|
||||
}
|
||||
// Merged matrix must itself be sorted by (page, top, left) — the
|
||||
// same key pdfPosRowLess uses when merging.
|
||||
if pdfPosRowLess(pos[1], pos[0]) {
|
||||
t.Errorf("merged _pdf_positions not sorted by (page, top, left): %v", pos)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("merged body group chunk not found in output")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSortRecordsByPosition_Unit pins the comparator directly: (page, top,
|
||||
// left) ascending, stable for equal keys, and records without positions keep
|
||||
// their original relative order (so a no-coordinate payload is untouched).
|
||||
func TestSortRecordsByPosition_Unit(t *testing.T) {
|
||||
recs := []lineRecord{
|
||||
{text: "a", pdfPositions: mustPosJSON(t, 1, 200, 300, 400, 500)},
|
||||
{text: "b", pdfPositions: mustPosJSON(t, 1, 10, 100, 300, 400)},
|
||||
{text: "c", pdfPositions: mustPosJSON(t, 1, 10, 100, 100, 200)},
|
||||
{text: "d"}, // no position
|
||||
}
|
||||
sortRecordsByPosition(recs)
|
||||
got := []string{recs[0].text, recs[1].text, recs[2].text, recs[3].text}
|
||||
want := []string{"c", "b", "a", "d"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("sorted order = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHasPdfPositions is a tiny white-box check for the gate that decides
|
||||
// whether a resort is performed at all.
|
||||
func TestHasPdfPositions(t *testing.T) {
|
||||
if hasPdfPositions([]lineRecord{{text: "x"}}) {
|
||||
t.Error("plain record should report no positions")
|
||||
}
|
||||
if !hasPdfPositions([]lineRecord{{text: "x", pdfPositions: mustPosJSON(t, 1, 1, 1, 1, 1)}}) {
|
||||
t.Error("record with pdf_positions should report positions")
|
||||
}
|
||||
if !hasPdfPositions([]lineRecord{{text: "x", positions: mustPosJSON(t, 1, 1, 1, 1, 1)}}) {
|
||||
t.Error("record with positions should report positions")
|
||||
}
|
||||
}
|
||||
|
||||
// parserShapedItem builds a chunk item that mirrors EXACTLY the JSON shape the
|
||||
// Go PDF parser emits via pdfParseResultToJSON (see
|
||||
// TestPDFParseResultToJSON_NormalizesCoreFields): _pdf_positions is a [][]any
|
||||
// with 1-based page numbers, accompanied by layout / page_number /
|
||||
// doc_type_kwd. Using [][]any — not [][]float64 — matters: that is the concrete
|
||||
// in-memory type the parser produces, and recordsFromStructured must consume it
|
||||
// after the chunksFromInputs JSON round-trip.
|
||||
func parserShapedItem(text string, page, left, right, top, bottom float64) map[string]any {
|
||||
return map[string]any{
|
||||
"text": text,
|
||||
"doc_type_kwd": "text",
|
||||
"layout": "text",
|
||||
"page_number": int(page),
|
||||
"_pdf_positions": [][]any{{page, left, right, top, bottom}},
|
||||
}
|
||||
}
|
||||
|
||||
// legacyPosItem is like posItem but uses the legacy `positions` key instead of
|
||||
// the structured `_pdf_positions` key.
|
||||
func legacyPosItem(text string, page, left, right, top, bottom float64) map[string]any {
|
||||
return map[string]any{
|
||||
"text": text,
|
||||
"doc_type_kwd": "text",
|
||||
"positions": [][]float64{{page, left, right, top, bottom}},
|
||||
}
|
||||
}
|
||||
|
||||
// TestManualChunker_RealParserShapedPayload exercises the EXACT payload shape
|
||||
// the Go PDF parser delivers to the chunker (pdfParseResultToJSON ->
|
||||
// recordsFromStructured). It closes the coverage gap where the template
|
||||
// integration test only feeds a .txt fixture (no coordinates): here a
|
||||
// coordinate-bearing, parser-shaped multi-column doc proves the (page, top,
|
||||
// left) resort actually fires and produces a different order from
|
||||
// GroupTitleChunker (which groups in parser reading order).
|
||||
func TestManualChunker_RealParserShapedPayload(t *testing.T) {
|
||||
input := manualChunkerInput("doc",
|
||||
// Parser reading order (column-by-column) is scrambled vs physical.
|
||||
parserShapedItem("right column, read first", 1, 400, 600, 100, 400),
|
||||
parserShapedItem("left column, read second", 1, 10, 200, 100, 200),
|
||||
)
|
||||
|
||||
gc, _ := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
|
||||
gOut, err := gc.Invoke(t.Context(), nil, input)
|
||||
if err != nil {
|
||||
t.Fatalf("group Invoke: %v", err)
|
||||
}
|
||||
mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
|
||||
mOut, err := mc.Invoke(t.Context(), nil, input)
|
||||
if err != nil {
|
||||
t.Fatalf("manual Invoke: %v", err)
|
||||
}
|
||||
if reflect.DeepEqual(gOut["chunks"], mOut["chunks"]) {
|
||||
t.Fatalf("ManualChunker did not reorder parser-shaped payload; both = %#v", mOut["chunks"])
|
||||
}
|
||||
// Physical (page, top, left) order must put the left column first.
|
||||
if texts := manualChunkTexts(mOut); len(texts) == 0 || !strings.Contains(texts[0], "left column") {
|
||||
t.Fatalf("expected left-column text first after resort, got %#v", texts)
|
||||
}
|
||||
}
|
||||
|
||||
// TestManualChunker_LegacyPositionsKeyTriggersSort pins that the resort fires
|
||||
// when records carry the LEGACY `positions` key rather than the structured
|
||||
// `_pdf_positions` key. firstPositionRow (and thus the shared pdfPosRowLess
|
||||
// comparator) must read both forms, otherwise a payload emitted with the old
|
||||
// key would silently skip the resort.
|
||||
func TestManualChunker_LegacyPositionsKeyTriggersSort(t *testing.T) {
|
||||
input := manualChunkerInput("doc",
|
||||
legacyPosItem("right column, read first", 1, 400, 600, 100, 400),
|
||||
legacyPosItem("left column, read second", 1, 10, 200, 100, 200),
|
||||
)
|
||||
mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
|
||||
out, err := mc.Invoke(t.Context(), nil, input)
|
||||
if err != nil {
|
||||
t.Fatalf("Invoke: %v", err)
|
||||
}
|
||||
// Without the legacy-key path the output would keep parser reading order
|
||||
// (right column before left). The resort must place the left column first.
|
||||
if texts := manualChunkTexts(out); len(texts) == 0 || !strings.Contains(texts[0], "left column") {
|
||||
t.Fatalf("legacy `positions` key did not trigger resort; got %#v", texts)
|
||||
}
|
||||
}
|
||||
|
||||
// TestManualChunker_MixedPositionedAndPlain exercises the boundary where a
|
||||
// payload mixes coordinate-bearing records with coordinate-free records. A
|
||||
// single document type will not usually do this (production payloads are
|
||||
// homogeneously positioned-or-plain), but the sort must not corrupt order when
|
||||
// it happens: the contiguous positioned run reorders by (page, top, left) and
|
||||
// coordinate-free records keep their ORIGINAL relative order. (Go's stable
|
||||
// sort only reorders within a contiguous positioned run — a coordinate-free
|
||||
// record between two positioned records correctly stops the reorder from
|
||||
// crossing it, which is why the positioned records here are kept adjacent.)
|
||||
func TestManualChunker_MixedPositionedAndPlain(t *testing.T) {
|
||||
input := manualChunkerInput("doc",
|
||||
plainItem("plain A"),
|
||||
posItem("right col", 1, 400, 600, 100, 400),
|
||||
posItem("left col", 1, 10, 200, 100, 200),
|
||||
plainItem("plain B"),
|
||||
)
|
||||
mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
|
||||
out, err := mc.Invoke(t.Context(), nil, input)
|
||||
if err != nil {
|
||||
t.Fatalf("Invoke: %v", err)
|
||||
}
|
||||
joined := strings.Join(manualChunkTexts(out), "\n")
|
||||
// The contiguous positioned run keeps physical order relative to itself.
|
||||
if i, j := strings.Index(joined, "left col"), strings.Index(joined, "right col"); i < 0 || j < 0 || i > j {
|
||||
t.Fatalf("positioned records not in physical order in %q", joined)
|
||||
}
|
||||
// Coordinate-free records keep their original relative order.
|
||||
if i, j := strings.Index(joined, "plain A"), strings.Index(joined, "plain B"); i < 0 || j < 0 || i > j {
|
||||
t.Fatalf("coordinate-free records lost original order in %q", joined)
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,7 @@
|
||||
},
|
||||
"Parser:HipSignsRhyme": {
|
||||
"downstream": [
|
||||
"TitleChunker:NineInsectsFind"
|
||||
"ManualChunker:NineInsectsFind"
|
||||
],
|
||||
"obj": {
|
||||
"component_name": "Parser",
|
||||
@@ -87,12 +87,12 @@
|
||||
"File"
|
||||
]
|
||||
},
|
||||
"TitleChunker:NineInsectsFind": {
|
||||
"ManualChunker:NineInsectsFind": {
|
||||
"downstream": [
|
||||
"Extractor:AutoExtractDefault"
|
||||
],
|
||||
"obj": {
|
||||
"component_name": "TitleChunker",
|
||||
"component_name": "ManualChunker",
|
||||
"params": {
|
||||
"hierarchy": 0,
|
||||
"include_heading_content": false,
|
||||
@@ -158,7 +158,7 @@
|
||||
},
|
||||
"Extractor:AutoExtractDefault": {
|
||||
"upstream": [
|
||||
"TitleChunker:NineInsectsFind"
|
||||
"ManualChunker:NineInsectsFind"
|
||||
],
|
||||
"downstream": [
|
||||
"Tokenizer:FunnyBalloonsGrin"
|
||||
@@ -197,15 +197,15 @@
|
||||
"targetHandle": "end"
|
||||
},
|
||||
{
|
||||
"id": "xy-edge__Parser:HipSignsRhymestart-TitleChunker:NineInsectsFindend",
|
||||
"id": "xy-edge__Parser:HipSignsRhymestart-ManualChunker:NineInsectsFindend",
|
||||
"source": "Parser:HipSignsRhyme",
|
||||
"sourceHandle": "start",
|
||||
"target": "TitleChunker:NineInsectsFind",
|
||||
"target": "ManualChunker:NineInsectsFind",
|
||||
"targetHandle": "end"
|
||||
},
|
||||
{
|
||||
"id": "xy-edge__TitleChunker:NineInsectsFindstart-Extractor:AutoExtractDefaultend",
|
||||
"source": "TitleChunker:NineInsectsFind",
|
||||
"id": "xy-edge__ManualChunker:NineInsectsFindstart-Extractor:AutoExtractDefaultend",
|
||||
"source": "ManualChunker:NineInsectsFind",
|
||||
"sourceHandle": "start",
|
||||
"target": "Extractor:AutoExtractDefault",
|
||||
"targetHandle": "end"
|
||||
@@ -407,10 +407,10 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"label": "TitleChunker",
|
||||
"name": "Title Chunker_0"
|
||||
"label": "ManualChunker",
|
||||
"name": "Manual Chunker_0"
|
||||
},
|
||||
"id": "TitleChunker:NineInsectsFind",
|
||||
"id": "ManualChunker:NineInsectsFind",
|
||||
"measured": {
|
||||
"height": 74,
|
||||
"width": 200
|
||||
|
||||
@@ -424,9 +424,9 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {
|
||||
t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)
|
||||
}
|
||||
}
|
||||
chunkerState, ok := state["TitleChunker:NineInsectsFind"]
|
||||
chunkerState, ok := state["ManualChunker:NineInsectsFind"]
|
||||
if !ok {
|
||||
t.Fatal("missing TitleChunker:NineInsectsFind state")
|
||||
t.Fatal("missing ManualChunker:NineInsectsFind state")
|
||||
}
|
||||
chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)
|
||||
if !ok || len(chunkerChunks) != len(wantChunkTexts) {
|
||||
|
||||
@@ -43,7 +43,7 @@ var builtinComponentParamsGolden = map[string]string{
|
||||
"email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
|
||||
"general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
|
||||
"laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\", \"[\\\\(（][0-9]{,2}[\\\\)）]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
|
||||
"manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\", \"[\\\\(（][0-9]{,2}[\\\\)）]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
|
||||
"manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\", \"[\\\\(（][0-9]{,2}[\\\\)）]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
|
||||
"one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
|
||||
"paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\", \"[\\\\(（][0-9]{,2}[\\\\)）]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
|
||||
"picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"image\": {\"output_format\": \"json\", \"parse_method\": \"ocr\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"TokenChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
|
||||
|
||||
@@ -62,7 +62,7 @@ func TestComponentsService_List_FilterIngestion(t *testing.T) {
|
||||
t.Fatalf("List(Ingestion) returned error: %v", err)
|
||||
}
|
||||
wantNames := []string{
|
||||
"extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
|
||||
"extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
|
||||
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",
|
||||
}
|
||||
assertComponentNameSet(t, "ingestion", namesOf(got), wantNames)
|
||||
@@ -85,7 +85,7 @@ func TestComponentsService_List_FilterIngestionAndShared(t *testing.T) {
|
||||
t.Fatalf("List(Ingestion,Shared) returned error: %v", err)
|
||||
}
|
||||
wantNames := []string{
|
||||
"extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
|
||||
"extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
|
||||
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",
|
||||
}
|
||||
assertComponentNameSet(t, "ingestion+shared", namesOf(got), wantNames)
|
||||
@@ -104,7 +104,7 @@ func TestComponentsService_List_FilterDuplicates(t *testing.T) {
|
||||
t.Fatalf("List(Ingestion x3) returned error: %v", err)
|
||||
}
|
||||
wantNames := []string{
|
||||
"extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
|
||||
"extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
|
||||
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",
|
||||
}
|
||||
if len(got) != len(wantNames) {
|
||||
|
||||
Reference in New Issue
Block a user