2024-10-11 08:45:10 +08:00
|
|
|
import { RunningStatus } from '@/constants/knowledge';
|
|
|
|
|
|
2024-08-14 17:26:47 +08:00
|
|
|
export interface IDocumentInfo {
|
2026-04-20 19:24:20 +08:00
|
|
|
// chunk_num: number;
|
2024-08-14 17:26:47 +08:00
|
|
|
create_date: string;
|
|
|
|
|
create_time: number;
|
|
|
|
|
created_by: string;
|
2025-10-09 12:36:19 +08:00
|
|
|
nickname: string;
|
2024-08-14 17:26:47 +08:00
|
|
|
id: string;
|
2026-04-20 14:54:40 +08:00
|
|
|
dataset_id: string;
|
2024-08-14 17:26:47 +08:00
|
|
|
location: string;
|
|
|
|
|
name: string;
|
2024-10-11 08:45:10 +08:00
|
|
|
parser_config: IParserConfig;
|
2026-04-20 19:24:20 +08:00
|
|
|
// parser_id: string;
|
2025-10-09 12:36:19 +08:00
|
|
|
pipeline_id: string;
|
|
|
|
|
pipeline_name: string;
|
2024-11-28 18:44:36 +08:00
|
|
|
process_begin_at?: string;
|
2025-07-07 14:11:47 +08:00
|
|
|
process_duration: number;
|
2024-08-14 17:26:47 +08:00
|
|
|
progress: number;
|
|
|
|
|
progress_msg: string;
|
2024-10-11 08:45:10 +08:00
|
|
|
run: RunningStatus;
|
2024-08-14 17:26:47 +08:00
|
|
|
size: number;
|
|
|
|
|
source_type: string;
|
|
|
|
|
status: string;
|
2025-10-09 12:36:19 +08:00
|
|
|
suffix: string;
|
2024-08-14 17:26:47 +08:00
|
|
|
thumbnail: string;
|
|
|
|
|
token_num: number;
|
|
|
|
|
type: string;
|
|
|
|
|
update_date: string;
|
|
|
|
|
update_time: number;
|
2025-01-13 17:13:37 +08:00
|
|
|
meta_fields?: Record<string, any>;
|
2026-04-20 19:24:20 +08:00
|
|
|
chunk_method: string;
|
|
|
|
|
chunk_count: number;
|
2024-08-14 17:26:47 +08:00
|
|
|
}
|
|
|
|
|
|
2024-10-11 08:45:10 +08:00
|
|
|
export interface IParserConfig {
|
2024-11-28 18:44:36 +08:00
|
|
|
delimiter?: string;
|
|
|
|
|
html4excel?: boolean;
|
2025-12-17 12:58:48 +08:00
|
|
|
layout_recognize?: string;
|
|
|
|
|
pages?: any[];
|
|
|
|
|
chunk_token_num?: number;
|
|
|
|
|
auto_keywords?: number;
|
|
|
|
|
auto_questions?: number;
|
|
|
|
|
toc_extraction?: boolean;
|
|
|
|
|
task_page_size?: number;
|
2024-11-28 18:44:36 +08:00
|
|
|
raptor?: Raptor;
|
2025-01-22 19:43:27 +08:00
|
|
|
graphrag?: GraphRag;
|
2025-12-24 17:21:36 +08:00
|
|
|
image_context_window?: number;
|
Refa: improve image table context (#12244)
### What problem does this PR solve?
Improve image table context.
Current strategy in attach_media_context:
- Order by position when possible: if any chunk has page/position info,
sort by (page, top, left), otherwise keep original order.
- Apply only to media chunks: images use image_context_size, tables use
table_context_size.
- Primary matching: on the same page, choose a text chunk whose vertical
span overlaps the media, then pick the one with the closest vertical
midpoint.
- Fallback matching: if no overlap on that page, choose the nearest text
chunk on the same page (page-head uses the next text; page-tail uses the
previous text).
- Context extraction: inside the chosen text chunk, find a mid-sentence
boundary near the text midpoint, then take context_size tokens split
before/after (total budget).
- No multi-chunk stitching: context comes from a single text chunk to
avoid mixing unrelated segments.
### Type of change
- [x] Refactoring
---------
Co-authored-by: Kevin Hu <kevinhu.sh@gmail.com>
2025-12-26 17:55:32 +08:00
|
|
|
image_table_context_window?: number;
|
|
|
|
|
image_context_size?: number;
|
|
|
|
|
table_context_size?: number;
|
2025-12-24 17:21:36 +08:00
|
|
|
mineru_parse_method?: 'auto' | 'txt' | 'ocr';
|
|
|
|
|
mineru_formula_enable?: boolean;
|
|
|
|
|
mineru_table_enable?: boolean;
|
|
|
|
|
mineru_lang?: string;
|
|
|
|
|
entity_types?: string[];
|
|
|
|
|
metadata?: Array<{
|
|
|
|
|
key?: string;
|
|
|
|
|
description?: string;
|
|
|
|
|
enum?: string[];
|
|
|
|
|
}>;
|
|
|
|
|
enable_metadata?: boolean;
|
2024-08-14 17:26:47 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface Raptor {
|
|
|
|
|
use_raptor: boolean;
|
|
|
|
|
}
|
2025-01-22 19:43:27 +08:00
|
|
|
|
|
|
|
|
interface GraphRag {
|
2026-05-18 16:10:21 +08:00
|
|
|
batch_chunk_token_size?: number;
|
2025-01-22 19:43:27 +08:00
|
|
|
community?: boolean;
|
|
|
|
|
entity_types?: string[];
|
|
|
|
|
method?: string;
|
|
|
|
|
resolution?: boolean;
|
|
|
|
|
use_graphrag?: boolean;
|
|
|
|
|
}
|
2025-07-08 19:19:07 +08:00
|
|
|
|
|
|
|
|
export type IDocumentInfoFilter = {
|
|
|
|
|
run_status: Record<number, number>;
|
|
|
|
|
suffix: Record<string, number>;
|
2025-12-22 17:35:12 +08:00
|
|
|
metadata: Record<string, Record<string, number>>;
|
2025-07-08 19:19:07 +08:00
|
|
|
};
|