From 995e405e8c493ec784deeeed2aa003c5bb4eb1b9 Mon Sep 17 00:00:00 2001 From: qinling0210 <88864212+qinling0210@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:40:09 +0800 Subject: [PATCH] Support pipeline DSL modification through dataset configuration (backend) (#16991) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …end) ### Summary Support pipeline DSL modification through dataset configuration (backend) Key modification: knowledgebase.parser_config --------- Co-authored-by: yzc --- .github/workflows/sep-tests.yml | 14 +- .../advanced_ingestion_pipeline.json | 1464 ++++++++--------- agent/templates/chunk_summary.json | 920 ++++++----- agent/templates/title_chunker.json | 232 ++- internal/agent/canvas/compile.go | 17 +- ...est.go => compile_override_params_test.go} | 57 +- internal/agent/canvas/node_body.go | 36 +- ...t.go => node_body_override_params_test.go} | 64 +- internal/common/parser_config.go | 111 +- internal/common/parser_config_test.go | 215 +++ internal/dao/canvas_template_seed.go | 111 +- internal/ingestion/component/chunker/qa.go | 50 +- .../ingestion/component/chunker/qa_test.go | 118 ++ internal/ingestion/component/extractor.go | 420 ++++- .../ingestion/component/extractor_test.go | 12 +- internal/ingestion/component/parser.go | 170 +- .../parser_dispatch_pdf_vision_cgo_test.go | 7 +- .../component/parser_dispatch_test.go | 103 +- internal/ingestion/component/parser_test.go | 16 +- .../component/pdf_vision_dispatch.go | 7 +- .../ingestion/component/schema/extractor.go | 32 +- internal/ingestion/component/schema/parser.go | 109 -- .../ingestion/component/schema/schema_test.go | 25 +- internal/ingestion/pipeline/pipeline.go | 53 + .../template/ingestion_pipeline_audio.json | 87 +- .../template/ingestion_pipeline_book.json | 171 +- .../template/ingestion_pipeline_email.json | 102 +- .../template/ingestion_pipeline_general.json | 233 +-- .../template/ingestion_pipeline_laws.json | 204 ++- .../template/ingestion_pipeline_manual.json | 118 +- .../template/ingestion_pipeline_one.json | 230 ++- .../template/ingestion_pipeline_paper.json | 77 +- .../template/ingestion_pipeline_picture.json | 112 +- .../ingestion_pipeline_presentation.json | 97 +- .../template/ingestion_pipeline_qa.json | 138 +- .../template/ingestion_pipeline_resume.json | 83 +- .../template/ingestion_pipeline_table.json | 83 +- .../template/ingestion_pipeline_tag.json | 62 +- internal/ingestion/task/chunk_process.go | 114 ++ internal/ingestion/task/embedder.go | 43 +- internal/ingestion/task/embedder_test.go | 79 +- internal/ingestion/task/pipeline_executor.go | 84 +- .../task/pipeline_executor_defaults.go | 94 -- .../ingestion/task/pipeline_executor_test.go | 12 +- internal/parser/parser/csv_parser.go | 8 +- internal/parser/parser/pdf_postprocess.go | 82 +- internal/parser/parser/xlsx_parser.go | 7 +- internal/service/dataset.go | 118 +- internal/service/dataset_update_test.go | 28 +- internal/service/model_service.go | 9 + .../tests/dsl_examples/title_chunker.json | 72 +- test/testcases/restful_api/conftest.py | 2 + 52 files changed, 4093 insertions(+), 2819 deletions(-) rename internal/agent/canvas/{compile_setup_override_test.go => compile_override_params_test.go} (50%) rename internal/agent/canvas/{node_body_setup_override_test.go => node_body_override_params_test.go} (57%) create mode 100644 internal/common/parser_config_test.go delete mode 100644 internal/ingestion/task/pipeline_executor_defaults.go diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml index 635a36472e..01f48fe928 100644 --- a/.github/workflows/sep-tests.yml +++ b/.github/workflows/sep-tests.yml @@ -385,7 +385,8 @@ jobs: # each container has its own /tmp and its own network namespace, and `ss` cannot # see host ports bound by other runners' compose stacks. Use the shared daemon for # both primitives: - # - offset mutex = a docker network created atomically on the shared daemon + # - offset mutex = a config-only docker network created atomically on the shared daemon + # without allocating a subnet from the daemon's address pools # (docker network create is NOT idempotent: it fails on name conflict, which # is exactly the cross-runner test-and-set we need); # - port availability = docker ps published ports (sees every container on the @@ -398,7 +399,7 @@ jobs: local base port used # Enumerate host-published tcp ports across ALL containers on the shared daemon. # Format example: "0.0.0.0:38217->4222/tcp". We extract the left side. - used=$(sudo docker ps --format '{{.Ports}}' | grep -oE '[0-9]+->' | tr -d '->' | sort -u) + used=$(sudo docker ps --format '{{.Ports}}' | grep -oE '[0-9]+->' | sed 's/->$//' | sort -u) for base in "${PORT_BASES[@]}"; do port=$((base + offset)) if grep -qx "${port}" <<< "${used}"; then @@ -436,7 +437,7 @@ jobs: # Atomic cross-runner test-and-set: docker network create fails if the name # already exists on the shared daemon, so even if two runners race here, only # one wins; the loser retries the next candidate. - if sudo docker network create --label ragflow-ci-created="$(date -u +%s)" "${guard}" >/dev/null 2>&1; then + if sudo docker network create --config-only --label ragflow-ci-created="$(date -u +%s)" "${guard}" >/dev/null 2>&1; then PORT_OFFSET=${candidate} PORT_RESERVATION="${guard}" return 0 @@ -895,7 +896,8 @@ jobs: # each container has its own /tmp and its own network namespace, and `ss` cannot # see host ports bound by other runners' compose stacks. Use the shared daemon for # both primitives: - # - offset mutex = a docker network created atomically on the shared daemon + # - offset mutex = a config-only docker network created atomically on the shared daemon + # without allocating a subnet from the daemon's address pools # (docker network create is NOT idempotent: it fails on name conflict, which # is exactly the cross-runner test-and-set we need); # - port availability = docker ps published ports (sees every container on the @@ -908,7 +910,7 @@ jobs: local base port used # Enumerate host-published tcp ports across ALL containers on the shared daemon. # Format example: "0.0.0.0:38217->4222/tcp". We extract the left side. - used=$(sudo docker ps --format '{{.Ports}}' | grep -oE '[0-9]+->' | tr -d '->' | sort -u) + used=$(sudo docker ps --format '{{.Ports}}' | grep -oE '[0-9]+->' | sed 's/->$//' | sort -u) for base in "${PORT_BASES[@]}"; do port=$((base + offset)) if grep -qx "${port}" <<< "${used}"; then @@ -946,7 +948,7 @@ jobs: # Atomic cross-runner test-and-set: docker network create fails if the name # already exists on the shared daemon, so even if two runners race here, only # one wins; the loser retries the next candidate. - if sudo docker network create --label ragflow-ci-created="$(date -u +%s)" "${guard}" >/dev/null 2>&1; then + if sudo docker network create --config-only --label ragflow-ci-created="$(date -u +%s)" "${guard}" >/dev/null 2>&1; then PORT_OFFSET=${candidate} PORT_RESERVATION="${guard}" return 0 diff --git a/agent/templates/advanced_ingestion_pipeline.json b/agent/templates/advanced_ingestion_pipeline.json index 27ba006df0..644ed97d38 100644 --- a/agent/templates/advanced_ingestion_pipeline.json +++ b/agent/templates/advanced_ingestion_pipeline.json @@ -12,181 +12,457 @@ }, "canvas_type": "Ingestion Pipeline", "canvas_category": "dataflow_canvas", - "dsl": { - "components": { - "Extractor:CurlyEmusJam": { - "downstream": [ - "Tokenizer:WittySunsListen" - ], - "obj": { - "component_name": "Extractor", - "params": { - "field_name": "metadata", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } + "dsl": { + "components": { + "Extractor:CurlyEmusJam": { + "downstream": [ + "Tokenizer:WittySunsListen" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "metadata", + "frequencyPenaltyEnabled": true, + "frequency_penalty": 0.7, + "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", + "maxTokensEnabled": false, + "max_tokens": 256, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "presencePenaltyEnabled": true, + "presence_penalty": 0.4, + "prompts": [ + { + "content": "Content:\n{Extractor:SmartWindowsHammer@chunks}", + "role": "user" + } + ], + "sys_prompt": "Extract important structured information from the given content. Output ONLY a valid JSON string with no additional text. If no important structured information is found, output an empty JSON object: {}.\n\nImportant structured information may include: names, dates, locations, events, key facts, numerical data, or other extractable entities.", + "temperature": 0.1, + "temperatureEnabled": true, + "tenant_llm_id": 63, + "topPEnabled": true, + "top_p": 0.3 + } + }, + "upstream": [ + "Extractor:SmartWindowsHammer" + ] + }, + "Extractor:LazyCarpetsKiss": { + "downstream": [ + "Extractor:LovelyPearsRest" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "summary", + "frequencyPenaltyEnabled": true, + "frequency_penalty": 0.7, + "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", + "maxTokensEnabled": false, + "max_tokens": 256, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "presencePenaltyEnabled": true, + "presence_penalty": 0.4, + "prompts": [ + { + "content": "Text to Summarize:\n{TokenChunker:BumpyStarsPress@chunks}", + "role": "user" + } + ], + "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.", + "temperature": 0.1, + "temperatureEnabled": true, + "tenant_llm_id": 63, + "topPEnabled": true, + "top_p": 0.3 + } + }, + "upstream": [ + "TokenChunker:BumpyStarsPress" + ] + }, + "Extractor:LovelyPearsRest": { + "downstream": [ + "Extractor:SmartWindowsHammer" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "keywords", + "frequencyPenaltyEnabled": true, + "frequency_penalty": 0.7, + "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", + "maxTokensEnabled": false, + "max_tokens": 256, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "presencePenaltyEnabled": true, + "presence_penalty": 0.4, + "prompts": [ + { + "content": "Text Content\n{Extractor:LazyCarpetsKiss@chunks}", + "role": "user" + } + ], + "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top 5 important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY.", + "temperature": 0.1, + "temperatureEnabled": true, + "tenant_llm_id": 63, + "topPEnabled": true, + "top_p": 0.3 + } + }, + "upstream": [ + "Extractor:LazyCarpetsKiss" + ] + }, + "Extractor:SmartWindowsHammer": { + "downstream": [ + "Extractor:CurlyEmusJam" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "questions", + "frequencyPenaltyEnabled": true, + "frequency_penalty": 0.7, + "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", + "maxTokensEnabled": false, + "max_tokens": 256, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "presencePenaltyEnabled": true, + "presence_penalty": 0.4, + "prompts": [ + { + "content": "Text Content\n{Extractor:LovelyPearsRest@chunks}", + "role": "user" + } + ], + "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose 3 questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top 3 important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY.", + "temperature": 0.1, + "temperatureEnabled": true, + "tenant_llm_id": 63, + "topPEnabled": true, + "top_p": 0.3 + } + }, + "upstream": [ + "Extractor:LovelyPearsRest" + ] + }, + "File": { + "downstream": [ + "Parser:HipSignsRhyme" + ], + "obj": { + "component_name": "File", + "params": {} + }, + "upstream": [] + }, + "Parser:HipSignsRhyme": { + "downstream": [ + "TokenChunker:BumpyStarsPress" + ], + "obj": { + "component_name": "Parser", + "params": { + "outputs": { + "html": { + "type": "string", + "value": "" }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": [ - { - "content": "Content:\n{Extractor:SmartWindowsHammer@chunks}", - "role": "user" - } - ], - "sys_prompt": "Extract important structured information from the given content. Output ONLY a valid JSON string with no additional text. If no important structured information is found, output an empty JSON object: {}.\n\nImportant structured information may include: names, dates, locations, events, key facts, numerical data, or other extractable entities.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": 63, - "topPEnabled": true, - "top_p": 0.3 - } - }, - "upstream": [ - "Extractor:SmartWindowsHammer" - ] - }, - "Extractor:LazyCarpetsKiss": { - "downstream": [ - "Extractor:LovelyPearsRest" - ], - "obj": { - "component_name": "Extractor", - "params": { - "field_name": "summary", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } + "json": { + "type": "Array", + "value": [] }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": [ - { - "content": "Text to Summarize:\n{TokenChunker:BumpyStarsPress@chunks}", - "role": "user" - } - ], - "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": 63, - "topPEnabled": true, - "top_p": 0.3 - } - }, - "upstream": [ - "TokenChunker:BumpyStarsPress" - ] - }, - "Extractor:LovelyPearsRest": { - "downstream": [ - "Extractor:SmartWindowsHammer" - ], - "obj": { - "component_name": "Extractor", - "params": { - "field_name": "keywords", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } + "markdown": { + "type": "string", + "value": "" }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": [ - { - "content": "Text Content\n{Extractor:LazyCarpetsKiss@chunks}", - "role": "user" - } + "text": { + "type": "string", + "value": "" + } + }, + "doc": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "doc" + ] + }, + "docx": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "docx" ], - "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top 5 important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": 63, - "topPEnabled": true, - "top_p": 0.3 - } - }, - "upstream": [ - "Extractor:LazyCarpetsKiss" - ] - }, - "Extractor:SmartWindowsHammer": { - "downstream": [ - "Extractor:CurlyEmusJam" - ], - "obj": { - "component_name": "Extractor", - "params": { - "field_name": "questions", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": [ - { - "content": "Text Content\n{Extractor:LovelyPearsRest@chunks}", - "role": "user" - } + "vlm": {} + }, + "email": { + "fields": [ + "from", + "to", + "cc", + "bcc", + "date", + "subject", + "body", + "attachments" ], - "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose 3 questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top 3 important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": 63, - "topPEnabled": true, - "top_p": 0.3 + "output_format": "text", + "preprocess": "main_content", + "suffix": [ + "eml", + "msg" + ] + }, + "html": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "htm", + "html" + ] + }, + "image": { + "output_format": "text", + "parse_method": "ocr", + "preprocess": "main_content", + "suffix": [ + "jpg", + "jpeg", + "png", + "gif" + ], + "system_prompt": "" + }, + "markdown": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "md", + "markdown", + "mdx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": "main_content", + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "slides": { + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": "main_content", + "suffix": [ + "pptx", + "ppt" + ] + }, + "spreadsheet": { + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": "main_content", + "suffix": [ + "xls", + "xlsx", + "csv" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "txt", + "py", + "js", + "java", + "c", + "cpp", + "h", + "php", + "go", + "ts", + "sh", + "cs", + "kt", + "sql" + ] } - }, - "upstream": [ - "Extractor:LovelyPearsRest" - ] + } }, - "File": { - "downstream": [ - "Parser:HipSignsRhyme" - ], - "obj": { - "component_name": "File", - "params": {} - }, - "upstream": [] + "upstream": [ + "File" + ] + }, + "TokenChunker:BumpyStarsPress": { + "downstream": [ + "Extractor:LazyCarpetsKiss" + ], + "obj": { + "component_name": "TokenChunker", + "params": { + "children_delimiters": [], + "chunk_token_size": 512, + "delimiter_mode": "token_size", + "delimiters": [], + "image_context_size": 0, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "overlapped_percent": 0, + "table_context_size": 0 + } }, - "Parser:HipSignsRhyme": { - "downstream": [ - "TokenChunker:BumpyStarsPress" - ], - "obj": { - "component_name": "Parser", - "params": { + "upstream": [ + "Parser:HipSignsRhyme" + ] + }, + "Tokenizer:WittySunsListen": { + "downstream": [], + "obj": { + "component_name": "Tokenizer", + "params": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + } + }, + "upstream": [ + "Extractor:CurlyEmusJam" + ] + } + }, + "globals": { + "sys.history": [] + }, + "graph": { + "edges": [ + { + "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", + "source": "File", + "sourceHandle": "start", + "target": "Parser:HipSignsRhyme", + "targetHandle": "end" + }, + { + "id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:BumpyStarsPressend", + "source": "Parser:HipSignsRhyme", + "sourceHandle": "start", + "target": "TokenChunker:BumpyStarsPress", + "targetHandle": "end" + }, + { + "id": "xy-edge__TokenChunker:BumpyStarsPressstart-Extractor:LazyCarpetsKissend", + "source": "TokenChunker:BumpyStarsPress", + "sourceHandle": "start", + "target": "Extractor:LazyCarpetsKiss", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__Extractor:LazyCarpetsKissstart-Extractor:LovelyPearsRestend", + "source": "Extractor:LazyCarpetsKiss", + "sourceHandle": "start", + "target": "Extractor:LovelyPearsRest", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__Extractor:LovelyPearsReststart-Extractor:SmartWindowsHammerend", + "selected": false, + "source": "Extractor:LovelyPearsRest", + "sourceHandle": "start", + "target": "Extractor:SmartWindowsHammer", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__Extractor:SmartWindowsHammerstart-Extractor:CurlyEmusJamend", + "selected": false, + "source": "Extractor:SmartWindowsHammer", + "sourceHandle": "start", + "target": "Extractor:CurlyEmusJam", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__Extractor:CurlyEmusJamstart-Tokenizer:WittySunsListenend", + "source": "Extractor:CurlyEmusJam", + "sourceHandle": "start", + "target": "Tokenizer:WittySunsListen", + "targetHandle": "end" + } + ], + "nodes": [ + { + "data": { + "label": "File", + "name": "File" + }, + "id": "File", + "measured": { + "height": 50, + "width": 200 + }, + "position": { + "x": 50, + "y": 200 + }, + "sourcePosition": "left", + "targetPosition": "right", + "type": "beginNode" + }, + { + "data": { + "form": { "outputs": { "html": { "type": "string", @@ -205,24 +481,29 @@ "value": "" } }, - "setups": { - "doc": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "doc" - ] - }, - "docx": { + "setups": [ + { + "fileFormat": "pdf", "flatten_media_to_text": false, "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "docx" - ], - "vlm": {} + "parse_method": "DeepDOC", + "preprocess": "main_content" }, - "email": { + { + "fileFormat": "spreadsheet", + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": "main_content" + }, + { + "fileFormat": "image", + "output_format": "text", + "parse_method": "ocr", + "preprocess": "main_content", + "system_prompt": "" + }, + { "fields": [ "from", "to", @@ -233,133 +514,274 @@ "body", "attachments" ], + "fileFormat": "email", "output_format": "text", - "preprocess": "main_content", - "suffix": [ - "eml", - "msg" - ] + "preprocess": "main_content" }, - "html": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "htm", - "html" - ] - }, - "image": { - "output_format": "text", - "parse_method": "ocr", - "preprocess": "main_content", - "suffix": [ - "jpg", - "jpeg", - "png", - "gif" - ], - "system_prompt": "" - }, - "markdown": { + { + "fileFormat": "markdown", "flatten_media_to_text": false, "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "md", - "markdown", - "mdx" - ], - "vlm": {} + "preprocess": "main_content" }, - "pdf": { + { + "fileFormat": "text&code", + "output_format": "json", + "preprocess": "main_content" + }, + { + "fileFormat": "html", + "output_format": "json", + "preprocess": "main_content" + }, + { + "fileFormat": "doc", + "output_format": "json", + "preprocess": "main_content" + }, + { + "fileFormat": "docx", "flatten_media_to_text": false, "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pdf" - ], - "vlm": {} + "preprocess": "main_content" }, - "slides": { + { + "fileFormat": "slides", "output_format": "json", "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pptx", - "ppt" - ] - }, - "spreadsheet": { - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} - }, - "text&code": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" - ] + "preprocess": "main_content" } - } - } + ] + }, + "label": "Parser", + "name": "Parser_0" }, - "upstream": [ - "File" - ] + "dragging": false, + "id": "Parser:HipSignsRhyme", + "measured": { + "height": 57, + "width": 200 + }, + "position": { + "x": 316.99524094206413, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "parserNode" }, - "TokenChunker:BumpyStarsPress": { - "downstream": [ - "Extractor:LazyCarpetsKiss" - ], - "obj": { - "component_name": "TokenChunker", - "params": { + { + "data": { + "form": { "children_delimiters": [], "chunk_token_size": 512, "delimiter_mode": "token_size", - "delimiters": [], - "image_context_size": 0, + "delimiters": [ + { + "value": "\n" + } + ], + "image_table_context_window": 0, "outputs": { "chunks": { "type": "Array", "value": [] } }, - "overlapped_percent": 0, - "table_context_size": 0 - } + "overlapped_percent": 0 + }, + "label": "TokenChunker", + "name": "Token Chunker_0" }, - "upstream": [ - "Parser:HipSignsRhyme" - ] + "id": "TokenChunker:BumpyStarsPress", + "measured": { + "height": 74, + "width": 200 + }, + "position": { + "x": 616.9952409420641, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "chunkerNode" }, - "Tokenizer:WittySunsListen": { - "downstream": [], - "obj": { - "component_name": "Tokenizer", - "params": { + { + "data": { + "form": { + "field_name": "summary", + "frequencyPenaltyEnabled": true, + "frequency_penalty": 0.7, + "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", + "maxTokensEnabled": false, + "max_tokens": 256, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "presencePenaltyEnabled": true, + "presence_penalty": 0.4, + "prompts": "Text to Summarize:\n{TokenChunker:BumpyStarsPress@chunks}", + "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.", + "temperature": 0.1, + "temperatureEnabled": true, + "tenant_llm_id": 63, + "topPEnabled": true, + "top_p": 0.3 + }, + "label": "Extractor", + "name": "Summarization" + }, + "id": "Extractor:LazyCarpetsKiss", + "measured": { + "height": 90, + "width": 200 + }, + "position": { + "x": 916.9952409420641, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "contextNode" + }, + { + "data": { + "form": { + "field_name": "keywords", + "frequencyPenaltyEnabled": true, + "frequency_penalty": 0.7, + "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", + "maxTokensEnabled": false, + "max_tokens": 256, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "presencePenaltyEnabled": true, + "presence_penalty": 0.4, + "prompts": "Text Content\n{Extractor:LazyCarpetsKiss@chunks}", + "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top 5 important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY.", + "temperature": 0.1, + "temperatureEnabled": true, + "tenant_llm_id": 63, + "topPEnabled": true, + "top_p": 0.3 + }, + "label": "Extractor", + "name": "Auto Keyword" + }, + "dragging": false, + "id": "Extractor:LovelyPearsRest", + "measured": { + "height": 90, + "width": 200 + }, + "position": { + "x": 983.5410692821999, + "y": 301.1557383781162 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "contextNode" + }, + { + "data": { + "form": { + "field_name": "questions", + "frequencyPenaltyEnabled": true, + "frequency_penalty": 0.7, + "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", + "maxTokensEnabled": false, + "max_tokens": 256, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "presencePenaltyEnabled": true, + "presence_penalty": 0.4, + "prompts": "Text Content\n{Extractor:LovelyPearsRest@chunks}", + "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose 3 questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top 3 important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY.", + "temperature": 0.1, + "temperatureEnabled": true, + "tenant_llm_id": 63, + "topPEnabled": true, + "top_p": 0.3 + }, + "label": "Extractor", + "name": "Auto Question" + }, + "dragging": false, + "id": "Extractor:SmartWindowsHammer", + "measured": { + "height": 90, + "width": 200 + }, + "position": { + "x": 1021.1009769800036, + "y": 421.67760363913044 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "contextNode" + }, + { + "data": { + "form": { + "field_name": "metadata", + "frequencyPenaltyEnabled": true, + "frequency_penalty": 0.7, + "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", + "maxTokensEnabled": false, + "max_tokens": 256, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "presencePenaltyEnabled": true, + "presence_penalty": 0.4, + "prompts": "Content:\n{Extractor:SmartWindowsHammer@chunks}", + "sys_prompt": "Extract important structured information from the given content. Output ONLY a valid JSON string with no additional text. If no important structured information is found, output an empty JSON object: {}.\n\nImportant structured information may include: names, dates, locations, events, key facts, numerical data, or other extractable entities.", + "temperature": 0.1, + "temperatureEnabled": true, + "tenant_llm_id": 63, + "topPEnabled": true, + "top_p": 0.3 + }, + "label": "Extractor", + "name": "Auto Metadata" + }, + "dragging": false, + "id": "Extractor:CurlyEmusJam", + "measured": { + "height": 90, + "width": 200 + }, + "position": { + "x": 1065.7115140232393, + "y": 527.4370438206126 + }, + "selected": true, + "sourcePosition": "right", + "targetPosition": "left", + "type": "contextNode" + }, + { + "data": { + "form": { "fields": "text", "filename_embd_weight": 0.1, "outputs": {}, @@ -367,456 +789,32 @@ "embedding", "full_text" ] - } + }, + "label": "Tokenizer", + "name": "Indexer_0" }, - "upstream": [ - "Extractor:CurlyEmusJam" - ] + "dragging": false, + "id": "Tokenizer:WittySunsListen", + "measured": { + "height": 114, + "width": 200 + }, + "position": { + "x": 1327.3247542536642, + "y": 164.72133416115918 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "tokenizerNode" } - }, - "globals": { - "sys.history": [] - }, - "graph": { - "edges": [ - { - "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", - "source": "File", - "sourceHandle": "start", - "target": "Parser:HipSignsRhyme", - "targetHandle": "end" - }, - { - "id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:BumpyStarsPressend", - "source": "Parser:HipSignsRhyme", - "sourceHandle": "start", - "target": "TokenChunker:BumpyStarsPress", - "targetHandle": "end" - }, - { - "id": "xy-edge__TokenChunker:BumpyStarsPressstart-Extractor:LazyCarpetsKissend", - "source": "TokenChunker:BumpyStarsPress", - "sourceHandle": "start", - "target": "Extractor:LazyCarpetsKiss", - "targetHandle": "end" - }, - { - "data": { - "isHovered": false - }, - "id": "xy-edge__Extractor:LazyCarpetsKissstart-Extractor:LovelyPearsRestend", - "source": "Extractor:LazyCarpetsKiss", - "sourceHandle": "start", - "target": "Extractor:LovelyPearsRest", - "targetHandle": "end" - }, - { - "data": { - "isHovered": false - }, - "id": "xy-edge__Extractor:LovelyPearsReststart-Extractor:SmartWindowsHammerend", - "selected": false, - "source": "Extractor:LovelyPearsRest", - "sourceHandle": "start", - "target": "Extractor:SmartWindowsHammer", - "targetHandle": "end" - }, - { - "data": { - "isHovered": false - }, - "id": "xy-edge__Extractor:SmartWindowsHammerstart-Extractor:CurlyEmusJamend", - "selected": false, - "source": "Extractor:SmartWindowsHammer", - "sourceHandle": "start", - "target": "Extractor:CurlyEmusJam", - "targetHandle": "end" - }, - { - "data": { - "isHovered": false - }, - "id": "xy-edge__Extractor:CurlyEmusJamstart-Tokenizer:WittySunsListenend", - "source": "Extractor:CurlyEmusJam", - "sourceHandle": "start", - "target": "Tokenizer:WittySunsListen", - "targetHandle": "end" - } - ], - "nodes": [ - { - "data": { - "label": "File", - "name": "File" - }, - "id": "File", - "measured": { - "height": 50, - "width": 200 - }, - "position": { - "x": 50, - "y": 200 - }, - "sourcePosition": "left", - "targetPosition": "right", - "type": "beginNode" - }, - { - "data": { - "form": { - "outputs": { - "html": { - "type": "string", - "value": "" - }, - "json": { - "type": "Array", - "value": [] - }, - "markdown": { - "type": "string", - "value": "" - }, - "text": { - "type": "string", - "value": "" - } - }, - "setups": [ - { - "fileFormat": "pdf", - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content" - }, - { - "fileFormat": "spreadsheet", - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": "main_content" - }, - { - "fileFormat": "image", - "output_format": "text", - "parse_method": "ocr", - "preprocess": "main_content", - "system_prompt": "" - }, - { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "fileFormat": "email", - "output_format": "text", - "preprocess": "main_content" - }, - { - "fileFormat": "markdown", - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "text&code", - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "html", - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "doc", - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "docx", - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "slides", - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content" - } - ] - }, - "label": "Parser", - "name": "Parser_0" - }, - "dragging": false, - "id": "Parser:HipSignsRhyme", - "measured": { - "height": 57, - "width": 200 - }, - "position": { - "x": 316.99524094206413, - "y": 195.39629819663406 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "parserNode" - }, - { - "data": { - "form": { - "children_delimiters": [], - "chunk_token_size": 512, - "delimiter_mode": "token_size", - "delimiters": [ - { - "value": "\n" - } - ], - "image_table_context_window": 0, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "overlapped_percent": 0 - }, - "label": "TokenChunker", - "name": "Token Chunker_0" - }, - "id": "TokenChunker:BumpyStarsPress", - "measured": { - "height": 74, - "width": 200 - }, - "position": { - "x": 616.9952409420641, - "y": 195.39629819663406 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "chunkerNode" - }, - { - "data": { - "form": { - "field_name": "summary", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": "Text to Summarize:\n{TokenChunker:BumpyStarsPress@chunks}", - "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": 63, - "topPEnabled": true, - "top_p": 0.3 - }, - "label": "Extractor", - "name": "Summarization" - }, - "id": "Extractor:LazyCarpetsKiss", - "measured": { - "height": 90, - "width": 200 - }, - "position": { - "x": 916.9952409420641, - "y": 195.39629819663406 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "contextNode" - }, - { - "data": { - "form": { - "field_name": "keywords", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": "Text Content\n{Extractor:LazyCarpetsKiss@chunks}", - "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top 5 important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": 63, - "topPEnabled": true, - "top_p": 0.3 - }, - "label": "Extractor", - "name": "Auto Keyword" - }, - "dragging": false, - "id": "Extractor:LovelyPearsRest", - "measured": { - "height": 90, - "width": 200 - }, - "position": { - "x": 983.5410692821999, - "y": 301.1557383781162 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "contextNode" - }, - { - "data": { - "form": { - "field_name": "questions", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": "Text Content\n{Extractor:LovelyPearsRest@chunks}", - "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose 3 questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top 3 important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": 63, - "topPEnabled": true, - "top_p": 0.3 - }, - "label": "Extractor", - "name": "Auto Question" - }, - "dragging": false, - "id": "Extractor:SmartWindowsHammer", - "measured": { - "height": 90, - "width": 200 - }, - "position": { - "x": 1021.1009769800036, - "y": 421.67760363913044 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "contextNode" - }, - { - "data": { - "form": { - "field_name": "metadata", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": "Content:\n{Extractor:SmartWindowsHammer@chunks}", - "sys_prompt": "Extract important structured information from the given content. Output ONLY a valid JSON string with no additional text. If no important structured information is found, output an empty JSON object: {}.\n\nImportant structured information may include: names, dates, locations, events, key facts, numerical data, or other extractable entities.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": 63, - "topPEnabled": true, - "top_p": 0.3 - }, - "label": "Extractor", - "name": "Auto Metadata" - }, - "dragging": false, - "id": "Extractor:CurlyEmusJam", - "measured": { - "height": 90, - "width": 200 - }, - "position": { - "x": 1065.7115140232393, - "y": 527.4370438206126 - }, - "selected": true, - "sourcePosition": "right", - "targetPosition": "left", - "type": "contextNode" - }, - { - "data": { - "form": { - "fields": "text", - "filename_embd_weight": 0.1, - "outputs": {}, - "search_method": [ - "embedding", - "full_text" - ] - }, - "label": "Tokenizer", - "name": "Indexer_0" - }, - "dragging": false, - "id": "Tokenizer:WittySunsListen", - "measured": { - "height": 114, - "width": 200 - }, - "position": { - "x": 1327.3247542536642, - "y": 164.72133416115918 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "tokenizerNode" - } - ] - }, - "history": [], - "messages": [], - "path": [], - "retrieval": [], - "variables": [] + ] }, + "history": [], + "messages": [], + "path": [], + "retrieval": [], + "variables": [] + }, "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAACXBIWXMAABYlAAAWJQFJUiTwAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAABpQSURBVHgBbXoJfFTluf5zzpw5syaZZLKRjZCEBEIgLAFBLYIW0apsUnFprUu1t3q1WqWl2Nvi36X6v7aIqNV760WttVotFVBZROAiEowIYQ1bQvY9mcxk9jlzzn2+E7Cx7eSX32xneb93ed7nfb6R8A+PVWvfm3ek4exih01ZAkjFsiShu7cf40uKcM9ty5DuSUEyqSMS03i0AYfNCsOA+fofH+LceEJDXNNhtylQZRmlbh1/6gHur3Mi3KgAHkCxAZqb1/PzvYXniasN6TCcwPIpftzvTdaHNLneZZEfnz8tvXn0PaSLL9as/Zvnk7qDv7Zb5YfSUlPMzwZ8fqS4nXjgrpsxc1oJurqDpkHCVJtVMc+OxTU+SVBVC3R9ZBESDReL1HQdikWGhf/FqoavYjpu2ZuKzrMKUjOA66/fi+BgLjbvKQdSdUg0XgrJ0LmWmpIgXqwII0WXYbVznW4rBjULWgLS82FH4vGl49KHvl7AvCUPeeJWdXd2tneqsC4cjWB4OIw7br4BixZeytcxRKJxGmTASmNketIw/m6seB1PJGFVaCy/i3KR4lkcN0bVEVcN3LLPir1nHYAziTG5Cn660MDjoWswxTseLetfRDeNT9Ka7JQoXpgSwFyngR4u5C/tKXir2YGOuASFp8u8nmzI9WEd83GnNGQu4OoHXlzrjvQ95EgOYyiiY/Ilc3D/HUu4FhlDgTCN12iMBKfd+rWXv/mQ+D3M45L0usOhIoXvc91JrGxQ8OIBJ6QgF58rIz+X6ZGWQCEXMS/Hj81bHDhySoVaoOGXU/24JU2HapWx/pQdvz3kRtZkA3fNiGKsR0ePy4FlvNFW3lEJRZ5/xO18WLJv8BVrcJ7XArxjMIiaTA1/+q4XhRjCSZ8ECz2sWkV6gLksPCsxCnw/KgIa00XnAVbFAoXflzo0vDkAPPhpKsLDMiR6zghrcOQr8DCvbTQy7Ad6j/KeKQZuqvbjiaI4OkMKitN1fHtzOhodFjywPI4qScNZw4pImhV3PPU31Gyvg89hx2sbfoLTuZ750sy1H21w2K13LJxbDU9RJnY06di0XcFUFtVHy8MwkqwFhk++mNs0VNNGvCQ8H9eSNJqv+V/m1HEiYWD5p3a09togpzMHBhTcXJlATZ4VH7bo2NXBnGa6JMIGqseG8PKEEJS4bKbIVSVcwB/S4Jss4en5UVgGk+gc48Llnx7Hktt/g2S/D4mqCTDOdSAlNIiz9928Tspoe+zwOE/hVCYqdqYvw2c+F1rtUTx9xInOLTL+dmsYc3M1tIe4CAlmwXIdCMXiJvA4bCqyrXxh0/H9Qyp2HHBAUnRY0mTYmW4nFvtQVOBBPCmDdY6T24GZ53X8flYfZtPLp8My0l0WXF4kYewrbrSWyXhkbgzpEQNX5Ngx+81dkO97BeHSPCAQgTKzDIna02bdWQP+ZnlI8k09G2xCn94Mb9dDOIZe9PpUrCECFN+rY+k7TmxsVpDnNEyPJ+hxUbAOVYWXuV7miuO5Lub766nYsceBBTURTBmTgOYz8Na1YYQkG+IGC5uRDEQNVC4EGs61IyvDik5DQVmOFVVZEu7fy6jZZMy+VMMsJYGITcKvB5KQHqbxHhcwFGTYElAWTIGcYodMoNBUe7H8Yfb9KLB60BruY157sCH5Borp1c6EgsecEaT90MA9G5046Seq6LwAkUbk+oQUHV9GdWS/n4r/3JUCKTWJVfcCp664CY7vPQIXDYi9k2QNKYi2EHyGgba6YWy4cRP0tx5DTmEG8jwKEqytYWbay/tUpN5g4AZZx511brz4Y8Lq1A5I6Q6inx3JuBVJxYXg6vdopw0J2KA7PLDsedS7JkDDch1p0JjK3dIJxNvnIZLuQr7LwLdsCWz3WvHOByp+My+BFKbPkEXHPHr71d1OMOWRyfCXVQCdxMHFhJnI8bk41piFnlWbMGNgEh5bvxq3PrwA7d1NaNryEfKWXIn078xCIhBDPov6V1/ZcchtQaJUwqGfJHDL6q+w4FiUKepDjfskOqMWdIfjCDpt6A0m0D8UhjyvEtq0cbAEV1yyJk9OQ2NvADkJD7xyDiJtDeieXY03vHZIBVZUlmg40WRB0JfEH70qHnzbCR8RQ04H0hgJm1uG0yrBn4jj021lOHogB94ZSVzPIiyak45ZN3rQuG8Q3YeAaMUw7vx4PSw9cdGqkWFJ4L5tLoRvl7BwcRcWbT2HbiLg5wTLNuUI5iQTrD8N5U8th1xTita6VoQdDsTzvLCW5kDyfPRzwykxH7uCSEt1wGF3YOn8ctz46BwceCWKTUU2fLl2LKQZdMjrOlauOYxX/rsKId1GSAWsRCVnKptfREZ4iJ1DMfBQtQ/3ZybRTIiMhqMo92ZDdzONND+amwYxFPegcmwGAsGYCbuTd6ZiWkM3rvttJ7bImxHVh6HAhWtvm457ssKo3XQKfd1DKCr2oul0P/sM6QuRMBqLQrEft2DlNVcyZRJ4/WgtBvQQDv3gHLb+tQ7lSiUq20qx5EYDv24uhDbHgq7ibDzy4+N4/L1q6F0MO+F0gKhiGMTvmgB+VxRjsSpoVW0YbO6Bt0DFF0fPIcTWyazG9o1voLLIiyn//lOkZ6o4OSiZaPad33bjbcvrSEtmwIlUsO8jSb610e+AlcFqjBL9+D/gYFNk3QRZxIXeDEhF33/MsFmt8BY5cOpgHyZPz0Lw/xH/mefWpIsXiiLPUoopzho8Wz8BEx5owdiPP8CpFfPRevUUGGxYxUVhvFoVRB7RppmYnpvGhkXy9qvCKCY5PVjNJjkYjrHhGTh1thHDwTDSvUlseZpXrynHBl82ap5+DD6oGKFyIyRHMzTE+GxnylxT6kSsL8jGGMOgnkRVSQYOikabaVyxJnYuAa4BatgC92YZMWKeypgnmYsJXjZuWCDH+6H4y9BcxoLddwbNegqMRWPxuyl9eLKIeRogJ7JbUEyvOtIlHHghgjMth9HRH0ORJQvFC9jBeZPSwmx4Ur2oGJePk5t0dMlWHPBKyPnkA/rc8002yz9WGoo9Ko62RxCmA149+RBO1XVje/0QnIK5FmWlID/HiZtn1eDfLr8UA4NhhojECXb63ocgS4pNHg1oQkFtN8IZNvi5rJv+fwWOVvZjgqygWVNQUeBAVqqKYIT4HDfIThV4b2nHwgdd6Not6tUwO3g4rsPFBf7urtM49VYBblu3HfrsFMSduZDkf+ZZ4pMgOdbciR4U5KXiscV/Ri9Z8fRxbuTSFjl+mkStXcKZXV04Wd8Fm2I1Twyjj/4ooVfCXMZZpEte1LsPIYdN5Y3+H2BFiR1nhi1sSAoK2JSS7LqC6NlpeJzEbepPZfQ9uwgnD7ehYEM7GaRi8idFFD69/smeZuRW7EV1/A+oWDmMw2+tJHU+byLT6AedjPPDUaSRTlfl2ZGd60ax14bLJzOFWvyQU6fZ4Kl2oMfhR2NLDxuG/nUADTaLbFQjKg/DafSi64dzUbPTh0/PB9nQgLJcG/m+4EPfJHZiDpAJs98/SFq8qBoWLYS/7T0Kj0NBU6cPv9+wA6+fnI87Dl2OprW/QM0Xh1H5bgx733seDr0XVsZ99ELSGeW36zqwaEk5Vqd2476bxqHuBLODjECed1UZ8qalIPNKF/oOhtk55Qs1JJlRUNkXZuhX48Tcy2Fz5eLKQ61YXJMNN9v+cDhhelW+cLPRw06UYS/JzkRH5SmMzy9FiTsTqdIGwm8vfrBiATraEwgO6Ch44DtocJ3H8ne7seRXPtR+/CLOLfs2VKKDHf1cRwK8CSJkwm8f6IO1bheije147VSfuTApU3vEuNVyGU5UnUbfSTJA48LKJUHadLj1NhxfvBShe6/HXdftRt5VY7BgVRHSqgmLMdIT5rUwXFBsk3aPGi8FSw0Sfc4NdbPGKhFiQ/JHIugNMJWUkftk5jqw9nsH8Mn7TaiUilCt52HjwjQcW5KDaFcL0vbVIauhEakDPvTFA7hh9mTsZjNzWKKIkD1IM431RtVfM/D58mNwy5zdiOdWI2SGsXN8FZrW3I7yWhWLX6wl9FWjgxeyEutXtZShuzlqzgEq8ymRTJp2jx4tTQOzHfj9bUew//QRyGQHcy6dittemIBYIGlGWbBKEcBrx76BchSQqqSC4wMuMUpQ6+lEYOpUtE60wUeAiitJJDkZJsmzxFEKGbJ0HVYbHYRLq+pgi07BUEkheucQ36uqUHxWxtWvNmBMzIZuwuaA0YMMHuctt+LOI2Mw3Jo0511hhMh/kU6JC/OBIH1iIYY9iUidDR8938aRUMbilUWwTYoh5E+aBTox34rjtRrWL2xDZtwN0e6OkxPzTKTQzBKpAL0Gew9yze/6yZbHgwiII3yeBCnnl1EjwanFzYxTWTsZrSFUfOVDdiuRg3KBx12ChqE2/PzDcTj+YQh2zor7/urDI+fGYqAt8U+wJ+pBzM6xWMKcn+NkiFGOlenVRCEW+9CxJGyTE5iSLuMo8eKe92KwPOXD4s4YfBYbmycHd/51op3mGkhnX7bT2yKmCvNC4V8cMbY8FSFmibL4yb0MnVjRUZRiItq4wgA7QEraRGJ6GAeHGvHS0SlYf3szrr03C80HImaOs3xN2eTiaGkaL1CINSGmNjdnhQEySG2QPaGIlDnA/sBDx5TyPH5/w5c27KrnkLR3APptaXCeSWDapmZ8yX4jC5VD0GVzCRYzGuI1k4emjzw0WiDmRHmIB+xj0KJcZx1D18nG1YsOdr0o2uJ+/PvaEjx73XmcqA/RbgkOj0gZySz2KCWVv6OQYaoRuIBCYlkC0VQy1bhFo9/Y/jnkvxxg09vqwd5aO6oK6QDmkdKroXaGG/VpjbAzClHT0ISpaoRptmSKJ9IFbJRGvafTwgwDfQM2aq7Hwj/FDBUbJ8aVOdCwPYTGthApggPj5zux+91BOEU3oreFWCUsDQpP0/MONjFxU7Eokf8p/N5iY1NLZbpoEopqXVj3uZNDvYxXfgH88DYV117lghZiLvUm0VGQj5ARwKKnZ2Lidfno1wPkQpwDpJhp2T8+xGirSJDwrx6m1jNInJ5iw8tPVCKFMshrS9vRTaqweEcuwj1itBxhoS6miyheIXJZhfe5KGISShw6WuwSLvncgTOH6ZhsGSXTOD/k0WHEcC+P6ajI5BjHmdSvY/ZcD258exlpdhKli/Ix9cdlOPzCOZzZ3wl/MMQ6tV2Igrg+0c/CPnA9NhkXFyFWJLKu1xjEPMcVGIqGGBkDk9noPjzUi6t/lo0Vz+TAoPGBoGYqExdFLhMSzTSicEWYczok3H6ESsRBzq9chKpoSC9QwKEKzowkjyPVOCBhiJmZVqnj0axuVNoVtPclUZ5vQ3GWii6fhpCdvYRjXx+7f6A5QqeyWTIY6cVOeCvckObjXUOsShhup/eyyzyoeqIQyivZOLmzWWQisq9zYMWfc0EGjH5OUjamhlDhkqPw3uzAdEBJmoEnTgNP1nNWjDHVqEaIhidIvI1YnpnD+aGfw0gfCSMHoQerfPiRN4oGvwrO/KguspOecHJLGOawYyqYrKVwVDNLN4MKhlPIG5ooa6bQt56dYobdzVV7Sh3IyLIhjfJf58c6jhVacPO2fEwe74CvLQYhRrpcKuJMlRhvoCojApcwfgKnr61UImZudNFTMmxMlxhh58qxUTwzmwoGi/n1Yxqe2MiDOfteM2MYz+SH2ZnJj+J25PN4j1OB0Iy1xIhjBC1TSKHzHVGcDFqwpd2NfZ2Umig4iGwZY+eIdKyT1FEgCQ9WWRE6T0gytweZv1lkfUY3wxhLmjThog5qNi1CYYzHlXHK72ESLieqHKdEaEljGtJbBnP84aoYnq1JwEh1E7Fgpk/b/xr4jCPjJG8YXUkVOakKHSYLh5p1JKIqHCPR++XuBN46o+CRfS4MM5UmjU9i0hgdXgpoYL/oZPpJx1oCxr9sRppmGijkc/F+ND0QofNaDAgR+646Ge+fcZAuizpiuognRkgllvU9GEDvgA1jGdWk8Cb5vtQq4fsfb8Uvr/42LGrMLPbR1xb3kpMaXKQNc9/zoJmOu3dxHDO9GvqpXCsJlZRHxnnlPB7ISGHHp5pwMWQXPSzQRKjLLtLf6IXXF3Oeh6OMUvjzbTJW/8WFhJ+5nMlruBj7OMycB8+z+UP4U4UfNx/Ox6dPtcOVA+x4fytecX6K6t9/GwX0YFsS/6Ryq9TWA5TUJ77sgftaA+unh6EFmZIh6kPsUHusn+GAvh9+TojP9boJENm6iUFi5QkiiMBzq3IBz/WRwhbfiw2N8S4dZMEYs9mFlW+loCpfxrXTOWnRcGmY1SZykp6WeUyYw/qfz32G8DoZ3/vNtzD3nkKo5D3xD/14UrsaLbGQyWJHlO2RexucdQ3ee8Z/pcK+Avjt5Ch6KaA7dRXjMo7gV/pqbA9vhT8axFrPXVTqXVA8Fhf6SVOFsaL9C4cYo3ZbNH5QSHoYYAQu+0zBoWa7qVDPuwm4ezoEY0fXm0B9gJ2XvCdJplE0No71i8JoMi6B9CMdO+dvw+ZX92HH9l14+Z3bYItnQFFls5uHo4kRR9lVjKeE7n2eMuL1wC/zoxhmPY7h+CkA44qu9XBxtr4urRLjLBl4L3QM2bZUyL3BYXqZ0MQLCCogeIyZ8zQ+lXlelpYkcaNq/GoKDp9yIsPLIbuGfYD6D/VXxHqBoy30JKVDGwXe564YwI5Jg7D20aD7ktjbcI4ClQPfeeYW7Gx/E2cpMyoWMlECBhVUjpcy8rLTkGON4e49doTzJCybGoeXmyJvDjnx6Os2vLKqj2GPI6xH0WMJ4LNEE/bHz6KHEpCcGFBGOhhG5llhuMa8r6TWudknI+PtFGxg+1c8pAY5TC86SKXuE2K3fJieX7WOZ2YpuGuWD4em92MyC7U1acfYigQO/9yJaYW5rCUdZ042o7ahH++8/gK6mo/CGiIz/UxFSUkaXn3rA+6TqXj7ICncVUBrixUPrpZQPO0MfnHfOUzs2wmpIw/VkQLsOd6GhrZeKM0uxBq5hWVIf0+XBEM5gchykJJ3wRYVPQHCouCGYsuMXTcRIUUmnW5vpcOJCMLrc+aE8dxYdghi8/mEHTkZXDR3Yg48qeLIuz5om3PxH8FUaBMNKhIJfLBlC8mTFfeWNuNSDuk/3fI7Ktyp+MvVN7ILMqKUhrxzmvBMj4H98nnsJa1umd6CiS2pONLUSXt0HPjuo7j5i//BALfBFJMCM8/zGLKYVceCOjv2HaBs6L3AOTJYkUNWEx416jq+RjJQyihjJibwwmV+yioGWmi4h1he6BQbCEQMbkXt23kEamEAp9qA5p1lyLmSewbkLiX5OZDZSGvWbcD/7KpD585z+PPuP2LpLjpyqYSfzehEb28r3pe/ouTfhstnVeHNX6zEmAdXIjs7w9xI+4/tW6B1C4mf7lWIueVMlzVNlEdeS8MXX9oxs5jNjNK5JISyAGUWAZGi0XBbSM2y4Kn5/dhbMQQHi3aQQ09JNpU9t2Jyd4EsQvMsySvGtHUBVJRmITZ0oX8YukkTztS24j/XrYOzK4ocdxzlkypxltPsQvaNjo79qMVmRPQgJ7IMHKfUs9S7DvPHjIdxkvXJ/9NHu9C+w4/gnjjkfWKHhHj+0ud2uFhA/02au/xOjn4kWGK7U3RoKT7CZ26dHkDdrE7MYkdtpkZflM1OShlRNCnNbEaGOQvEWTtX/UHBl1QX7HcdRtEy7o/R8jSStf1H+/GjSw/hBstP0HRiK55b/xKG+ukcyjNFGw/jS8s+uIx0XCSYGvlEZIhOWR3EtPQsyMcJAueJmA0k/sf4+u63U5qt+VJxxVReJCuBADfUCnji2SG6ilOULnB5SgTrCqgYk0G2ay54WRMuu2ymSyJpjOLnI3tmoiGl6k7cevY8yjOrcYm6ET/7YxlSCmUTScIM5xVZK/DSsXvgirhxXecPIMXeQKO+By49/Rswbi5E9BaPBf0vhzDzllxMKivEno5TcFvVesVRkNzkyVV+IkhUJrvjs7U6Al9Q80lIyGJRra3oxTQCVSOrKyvFwp12i8lbBHbHEyO7loK7CHlFYyMa2Ue2jOhCtgJ8susM6mI3mtKiws9bOwewcg+lwstySBDjGFIG8Xn7u6QEb1B55qRHbDb0b1Ibk534NRa3E9GjGq65uwrbXjrGvTjUc5vVmOcwtN0qJcIhzhUxcjvVbeDn1PhvdUVNpuimipqVajEvlfwH3iI6t6AbKhudoNnJURERFEQcv7/+FL63aDKvP4wz56MoLM9GhPsGpnzL1v1vPXciev9rOLYsgJmrHkdQLoa5rzvqId65OE/k3ZiK3tNUClNYoBZlnBy9U9ozzAzpOUJ2SUK+rGYYX13Ri/lEpTbJhdx0i0mFdWO08SM7lYJWmx3cqTKb5Au/n8DX+SuaYkGREzvuDmKy9Q8oz3sHNksKIThi2id2Z8Ult036K6bU7Yc7twgNt9xBMa2ZbFb+xgLE2O1rj0LmODrru2Ox8EfV67a983DzyJ3WGp4pxbHd60p93B6W0UMmmUua66aOw5T+ht4jSF3swigpNvsukrCLNZC4MNgLddrQRwZ9uTEFDccHMGacE1nc8FDGJhALj6gXIv3yily4e9w2TGhOwTu/uQztjnbMeGQtWSmdKmVAzHsmxxELtmlYEJ5Z/xw2zIf0wdDIpLz98ei2VXe/2685HKpLnV1Itdlikb5OB7FKcSONixBeFosQ84HxL351oFgsI9HgABClgYl+FTEjgcwyG7jjijANj3Hj0MUFWq0jURO/kehv82M3N1iu3TmAzNY01K5dgfayfDhaW5Hq76RLIxQamK5aYl16S8qd9Uuf/fuPPUY/Gs77ijlGriHAVPPrqRc/F+giclrkemykii8M8MY3yk2jxDgi8BI1hknUQkIHtZh7ZyBNFlGh0g7VzlqjRxUzikRAwuVNZX/BGEqLU6XJXHwz9IKpqFuYi/PjYs26v2dT+rFzH7RvW7BntL3/B41Ezp+M4ooqAAAAAElFTkSuQmCC" } diff --git a/agent/templates/chunk_summary.json b/agent/templates/chunk_summary.json index 8001576520..a9605fd8e0 100644 --- a/agent/templates/chunk_summary.json +++ b/agent/templates/chunk_summary.json @@ -12,64 +12,311 @@ }, "canvas_type": "Ingestion Pipeline", "canvas_category": "dataflow_canvas", - "dsl": { - "components": { - "Extractor:SharpTaxisSay": { - "downstream": [ - "Tokenizer:ShaggyShrimpsLose" - ], - "obj": { - "component_name": "Extractor", - "params": { - "field_name": "summary", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } + "dsl": { + "components": { + "Extractor:SharpTaxisSay": { + "downstream": [ + "Tokenizer:ShaggyShrimpsLose" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "summary", + "frequencyPenaltyEnabled": true, + "frequency_penalty": 0.7, + "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", + "maxTokensEnabled": false, + "max_tokens": 256, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "presencePenaltyEnabled": true, + "presence_penalty": 0.4, + "prompts": [ + { + "content": "Text to Summarize:\n{TokenChunker:ModernPetsKneel@chunks}", + "role": "user" + } + ], + "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.", + "temperature": 0.1, + "temperatureEnabled": true, + "tenant_llm_id": 63, + "topPEnabled": true, + "top_p": 0.3 + } + }, + "upstream": [ + "TokenChunker:ModernPetsKneel" + ] + }, + "File": { + "downstream": [ + "Parser:HipSignsRhyme" + ], + "obj": { + "component_name": "File", + "params": {} + }, + "upstream": [] + }, + "Parser:HipSignsRhyme": { + "downstream": [ + "TokenChunker:ModernPetsKneel" + ], + "obj": { + "component_name": "Parser", + "params": { + "outputs": { + "html": { + "type": "string", + "value": "" }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": [ - { - "content": "Text to Summarize:\n{TokenChunker:ModernPetsKneel@chunks}", - "role": "user" - } + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "doc": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "doc" + ] + }, + "docx": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "docx" ], - "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": 63, - "topPEnabled": true, - "top_p": 0.3 + "vlm": {} + }, + "email": { + "fields": [ + "from", + "to", + "cc", + "bcc", + "date", + "subject", + "body", + "attachments" + ], + "output_format": "text", + "preprocess": "main_content", + "suffix": [ + "eml", + "msg" + ] + }, + "html": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "htm", + "html" + ] + }, + "image": { + "output_format": "text", + "parse_method": "ocr", + "preprocess": "main_content", + "suffix": [ + "jpg", + "jpeg", + "png", + "gif" + ], + "system_prompt": "" + }, + "markdown": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "md", + "markdown", + "mdx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": "main_content", + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "slides": { + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": "main_content", + "suffix": [ + "pptx", + "ppt" + ] + }, + "spreadsheet": { + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": "main_content", + "suffix": [ + "xls", + "xlsx", + "csv" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "txt", + "py", + "js", + "java", + "c", + "cpp", + "h", + "php", + "go", + "ts", + "sh", + "cs", + "kt", + "sql" + ] } - }, - "upstream": [ - "TokenChunker:ModernPetsKneel" - ] + } }, - "File": { - "downstream": [ - "Parser:HipSignsRhyme" - ], - "obj": { - "component_name": "File", - "params": {} - }, - "upstream": [] + "upstream": [ + "File" + ] + }, + "TokenChunker:ModernPetsKneel": { + "downstream": [ + "Extractor:SharpTaxisSay" + ], + "obj": { + "component_name": "TokenChunker", + "params": { + "children_delimiters": [], + "chunk_token_size": 512, + "delimiter_mode": "token_size", + "delimiters": [], + "image_context_size": 0, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "overlapped_percent": 0, + "table_context_size": 0 + } }, - "Parser:HipSignsRhyme": { - "downstream": [ - "TokenChunker:ModernPetsKneel" - ], - "obj": { - "component_name": "Parser", - "params": { + "upstream": [ + "Parser:HipSignsRhyme" + ] + }, + "Tokenizer:ShaggyShrimpsLose": { + "downstream": [], + "obj": { + "component_name": "Tokenizer", + "params": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + } + }, + "upstream": [ + "Extractor:SharpTaxisSay" + ] + } + }, + "globals": { + "sys.history": [] + }, + "graph": { + "edges": [ + { + "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", + "source": "File", + "sourceHandle": "start", + "target": "Parser:HipSignsRhyme", + "targetHandle": "end" + }, + { + "id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:ModernPetsKneelend", + "source": "Parser:HipSignsRhyme", + "sourceHandle": "start", + "target": "TokenChunker:ModernPetsKneel", + "targetHandle": "end" + }, + { + "id": "xy-edge__TokenChunker:ModernPetsKneelstart-Extractor:SharpTaxisSayend", + "source": "TokenChunker:ModernPetsKneel", + "sourceHandle": "start", + "target": "Extractor:SharpTaxisSay", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__Extractor:SharpTaxisSaystart-Tokenizer:ShaggyShrimpsLoseend", + "markerEnd": "logo", + "source": "Extractor:SharpTaxisSay", + "sourceHandle": "start", + "target": "Tokenizer:ShaggyShrimpsLose", + "targetHandle": "end", + "type": "buttonEdge", + "zIndex": 1001 + } + ], + "nodes": [ + { + "data": { + "label": "File", + "name": "File" + }, + "id": "File", + "measured": { + "height": 50, + "width": 200 + }, + "position": { + "x": 50, + "y": 200 + }, + "sourcePosition": "left", + "targetPosition": "right", + "type": "beginNode" + }, + { + "data": { + "form": { "outputs": { "html": { "type": "string", @@ -88,24 +335,29 @@ "value": "" } }, - "setups": { - "doc": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "doc" - ] - }, - "docx": { + "setups": [ + { + "fileFormat": "pdf", "flatten_media_to_text": false, "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "docx" - ], - "vlm": {} + "parse_method": "DeepDOC", + "preprocess": "main_content" }, - "email": { + { + "fileFormat": "spreadsheet", + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": "main_content" + }, + { + "fileFormat": "image", + "output_format": "text", + "parse_method": "ocr", + "preprocess": "main_content", + "system_prompt": "" + }, + { "fields": [ "from", "to", @@ -116,133 +368,101 @@ "body", "attachments" ], + "fileFormat": "email", "output_format": "text", - "preprocess": "main_content", - "suffix": [ - "eml", - "msg" - ] + "preprocess": "main_content" }, - "html": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "htm", - "html" - ] - }, - "image": { - "output_format": "text", - "parse_method": "ocr", - "preprocess": "main_content", - "suffix": [ - "jpg", - "jpeg", - "png", - "gif" - ], - "system_prompt": "" - }, - "markdown": { + { + "fileFormat": "markdown", "flatten_media_to_text": false, "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "md", - "markdown", - "mdx" - ], - "vlm": {} + "preprocess": "main_content" }, - "pdf": { + { + "fileFormat": "text&code", + "output_format": "json", + "preprocess": "main_content" + }, + { + "fileFormat": "html", + "output_format": "json", + "preprocess": "main_content" + }, + { + "fileFormat": "doc", + "output_format": "json", + "preprocess": "main_content" + }, + { + "fileFormat": "docx", "flatten_media_to_text": false, "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pdf" - ], - "vlm": {} + "preprocess": "main_content" }, - "slides": { + { + "fileFormat": "slides", "output_format": "json", "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pptx", - "ppt" - ] - }, - "spreadsheet": { - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} - }, - "text&code": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" - ] + "preprocess": "main_content" } - } - } + ] + }, + "label": "Parser", + "name": "Parser_0" }, - "upstream": [ - "File" - ] + "dragging": false, + "id": "Parser:HipSignsRhyme", + "measured": { + "height": 57, + "width": 200 + }, + "position": { + "x": 316.99524094206413, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "parserNode" }, - "TokenChunker:ModernPetsKneel": { - "downstream": [ - "Extractor:SharpTaxisSay" - ], - "obj": { - "component_name": "TokenChunker", - "params": { - "children_delimiters": [], + { + "data": { + "form": { "chunk_token_size": 512, "delimiter_mode": "token_size", - "delimiters": [], - "image_context_size": 0, + "delimiters": [ + { + "value": "\n" + } + ], + "image_table_context_window": 0, "outputs": { "chunks": { "type": "Array", "value": [] } }, - "overlapped_percent": 0, - "table_context_size": 0 - } + "overlapped_percent": 0 + }, + "label": "TokenChunker", + "name": "Token Chunker_0" }, - "upstream": [ - "Parser:HipSignsRhyme" - ] + "id": "TokenChunker:ModernPetsKneel", + "measured": { + "height": 74, + "width": 200 + }, + "position": { + "x": 616.9952409420641, + "y": 195.39629819663406 + }, + "sourcePosition": "right", + "targetPosition": "left", + "type": "chunkerNode" }, - "Tokenizer:ShaggyShrimpsLose": { - "downstream": [], - "obj": { - "component_name": "Tokenizer", - "params": { + { + "data": { + "form": { "fields": "text", "filename_embd_weight": 0.1, "outputs": {}, @@ -250,297 +470,75 @@ "embedding", "full_text" ] - } + }, + "label": "Tokenizer", + "name": "Indexer_0" }, - "upstream": [ - "Extractor:SharpTaxisSay" - ] + "dragging": false, + "id": "Tokenizer:ShaggyShrimpsLose", + "measured": { + "height": 114, + "width": 200 + }, + "position": { + "x": 1188.9891545215792, + "y": 159.26426539640332 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "tokenizerNode" + }, + { + "data": { + "form": { + "field_name": "summary", + "frequencyPenaltyEnabled": true, + "frequency_penalty": 0.7, + "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", + "maxTokensEnabled": false, + "max_tokens": 256, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "presencePenaltyEnabled": true, + "presence_penalty": 0.4, + "prompts": "Text to Summarize:\n{TokenChunker:ModernPetsKneel@chunks}", + "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.", + "temperature": 0.1, + "temperatureEnabled": true, + "tenant_llm_id": 63, + "topPEnabled": true, + "top_p": 0.3 + }, + "label": "Extractor", + "name": "Summarization" + }, + "dragging": false, + "id": "Extractor:SharpTaxisSay", + "measured": { + "height": 90, + "width": 200 + }, + "position": { + "x": 878.855872986265, + "y": 177.33028179651868 + }, + "selected": true, + "sourcePosition": "right", + "targetPosition": "left", + "type": "contextNode" } - }, - "globals": { - "sys.history": [] - }, - "graph": { - "edges": [ - { - "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", - "source": "File", - "sourceHandle": "start", - "target": "Parser:HipSignsRhyme", - "targetHandle": "end" - }, - { - "id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:ModernPetsKneelend", - "source": "Parser:HipSignsRhyme", - "sourceHandle": "start", - "target": "TokenChunker:ModernPetsKneel", - "targetHandle": "end" - }, - { - "id": "xy-edge__TokenChunker:ModernPetsKneelstart-Extractor:SharpTaxisSayend", - "source": "TokenChunker:ModernPetsKneel", - "sourceHandle": "start", - "target": "Extractor:SharpTaxisSay", - "targetHandle": "end" - }, - { - "data": { - "isHovered": false - }, - "id": "xy-edge__Extractor:SharpTaxisSaystart-Tokenizer:ShaggyShrimpsLoseend", - "markerEnd": "logo", - "source": "Extractor:SharpTaxisSay", - "sourceHandle": "start", - "target": "Tokenizer:ShaggyShrimpsLose", - "targetHandle": "end", - "type": "buttonEdge", - "zIndex": 1001 - } - ], - "nodes": [ - { - "data": { - "label": "File", - "name": "File" - }, - "id": "File", - "measured": { - "height": 50, - "width": 200 - }, - "position": { - "x": 50, - "y": 200 - }, - "sourcePosition": "left", - "targetPosition": "right", - "type": "beginNode" - }, - { - "data": { - "form": { - "outputs": { - "html": { - "type": "string", - "value": "" - }, - "json": { - "type": "Array", - "value": [] - }, - "markdown": { - "type": "string", - "value": "" - }, - "text": { - "type": "string", - "value": "" - } - }, - "setups": [ - { - "fileFormat": "pdf", - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content" - }, - { - "fileFormat": "spreadsheet", - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": "main_content" - }, - { - "fileFormat": "image", - "output_format": "text", - "parse_method": "ocr", - "preprocess": "main_content", - "system_prompt": "" - }, - { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "fileFormat": "email", - "output_format": "text", - "preprocess": "main_content" - }, - { - "fileFormat": "markdown", - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "text&code", - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "html", - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "doc", - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "docx", - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "slides", - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content" - } - ] - }, - "label": "Parser", - "name": "Parser_0" - }, - "dragging": false, - "id": "Parser:HipSignsRhyme", - "measured": { - "height": 57, - "width": 200 - }, - "position": { - "x": 316.99524094206413, - "y": 195.39629819663406 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "parserNode" - }, - { - "data": { - "form": { - "chunk_token_size": 512, - "delimiter_mode": "token_size", - "delimiters": [ - { - "value": "\n" - } - ], - "image_table_context_window": 0, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "overlapped_percent": 0 - }, - "label": "TokenChunker", - "name": "Token Chunker_0" - }, - "id": "TokenChunker:ModernPetsKneel", - "measured": { - "height": 74, - "width": 200 - }, - "position": { - "x": 616.9952409420641, - "y": 195.39629819663406 - }, - "sourcePosition": "right", - "targetPosition": "left", - "type": "chunkerNode" - }, - { - "data": { - "form": { - "fields": "text", - "filename_embd_weight": 0.1, - "outputs": {}, - "search_method": [ - "embedding", - "full_text" - ] - }, - "label": "Tokenizer", - "name": "Indexer_0" - }, - "dragging": false, - "id": "Tokenizer:ShaggyShrimpsLose", - "measured": { - "height": 114, - "width": 200 - }, - "position": { - "x": 1188.9891545215792, - "y": 159.26426539640332 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "tokenizerNode" - }, - { - "data": { - "form": { - "field_name": "summary", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": "Text to Summarize:\n{TokenChunker:ModernPetsKneel@chunks}", - "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": 63, - "topPEnabled": true, - "top_p": 0.3 - }, - "label": "Extractor", - "name": "Summarization" - }, - "dragging": false, - "id": "Extractor:SharpTaxisSay", - "measured": { - "height": 90, - "width": 200 - }, - "position": { - "x": 878.855872986265, - "y": 177.33028179651868 - }, - "selected": true, - "sourcePosition": "right", - "targetPosition": "left", - "type": "contextNode" - } - ] - }, - "history": [], - "messages": [], - "path": [], - "retrieval": [], - "variables": [] + ] }, + "history": [], + "messages": [], + "path": [], + "retrieval": [], + "variables": [] + }, "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAACXBIWXMAABYlAAAWJQFJUiTwAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAA7ESURBVHgBvVpLrF1lFV7/v/c+j3t729tSClZCigXFF/JQE21CaKITHcAEica5UwcmDsW5A4mJTIwzB5iQiMTIwAhGQzQSrKJIomADWkpLuX3cx3ns/f9+31rrP4/b2xqNsJtzzzn7/I/1r/Wtbz12g+y6Nq7kB1KQB0OSh3KQY/IeXV0QyXn5Xox4JZEQ5BR+OhUr+fbBYTi9OCaUDxs5r8umfAvjvy7v8pX9lfRvuP5g/Fznpe/fjVMc5GC4KGU2hU9b8izWu1vepUsFVi1nCD4XOha5YPLFw+S88Bkfe9nWcKFPhVZO8hA2H5p/t4RvQ5YJ3qfYfdpl6Vz9lQsfY1aoVPhWQdJQXrgfgonMt2mYmwGf7k4NZOZvGzv5WGrlH/J/uqjlhB06mWOaugSOZ9qm0LxbJ8LILvqAyohBwXCv62Q4QU42M0L0uGAlDDtZp85O8r9cXIp7Usu0c2vAVqEqOuCS0PauQuOUHDNeBEV2APl8npFTGgg/4RedE0wZZU2Rh+r/FjozocWwnJLdzS4kNVfPhM66ZcX9k1mHcGqp2jAXRaG0hy9z6YkeIsjU77X8PhfmwXDhym7yuvbVFngk8yjOrKqwDA+auUiTjGlaP2hKjulg88QhwbkhOjtxUVi0wEatgVfNw7uV6B88FMfX1xO44Jnazp1tqJMqM3mRMwY6XFDOznhvk2mc83QRh0oVDehVsIl1DDNqad25GzEnnkB0Qo2bFAYjE9EiAV86P3x9TaFJd50oBExos9lM08BF5OBg9yn0GBNVyXl2MmcZ+175IdxlwEpirOTxgHPH2SxF8fqYMB4lxeYA0jcucKuWCdIrFpgJzYmFOajRmSMGv5cxwTWSDZedUmPRpMPJfUEVTh/wzx1kmXTcJ8FJo2qRXpI786cpBnBfKivAnBvjIOcuiawNsxwAfvfvMxKgEinrlMp4Ez7Qdtmpzk0splmFqdj37EFzUhwyzSNqED9oMCwbv5vGqWGu35JFqCAeBHDsctZgRX+qYKVhL8qwMcEmrej4BlZeGwR55WySmw9EaXF/fdXSCyqP64Uzl7N6ZMGllM2doAn9tgjtjut0rRYhPConlMoRxbHTZBrvDFMQymiQkyk89xjA/isQsAWwt6dZrVpR+zkBMkEPOMXmB/cF+dvZVo6s1Wq1tZ6xhir+LVhAFU8BHD6dzCGVHB5pJrQf0IWecb36gEVbHjLhz1QZK+piWcwCBODKIOohtzCAAtKzAjRGeDYYvn8l43BRxnCsGPieZf8wyN/fyrK+EmTYFxlWJqPSaHQtU/DWBciO9Rk8qmWoRJMZZnXtQmsJuMZffI8KxZD8EHj1ENkq3NuBSSdtmlmRizbQCK3Zg1aGyNzWIewTL07lCx9vIHSS22+MOjbjkJvbQf3q0JpRaz0N5ojZo2NwKFWqbjhtMFyQ3wMpjfQILE5oHTofnZH01kXzF2iMTjrGepzbRIvQV8aGfVULxkQIUymMIiABrQNKPz7Vyi0Hgvz0pSSPP9LI9389kc1JlHtviXJxBMvhoJex9irGXt6GVWCNOHWHyTErLGoKDS1gXekVGqSFIPgOKG0TPDfiAfDaxOm326xQ6/C+Bc+6MqElgzrwCL9f3AFUxjZeU2PgftAkObwSsb7IX8508qMXkzz6zES+fF+Uc5tBHr6vkm8+PZGvneiZM6s8QXbAIO87FOXtzaQ+SxoO/7oECMU5vosvkFrbzpyxhTdmwiOb8MRe7TzO74wZg8ZwrTRJ1oG26T4NhcaC+oJW1gdZXt/I8uQfsjxyT5ZfvRp1j2+cFPnOLzusGeTkHUHuOhrV/146k+SeW4OMp2J0jc37sNiFTdDrwJ24MshajpONPaaOU0bD7Za0lzRokBm2W7NI7XzPg6gfuNNTIfztMCjvZVDg8SMRB8ny8psQ/I+dfOneCrSZ5Zk/Z/nM8SB3Hgny+9eT3H+8lgNDUQ1fHInccRj+0mlKKk2TFYK09hBauQQk7Ou5EzPkt0p55OVkQQpCjTtSpGl/GxQz4UIVgw1wPqUf2EGUofBnMAAt4pSn32nl+OFavvdcJ1/5dCUHEYieBExuWs9y4rZKD0EK/eJHKnl7K8udNxnUpmQprH3DEEFsC76B9wGEpUWraIqhr049dqyCTsMbF3NOHr5png4LVDgQnW6nNUgQf8UPxq79XmW/E8cNPlMbv3illRMfrORpCPj5D1WyD3T3879iLfjO7UdQ+W1GOXulk4fvqbBmVM4/OEhyaWRr0KEHdVKokhTqaL5I4Sk0uWjcJSWKXmMyRI2oyQISMbqGTQErOKOnD/i3A23TpJt4TaeEW5bHf9NCU9A4TYv5F7aYo1Tywmut3H97lB/8tlPtnTgGwXeyvH9/LZ+7MwD3lfpQhMnWeknxf8MKIiys11fhbde+sxSVtQViGIGNyHyHVk14yaZcdWLNF8k+oKERoPLaBct+Jh5NYwkIGHMzaO4izP4aHPED6wFMZEU3zUtn/uHvRD55q6gVD8IHju4PeIE5trMqatgPEDwr1GgB8nIuuVYMnucUMoBFIOwQSqIfMq4wcvdqMpwF3VppUgwipLoLW3AQaGzQD1ra0bRqKqw8BPU8+rOJfPVTNWjQNEPqHSUmeVExedfRTj57rDIYtBauzm1mzfNvQDI2AButNpaY55IAegozcXqO0QIX/Y1WzErLIIhoySJnU9kMospCvLU98fQWwrz6TlDWWEVE7DrDPVOEd7aSvHIOGoQAq4Dah4HrTXDzPqQGhNwAsIg134N2GTYn5hs0eZ9FCBSxqgLNk0diewxNEwfRqznmpoR0v7YYwFgwmVoKvor1eI/s2JZoTmpidOWiByD0QfAr04pLgMpl4h4njZhNS3wUjPH86SQfOwptwoOfeDHIC/9kbeAlJg46gWM1cDQ0oaSPoNWIpQn7Gk31VPxRSyrke+UpylSjfQU/GEIh+wCVqjZmZPQnjRrsklVtXguoBUYTOhPZB45TWyHx/GnRJGp9aFYZtRY5iatDQ4T2cVJTHx6SsbI6OrVNndRaxFgE56H7zDobsySxramEBp7O0xbs2US1UMhWsDBDoJxkoaaxd0tWli87ALxmMuUilqjxhOe3wefnLUkb9ixl6HtlRV9IweJBla0HUkdbmokZWyHcsIZCmrrS36ggK947nUu/6yGvYDwoiSHxT1ailAo7T2lK8a81BKuyMK9F+Fk5OcXCBpaFruOX1T5xbKXeIEaLgtgwQaDR2MpKmrUO3udhskc/8Hq5Bga0WM8UxKqgCG5cqRvVNq9xZ8lhciZb7RncyIrZy0zN6MmGwQorjQmV5VvhzJWUmTWOJ97PsXRDg08PePzT2VqmgBgZJCEyM73vBbeA5u8mfPQqnywxxMGowdJXUCjBEk1txQ+zA8KzbS0BIy0OqG2vu5OIt2vs8FY2m2IZGzguutK0IgteZe8A1wxmXOfyyALUDjR05lJE8gTsr4imsBq6xTJVa51YK3CtD+1WSe+TRusqanSNtWGbTDL29gq1TMg2XvSr0F4IKaSy5UC+lflCtFqZjFV6p15SWnlIjTC7JMan2WBCLW/AGm9dtiKc3EvaLO3BFZicGSb5nRilX/Qb1rPWfbDE0DoWTAnUqRurLcjtTL1zMkuVJpl17uxgjacq4jnZ7i5WeGPD0ungNe32xIJOD0dmXLCSLyg1nt+MyiRUEdMIpgIkT0KiD4ekcApBKZHUDqB5TmN1MAOQp10e5oMmaFrGeuOrroOTQNY0IKUou6/sVgpvMJVIppHoPLU1TnoAb/nA9Eyu6EmtLsy4q/UtoNFnzq+sJNpHmpQEkGZvsrHMQt+wYDvxA6yl1SBhUxmZKJWWdku+Sm47fLBUhaShNMpNK2cF1q1TBJkdUKvi14v3WZhXdgE9At8hF/orbROvuGp7N3osjQHzC7p7l60c5Td17Eo08nMf7cZdJTTGBzMt02iSju6t2aqnrKygqsoaVnSuDhuxqGFgI9zryoOJrh8N297vYUrATLLBAGK2OGSbC6azYpQsxnqD1Nproq9nvSIVPO+CSJo/xWnckSmAte/5YzQLaCLXWYJFrq1cCaOJZaRNLF0E6zp0zF10QUuqepW3Bl1wyd7ZyFb0U4NkLiqjV3vwCdZI2I2S4guWG9EXSztx7/G1ndYgwbqTuFYqIz8jNYidWad04gLpr2f4Y2HdLbXZDc9aP8/KSz9kP+jx1RKq7WWotLlQpzk6g6MGrmQN3flTgeVrlo3ytJ22+mTeTpH584Au2XMsNpo0Z59p23GarPXVOpOQMpva1o3XwrZr2xBmlKkw0edoMiturnUZU5XFskdJb8gWK+SFwdl0aB3l4Bsl8a5bVGyTAntaY2T1YuX0XXZvHV6aHmDTQaFfZsX5aussabz0YLOlF8vtdYb8yHTYqKfNngGGmbx6aeuwk9nTRgrL4EUGC154e2zaU9sxWN6kTJVZhwdL4iRcnW7u0jgX0v2jDa8h3Gnse0w3ScbZtQcjzd7zvOViEHUiFndiL/ipDuaYeUHogjCmJNnX7lU2j1rpsiy05pevsr+25sXK2tYF96Yh752KiDVPLU7M2R5vUpuWf2Q1m2nfn3kh8Rr0LM+3p4lBhV+8Zm31TtTZ2W1mJquMkgqMrhbas2lVYs8F1w65a1wfkiQpD0lOhfNX8gNoKz6b93AyfUiXwsyRzdt9s3S11mba9uc6zBybELz+Ddd9Mk/6pHC1zIkj+XMz1/bV8rVyW7xxLTxXpfBYWTcsPFBOnqN0IdtiIc8ebuzWXOnoaS2rPZ6gwrcOE5uyNxPxGrjw3Id9Yr4rhNLewuPWY3hSf1pX3NjI67mRZ6f+yLUcIl+HDWxzOxApsCpPZoLxf8rlicJe8+ZPcbxKVaFdMBU4XMeZcbJTcbrwXw34ATX1SYj72NK4kJcssiS4czSDDjVeOzW2M9qM19rcfMsFTQvCc6uC72td1HwR3pdbvt7cyceqLj2Knz6Br2qRRUvMtBesn6S4l3xd7i6CK6MkrVPcYa3bEfN/mJrlNPTyFIb9hJBf/O3fQ3B6D7564aoAAAAASUVORK5CYII=" } diff --git a/agent/templates/title_chunker.json b/agent/templates/title_chunker.json index 91e574e05c..42eb891451 100644 --- a/agent/templates/title_chunker.json +++ b/agent/templates/title_chunker.json @@ -88,123 +88,121 @@ "value": "" } }, - "setups": { - "doc": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "doc" - ] - }, - "docx": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "docx" - ], - "vlm": {} - }, - "email": { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "output_format": "text", - "preprocess": "main_content", - "suffix": [ - "eml", - "msg" - ] - }, - "html": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "htm", - "html" - ] - }, - "image": { - "output_format": "text", - "parse_method": "ocr", - "preprocess": "main_content", - "suffix": [ - "jpg", - "jpeg", - "png", - "gif" - ], - "system_prompt": "" - }, - "markdown": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "md", - "markdown", - "mdx" - ], - "vlm": {} - }, - "pdf": { - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pdf" - ], - "vlm": {} - }, - "slides": { - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pptx", - "ppt" - ] - }, - "spreadsheet": { - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} - }, - "text&code": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" - ] - } + "doc": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "doc" + ] + }, + "docx": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "docx" + ], + "vlm": {} + }, + "email": { + "fields": [ + "from", + "to", + "cc", + "bcc", + "date", + "subject", + "body", + "attachments" + ], + "output_format": "text", + "preprocess": "main_content", + "suffix": [ + "eml", + "msg" + ] + }, + "html": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "htm", + "html" + ] + }, + "image": { + "output_format": "text", + "parse_method": "ocr", + "preprocess": "main_content", + "suffix": [ + "jpg", + "jpeg", + "png", + "gif" + ], + "system_prompt": "" + }, + "markdown": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "md", + "markdown", + "mdx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": "main_content", + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "slides": { + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": "main_content", + "suffix": [ + "pptx", + "ppt" + ] + }, + "spreadsheet": { + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": "main_content", + "suffix": [ + "xls", + "xlsx", + "csv" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "txt", + "py", + "js", + "java", + "c", + "cpp", + "h", + "php", + "go", + "ts", + "sh", + "cs", + "kt", + "sql" + ] } } }, diff --git a/internal/agent/canvas/compile.go b/internal/agent/canvas/compile.go index cf53976064..88cdd9e080 100644 --- a/internal/agent/canvas/compile.go +++ b/internal/agent/canvas/compile.go @@ -78,11 +78,10 @@ type CompileOptions struct { // OverrideParams is a run-level override map keyed by cpnID. Each // component's `params` is merged only with its own entry // (an arbitrary string-keyed map); the override wins on top-level key - // collision (see node_body.go mergeSetups). Components absent from the + // collision. Components absent from the // map are left untouched. Used by the ingestion pipeline so a single - // Pipeline.Run can override the DSL-baked component setups without - // mutating the shared *Canvas (see node_body.go applyOverrideParams / - // mergeSetups). + // Pipeline.Run can override the DSL-baked component params without + // mutating the shared *Canvas (see node_body.go applyOverrideParams). OverrideParams map[string]any } @@ -130,10 +129,10 @@ func WithInterruptAfterNonTerminalCpn() CompileOption { return func(o *CompileOptions) { o.InterruptAfterNonTerminal = true } } -// WithOverrideParams attaches a run-level setups override map (keyed by -// cpnID) to the compile. Each component's `params["setups"]` is merged with +// WithOverrideParams attaches a run-level override map (keyed by +// cpnID) to the compile. Each component's params are merged with // its own entry at compile time (run-level wins on key collision, see -// node_body.go mergeSetups). Passing nil is a no-op. +// node_body.go applyOverrideParams). Passing nil is a no-op. func WithOverrideParams(m map[string]any) CompileOption { return func(o *CompileOptions) { o.OverrideParams = m } } @@ -213,8 +212,8 @@ func Compile(ctx context.Context, c *Canvas, opts ...CompileOption) (*CompiledCa } } - // Thread the run-level setups override (if any) into ctx so each - // component's `params["setups"]` is merged with its own entry inside + // Thread the run-level override (if any) into ctx so each + // component's params is merged with its own entry inside // buildNodeBody. The override is keyed by cpnID; the canvas package // never imports ingestion. if cfg.OverrideParams != nil { diff --git a/internal/agent/canvas/compile_setup_override_test.go b/internal/agent/canvas/compile_override_params_test.go similarity index 50% rename from internal/agent/canvas/compile_setup_override_test.go rename to internal/agent/canvas/compile_override_params_test.go index dfea2edef6..a0cd1f7efe 100644 --- a/internal/agent/canvas/compile_setup_override_test.go +++ b/internal/agent/canvas/compile_override_params_test.go @@ -1,18 +1,3 @@ -// -// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package canvas import ( @@ -22,20 +7,9 @@ import ( "ragflow/internal/agent/runtime" ) -// TestCompile_OverrideParams exercises the full canvas-level wiring: -// -// canvas.Compile(ctx, dsl, WithOverrideParams(override)) -// -// threads the cpnID-keyed override through ctx into -// BuildWorkflow → buildNodeBody → applyOverrideParams → mergeSetups, so -// each component's factory receives its own merged params["setups"]. Only -// the entry for a component's own cpnID applies; components absent from the -// override map keep their base params (no spurious "setups" injected). func TestCompile_OverrideParams(t *testing.T) { captured := map[string]map[string]any{} // component_name -> params received by factory factory := func(name string, params map[string]any) (runtime.Component, error) { - // Deep-shallow copy so later mutations don't hide what the - // factory actually received. cp := make(map[string]any, len(params)) for k, v := range params { cp[k] = v @@ -50,14 +24,12 @@ func TestCompile_OverrideParams(t *testing.T) { Obj: CanvasComponentObj{ ComponentName: "Parser", Params: map[string]any{ - "setups": map[string]any{ - "pdf": map[string]any{ - "output_format": "one", - "parse_method": "naive", - }, - "doc": map[string]any{ - "output_format": "one", - }, + "pdf": map[string]any{ + "output_format": "one", + "parse_method": "naive", + }, + "doc": map[string]any{ + "output_format": "one", }, }, }, @@ -76,10 +48,6 @@ func TestCompile_OverrideParams(t *testing.T) { Path: []string{"parser_0", "sink_0"}, } - // Override is keyed by cpnID. Only "parser_0" is present, so its - // "pdf" entry is fully replaced (parse_method dropped) and "docx" is - // injected; "doc" survives from the base. "sink_0" is absent and must - // keep its base params untouched. override := map[string]any{ "parser_0": map[string]any{ "pdf": map[string]any{ @@ -96,10 +64,9 @@ func TestCompile_OverrideParams(t *testing.T) { t.Fatalf("Compile: %v", err) } - // parser_0: override merged into params["setups"] before the factory. - got, ok := captured["Parser"]["setups"].(map[string]any) - if !ok { - t.Fatalf("Parser factory did not receive a setups map: %#v", captured["Parser"]) + got := captured["Parser"] + if got == nil { + t.Fatalf("Parser factory not called") } // doc preserved from base. if v, _ := got["doc"].(map[string]any); v == nil || v["output_format"] != "one" { @@ -121,8 +88,8 @@ func TestCompile_OverrideParams(t *testing.T) { t.Errorf("pdf.parse_method should be dropped by shallow merge: %#v", pdf) } - // sink_0: absent from the override → no setups key injected. - if _, ok := captured["Sink"]["setups"]; ok { - t.Errorf("Sink should not receive a setups key: %#v", captured["Sink"]) + // sink_0: absent from the override → no file-format keys injected. + if _, ok := captured["Sink"]["pdf"]; ok { + t.Errorf("Sink should not have pdf injected: %#v", captured["Sink"]) } } diff --git a/internal/agent/canvas/node_body.go b/internal/agent/canvas/node_body.go index 7d74711274..c8859811ee 100644 --- a/internal/agent/canvas/node_body.go +++ b/internal/agent/canvas/node_body.go @@ -74,8 +74,8 @@ type nodeBodyFn = func(ctx context.Context, in map[string]any) (map[string]any, // Outputs bucket. UserFillUpNodeBody tags its output itself so the // interrupt-driven branch still attributes the resume payload to the // right cpn. -// ctxKeyOverrideParams carries the run-level setups override map into -// BuildWorkflow so a component's `params["setups"]` can be merged with it +// ctxKeyOverrideParams carries the run-level override map into +// BuildWorkflow so a component's params can be merged with it // at compile time. The map is keyed by cpnID; each component only sees the // entry for its own id (an arbitrary string-keyed map). It mirrors the ctx // plumbing used for the per-run component factory @@ -84,7 +84,7 @@ type nodeBodyFn = func(ctx context.Context, in map[string]any) (map[string]any, // package ever importing the ingestion layer. const ctxKeyOverrideParams ctxKey = "canvas_override_params" -// withOverrideParams attaches a run-level setups override map to ctx. It is +// withOverrideParams attaches a run-level override map to ctx. It is // a no-op when m is nil so callers can pass a possibly-nil run parameter // straight through. func withOverrideParams(ctx context.Context, m map[string]any) context.Context { @@ -100,41 +100,25 @@ func overrideParamsFromContext(ctx context.Context) map[string]any { } // applyOverrideParams returns a clone of params with the per-component -// setups override (already resolved for this cpnID by the caller) merged -// into params["setups"]. The override wins on top-level key collisions. The -// original params map is never mutated — the merge result is a fresh map — +// override (already resolved for this cpnID by the caller) merged into +// params. The override wins on top-level key collisions. The original +// params map is never mutated — the merge result is a fresh map — // because the params come from the shared *Canvas and a per-run override // must not leak into the next Run on the same Pipeline. func applyOverrideParams(params, cpnOverride map[string]any) map[string]any { if len(cpnOverride) == 0 { return params } - out := make(map[string]any, len(params)+1) + out := make(map[string]any, len(params)+len(cpnOverride)) for k, v := range params { out[k] = v } - base, _ := out["setups"].(map[string]any) - out["setups"] = mergeSetups(base, cpnOverride) + for k, v := range cpnOverride { + out[k] = v + } return out } -// mergeSetups merges a component-level setups map (base) with a run-level -// override map. The maps are arbitrary string-keyed maps; when the same -// top-level key exists in both, the override value wins (a full replacement -// of that entry). The merge is shallow: only the top-level key-value pairs -// are considered. (The Parser component happens to use file-type keys such -// as "pdf"/"docx" as one example, but that is not required by this merge.) -func mergeSetups(base, override map[string]any) map[string]any { - merged := make(map[string]any, len(base)+len(override)) - for k, v := range base { - merged[k] = v - } - for k, ov := range override { - merged[k] = ov - } - return merged -} - func buildNodeBody(ctx context.Context, cpnID, name string, params map[string]any) (nodeBodyFn, error) { if overrides := overrideParamsFromContext(ctx); len(overrides) > 0 { // overrides is keyed by cpnID; a component only sees its own diff --git a/internal/agent/canvas/node_body_setup_override_test.go b/internal/agent/canvas/node_body_override_params_test.go similarity index 57% rename from internal/agent/canvas/node_body_setup_override_test.go rename to internal/agent/canvas/node_body_override_params_test.go index 5a90b4db6e..4462437fb3 100644 --- a/internal/agent/canvas/node_body_setup_override_test.go +++ b/internal/agent/canvas/node_body_override_params_test.go @@ -1,18 +1,3 @@ -// -// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package canvas import ( @@ -22,9 +7,6 @@ import ( "ragflow/internal/agent/runtime" ) -// stubComponent records the params it was constructed with so a test can -// assert that buildNodeBody forwarded the merged setups. Its Invoke is a -// no-op echo. type stubComponent struct { params map[string]any } @@ -33,17 +15,15 @@ func (s *stubComponent) Invoke(_ context.Context, in map[string]any) (map[string return in, nil } -// TestBuildNodeBody_OverrideParams asserts that a run-level setups override +// TestBuildNodeBody_OverrideParams asserts that a run-level override // (threaded via ctx, keyed by cpnID) is merged into the component's -// `params["setups"]` before the factory is called. Only the entry for the +// params before the factory is called. Only the entry for the // component's own cpnID applies. The merge is shallow: a top-level key // present in the override fully replaces the base entry for that key // (no inner deep-merge), while base keys absent from the override survive. func TestBuildNodeBody_OverrideParams(t *testing.T) { captured := map[string]any{} factory := func(name string, params map[string]any) (runtime.Component, error) { - // Deep-shallow copy so later mutations by the builder don't - // hide what the factory actually received. cp := make(map[string]any, len(params)) for k, v := range params { cp[k] = v @@ -53,14 +33,12 @@ func TestBuildNodeBody_OverrideParams(t *testing.T) { } baseParams := map[string]any{ - "setups": map[string]any{ - "pdf": map[string]any{ - "output_format": "one", - "parse_method": "naive", - }, - "doc": map[string]any{ - "output_format": "one", - }, + "pdf": map[string]any{ + "output_format": "one", + "parse_method": "naive", + }, + "doc": map[string]any{ + "output_format": "one", }, } // Run-level override is keyed by cpnID. For "cpn-parser": the whole @@ -94,23 +72,19 @@ func TestBuildNodeBody_OverrideParams(t *testing.T) { t.Fatalf("body: %v", err) } - got, ok := captured["setups"].(map[string]any) - if !ok { - t.Fatalf("factory did not receive a setups map: %#v", captured["setups"]) - } // doc is untouched (absent from the override). - if v, _ := got["doc"].(map[string]any); v == nil || v["output_format"] != "one" { - t.Errorf("doc should be preserved from base: %#v", got["doc"]) + if v, _ := captured["doc"].(map[string]any); v == nil || v["output_format"] != "one" { + t.Errorf("doc should be preserved from base: %#v", captured["doc"]) } // docx injected by the override (was absent in base). - if v, _ := got["docx"].(map[string]any); v == nil || v["output_format"] != "one" { - t.Errorf("docx injection missing: %#v", got["docx"]) + if v, _ := captured["docx"].(map[string]any); v == nil || v["output_format"] != "one" { + t.Errorf("docx injection missing: %#v", captured["docx"]) } // pdf: the override fully replaces the base entry, so output_format is // overridden and the base-only parse_method is dropped (shallow merge). - pdf, _ := got["pdf"].(map[string]any) + pdf, _ := captured["pdf"].(map[string]any) if pdf == nil { - t.Fatalf("pdf setup missing: %#v", got) + t.Fatalf("pdf setup missing: %#v", captured) } if pdf["output_format"] != "detailed" { t.Errorf("pdf.output_format not overridden: %#v", pdf) @@ -120,9 +94,6 @@ func TestBuildNodeBody_OverrideParams(t *testing.T) { } } -// TestBuildNodeBody_OverrideParamsNilIsNoOp asserts that with no override in -// ctx the component receives exactly its base params (no spurious setups key -// injected, and the original map is untouched). func TestBuildNodeBody_OverrideParamsNilIsNoOp(t *testing.T) { captured := map[string]any{} factory := func(name string, params map[string]any) (runtime.Component, error) { @@ -143,10 +114,7 @@ func TestBuildNodeBody_OverrideParamsNilIsNoOp(t *testing.T) { if _, err := body(context.Background(), map[string]any{}); err != nil { t.Fatalf("body: %v", err) } - if _, ok := captured["setups"]; ok { - t.Errorf("setups key should not be injected when no override is present: %#v", captured) - } - if _, ok := baseParams["setups"]; ok { - t.Errorf("base params map must not be mutated: %#v", baseParams) + if captured["name"] != "x" { + t.Errorf("base params should be preserved: %#v", captured) } } diff --git a/internal/common/parser_config.go b/internal/common/parser_config.go index 50cbb24ddd..86e8ef913b 100644 --- a/internal/common/parser_config.go +++ b/internal/common/parser_config.go @@ -1,5 +1,29 @@ package common +import "strings" + +// InjectExtractorLLMID finds all Extractor component entries (keys prefixed +// with "extractor:" or "extractor_") in parserConfig and sets their llm_id +// to the given value. Returns whether any entry was updated. +func InjectExtractorLLMID(parserConfig map[string]interface{}, llmID string) bool { + if parserConfig == nil || llmID == "" { + return false + } + updated := false + for cid, raw := range parserConfig { + compMap, ok := raw.(map[string]interface{}) + if !ok { + continue + } + cidLower := strings.ToLower(cid) + if strings.HasPrefix(cidLower, "extractor:") || strings.HasPrefix(cidLower, "extractor_") { + compMap["llm_id"] = llmID + updated = true + } + } + return updated +} + // deepCopyMap duplicates a JSON-like map so later merges do not mutate shared defaults. func deepCopyMap(source map[string]interface{}) map[string]interface{} { if source == nil { @@ -68,47 +92,58 @@ func GetParserConfig(parserID string, parserConfig map[string]interface{}) map[s "auto_questions": 0, "html4excel": false, "topn_tags": 3, - "raptor": map[string]interface{}{ - "use_raptor": true, - "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.", - "max_token": 256, - "threshold": 0.1, - "max_cluster": 64, - "random_seed": 0, - }, - "graphrag": map[string]interface{}{ - "use_graphrag": true, - "entity_types": []interface{}{"organization", "person", "geo", "event", "category"}, - "method": "light", - }, - }, - "qa": { - "raptor": map[string]interface{}{"use_raptor": false}, - "graphrag": map[string]interface{}{"use_graphrag": false}, - }, - "resume": nil, - "manual": { - "raptor": map[string]interface{}{"use_raptor": false}, - "graphrag": map[string]interface{}{"use_graphrag": false}, - }, - "paper": { - "raptor": map[string]interface{}{"use_raptor": false}, - "graphrag": map[string]interface{}{"use_graphrag": false}, - }, - "book": { - "raptor": map[string]interface{}{"use_raptor": false}, - "graphrag": map[string]interface{}{"use_graphrag": false}, - }, - "laws": { - "raptor": map[string]interface{}{"use_raptor": false}, - "graphrag": map[string]interface{}{"use_graphrag": false}, - }, - "presentation": { - "raptor": map[string]interface{}{"use_raptor": false}, - "graphrag": map[string]interface{}{"use_graphrag": false}, }, + "qa": nil, + "resume": nil, + "manual": nil, + "paper": nil, + "book": nil, + "laws": nil, + "presentation": nil, } merged := DeepMergeMaps(baseDefaults, defaultConfigs[parserID]) return DeepMergeMaps(merged, parserConfig) } + +func ExtractPipelineDefaults(dsl map[string]interface{}) map[string]interface{} { + if dsl == nil { + return nil + } + if inner, ok := dsl["dsl"].(map[string]interface{}); ok { + dsl = inner + } + components, _ := dsl["components"].(map[string]interface{}) + if components == nil { + return nil + } + + result := make(map[string]interface{}) + hasAny := false + for cid, compVal := range components { + compMap, ok := compVal.(map[string]interface{}) + if !ok { + continue + } + obj, _ := compMap["obj"].(map[string]interface{}) + if obj == nil { + continue + } + name, _ := obj["component_name"].(string) + if name == "" || name == "File" { + continue + } + params, _ := obj["params"].(map[string]interface{}) + if params == nil { + continue + } + copy_ := deepCopyMap(params) + delete(copy_, "outputs") + result[cid] = copy_ + hasAny = true + } + if !hasAny { + return nil + } + return result +} diff --git a/internal/common/parser_config_test.go b/internal/common/parser_config_test.go new file mode 100644 index 0000000000..cf2aff2b2d --- /dev/null +++ b/internal/common/parser_config_test.go @@ -0,0 +1,215 @@ +package common + +import ( + "testing" +) + +func TestDeepMergeMaps_Nil(t *testing.T) { + result := DeepMergeMaps(nil, nil) + if result == nil { + t.Fatal("expected non-nil for nil inputs") + } +} + +func TestDeepMergeMaps_Override(t *testing.T) { + base := map[string]interface{}{ + "a": 1, + "b": "hello", + } + override := map[string]interface{}{ + "a": 2, + "c": "world", + } + result := DeepMergeMaps(base, override) + if result["a"] != 2 { + t.Fatalf("expected a=2, got %v", result["a"]) + } + if result["b"] != "hello" { + t.Fatalf("expected b=hello, got %v", result["b"]) + } + if result["c"] != "world" { + t.Fatalf("expected c=world, got %v", result["c"]) + } +} + +func TestDeepMergeMaps_Nested(t *testing.T) { + base := map[string]interface{}{ + "parser": map[string]interface{}{ + "lang": "en", + "model": "gpt4", + }, + } + override := map[string]interface{}{ + "parser": map[string]interface{}{ + "lang": "zh", + }, + } + result := DeepMergeMaps(base, override) + parser, _ := result["parser"].(map[string]interface{}) + if parser["lang"] != "zh" { + t.Fatalf("expected lang=zh, got %v", parser["lang"]) + } + if parser["model"] != "gpt4" { + t.Fatalf("expected model=gpt4, got %v", parser["model"]) + } +} + +func TestGetParserConfig_Naive(t *testing.T) { + result := GetParserConfig("naive", nil) + if result == nil { + t.Fatal("expected non-nil") + } + if result["chunk_token_num"] != 512 { + t.Fatalf("expected chunk_token_num=512, got %v", result["chunk_token_num"]) + } + if result["layout_recognize"] != "DeepDOC" { + t.Fatalf("expected layout_recognize=DeepDOC, got %v", result["layout_recognize"]) + } +} + +func TestGetParserConfig_QA(t *testing.T) { + result := GetParserConfig("qa", nil) + if result == nil { + t.Fatal("expected non-nil") + } +} + +func TestGetParserConfig_Override(t *testing.T) { + override := map[string]interface{}{ + "chunk_token_num": 256, + } + result := GetParserConfig("naive", override) + if result["chunk_token_num"] != 256 { + t.Fatalf("expected chunk_token_num=256, got %v", result["chunk_token_num"]) + } + if result["layout_recognize"] != "DeepDOC" { + t.Fatalf("expected layout_recognize preserved, got %v", result["layout_recognize"]) + } +} + +func TestExtractPipelineDefaults_Nil(t *testing.T) { + if result := ExtractPipelineDefaults(nil); result != nil { + t.Fatalf("expected nil for nil input, got %v", result) + } +} + +func TestExtractPipelineDefaults_Basic(t *testing.T) { + dsl := map[string]interface{}{ + "components": map[string]interface{}{ + "Parser:abc": map[string]interface{}{ + "obj": map[string]interface{}{ + "component_name": "Parser", + "params": map[string]interface{}{ + "pdf": map[string]interface{}{ + "parse_method": "DeepDOC", + "lang": "en", + }, + }, + }, + }, + }, + } + result := ExtractPipelineDefaults(dsl) + if result == nil { + t.Fatal("expected non-nil") + } + params, ok := result["Parser:abc"].(map[string]interface{}) + if !ok { + t.Fatal("expected Parser:abc in result") + } + pdf, _ := params["pdf"].(map[string]interface{}) + if pdf["parse_method"] != "DeepDOC" { + t.Fatalf("expected parse_method=DeepDOC, got %v", pdf["parse_method"]) + } +} + +func TestExtractPipelineDefaults_SkipsFile(t *testing.T) { + dsl := map[string]interface{}{ + "components": map[string]interface{}{ + "File:xyz": map[string]interface{}{ + "obj": map[string]interface{}{ + "component_name": "File", + "params": map[string]interface{}{ + "path": "/tmp/test", + }, + }, + }, + }, + } + result := ExtractPipelineDefaults(dsl) + if result != nil { + t.Fatal("expected nil when only File component exists") + } +} + +func TestExtractPipelineDefaults_StripsOutputs(t *testing.T) { + dsl := map[string]interface{}{ + "components": map[string]interface{}{ + "Tokenizer:abc": map[string]interface{}{ + "obj": map[string]interface{}{ + "component_name": "Tokenizer", + "params": map[string]interface{}{ + "fields": "text", + "search_method": []interface{}{"embedding"}, + "outputs": map[string]interface{}{"chunks": "data"}, + }, + }, + }, + }, + } + result := ExtractPipelineDefaults(dsl) + params := result["Tokenizer:abc"].(map[string]interface{}) + if _, ok := params["outputs"]; ok { + t.Fatal("expected outputs to be stripped") + } + if params["fields"] != "text" { + t.Fatalf("expected fields=text, got %v", params["fields"]) + } +} + +func TestExtractPipelineDefaults_WithDSLWrapper(t *testing.T) { + dsl := map[string]interface{}{ + "dsl": map[string]interface{}{ + "components": map[string]interface{}{ + "Extractor:xyz": map[string]interface{}{ + "obj": map[string]interface{}{ + "component_name": "Extractor", + "params": map[string]interface{}{ + "auto_keywords": float64(5), + "auto_questions": float64(3), + }, + }, + }, + }, + }, + } + result := ExtractPipelineDefaults(dsl) + raw := result["Extractor:xyz"].(map[string]interface{}) + if raw["auto_keywords"] != float64(5) { + t.Fatalf("expected auto_keywords=5, got %v", raw["auto_keywords"]) + } + if raw["auto_questions"] != float64(3) { + t.Fatalf("expected auto_questions=3, got %v", raw["auto_questions"]) + } +} + +func TestExtractPipelineDefaults_Delimiters(t *testing.T) { + dsl := map[string]interface{}{ + "components": map[string]interface{}{ + "TokenChunker:abc": map[string]interface{}{ + "obj": map[string]interface{}{ + "component_name": "TokenChunker", + "params": map[string]interface{}{ + "delimiters": []interface{}{"\n", ".", " "}, + }, + }, + }, + }, + } + result := ExtractPipelineDefaults(dsl) + raw := result["TokenChunker:abc"].(map[string]interface{}) + delims, _ := raw["delimiters"].([]interface{}) + if len(delims) != 3 { + t.Fatalf("expected 3 delimiters, got %v", delims) + } +} diff --git a/internal/dao/canvas_template_seed.go b/internal/dao/canvas_template_seed.go index 471bb50109..debf7fe62f 100644 --- a/internal/dao/canvas_template_seed.go +++ b/internal/dao/canvas_template_seed.go @@ -34,41 +34,107 @@ import ( ) // SeedCanvasTemplates seeds the canvas_template table from the built-in -// agent/templates/*.json files. This mirrors Python's -// init_data.add_graph_templates() so that the Go backend serves the same -// template catalogue without relying on Python-side initialization. +// agent/templates/*.json and internal/ingestion/pipeline/template/*.json files. func SeedCanvasTemplates() error { - dir := findAgentTemplatesDir() - if dir == "" { - common.Warn("Agent templates directory not found, skipping canvas template seeding") - return nil - } - - // The canvas_template table may have been created by the Python backend, - // which has no parser_ids column, and the Go server is often started with - // migration disabled so GORM AutoMigrate never adds it. Ensure the column - // exists before inserting, otherwise every INSERT below fails with - // "Unknown column 'parser_ids'" and the catalogue ends up empty. if err := addColumnIfNotExists(DB, "canvas_template", "parser_ids", "LONGTEXT NULL"); err != nil { return fmt.Errorf("failed to ensure canvas_template.parser_ids column: %w", err) } - entries, err := os.ReadDir(dir) - if err != nil { - return fmt.Errorf("failed to read agent templates directory %s: %w", dir, err) + var allTemplates []*entity.CanvasTemplate + var allIDs []string + for _, dir := range findTemplateDirs() { + if dir == "" { + continue + } + entries, err := os.ReadDir(dir) + if err != nil { + common.Warn("Failed to read template directory", zap.String("dir", dir), zap.Error(err)) + continue + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + templates, ids := loadTemplatesFromDir(dir, entries) + allTemplates = append(allTemplates, templates...) + allIDs = append(allIDs, ids...) } - sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + if len(allTemplates) == 0 { + common.Warn("No template directories found, skipping canvas template seeding") + return nil + } - seeded, err := seedCanvasTemplates(DB, dir, entries) + err := DB.Transaction(func(tx *gorm.DB) error { + for _, tmpl := range allTemplates { + if err := tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "id"}}, + DoUpdates: clause.AssignmentColumns([]string{ + "avatar", "title", "description", "canvas_type", "canvas_types", "canvas_category", "dsl", + }), + }).Create(tmpl).Error; err != nil { + return fmt.Errorf("failed to save agent template %s: %w", tmpl.ID, err) + } + } + if err := tx.Where("id NOT IN ?", allIDs).Delete(&entity.CanvasTemplate{}).Error; err != nil { + return fmt.Errorf("failed to remove stale agent templates: %w", err) + } + return nil + }) if err != nil { return err } - - common.Info("Seeded canvas templates", zap.Int("count", seeded), zap.String("dir", dir)) + common.Info("Seeded canvas templates", zap.Int("count", len(allTemplates))) return nil } +func loadTemplatesFromDir(dir string, entries []os.DirEntry) ([]*entity.CanvasTemplate, []string) { + templates := make([]*entity.CanvasTemplate, 0, len(entries)) + ids := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + path := filepath.Join(dir, entry.Name()) + raw, err := os.ReadFile(path) + if err != nil { + common.Warn("Failed to read agent template", zap.String("file", path), zap.Error(err)) + continue + } + tmpl, err := parseCanvasTemplateFile(raw) + if err != nil { + common.Warn("Failed to parse agent template", zap.String("file", path), zap.Error(err)) + continue + } + templates = append(templates, tmpl) + ids = append(ids, tmpl.ID) + } + return templates, ids +} + +func findTemplateDirs() []string { + var dirs []string + if d := findAgentTemplatesDir(); d != "" { + dirs = append(dirs, d) + } + if d := findIngestionTemplatesDir(); d != "" { + dirs = append(dirs, d) + } + return dirs +} + +func findIngestionTemplatesDir() string { + candidates := []string{ + "internal/ingestion/pipeline/template", + filepath.Join("..", "internal", "ingestion", "pipeline", "template"), + filepath.Join("..", "..", "internal", "ingestion", "pipeline", "template"), + filepath.Join("..", "..", "..", "internal", "ingestion", "pipeline", "template"), + } + for _, candidate := range candidates { + if info, err := os.Stat(candidate); err == nil && info.IsDir() { + return candidate + } + } + return "" +} + func seedCanvasTemplates(db *gorm.DB, dir string, entries []os.DirEntry) (int, error) { templates := make([]*entity.CanvasTemplate, 0, len(entries)) ids := make([]string, 0, len(entries)) @@ -90,11 +156,9 @@ func seedCanvasTemplates(db *gorm.DB, dir string, entries []os.DirEntry) (int, e templates = append(templates, tmpl) ids = append(ids, tmpl.ID) } - if len(templates) == 0 { return 0, nil } - err := db.Transaction(func(tx *gorm.DB) error { for _, tmpl := range templates { if err := tx.Clauses(clause.OnConflict{ @@ -106,7 +170,6 @@ func seedCanvasTemplates(db *gorm.DB, dir string, entries []os.DirEntry) (int, e return fmt.Errorf("failed to save agent template %s: %w", tmpl.ID, err) } } - if err := tx.Where("id NOT IN ?", ids).Delete(&entity.CanvasTemplate{}).Error; err != nil { return fmt.Errorf("failed to remove stale agent templates: %w", err) } diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go index 1897abac57..866b8e185d 100644 --- a/internal/ingestion/component/chunker/qa.go +++ b/internal/ingestion/component/chunker/qa.go @@ -34,6 +34,9 @@ import ( "regexp" "strings" + "github.com/gomarkdown/markdown" + "github.com/gomarkdown/markdown/parser" + "ragflow/internal/agent/runtime" "ragflow/internal/ingestion/component/schema" "ragflow/internal/tokenizer" @@ -41,9 +44,17 @@ import ( const ComponentNameQAChunker = "QAChunker" -type qaChunkerParam struct{} +type qaChunkerParam struct { + Lang string `json:"lang,omitempty"` +} -func (p *qaChunkerParam) Update(conf map[string]any) {} +func (p *qaChunkerParam) Update(conf map[string]any) { + if v, ok := conf["lang"]; ok { + if s, ok := v.(string); ok { + p.Lang = s + } + } +} func (qaChunkerParam) Defaults() qaChunkerParam { return qaChunkerParam{} } @@ -86,12 +97,20 @@ func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (m }, nil } + qPrefix, aPrefix := "问题：", "回答：" + eng := strings.EqualFold(c.param.Lang, "english") || c.param.Lang == "" + if eng { + qPrefix, aPrefix = "Question: ", "Answer: " + } + var qaPairs []qaPair + var isMarkdown bool switch upstream.OutputFormat { case schema.PayloadFormatHTML: qaPairs = extractQATable(stringPtrVal(upstream.HTMLResult)) case schema.PayloadFormatMarkdown: qaPairs = extractQAMarkdown(stringPtrVal(upstream.MarkdownResult)) + isMarkdown = true case schema.PayloadFormatText: qaPairs = extractQAText(stringPtrVal(upstream.TextResult)) default: @@ -102,8 +121,12 @@ func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (m for _, pair := range qaPairs { contentLTKS, _ := tokenizer.Tokenize(pair.Question) contentSMLTKS, _ := tokenizer.FineGrainedTokenize(contentLTKS) + answer := rmQAPrefix(pair.Answer) + if isMarkdown { + answer = renderMarkdown(answer) + } chunk := schema.ChunkDoc{ - ContentWithWeight: fmt.Sprintf("Question: %s\tAnswer: %s", rmQAPrefix(pair.Question), rmQAPrefix(pair.Answer)), + ContentWithWeight: fmt.Sprintf("%s%s\t%s%s", qPrefix, rmQAPrefix(pair.Question), aPrefix, answer), DocType: "text", ContentLtks: contentLTKS, ContentSmLtks: contentSMLTKS, @@ -114,12 +137,18 @@ func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (m return chunkOutputs(chunks), nil } +func renderMarkdown(s string) string { + mdParser := parser.NewWithExtensions(parser.CommonExtensions | parser.Tables) + output := markdown.ToHTML([]byte(s), mdParser, nil) + return string(output) +} + type qaPair struct { Question string Answer string } -var rmQAPrefixRe = regexp.MustCompile(`^(问题|答案|回答|user|assistant|Q|A|Question|Answer|问|答)[\t:： ]+`) +var rmQAPrefixRe = regexp.MustCompile(`(?i)^(问题|答案|回答|user|assistant|Q|A|Question|Answer|问|答)[ \t]*(?:[:：]|\t)[ \t]*`) func rmQAPrefix(txt string) string { return strings.TrimSpace(rmQAPrefixRe.ReplaceAllString(txt, "")) @@ -167,7 +196,7 @@ func extractQATable(htmlStr string) []qaPair { // Markdown QA extraction // --------------------------------------------------------------------------- -var mdHeading = regexp.MustCompile(`^(#{1,6})\s+`) +var mdHeading = regexp.MustCompile(`^(#*)`) func extractQAMarkdown(md string) []qaPair { if md == "" { @@ -175,7 +204,8 @@ func extractQAMarkdown(md string) []qaPair { } lines := strings.Split(md, "\n") var pairs []qaPair - var questionStack, levelStack []string + var questionStack []string + var levelStack []int var answer []string codeBlock := false @@ -199,16 +229,16 @@ func extractQAMarkdown(md string) []qaPair { } m := mdHeading.FindStringSubmatch(line) - if m == nil || len(m[1]) > 6 { + level := len(m[1]) + if level == 0 || level > 6 { answer = append(answer, line) continue } flushAnswer() - level := m[1] - question := strings.TrimSpace(line[len(m[0]):]) + question := strings.TrimSpace(line[level:]) - for len(levelStack) > 0 && len(level) <= len(levelStack[len(levelStack)-1]) { + for len(levelStack) > 0 && level <= levelStack[len(levelStack)-1] { questionStack = questionStack[:len(questionStack)-1] levelStack = levelStack[:len(levelStack)-1] } diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.go index 821d261540..a305b00e38 100644 --- a/internal/ingestion/component/chunker/qa_test.go +++ b/internal/ingestion/component/chunker/qa_test.go @@ -18,6 +18,7 @@ package chunker import ( "context" + "strings" "testing" "ragflow/internal/agent/runtime" @@ -167,3 +168,120 @@ func TestQAChunker_Empty(t *testing.T) { t.Fatalf("expected 0 chunks, got %d", len(chunks)) } } + +func TestQAChunker_CaseInsensitivePrefix(t *testing.T) { + comp, err := NewQAChunker(nil) + if err != nil { + t.Fatal(err) + } + inputs := map[string]any{ + "name": "test.txt", + "output_format": "text", + "text": "QUESTION: Hello\tANSWER: World", + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("Invoke failed: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) != 1 { + t.Fatalf("expected 1 chunk, got %d", len(chunks)) + } + cww, _ := chunks[0]["content_with_weight"].(string) + if cww != "Question: Hello\tAnswer: World" { + t.Fatalf("case-insensitive prefix not stripped: %q", cww) + } +} + +func TestQAChunker_PrefixRequiresColonOrTab(t *testing.T) { + comp, err := NewQAChunker(nil) + if err != nil { + t.Fatal(err) + } + inputs := map[string]any{ + "name": "test.txt", + "output_format": "text", + "text": "A language model is useful\tQ How does it work", + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("Invoke failed: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) != 1 { + t.Fatalf("expected 1 chunk, got %d", len(chunks)) + } + cww, _ := chunks[0]["content_with_weight"].(string) + if cww != "Question: A language model is useful\tAnswer: Q How does it work" { + t.Fatalf("space-only separator should not strip prefix: %q", cww) + } +} + +func TestQAChunker_HeadingNoTrailingSpace(t *testing.T) { + comp, err := NewQAChunker(nil) + if err != nil { + t.Fatal(err) + } + inputs := map[string]any{ + "name": "test.md", + "output_format": "markdown", + "markdown": "#Hello\nWorld\n", + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("Invoke failed: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) != 1 { + t.Fatalf("expected 1 chunk, got %d", len(chunks)) + } +} + +func TestQAChunker_ChineseLang(t *testing.T) { + comp, err := NewQAChunker(map[string]any{"lang": "Chinese"}) + if err != nil { + t.Fatal(err) + } + inputs := map[string]any{ + "name": "test.txt", + "output_format": "text", + "text": "什么是Go？\tGo是一种编程语言。", + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("Invoke failed: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) != 1 { + t.Fatalf("expected 1 chunk, got %d", len(chunks)) + } + cww, _ := chunks[0]["content_with_weight"].(string) + if want := "问题：什么是Go？\t回答：Go是一种编程语言。"; cww != want { + t.Fatalf("unexpected content: %q, want %q", cww, want) + } +} + +func TestQAChunker_MarkdownRendersHTML(t *testing.T) { + comp, err := NewQAChunker(nil) + if err != nil { + t.Fatal(err) + } + inputs := map[string]any{ + "name": "test.md", + "output_format": "markdown", + "markdown": "# Title\nThis is **bold** text.\n", + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("Invoke failed: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) != 1 { + t.Fatalf("expected 1 chunk, got %d", len(chunks)) + } + cww, _ := chunks[0]["content_with_weight"].(string) + if !strings.Contains(cww, "bold") && + !strings.Contains(cww, "bold") { + t.Fatalf("markdown not rendered to HTML: %q", cww) + } +} diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go index 2473ae0a8c..7a620553dd 100644 --- a/internal/ingestion/component/extractor.go +++ b/internal/ingestion/component/extractor.go @@ -71,10 +71,15 @@ import ( "time" eschema "github.com/cloudwego/eino/schema" + "go.uber.org/zap" "ragflow/internal/agent/runtime" + "ragflow/internal/common" + "ragflow/internal/dao" + "ragflow/internal/entity" "ragflow/internal/entity/models" "ragflow/internal/ingestion/component/schema" + "ragflow/internal/tokenizer" ) const componentNameExtractor = "Extractor" @@ -85,6 +90,44 @@ const componentNameExtractor = "Extractor" // is configured. const extractorTimeout = 60 * time.Second +const ( + autoKeywordPrompt = `## Role +You are a text analyzer. + +## Task +Extract the most important keywords/phrases of a given piece of text content. + +## Requirements +- Summarize the text content, and give the top %d important keywords/phrases. +- The keywords MUST be in the same language as the given piece of text content. +- The keywords are delimited by ENGLISH COMMA. +- Output keywords ONLY. + +--- + +## Text Content +%s` + + autoQuestionPrompt = `## Role +You are a text analyzer. + +## Task +Propose questions about a given piece of text content. + +## Requirements +- Understand and summarize the text content, and propose the top %d important questions. +- The questions SHOULD NOT have overlapping meanings. +- The questions SHOULD cover the main content of the text as much as possible. +- The questions MUST be in the same language as the given piece of text content. +- One question per line. +- Output questions ONLY. + +--- + +## Text Content +%s` +) + // ExtractorComponent performs LLM-based extraction over a chunk // list (or a single empty call when no chunks are wired in). // @@ -98,14 +141,12 @@ type ExtractorComponent struct { } // NewExtractorComponent constructs an Extractor from a DSL param -// map. Missing keys fall back to schema.ExtractorParam.Defaults(); -// an empty FieldName is rejected (matches python -// `check_empty(self.field_name, "Result Destination")`). +// map. Missing keys fall back to schema.ExtractorParam.Defaults(). // // Param map shape (all keys optional; missing → Defaults()): // // { -// "field_name": string, — required; key the extraction lands under +// "field_name": string, — optional; key the extraction lands under // "llm_id": string, — optional; resolves via models.NewModelFactory // "system_prompt": string, — optional override // "prompt": string, — optional user prompt @@ -128,6 +169,12 @@ func NewExtractorComponent(params map[string]any) (runtime.Component, error) { if v, ok := params["prompt"].(string); ok { p.Prompt = v } + if v, ok := params["auto_keywords"]; ok { + p.AutoKeywords = mapInt(v) + } + if v, ok := params["auto_questions"]; ok { + p.AutoQuestions = mapInt(v) + } } if err := p.Validate(); err != nil { return nil, fmt.Errorf("extractor: param check: %w", err) @@ -191,11 +238,12 @@ type extractorChatInvoker interface { // "model@provider". APIKey / BaseURL are passed through so the // driver can authenticate without re-reading the tenant config. type extractorChatRequest struct { - Driver string - ModelName string - APIKey string - BaseURL string - Messages []eschema.Message + Driver string + ModelName string + APIKey string + BaseURL string + Messages []eschema.Message + Temperature *float64 } // extractorChatResponse holds the LLM's text answer. Token / @@ -270,7 +318,7 @@ func (e *einoExtractorChatInvoker) Chat(ctx context.Context, req extractorChatRe driver := strings.ToLower(strings.TrimSpace(req.Driver)) modelName := req.ModelName if driver == "" && modelName != "" { - if bare, provider, ok := splitExtractorLLID(modelName); ok { + if bare, provider, ok := splitExtractorLLIDPair(modelName); ok { driver = provider modelName = bare } @@ -293,7 +341,12 @@ func (e *einoExtractorChatInvoker) Chat(ctx context.Context, req extractorChatRe apiKey := req.APIKey cfg := &models.APIConfig{ApiKey: &apiKey} cm := models.NewChatModel(d, &modelName, cfg) - wrapper := models.NewEinoChatModel(cm, nil) + var chatCfg *models.ChatConfig + if req.Temperature != nil { + temp := *req.Temperature + chatCfg = &models.ChatConfig{Temperature: &temp} + } + wrapper := models.NewEinoChatModel(cm, chatCfg) // Honour ctx cancel up front so the caller's WithTimeout(...) // is observed even when the driver layer doesn't take a ctx. if err := ctx.Err(); err != nil { @@ -306,7 +359,7 @@ func (e *einoExtractorChatInvoker) Chat(ctx context.Context, req extractorChatRe return &extractorChatResponse{Content: out.Content}, nil } -// splitExtractorLLID parses a composite llm_id "model@provider" +// splitExtractorLLIDPair parses a composite llm_id "model@provider" // mirroring agent/component/llm_credentials.go:parseLLMIDParts // (the canonical composite form throughout the codebase). Returns // ok=false when no "@" is present or the id is malformed. @@ -316,7 +369,7 @@ func (e *einoExtractorChatInvoker) Chat(ctx context.Context, req extractorChatRe // // Kept local so the ingestion package doesn't import // agent/component. -func splitExtractorLLID(s string) (modelName, provider string, ok bool) { +func splitExtractorLLIDPair(s string) (modelName, provider string, ok bool) { parts := strings.Split(strings.TrimSpace(s), "@") switch len(parts) { case 2: @@ -468,19 +521,11 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, inputs map[string]any) } in := c.resolveInputs(inputs) if in.fieldName == "toc" { - // TODO(parity-gap): _build_TOC is not ported yet — surface - // a clear error rather than silently emitting empty chunks. return nil, fmt.Errorf("extractor: field_name %q requires the TOC prompt generator which is not yet ported to Go", "toc") } - // Progress (_created_time / _elapsed_time stamping, start/done - // callbacks) is owned by the canvas framework (realComponentBody), - // not by this component. The per-chunk LLM timeout below is a - // business concern that stays here. if err := runtime.WithTimeout(ctx, extractorTimeout, func(timeoutCtx context.Context) error { if len(in.chunks) == 0 { - // Fast path (python _invoke line 108): one - // call with the resolved args directly. ans, callErr := c.call(timeoutCtx, in, "") if callErr != nil { return callErr @@ -490,32 +535,141 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, inputs map[string]any) } for i, ck := range in.chunks { text, _ := ck["text"].(string) - ans, callErr := c.call(timeoutCtx, in, text) - if callErr != nil { - return fmt.Errorf("chunk %d: %w", i, callErr) + if strings.TrimSpace(text) == "" { + text, _ = ck["content_with_weight"].(string) } - ck[in.fieldName] = ans + + if c.Param.AutoKeywords > 0 { + if err := c.runAutoKeywords(timeoutCtx, in, ck, text); err != nil { + return fmt.Errorf("chunk %d keywords: %w", i, err) + } + } + if c.Param.AutoQuestions > 0 { + if err := c.runAutoQuestions(timeoutCtx, in, ck, text); err != nil { + return fmt.Errorf("chunk %d questions: %w", i, err) + } + } + + if in.fieldName != "" { + ans, callErr := c.call(timeoutCtx, in, text) + if callErr != nil { + return fmt.Errorf("chunk %d: %w", i, callErr) + } + ck[in.fieldName] = ans + } + in.chunks[i] = ck } return nil }); err != nil { return nil, fmt.Errorf("extractor: %w", err) } - // Run-level metadata (name, tenant_id, kb_id, ...) is read by - // downstream components from the workflow-wide CanvasState.Globals - // bag (seeded at pipeline start), so it is not re-emitted here. return map[string]any{ "chunks": in.chunks, "output_format": "chunks", }, nil } +func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, in extractorInputs, ck map[string]any, chunkText string) error { + if _, exists := ck["important_kwd"]; exists { + return nil + } + kwIn := in + kwIn.prompt = "Output: " + kwIn.systemPrompt = fmt.Sprintf(autoKeywordPrompt, c.Param.AutoKeywords, chunkText) + kwIn.fieldName = "" + result, err := c.call(ctx, kwIn, "") + if err != nil { + return err + } + resultStr, _ := result.(string) + resultStr = cleanExtractionResult(resultStr) + if resultStr == "" { + return nil + } + kwds := splitKeywords(resultStr) + if len(kwds) == 0 { + return nil + } + ck["important_kwd"] = kwds + tks, tkErr := tokenizer.Tokenize(strings.Join(kwds, " ")) + if tkErr == nil { + ck["important_tks"] = tks + } + return nil +} + +func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, in extractorInputs, ck map[string]any, chunkText string) error { + if _, exists := ck["question_kwd"]; exists { + return nil + } + qIn := in + qIn.prompt = "Output: " + qIn.systemPrompt = fmt.Sprintf(autoQuestionPrompt, c.Param.AutoQuestions, chunkText) + qIn.fieldName = "" + result, err := c.call(ctx, qIn, "") + if err != nil { + return err + } + resultStr, _ := result.(string) + resultStr = cleanExtractionResult(resultStr) + if resultStr == "" { + return nil + } + qs := strings.Split(resultStr, "\n") + // Filter empty lines + var filtered []string + for _, q := range qs { + q = strings.TrimSpace(q) + if q != "" { + filtered = append(filtered, q) + } + } + if len(filtered) == 0 { + return nil + } + ck["question_kwd"] = filtered + tks, tkErr := tokenizer.Tokenize(strings.Join(filtered, "\n")) + if tkErr == nil { + ck["question_tks"] = tks + } + return nil +} + +// cleanExtractionResult strips `` tags and rejects `**ERROR**` responses, +// matching Python's keyword_extraction and question_proposal post-processing. +func cleanExtractionResult(s string) string { + if i := strings.Index(s, ""); i >= 0 { + s = s[i+len(""):] + } + s = strings.TrimSpace(s) + if strings.Contains(s, "**ERROR**") { + return "" + } + return s +} + +// splitKeywords splits a comma-delimited keyword string. +func splitKeywords(s string) []string { + parts := strings.FieldsFunc(s, func(r rune) bool { + return r == ',' || r == '，' || r == ';' || r == '；' || r == '、' || r == '\r' || r == '\n' + }) + result := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + result = append(result, p) + } + } + return result +} + // call dispatches one LLM chat call for the supplied chunk text // (empty string in the no-chunk fast path). The result is the // raw string from the model — JSON parsing happens here so // callers can rely on a structured value downstream. func (c *ExtractorComponent) call(ctx context.Context, in extractorInputs, chunkText string) (any, error) { - driver, modelName, apiKey, baseURL := resolveExtractorChatTarget(in.llmID) + driver, modelName, apiKey, baseURL := resolveExtractorChatTarget(ctx, in.llmID) msgs := buildExtractorMessages(in.systemPrompt, in.prompt, chunkText, in.chunks) inv := getExtractorChatInvoker() resp, err := inv.Chat(ctx, extractorChatRequest{ @@ -545,14 +699,9 @@ func (c *ExtractorComponent) call(ctx context.Context, in extractorInputs, chunk } // resolveExtractorChatTarget splits a composite llm_id -// "model@provider" or "openai/model@provider" into driver / -// model / api_key / base_url. Today the Extractor has no tenant- -// scoped credential lookup — credentials are read from the -// per-call inputs map only. Future iterations can fill that gap -// with the same pattern internal/agent/component/llm_credentials.go -// uses (resolveTenantLLMConfig). For Phase 2.5 the test seam -// (SetExtractorChatInvoker) carries the wire-level signals. -func resolveExtractorChatTarget(llmID string) (driver, modelName, apiKey, baseURL string) { +// "model@provider" into driver / model / api_key / base_url +// using the tenant-scoped provider → instance → model lookup. +func resolveExtractorChatTarget(ctx context.Context, llmID string) (driver, modelName, apiKey, baseURL string) { if override := getExtractorChatTargetResolverOverride(); override != nil { if driver, modelName, apiKey, baseURL, ok := override(llmID); ok { return driver, modelName, apiKey, baseURL @@ -561,12 +710,181 @@ func resolveExtractorChatTarget(llmID string) (driver, modelName, apiKey, baseUR if llmID == "" { return "", "", "", "" } + + // Derive driver and model name from the composite llm_id. modelName = llmID - if bare, provider, ok := splitExtractorLLID(llmID); ok { - modelName = bare - driver = strings.ToLower(provider) + parsedModel, _, parsedProvider := splitExtractorLLID(llmID) + if parsedProvider != "" { + modelName = parsedModel + driver = strings.ToLower(parsedProvider) } - return driver, modelName, "", "" + + // Override with tenant-scoped credentials when available. + cfg := resolveExtractorChatConfig(ctx, llmID) + if cfg.driver != "" { + driver = cfg.driver + } + if cfg.modelName != "" { + modelName = cfg.modelName + } + if cfg.apiKey != "" { + apiKey = cfg.apiKey + } + if cfg.baseURL != "" { + baseURL = cfg.baseURL + } + + return driver, modelName, apiKey, baseURL +} + +// extractorChatConfig holds the resolved chat model configuration. +type extractorChatConfig struct { + driver string // llm_factory + modelName string // llm_name + apiKey string + baseURL string // api_base +} + +// resolveExtractorChatConfig resolves tenant-scoped credentials for +// the given composite llm_id using tenant_model_provider → instance → model. +func resolveExtractorChatConfig(ctx context.Context, compositeLLMID string) extractorChatConfig { + state, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx) + if err != nil || state == nil { + return extractorChatConfig{} + } + tidVal, _ := state.GetGlobal("tenant_id") + tid, _ := tidVal.(string) + if tid == "" { + return extractorChatConfig{} + } + + // Split the composite id with rsplit from the right + // (preserves embedded '@' in model names). + pureModelName, instanceName, providerName := splitExtractorLLID(compositeLLMID) + if providerName == "" { + return extractorChatConfig{} + } + + // 1. Lookup provider (case-insensitive). + provider, err := dao.NewTenantModelProviderDAO().GetByTenantIDAndProviderName(tid, providerName) + if err != nil || provider == nil { + common.Debug("extractor credentials: provider not found", + zap.String("provider", providerName), + zap.Error(err)) + return extractorChatConfig{} + } + + // 2. Lookup instance (with "default" → sole active fallback). + instance, err := dao.NewTenantModelInstanceDAO().GetByProviderIDAndInstanceName(provider.ID, instanceName) + if err != nil || instance == nil { + if instanceName == "default" { + if fallback := findExtractorSoleActiveInstance(provider.ID); fallback != nil { + common.Debug("extractor credentials: remapped default instance to sole active", + zap.String("instance", fallback.InstanceName), + zap.String("provider", providerName)) + instance = fallback + err = nil + } + } + } + if err != nil || instance == nil { + common.Debug("extractor credentials: instance not found", + zap.String("instance", instanceName), + zap.String("provider", providerName)) + return extractorChatConfig{} + } + + // 3. Lookup tenant_model to validate the model is registered. + model, modelErr := dao.NewTenantModelDAO().GetByProviderIDAndInstanceIDAndModelTypeAndModelName( + provider.ID, instance.ID, int(entity.ModelTypeChat), pureModelName) + if modelErr != nil || model == nil { + common.Debug("extractor credentials: tenant_model not found", + zap.String("model", pureModelName), + zap.Error(modelErr)) + return extractorChatConfig{} + } + canonicalModelName := pureModelName + if model.ModelName != "" { + canonicalModelName = model.ModelName + } + common.Debug("extractor credentials: tenant_model found", + zap.String("model", canonicalModelName)) + + // 4. Assemble config from instance. + if instance.APIKey == "" { + common.Debug("extractor credentials: instance has no api_key", + zap.String("instance", instance.InstanceName), + zap.String("provider", providerName)) + return extractorChatConfig{} + } + baseURL := "" + if instance.Extra != "" { + var extra map[string]string + if err := json.Unmarshal([]byte(instance.Extra), &extra); err == nil { + baseURL = extra["base_url"] + } + } + + common.Debug("extractor credentials: resolved", + zap.String("llm_factory", provider.ProviderName), + zap.String("instance", instance.InstanceName), + zap.String("model", canonicalModelName), + zap.Bool("api_key_present", true), + zap.Bool("base_url_present", baseURL != "")) + return extractorChatConfig{ + driver: provider.ProviderName, + modelName: canonicalModelName, + apiKey: instance.APIKey, + baseURL: baseURL, + } +} + +// splitExtractorLLID splits a composite llm_id using rsplit("@", 2) +// from the right to preserve embedded '@' in model names. +// +// "model@provider" → ("model", "default", "provider") +// "model@instance@provider" → ("model", "instance", "provider") +// "a@b@c@d" → ("a@b", "c", "d") +// "plain" → ("plain", "", "") +func splitExtractorLLID(s string) (modelName, instanceName, providerName string) { + parts := strings.SplitN(strings.TrimSpace(s), "@", -1) + // rsplit("@", 2) from the right + n := len(parts) + if n >= 3 { + // Last segment = provider, second-to-last = instance, + // everything to the left = model name (rejoined with @). + providerName = parts[n-1] + instanceName = parts[n-2] + modelName = strings.Join(parts[:n-2], "@") + return + } + if n == 2 { + return parts[0], "default", parts[1] + } + return s, "", "" +} + +// findExtractorSoleActiveInstance returns the sole active instance +// for a provider when the "default" instance is not found. +func findExtractorSoleActiveInstance(providerID string) *entity.TenantModelInstance { + instances, err := dao.NewTenantModelInstanceDAO().GetAllInstancesByProviderID(providerID) + if err != nil { + return nil + } + var active []*entity.TenantModelInstance + for _, inst := range instances { + if inst == nil { + continue + } + if strings.EqualFold(strings.TrimSpace(inst.Status), "inactive") { + continue + } + active = append(active, inst) + } + if len(active) != 1 { + return nil + } + return active[0] } // buildExtractorMessages assembles system + user messages for @@ -638,6 +956,9 @@ func substitutePromptPlaceholders(prompt string, chunks []map[string]any) string var b strings.Builder for i, ck := range chunks { t, _ := ck["text"].(string) + if t == "" { + t, _ = ck["content_with_weight"].(string) + } if t == "" { continue } @@ -691,6 +1012,23 @@ func tryParseJSONObject(s string) (map[string]any, bool) { return out, true } +// init registers Extractor under CategoryIngestion (per plan §4 +// Phase 2.5). Metadata is derived from the Inputs()/Outputs() +// methods on ExtractorComponent so the API layer (Phase 4) can +// enumerate the catalog without instantiating the component. +// mapInt converts a JSON-compatible value to int. +func mapInt(v interface{}) int { + switch n := v.(type) { + case float64: + return int(n) + case int: + return n + case int64: + return int(n) + } + return 0 +} + // init registers Extractor under CategoryIngestion (per plan §4 // Phase 2.5). Metadata is derived from the Inputs()/Outputs() // methods on ExtractorComponent so the API layer (Phase 4) can diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.go index fc05d0a1fa..1ffb5b700b 100644 --- a/internal/ingestion/component/extractor_test.go +++ b/internal/ingestion/component/extractor_test.go @@ -476,12 +476,12 @@ func TestExtractorComponent_Invoke_ChunkIndexInError(t *testing.T) { // the construction-time Validate() rejection of an empty // field_name (matches python check_empty "Result Destination"). func TestExtractorComponent_NewExtractorComponent_ParamCheck(t *testing.T) { - _, err := NewExtractorComponent(map[string]any{}) - if err == nil { - t.Fatal("expected error for missing field_name, got nil") + c, err := NewExtractorComponent(map[string]any{}) + if err != nil { + t.Fatalf("expected nil error, got %v", err) } - if !strings.Contains(err.Error(), "field_name") { - t.Errorf("error should mention field_name: %v", err) + if c == nil { + t.Fatal("expected non-nil component") } } @@ -546,7 +546,7 @@ func TestSplitExtractorLLID(t *testing.T) { } for _, tc := range cases { t.Run(tc.in, func(t *testing.T) { - model, provider, ok := splitExtractorLLID(tc.in) + model, provider, ok := splitExtractorLLIDPair(tc.in) if ok != tc.wantOK { t.Errorf("ok = %v, want %v", ok, tc.wantOK) } diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go index 29453a4f00..fb0c10d6d8 100644 --- a/internal/ingestion/component/parser.go +++ b/internal/ingestion/component/parser.go @@ -104,10 +104,8 @@ const pageFormFeed = '\f' // the goroutine that returned from Invoke. The static Param is // read-only after construction. type ParserComponent struct { - // Param is the static configuration from schema.ParserParam. - // Kept as a value (not a pointer) so callers can pass literals - // and the component makes its own copy. - Param schema.ParserParam + Setups map[string]schema.ParserSetup + Param schema.ParserParam } // NewParserComponent constructs a Parser from a DSL param map. @@ -119,7 +117,9 @@ type ParserComponent struct { // schema.ParserParam.Defaults() values): // // { -// "setups": map[string]map[string]any, +// "pdf": map[string]any, +// "docx": map[string]any, +// ... // "allowed_output_format": map[string][]string, // } // @@ -127,32 +127,23 @@ type ParserComponent struct { // param is caught at build time rather than mid-run. func NewParserComponent(params map[string]any) (runtime.Component, error) { p := schema.ParserParam{}.Defaults() + s := defaultSetups() if params == nil { - return &ParserComponent{Param: p}, nil + return &ParserComponent{Setups: s, Param: p}, nil } - // Setups — best-effort decode. A type mismatch in a single - // setup entry drops just that entry; the rest of the table - // remains usable. This matches the python behaviour of - // accepting whatever shape the JSON loader hands back. - if rawSetups, ok := params["setups"].(map[string]any); ok { - for fileType, raw := range rawSetups { - setupMap, ok := raw.(map[string]any) - if !ok { - continue - } - if p.Setups == nil { - p.Setups = make(map[string]schema.ParserSetup) - } - // Normalise to the canonical family name ("picture", - // "video", …) so downstream lookups via - // resolveParserFamily, maybeDispatchImage, etc. find - // the entry regardless of what key the template DSL - // used (e.g. "image", "visual", "picture"). - family := pythonFamilyName(fileType) - if family == "" { - family = fileType - } - p.Setups[family] = schema.ParserSetup(setupMap) + for k, raw := range params { + if k == "outputs" { + continue + } + ftCfg, ok := raw.(map[string]any) + if !ok { + continue + } + if _, exists := s[k]; !exists { + s[k] = schema.ParserSetup{} + } + for fk, fv := range ftCfg { + s[k][fk] = fv } } if rawAllowed, ok := params["allowed_output_format"].(map[string]any); ok { @@ -172,7 +163,101 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) { } p.AllowedOutputFormat = allowed } - return &ParserComponent{Param: p}, nil + return &ParserComponent{Setups: s, Param: p}, nil +} + +func defaultSetups() map[string]schema.ParserSetup { + return map[string]schema.ParserSetup{ + "pdf": { + "parse_method": "deepdoc", + "lang": "Chinese", + "flatten_media_to_text": false, + "remove_toc": false, + "remove_header_footer": false, + "suffix": []string{"pdf"}, + "output_format": "json", + }, + "spreadsheet": { + "parse_method": "deepdoc", + "flatten_media_to_text": false, + "output_format": "html", + "suffix": []string{"xls", "xlsx", "csv"}, + }, + "doc": { + "remove_toc": false, + "remove_header_footer": false, + "suffix": []string{"doc"}, + "output_format": "json", + }, + "docx": { + "flatten_media_to_text": false, + "remove_toc": false, + "remove_header_footer": false, + "suffix": []string{"docx"}, + "output_format": "json", + }, + "markdown": { + "flatten_media_to_text": false, + "suffix": []string{"md", "markdown", "mdx"}, + "remove_toc": false, + "output_format": "json", + }, + "text&code": { + "suffix": []string{ + "txt", "py", "js", "java", "c", "cpp", "h", "php", + "go", "ts", "sh", "cs", "kt", "sql", + }, + "output_format": "json", + }, + "html": { + "suffix": []string{"htm", "html"}, + "remove_toc": false, + "remove_header_footer": false, + "output_format": "json", + }, + "slides": { + "parse_method": "deepdoc", + "suffix": []string{"pptx", "ppt"}, + "output_format": "json", + }, + "image": { + "parse_method": "ocr", + "llm_id": "", + "lang": "Chinese", + "system_prompt": "", + "suffix": []string{"jpg", "jpeg", "png", "gif"}, + "output_format": "json", + }, + "email": { + "suffix": []string{"eml", "msg"}, + "fields": []string{ + "from", "to", "cc", "bcc", "date", "subject", + "body", "attachments", "metadata", + }, + "output_format": "text", + }, + "audio": { + "suffix": []string{ + "da", "wave", "wav", "mp3", "aac", "flac", "ogg", + "aiff", "au", "midi", "wma", "realaudio", "vqf", + "oggvorbis", "ape", + }, + "output_format": "text", + }, + "video": { + "suffix": []string{"mp4", "avi", "mkv"}, + "output_format": "text", + "prompt": "", + }, + "epub": { + "suffix": []string{"epub"}, + "output_format": "json", + }, + "json": { + "suffix": []string{"json", "jsonl", "ldjson"}, + "output_format": "json", + }, + } } // Inputs returns the static parameter metadata. The component @@ -246,6 +331,15 @@ func (c *ParserComponent) Invoke(ctx context.Context, inputs map[string]any) (ma docID, _ := inputs["doc_id"].(string) filename := parserInputName(inputs, docID) + // Inject run-level metadata from Globals into inputs so media + // dispatch branches (audio/image/video) can resolve tenant_id. + // The File component upstream does not emit tenant_id; the pipeline + // runner seeds it into CanvasState.Globals, and the Parser must pull + // it back into the local inputs map for the dispatch functions. + if tid := globals.GlobalOrInput(ctx, inputs, "tenant_id", ""); tid != "" { + inputs["tenant_id"] = tid + } + // 2. Resolve the file family from the inputs. When the family // is known, dispatchParse returns a typed parser payload. // Otherwise the component stays in text-page mode. @@ -272,13 +366,13 @@ func (c *ParserComponent) Invoke(ctx context.Context, inputs map[string]any) (ma // family-specific allowed_output_format whitelist. We do // this even when no setups entry exists so a misconfigured // DSL surfaces as _ERROR instead of a silent fallback. - if _, hasSetup := c.Param.Setups[fileTypeFam]; hasSetup { - if _, verr := resolveOutputFormat(fileTypeFam, c.Param.Setups, c.Param.AllowedOutputFormat); verr != nil { + if _, hasSetup := c.Setups[fileTypeFam]; hasSetup { + if _, verr := resolveOutputFormat(fileTypeFam, c.Setups, c.Param.AllowedOutputFormat); verr != nil { return nil, verr } } - dispatched, handledVision, visionErr := maybeDispatchPDFVision(fileTypeExt, filename, binary, inputs, c.Param.Setups) + dispatched, handledVision, visionErr := maybeDispatchPDFVision(fileTypeExt, filename, binary, inputs, c.Setups) if visionErr != nil { return nil, visionErr } @@ -287,7 +381,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, inputs map[string]any) (ma if !handledVision { // Video dispatch: IMAGE2TEXT vision chat. // Mirrors Python's _video(). - dispatched, handledMedia, visionErr = maybeDispatchVideo(fileTypeExt, filename, binary, inputs, c.Param.Setups) + dispatched, handledMedia, visionErr = maybeDispatchVideo(fileTypeExt, filename, binary, inputs, c.Setups) if visionErr != nil { return nil, visionErr } @@ -296,7 +390,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, inputs map[string]any) (ma if !handledVision && !handledMedia { // Image/Picture dispatch: OCR + IMAGE2TEXT vision describe. // Mirrors Python's rag/app/picture.py:chunk() image branch. - dispatched, handledImage, visionErr = maybeDispatchImage(fileTypeExt, filename, binary, inputs, c.Param.Setups) + dispatched, handledImage, visionErr = maybeDispatchImage(fileTypeExt, filename, binary, inputs, c.Setups) if visionErr != nil { return nil, visionErr } @@ -305,19 +399,19 @@ func (c *ParserComponent) Invoke(ctx context.Context, inputs map[string]any) (ma if !handledVision && !handledMedia && !handledImage { // Audio dispatch: SPEECH2TEXT transcription. // Mirrors Python's rag/app/audio.py:chunk(). - dispatched, handledAudio, visionErr = maybeDispatchAudio(fileTypeExt, filename, binary, inputs, c.Param.Setups) + dispatched, handledAudio, visionErr = maybeDispatchAudio(fileTypeExt, filename, binary, inputs, c.Setups) if visionErr != nil { return nil, visionErr } } if !handledVision && !handledMedia && !handledImage && !handledAudio { - dispatched = dispatchParse(fileTypeExt, filename, binary, c.Param.Setups) + dispatched = dispatchParse(fileTypeExt, filename, binary, c.Setups) dispatched = hydrateEmptyDispatchPayload(dispatched, binary) // DOCX vision figure enhancement: enrich the markdown // with LLM-generated descriptions of embedded images. // Mirrors Python's vision_figure_parser_docx_wrapper_naive. - dispatched, _, _ = maybeDispatchDOCXVision(fileTypeExt, dispatched, inputs, c.Param.Setups) + dispatched, _, _ = maybeDispatchDOCXVision(fileTypeExt, dispatched, inputs, c.Setups) // Markdown vision figure enhancement: enrich parsed // markdown JSON items with LLM-generated descriptions of diff --git a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go index e10259b8ca..52874131e1 100644 --- a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go +++ b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go @@ -60,9 +60,10 @@ func TestDispatch_PDFVisionJSON_RealPDFFixture(t *testing.T) { } param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["parse_method"] = "CustomVLM" - param.Setups["pdf"]["output_format"] = "json" - c := &ParserComponent{Param: param} + setups := defaultSetups() + setups["pdf"]["parse_method"] = "CustomVLM" + setups["pdf"]["output_format"] = "json" + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": data, diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go index ea01e74143..1cbfcd523b 100644 --- a/internal/ingestion/component/parser_dispatch_test.go +++ b/internal/ingestion/component/parser_dispatch_test.go @@ -57,8 +57,9 @@ func (c *captureSetupConfigurer) ConfigureFromSetup(setup map[string]any) { // allowed_output_format check and runs the structured dispatch. func TestDispatch_OutputFormatValidation_Allowed(t *testing.T) { param := schema.ParserParam{}.Defaults() + setups := defaultSetups() // Defaults already include markdown → {text, json}. - c := &ParserComponent{Param: param} + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("# Title\n\nbody\n"), @@ -102,16 +103,17 @@ func TestDispatch_OutputFormatValidation_Allowed(t *testing.T) { // degrade. func TestDispatch_OutputFormatValidation_Rejection(t *testing.T) { param := schema.ParserParam{}.Defaults() + setups := defaultSetups() // Override the markdown setup to ask for an unsupported format. // The key is "markdown" (the python-side family identifier), // NOT "md" — utility.FileTypeMarkdown happens to be the string // "md" but the setup key is the family name. resolveOutputFormat // looks up setups[string(fileType)], so the fileType passed in // here must match the setup key. - param.Setups["markdown"] = schema.ParserSetup{"output_format": "html"} + setups["markdown"] = schema.ParserSetup{"output_format": "html"} // inputs["file_type"] must also be "markdown" so fileTypeFromInputs // returns a FileType whose string form matches the setup key. - c := &ParserComponent{Param: param} + c := &ParserComponent{Param: param, Setups: setups} _, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("# Title\n"), @@ -136,7 +138,8 @@ func TestDispatch_OutputFormatValidation_Rejection(t *testing.T) { // a family hint. func TestDispatch_TextPageMode_NoFileType(t *testing.T) { param := schema.ParserParam{}.Defaults() - c := &ParserComponent{Param: param} + setups := defaultSetups() + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("plain content\n"), @@ -160,7 +163,8 @@ func TestDispatch_TextPageMode_NoFileType(t *testing.T) { // silently degrading to text-page mode. func TestDispatch_SupportedFamilyFailure_HardErrors(t *testing.T) { param := schema.ParserParam{}.Defaults() - c := &ParserComponent{Param: param} + setups := defaultSetups() + c := &ParserComponent{Param: param, Setups: setups} _, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("PDF payload as bytes (not a real PDF — stub test)\n"), @@ -284,7 +288,7 @@ func TestResolveOutputFormat_DefaultsAndWhitelist(t *testing.T) { } func TestConfigureParserFromSetups_UsesPythonFamilySetup(t *testing.T) { - setups := schema.ParserParam{}.Defaults().Setups + setups := defaultSetups() got := &captureSetupConfigurer{} configureParserFromSetups(got, utility.FileTypePDF, setups) @@ -306,8 +310,9 @@ func TestDispatch_PDFMarkdown_UsesConfiguredOutputFormat(t *testing.T) { } param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["output_format"] = "markdown" - c := &ParserComponent{Param: param} + setups := defaultSetups() + setups["pdf"]["output_format"] = "markdown" + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": data, @@ -337,9 +342,10 @@ func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) { } param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["parse_method"] = "plain_text" - param.Setups["pdf"]["output_format"] = "json" - c := &ParserComponent{Param: param} + setups := defaultSetups() + setups["pdf"]["parse_method"] = "plain_text" + setups["pdf"]["output_format"] = "json" + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": data, @@ -360,8 +366,9 @@ func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) { func TestDispatch_PDFUnsupportedParseMethod_HardErrors(t *testing.T) { param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["parse_method"] = "CustomVLM" - c := &ParserComponent{Param: param} + setups := defaultSetups() + setups["pdf"]["parse_method"] = "CustomVLM" + c := &ParserComponent{Param: param, Setups: setups} _, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("%PDF-1.4"), @@ -429,9 +436,10 @@ func TestDispatch_PDFVisionJSON_UsesTenantAwareModel(t *testing.T) { } param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["parse_method"] = "CustomVLM" - param.Setups["pdf"]["output_format"] = "json" - c := &ParserComponent{Param: param} + setups := defaultSetups() + setups["pdf"]["parse_method"] = "CustomVLM" + setups["pdf"]["output_format"] = "json" + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("%PDF-1.4"), @@ -494,9 +502,10 @@ func TestDispatch_PDFVisionJSON_PreservesEmptyPages(t *testing.T) { } param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["parse_method"] = "CustomVLM" - param.Setups["pdf"]["output_format"] = "json" - c := &ParserComponent{Param: param} + setups := defaultSetups() + setups["pdf"]["parse_method"] = "CustomVLM" + setups["pdf"]["output_format"] = "json" + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("%PDF-1.4"), @@ -542,9 +551,10 @@ func TestDispatch_PDFMinerUMarkdown_UsesConfiguredBackend(t *testing.T) { } param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["parse_method"] = "mineru" - param.Setups["pdf"]["output_format"] = "markdown" - c := &ParserComponent{Param: param} + setups := defaultSetups() + setups["pdf"]["parse_method"] = "mineru" + setups["pdf"]["output_format"] = "markdown" + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("%PDF-1.4"), @@ -631,11 +641,12 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesConfiguredBackend(t *testing.T) { defer server.Close() param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["parse_method"] = "PaddleOCR" - param.Setups["pdf"]["output_format"] = "markdown" - param.Setups["pdf"]["paddleocr_base_url"] = server.URL - param.Setups["pdf"]["paddleocr_api_key"] = "paddle-secret" - c := &ParserComponent{Param: param} + setups := defaultSetups() + setups["pdf"]["parse_method"] = "PaddleOCR" + setups["pdf"]["output_format"] = "markdown" + setups["pdf"]["paddleocr_base_url"] = server.URL + setups["pdf"]["paddleocr_api_key"] = "paddle-secret" + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("%PDF-1.4"), @@ -679,11 +690,12 @@ func TestDispatch_PDFDoclingMarkdown_UsesConfiguredBackend(t *testing.T) { defer server.Close() param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["parse_method"] = "Docling" - param.Setups["pdf"]["output_format"] = "markdown" - param.Setups["pdf"]["docling_server_url"] = server.URL - param.Setups["pdf"]["docling_api_key"] = "doc-secret" - c := &ParserComponent{Param: param} + setups := defaultSetups() + setups["pdf"]["parse_method"] = "Docling" + setups["pdf"]["output_format"] = "markdown" + setups["pdf"]["docling_server_url"] = server.URL + setups["pdf"]["docling_api_key"] = "doc-secret" + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("%PDF-1.4"), @@ -717,10 +729,11 @@ func TestDispatch_PDFOpenDataLoaderMarkdown_UsesConfiguredBackend(t *testing.T) defer server.Close() param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["parse_method"] = "OpenDataLoader" - param.Setups["pdf"]["output_format"] = "markdown" - param.Setups["pdf"]["opendataloader_apiserver"] = server.URL - c := &ParserComponent{Param: param} + setups := defaultSetups() + setups["pdf"]["parse_method"] = "OpenDataLoader" + setups["pdf"]["output_format"] = "markdown" + setups["pdf"]["opendataloader_apiserver"] = server.URL + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("%PDF-1.4"), @@ -750,10 +763,11 @@ func TestDispatch_PDFSoMarkMarkdown_UsesConfiguredBackend(t *testing.T) { defer server.Close() param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["parse_method"] = "SoMark" - param.Setups["pdf"]["output_format"] = "markdown" - param.Setups["pdf"]["somark_base_url"] = server.URL - c := &ParserComponent{Param: param} + setups := defaultSetups() + setups["pdf"]["parse_method"] = "SoMark" + setups["pdf"]["output_format"] = "markdown" + setups["pdf"]["somark_base_url"] = server.URL + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("%PDF-1.4"), @@ -785,10 +799,11 @@ func TestDispatch_PDFTCADPMarkdown_UsesConfiguredBackend(t *testing.T) { defer server.Close() param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["parse_method"] = "TCADP parser" - param.Setups["pdf"]["output_format"] = "markdown" - param.Setups["pdf"]["tcadp_apiserver"] = server.URL - c := &ParserComponent{Param: param} + setups := defaultSetups() + setups["pdf"]["parse_method"] = "TCADP parser" + setups["pdf"]["output_format"] = "markdown" + setups["pdf"]["tcadp_apiserver"] = server.URL + c := &ParserComponent{Param: param, Setups: setups} out, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("%PDF-1.4"), diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.go index 228068ac87..8c7a0218c4 100644 --- a/internal/ingestion/component/parser_test.go +++ b/internal/ingestion/component/parser_test.go @@ -196,14 +196,12 @@ func TestParserComponent_New_Defaults(t *testing.T) { } // TestParserComponent_New_Overrides verifies that a non-nil -// param map with a "setups" entry is layered on top of the +// param map with a file-type entry is layered on top of the // defaults. func TestParserComponent_New_Overrides(t *testing.T) { c, err := NewParserComponent(map[string]any{ - "setups": map[string]any{ - "text&code": map[string]any{ - "chunk_token_num": 256, - }, + "text&code": map[string]any{ + "chunk_token_size": 256, }, }) if err != nil { @@ -213,15 +211,15 @@ func TestParserComponent_New_Overrides(t *testing.T) { if !ok { t.Fatalf("NewParserComponent returned %T", c) } - setup, ok := pc.Param.Setups["text&code"] + setup, ok := pc.Setups["text&code"] if !ok { t.Fatalf("Setups[text&code] missing after override") } - if got, _ := setup["chunk_token_num"].(int); got != 256 { - t.Errorf("Setups[text&code][chunk_token_num] = %v, want 256", setup["chunk_token_num"]) + if got, _ := setup["chunk_token_size"].(int); got != 256 { + t.Errorf("Setups[text&code][chunk_token_size] = %v, want 256", setup["chunk_token_size"]) } // Defaults must still be present for other file types. - if _, ok := pc.Param.Setups["pdf"]; !ok { + if _, ok := pc.Setups["pdf"]; !ok { t.Errorf("Setups[pdf] missing; override should not erase defaults") } } diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go index 6addd7f732..1240a20f2b 100644 --- a/internal/ingestion/component/pdf_vision_dispatch.go +++ b/internal/ingestion/component/pdf_vision_dispatch.go @@ -61,7 +61,10 @@ func maybeDispatchPDFVision( layout := getStringOr(setup, "layout_recognizer", "") // MinerU dispatch: parse_method "mineru" or layout_recognizer "@MinerU" - if method == "mineru" || strings.HasPrefix(layout, "mineru") || strings.Contains(layout, "@MinerU") { + layoutLower := strings.ToLower(strings.TrimSpace(layout)) + if strings.EqualFold(strings.TrimSpace(method), "mineru") || + strings.HasPrefix(layoutLower, "mineru") || + strings.Contains(layoutLower, "@mineru") { tenantID := getStringOr(inputs, "tenant_id", "") if tenantID == "" { return parserDispatchResult{}, true, @@ -363,7 +366,7 @@ func isNamedPDFParseMethod(raw string) bool { return true } switch method { - case "deepdoc", "plain_text", "plaintext", "paddleocr", "docling", "opendataloader", "somark", "tcadp", "tcadp parser": + case "deepdoc", "mineru", "plain_text", "plain text", "plaintext", "paddleocr", "docling", "opendataloader", "somark", "tcadp", "tcadp parser": return true } return false diff --git a/internal/ingestion/component/schema/extractor.go b/internal/ingestion/component/schema/extractor.go index 2bfe216ef5..5c898cbbf1 100644 --- a/internal/ingestion/component/schema/extractor.go +++ b/internal/ingestion/component/schema/extractor.go @@ -66,9 +66,8 @@ func (ExtractorFromUpstream) Validate() error { return nil } // the wiring is explicit. type ExtractorParam struct { // FieldName is the chunk key the LLM extraction result is written - // to (Python: `self._param.field_name`). Required — `check()` - // raises when empty. Mapped to "Result Destination" in the - // frontend. + // to (Python: `self._param.field_name`). Optional — when empty, + // auto_keywords or auto_questions may still be used. FieldName string `json:"field_name"` // LLMID identifies the LLM model used for extraction. This is the @@ -81,25 +80,30 @@ type ExtractorParam struct { // Prompt is the user-side template passed to the LLM. Prompt string `json:"prompt,omitempty"` + + // AutoKeywords enables automatic keyword extraction with a fixed + // prompt. The value determines the top-N count. + AutoKeywords int `json:"auto_keywords,omitempty"` + + // AutoQuestions enables automatic question generation with a fixed + // prompt. The value determines the top-N count. + AutoQuestions int `json:"auto_questions,omitempty"` } -// Defaults returns the Python default ExtractorParam: FieldName is -// the empty string and is meant to be supplied at runtime. +// Defaults returns the default ExtractorParam. func (ExtractorParam) Defaults() ExtractorParam { return ExtractorParam{ - FieldName: "", - LLMID: "", - SystemPrompt: "", - Prompt: "", + FieldName: "", + LLMID: "", + SystemPrompt: "", + Prompt: "", + AutoKeywords: 0, + AutoQuestions: 0, } } -// Validate enforces the Python `check()` invariant: FieldName must -// be non-empty. +// Validate always returns nil. func (p *ExtractorParam) Validate() error { - if p.FieldName == "" { - return errRequiredField{Field: "field_name"} - } return nil } diff --git a/internal/ingestion/component/schema/parser.go b/internal/ingestion/component/schema/parser.go index ec6fa17d00..b74f59b4b7 100644 --- a/internal/ingestion/component/schema/parser.go +++ b/internal/ingestion/component/schema/parser.go @@ -64,25 +64,7 @@ type Page map[string]any type ParserSetup map[string]any // ParserParam is the static configuration for the Parser component. -// Mirrors rag/flow/parser/parser.py:ParserParam. -// -// Two top-level fields are configured in the Python class: -// -// setups: dict[str, dict] (one entry per file type: pdf, docx, ...) -// allowed_output_format: dict[str, list[str]] (per-file-type formats) -// -// `check()` runs further validation that is intentionally NOT -// replicated here — Validate() enforces wire-shape only; business-rule -// validation lives in the component implementation (Phase 2.2). type ParserParam struct { - // Setups holds the per-file-type parser config. Keys are file-type - // identifiers ("pdf", "docx", "markdown", "spreadsheet", "image", - // "audio", "video", "email", "epub", "doc", "text&code", "html", - // "slides"); values are free-form config blobs. - Setups map[string]ParserSetup `json:"setups"` - - // AllowedOutputFormat mirrors `allowed_output_format` from the - // Python class. Used for client-side input-form validation. AllowedOutputFormat map[string][]string `json:"allowed_output_format"` } @@ -108,97 +90,6 @@ func (ParserParam) Defaults() ParserParam { "epub": {"text", "json"}, "json": {"json"}, }, - Setups: map[string]ParserSetup{ - "pdf": { - "parse_method": "deepdoc", - "lang": "Chinese", - "flatten_media_to_text": false, - "remove_toc": false, - "remove_header_footer": false, - "suffix": []string{"pdf"}, - "output_format": "json", - }, - "spreadsheet": { - "parse_method": "deepdoc", - "flatten_media_to_text": false, - "output_format": "html", - "suffix": []string{"xls", "xlsx", "csv"}, - }, - "doc": { - "remove_toc": false, - "remove_header_footer": false, - "suffix": []string{"doc"}, - "output_format": "json", - }, - "docx": { - "flatten_media_to_text": false, - "remove_toc": false, - "remove_header_footer": false, - "suffix": []string{"docx"}, - "output_format": "json", - }, - "markdown": { - "flatten_media_to_text": false, - "suffix": []string{"md", "markdown", "mdx"}, - "remove_toc": false, - "output_format": "json", - }, - "text&code": { - "suffix": []string{ - "txt", "py", "js", "java", "c", "cpp", "h", "php", - "go", "ts", "sh", "cs", "kt", "sql", - }, - "output_format": "json", - }, - "html": { - "suffix": []string{"htm", "html"}, - "remove_toc": false, - "remove_header_footer": false, - "output_format": "json", - }, - "slides": { - "parse_method": "deepdoc", - "suffix": []string{"pptx", "ppt"}, - "output_format": "json", - }, - "image": { - "parse_method": "ocr", - "llm_id": "", - "lang": "Chinese", - "system_prompt": "", - "suffix": []string{"jpg", "jpeg", "png", "gif"}, - "output_format": "json", - }, - "email": { - "suffix": []string{"eml", "msg"}, - "fields": []string{ - "from", "to", "cc", "bcc", "date", "subject", - "body", "attachments", "metadata", - }, - "output_format": "text", - }, - "audio": { - "suffix": []string{ - "da", "wave", "wav", "mp3", "aac", "flac", "ogg", - "aiff", "au", "midi", "wma", "realaudio", "vqf", - "oggvorbis", "ape", - }, - "output_format": "text", - }, - "video": { - "suffix": []string{"mp4", "avi", "mkv"}, - "output_format": "text", - "prompt": "", - }, - "epub": { - "suffix": []string{"epub"}, - "output_format": "json", - }, - "json": { - "suffix": []string{"json", "jsonl", "ldjson"}, - "output_format": "json", - }, - }, } } diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.go index d59687a38d..cdb3dc4474 100644 --- a/internal/ingestion/component/schema/schema_test.go +++ b/internal/ingestion/component/schema/schema_test.go @@ -135,15 +135,6 @@ func TestParserParamDefaults(t *testing.T) { if err := p.Validate(); err != nil { t.Fatalf("default ParserParam failed Validate: %v", err) } - // Spot-check a few entries that the Python class initializes. - for _, key := range []string{"pdf", "docx", "image", "audio", "video", "email", "epub"} { - if _, ok := p.Setups[key]; !ok { - t.Errorf("default Setups missing key %q", key) - } - } - if got := p.Setups["pdf"]["parse_method"]; got != "deepdoc" { - t.Errorf("default pdf parse_method = %v, want deepdoc", got) - } if got := p.AllowedOutputFormat["pdf"]; len(got) != 2 || got[0] != "json" || got[1] != "markdown" { t.Errorf("default pdf allowed_output_format = %v, want [json markdown]", got) } @@ -155,15 +146,15 @@ func TestParserParamJSONRoundTrip(t *testing.T) { if err != nil { t.Fatalf("marshal: %v", err) } - if !strings.Contains(string(data), `"setups"`) { - t.Errorf("expected setups in JSON, got %s", data) + if !strings.Contains(string(data), `"allowed_output_format"`) { + t.Errorf("expected allowed_output_format in JSON, got %s", data) } var decoded ParserParam if err := json.Unmarshal(data, &decoded); err != nil { t.Fatalf("unmarshal: %v", err) } - if decoded.Setups["pdf"]["parse_method"] != "deepdoc" { - t.Errorf("round-trip lost default pdf parse_method: got %v", decoded.Setups["pdf"]["parse_method"]) + if got := decoded.AllowedOutputFormat["pdf"]; len(got) != 2 || got[0] != "json" || got[1] != "markdown" { + t.Errorf("round-trip lost pdf allowed_output_format: got %v", got) } } @@ -516,14 +507,14 @@ func TestExtractorParamDefaults(t *testing.T) { if p.FieldName != "" { t.Errorf("default field_name should be empty, got %q", p.FieldName) } - if err := p.Validate(); err == nil { - t.Fatal("default ExtractorParam should fail Validate (field_name required)") + if err := p.Validate(); err != nil { + t.Fatalf("default ExtractorParam should pass Validate, got %v", err) } } func TestExtractorParamValidate(t *testing.T) { - if err := (&ExtractorParam{}).Validate(); err == nil { - t.Fatal("expected error for empty field_name") + if err := (&ExtractorParam{}).Validate(); err != nil { + t.Fatalf("expected no error, got %v", err) } if err := (&ExtractorParam{FieldName: "summary"}).Validate(); err != nil { t.Fatalf("Validate with field_name should pass, got %v", err) diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.go index e1c1d7acbf..2878f29701 100644 --- a/internal/ingestion/pipeline/pipeline.go +++ b/internal/ingestion/pipeline/pipeline.go @@ -486,3 +486,56 @@ func (p *Pipeline) componentProgressCallback() runtime.ProgressCallback { }) } } + +func PatchDSL(raw string, parserConfig map[string]interface{}) (string, error) { + if len(parserConfig) == 0 { + return raw, nil + } + + var dslMap map[string]any + if err := json.Unmarshal([]byte(raw), &dslMap); err != nil { + return raw, fmt.Errorf("parse dsl: %w", err) + } + + components, _ := dslMap["components"].(map[string]any) + if components == nil { + if inner, ok := dslMap["dsl"].(map[string]any); ok { + components, _ = inner["components"].(map[string]any) + } + if components == nil { + return raw, nil + } + } + for cid, extraVal := range parserConfig { + compVal, ok := components[cid] + if !ok { + continue + } + extraParams, _ := extraVal.(map[string]any) + if extraParams == nil { + continue + } + compMap, _ := compVal.(map[string]any) + if compMap == nil { + continue + } + obj, _ := compMap["obj"].(map[string]any) + if obj == nil { + continue + } + existing, _ := obj["params"].(map[string]any) + if existing == nil { + obj["params"] = extraParams + } else { + for k, v := range extraParams { + existing[k] = v + } + } + } + + patched, err := json.Marshal(dslMap) + if err != nil { + return raw, fmt.Errorf("marshal patched dsl: %w", err) + } + return string(patched), nil +} diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json b/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json index 13cfa7be58..8a7c88a376 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_audio.json @@ -49,20 +49,28 @@ "value": "" } }, - "setups": { - "audio": { - "output_format": "text", - "preprocess": [ - "main_content" - ], - "suffix": [ - "wav", - "mp3", - "aac", - "flac", - "ogg" - ] - } + "audio": { + "output_format": "text", + "preprocess": [ + "main_content" + ], + "suffix": [ + "aac", + "aiff", + "ape", + "au", + "da", + "flac", + "midi", + "mp3", + "ogg", + "oggvorbis", + "realaudio", + "vqf", + "wav", + "wave", + "wma" + ] } } }, @@ -72,7 +80,7 @@ }, "TokenChunker:BlueSkiesLaugh": { "downstream": [ - "Tokenizer:KindEyesWatch" + "Extractor:AutoExtractDefault" ], "obj": { "component_name": "OneChunker", @@ -104,8 +112,31 @@ } }, "upstream": [ - "TokenChunker:BlueSkiesLaugh" + "Extractor:AutoExtractDefault" ] + }, + "Extractor:AutoExtractDefault": { + "upstream": [ + "TokenChunker:BlueSkiesLaugh" + ], + "downstream": [ + "Tokenizer:KindEyesWatch" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "auto_keywords": 0, + "auto_questions": 0, + "llm_id": "" + } + } } }, "globals": { @@ -128,12 +159,16 @@ "targetHandle": "end" }, { - "data": { - "isHovered": false - }, - "id": "xy-edge__TokenChunker:BlueSkiesLaughstart-Tokenizer:KindEyesWatchend", + "id": "xy-edge__TokenChunker:BlueSkiesLaughstart-Extractor:AutoExtractDefaultend", "source": "TokenChunker:BlueSkiesLaugh", "sourceHandle": "start", + "target": "Extractor:AutoExtractDefault", + "targetHandle": "end" + }, + { + "id": "xy-edge__Extractor:AutoExtractDefaultstart-Tokenizer:KindEyesWatchend", + "source": "Extractor:AutoExtractDefault", + "sourceHandle": "start", "target": "Tokenizer:KindEyesWatch", "targetHandle": "end" } @@ -259,6 +294,18 @@ "sourcePosition": "right", "targetPosition": "left", "type": "tokenizerNode" + }, + { + "id": "Extractor:AutoExtractDefault", + "type": "contextNode", + "position": { + "x": 796.9952409420641, + "y": 195.39629819663406 + }, + "data": { + "label": "Extractor", + "name": "Extractor_0" + } } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json b/internal/ingestion/pipeline/template/ingestion_pipeline_book.json index 973611b204..95e9ce3ba0 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_book.json @@ -49,72 +49,57 @@ "value": "" } }, - "setups": { - "doc": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "doc" - ] - }, - "docx": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "docx" - ], - "vlm": {} - }, - "html": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "htm", - "html" - ] - }, - "pdf": { - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "remove_toc": true, - "suffix": [ - "pdf" - ], - "vlm": {} - }, - "text&code": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" - ] - } + "doc": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "doc" + ] + }, + "docx": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "docx" + ], + "vlm": {} + }, + "html": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "htm", + "html" + ] + }, + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "remove_toc": true, + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "txt" + ] } } }, @@ -124,7 +109,7 @@ }, "TitleChunker:GrumpyGarlicsBake": { "downstream": [ - "Tokenizer:HotDonutsRing" + "Extractor:AutoExtractDefault" ], "obj": { "component_name": "TitleChunker", @@ -188,8 +173,31 @@ } }, "upstream": [ - "TitleChunker:GrumpyGarlicsBake" + "Extractor:AutoExtractDefault" ] + }, + "Extractor:AutoExtractDefault": { + "upstream": [ + "TitleChunker:GrumpyGarlicsBake" + ], + "downstream": [ + "Tokenizer:HotDonutsRing" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "auto_keywords": 0, + "auto_questions": 0, + "llm_id": "" + } + } } }, "globals": { @@ -212,9 +220,16 @@ "targetHandle": "end" }, { - "id": "xy-edge__TitleChunker:GrumpyGarlicsBakestart-Tokenizer:HotDonutsRingend", + "id": "xy-edge__TitleChunker:GrumpyGarlicsBakestart-Extractor:AutoExtractDefaultend", "source": "TitleChunker:GrumpyGarlicsBake", "sourceHandle": "start", + "target": "Extractor:AutoExtractDefault", + "targetHandle": "end" + }, + { + "id": "xy-edge__Extractor:AutoExtractDefaultstart-Tokenizer:HotDonutsRingend", + "source": "Extractor:AutoExtractDefault", + "sourceHandle": "start", "target": "Tokenizer:HotDonutsRing", "targetHandle": "end" } @@ -425,8 +440,7 @@ } ] } - ], - "setups": [] + ] }, "label": "TitleChunker", "name": "Title Chunker_0" @@ -454,8 +468,7 @@ "search_method": [ "embedding", "full_text" - ], - "setups": [] + ] }, "label": "Tokenizer", "name": "Indexer_0" @@ -472,6 +485,18 @@ "sourcePosition": "right", "targetPosition": "left", "type": "tokenizerNode" + }, + { + "id": "Extractor:AutoExtractDefault", + "type": "contextNode", + "position": { + "x": 796.9952409420641, + "y": 195.39629819663406 + }, + "data": { + "label": "Extractor", + "name": "Extractor_0" + } } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json b/internal/ingestion/pipeline/template/ingestion_pipeline_email.json index ddf2d348fc..af7bf5d56d 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_email.json @@ -49,27 +49,24 @@ "value": "" } }, - "setups": { - "email": { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "output_format": "text", - "preprocess": [ - "main_content" - ], - "suffix": [ - "eml", - "msg" - ] - } + "email": { + "fields": [ + "from", + "to", + "cc", + "bcc", + "date", + "subject", + "body", + "attachments" + ], + "output_format": "text", + "preprocess": [ + "main_content" + ], + "suffix": [ + "eml" + ] } } }, @@ -79,7 +76,7 @@ }, "TokenChunker:WarmBreadSmells": { "downstream": [ - "Tokenizer:NiceWordsSpoken" + "Extractor:AutoExtractDefault" ], "obj": { "component_name": "TokenChunker", @@ -97,12 +94,6 @@ "？" ], "image_context_size": 0, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, "overlapped_percent": 0, "table_context_size": 0 } @@ -126,8 +117,31 @@ } }, "upstream": [ - "TokenChunker:WarmBreadSmells" + "Extractor:AutoExtractDefault" ] + }, + "Extractor:AutoExtractDefault": { + "upstream": [ + "TokenChunker:WarmBreadSmells" + ], + "downstream": [ + "Tokenizer:NiceWordsSpoken" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "auto_keywords": 0, + "auto_questions": 0, + "llm_id": "" + } + } } }, "globals": { @@ -150,12 +164,16 @@ "targetHandle": "end" }, { - "data": { - "isHovered": false - }, - "id": "xy-edge__TokenChunker:WarmBreadSmellsstart-Tokenizer:NiceWordsSpokenend", + "id": "xy-edge__TokenChunker:WarmBreadSmellsstart-Extractor:AutoExtractDefaultend", "source": "TokenChunker:WarmBreadSmells", "sourceHandle": "start", + "target": "Extractor:AutoExtractDefault", + "targetHandle": "end" + }, + { + "id": "xy-edge__Extractor:AutoExtractDefaultstart-Tokenizer:NiceWordsSpokenend", + "source": "Extractor:AutoExtractDefault", + "sourceHandle": "start", "target": "Tokenizer:NiceWordsSpoken", "targetHandle": "end" } @@ -268,12 +286,6 @@ } ], "image_table_context_window": 0, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, "overlapped_percent": 0 }, "label": "TokenChunker", @@ -319,6 +331,18 @@ "sourcePosition": "right", "targetPosition": "left", "type": "tokenizerNode" + }, + { + "id": "Extractor:AutoExtractDefault", + "type": "contextNode", + "position": { + "x": 796.9952409420641, + "y": 195.39629819663406 + }, + "data": { + "label": "Extractor", + "name": "Extractor_0" + } } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json index 1851aa45c4..fd96981a1e 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json @@ -49,98 +49,96 @@ "value": "" } }, - "setups": { - "doc": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "doc" - ] - }, - "docx": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "docx" - ], - "vlm": {} - }, - "html": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "htm", - "html" - ] - }, - "markdown": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "md", - "markdown", - "mdx" - ], - "vlm": {} - }, - "pdf": { - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "pdf" - ], - "vlm": {} - }, - "spreadsheet": { - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} - }, - "text&code": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" - ] - } + "doc": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "doc" + ] + }, + "docx": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "docx" + ], + "vlm": {} + }, + "html": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "htm", + "html" + ] + }, + "markdown": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "md", + "markdown", + "mdx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "spreadsheet": { + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "xls", + "xlsx", + "csv" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "txt", + "py", + "js", + "java", + "c", + "cpp", + "h", + "php", + "go", + "ts", + "sh", + "cs", + "kt", + "sql" + ] } } }, @@ -150,7 +148,7 @@ }, "TokenChunker:SixApplesFall": { "downstream": [ - "Tokenizer:LegalReadersDecide" + "Extractor:AutoExtractDefault" ], "obj": { "component_name": "TokenChunker", @@ -197,8 +195,31 @@ } }, "upstream": [ - "TokenChunker:SixApplesFall" + "Extractor:AutoExtractDefault" ] + }, + "Extractor:AutoExtractDefault": { + "upstream": [ + "TokenChunker:SixApplesFall" + ], + "downstream": [ + "Tokenizer:LegalReadersDecide" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "auto_keywords": 0, + "auto_questions": 0, + "llm_id": "" + } + } } }, "globals": { @@ -221,12 +242,16 @@ "targetHandle": "end" }, { - "data": { - "isHovered": false - }, - "id": "xy-edge__TokenChunker:SixApplesFallstart-Tokenizer:LegalReadersDecideend", + "id": "xy-edge__TokenChunker:SixApplesFallstart-Extractor:AutoExtractDefaultend", "source": "TokenChunker:SixApplesFall", "sourceHandle": "start", + "target": "Extractor:AutoExtractDefault", + "targetHandle": "end" + }, + { + "id": "xy-edge__Extractor:AutoExtractDefaultstart-Tokenizer:LegalReadersDecideend", + "source": "Extractor:AutoExtractDefault", + "sourceHandle": "start", "target": "Tokenizer:LegalReadersDecide", "targetHandle": "end" } @@ -432,6 +457,18 @@ "sourcePosition": "right", "targetPosition": "left", "type": "tokenizerNode" + }, + { + "id": "Extractor:AutoExtractDefault", + "type": "contextNode", + "position": { + "x": 796.9952409420641, + "y": 195.39629819663406 + }, + "data": { + "label": "Extractor", + "name": "Extractor_0" + } } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json index d9e1a5f4ef..3525c8bc12 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json @@ -49,72 +49,69 @@ "value": "" } }, - "setups": { - "doc": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "doc" - ] - }, - "docx": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "docx" - ], - "vlm": {} - }, - "html": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "htm", - "html" - ] - }, - "markdown": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "md", - "markdown", - "mdx" - ], - "vlm": {} - }, - "pdf": { - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pdf" - ], - "vlm": {} - }, - "text&code": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" - ] - } + "doc": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "doc" + ] + }, + "docx": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "docx" + ], + "vlm": {} + }, + "html": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "htm", + "html" + ] + }, + "markdown": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "md", + "markdown", + "mdx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "txt" + ] } } }, @@ -124,7 +121,7 @@ }, "TitleChunker:SpicyKeysKick": { "downstream": [ - "Tokenizer:PublicJobsTake" + "Extractor:AutoExtractDefault" ], "obj": { "component_name": "TitleChunker", @@ -188,8 +185,31 @@ } }, "upstream": [ - "TitleChunker:SpicyKeysKick" + "Extractor:AutoExtractDefault" ] + }, + "Extractor:AutoExtractDefault": { + "upstream": [ + "TitleChunker:SpicyKeysKick" + ], + "downstream": [ + "Tokenizer:PublicJobsTake" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "auto_keywords": 0, + "auto_questions": 0, + "llm_id": "" + } + } } }, "globals": { @@ -215,12 +235,16 @@ "targetHandle": "end" }, { - "data": { - "isHovered": false - }, - "id": "xy-edge__TitleChunker:SpicyKeysKickstart-Tokenizer:PublicJobsTakeend", + "id": "xy-edge__TitleChunker:SpicyKeysKickstart-Extractor:AutoExtractDefaultend", "source": "TitleChunker:SpicyKeysKick", "sourceHandle": "start", + "target": "Extractor:AutoExtractDefault", + "targetHandle": "end" + }, + { + "id": "xy-edge__Extractor:AutoExtractDefaultstart-Tokenizer:PublicJobsTakeend", + "source": "Extractor:AutoExtractDefault", + "sourceHandle": "start", "target": "Tokenizer:PublicJobsTake", "targetHandle": "end" } @@ -271,34 +295,46 @@ "flatten_media_to_text": false, "output_format": "json", "parse_method": "DeepDOC", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "markdown", "flatten_media_to_text": false, "output_format": "json", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "text&code", "output_format": "json", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "html", "output_format": "json", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "doc", "output_format": "json", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "docx", "flatten_media_to_text": false, "output_format": "json", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] } ] }, @@ -468,6 +504,18 @@ "sourcePosition": "right", "targetPosition": "left", "type": "tokenizerNode" + }, + { + "id": "Extractor:AutoExtractDefault", + "type": "contextNode", + "position": { + "x": 796.9952409420641, + "y": 195.39629819663406 + }, + "data": { + "label": "Extractor", + "name": "Extractor_0" + } } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json index 56a5ec9dbe..7a26f7ce64 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json @@ -49,33 +49,37 @@ "value": "" } }, - "setups": { - "doc": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "doc" - ] - }, - "docx": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "docx" - ], - "vlm": {} - }, - "pdf": { - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pdf" - ], - "vlm": {} - } + "doc": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "doc" + ] + }, + "docx": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "docx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pdf" + ], + "vlm": {} } } }, @@ -85,7 +89,7 @@ }, "TitleChunker:NineInsectsFind": { "downstream": [ - "Tokenizer:FunnyBalloonsGrin" + "Extractor:AutoExtractDefault" ], "obj": { "component_name": "TitleChunker", @@ -149,8 +153,31 @@ } }, "upstream": [ - "TitleChunker:NineInsectsFind" + "Extractor:AutoExtractDefault" ] + }, + "Extractor:AutoExtractDefault": { + "upstream": [ + "TitleChunker:NineInsectsFind" + ], + "downstream": [ + "Tokenizer:FunnyBalloonsGrin" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "auto_keywords": 0, + "auto_questions": 0, + "llm_id": "" + } + } } }, "globals": { @@ -173,9 +200,16 @@ "targetHandle": "end" }, { - "id": "xy-edge__TitleChunker:NineInsectsFindstart-Tokenizer:FunnyBalloonsGrinend", + "id": "xy-edge__TitleChunker:NineInsectsFindstart-Extractor:AutoExtractDefaultend", "source": "TitleChunker:NineInsectsFind", "sourceHandle": "start", + "target": "Extractor:AutoExtractDefault", + "targetHandle": "end" + }, + { + "id": "xy-edge__Extractor:AutoExtractDefaultstart-Tokenizer:FunnyBalloonsGrinend", + "source": "Extractor:AutoExtractDefault", + "sourceHandle": "start", "target": "Tokenizer:FunnyBalloonsGrin", "targetHandle": "end" } @@ -226,18 +260,24 @@ "flatten_media_to_text": false, "output_format": "json", "parse_method": "DeepDOC", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "doc", "output_format": "json", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "docx", "flatten_media_to_text": false, "output_format": "json", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] } ] }, @@ -406,6 +446,18 @@ "sourcePosition": "right", "targetPosition": "left", "type": "tokenizerNode" + }, + { + "id": "Extractor:AutoExtractDefault", + "type": "contextNode", + "position": { + "x": 796.9952409420641, + "y": 195.39629819663406 + }, + "data": { + "label": "Extractor", + "name": "Extractor_0" + } } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_one.json b/internal/ingestion/pipeline/template/ingestion_pipeline_one.json index f3509ca0a7..da57f64a0b 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_one.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_one.json @@ -49,84 +49,82 @@ "value": "" } }, - "setups": { - "doc": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "doc" - ] - }, - "docx": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "docx" - ], - "vlm": {} - }, - "html": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "htm", - "html" - ] - }, - "markdown": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "md", - "markdown", - "mdx" - ], - "vlm": {} - }, - "pdf": { - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pdf" - ], - "vlm": {} - }, - "spreadsheet": { - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} - }, - "text&code": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" - ] - } + "doc": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "doc" + ] + }, + "docx": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "docx" + ], + "vlm": {} + }, + "html": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "htm", + "html" + ] + }, + "markdown": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "md", + "markdown", + "mdx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "spreadsheet": { + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "xls", + "xlsx" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "txt" + ] } } }, @@ -136,7 +134,7 @@ }, "OneChunker:DryDrinksVisit": { "downstream": [ - "Tokenizer:FrankWeeksListen" + "Extractor:AutoExtractDefault" ], "obj": { "component_name": "OneChunker", @@ -168,8 +166,31 @@ } }, "upstream": [ - "OneChunker:DryDrinksVisit" + "Extractor:AutoExtractDefault" ] + }, + "Extractor:AutoExtractDefault": { + "upstream": [ + "OneChunker:DryDrinksVisit" + ], + "downstream": [ + "Tokenizer:FrankWeeksListen" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "auto_keywords": 0, + "auto_questions": 0, + "llm_id": "" + } + } } }, "globals": { @@ -192,9 +213,16 @@ "targetHandle": "end" }, { - "id": "xy-edge__OneChunker:DryDrinksVisitstart-Tokenizer:FrankWeeksListenend", + "id": "xy-edge__OneChunker:DryDrinksVisitstart-Extractor:AutoExtractDefaultend", "source": "OneChunker:DryDrinksVisit", "sourceHandle": "start", + "target": "Extractor:AutoExtractDefault", + "targetHandle": "end" + }, + { + "id": "xy-edge__Extractor:AutoExtractDefaultstart-Tokenizer:FrankWeeksListenend", + "source": "Extractor:AutoExtractDefault", + "sourceHandle": "start", "target": "Tokenizer:FrankWeeksListen", "targetHandle": "end" } @@ -245,41 +273,55 @@ "flatten_media_to_text": false, "output_format": "json", "parse_method": "DeepDOC", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "spreadsheet", "flatten_media_to_text": false, "output_format": "html", "parse_method": "DeepDOC", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "markdown", "flatten_media_to_text": false, "output_format": "json", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "text&code", "output_format": "json", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "html", "output_format": "json", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "doc", "output_format": "json", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] }, { "fileFormat": "docx", "flatten_media_to_text": false, "output_format": "json", - "preprocess": "main_content" + "preprocess": [ + "main_content" + ] } ] }, @@ -354,6 +396,18 @@ "sourcePosition": "right", "targetPosition": "left", "type": "tokenizerNode" + }, + { + "id": "Extractor:AutoExtractDefault", + "type": "contextNode", + "position": { + "x": 796.9952409420641, + "y": 195.39629819663406 + }, + "data": { + "label": "Extractor", + "name": "Extractor_0" + } } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json index e3f655896e..107b3a11f8 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json @@ -49,20 +49,18 @@ "value": "" } }, - "setups": { - "pdf": { - "enable_multi_column": true, - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "pdf" - ], - "vlm": {} - } + "pdf": { + "enable_multi_column": true, + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pdf" + ], + "vlm": {} } } }, @@ -72,7 +70,7 @@ }, "TitleChunker:SparklySchoolsTravel": { "downstream": [ - "Tokenizer:GreatCarsWash" + "Extractor:AutoExtractDefault" ], "obj": { "component_name": "TitleChunker", @@ -136,8 +134,31 @@ } }, "upstream": [ - "TitleChunker:SparklySchoolsTravel" + "Extractor:AutoExtractDefault" ] + }, + "Extractor:AutoExtractDefault": { + "upstream": [ + "TitleChunker:SparklySchoolsTravel" + ], + "downstream": [ + "Tokenizer:GreatCarsWash" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "auto_keywords": 0, + "auto_questions": 0, + "llm_id": "" + } + } } }, "globals": { @@ -163,12 +184,16 @@ "targetHandle": "end" }, { - "data": { - "isHovered": false - }, - "id": "xy-edge__TitleChunker:SparklySchoolsTravelstart-Tokenizer:GreatCarsWashend", + "id": "xy-edge__TitleChunker:SparklySchoolsTravelstart-Extractor:AutoExtractDefaultend", "source": "TitleChunker:SparklySchoolsTravel", "sourceHandle": "start", + "target": "Extractor:AutoExtractDefault", + "targetHandle": "end" + }, + { + "id": "xy-edge__Extractor:AutoExtractDefaultstart-Tokenizer:GreatCarsWashend", + "source": "Extractor:AutoExtractDefault", + "sourceHandle": "start", "target": "Tokenizer:GreatCarsWash", "targetHandle": "end" } @@ -393,6 +418,18 @@ "sourcePosition": "right", "targetPosition": "left", "type": "tokenizerNode" + }, + { + "id": "Extractor:AutoExtractDefault", + "type": "contextNode", + "position": { + "x": 796.9952409420641, + "y": 195.39629819663406 + }, + "data": { + "label": "Extractor", + "name": "Extractor_0" + } } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json b/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json index 0f42843f07..7d004058bd 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json @@ -49,31 +49,42 @@ "value": "" } }, - "setups": { - "image": { - "output_format": "text", - "parse_method": "ocr", - "preprocess": [ - "main_content" - ], - "suffix": [ - "jpg", - "jpeg", - "png", - "gif" - ] - }, - "video": { - "output_format": "text", - "preprocess": [ - "main_content" - ], - "suffix": [ - "mp4", - "avi", - "mkv" - ] - } + "image": { + "output_format": "text", + "parse_method": "ocr", + "preprocess": [ + "main_content" + ], + "suffix": [ + "bmp", + "gif", + "jpeg", + "jpg", + "png", + "svg", + "tif", + "tiff", + "webp" + ] + }, + "video": { + "output_format": "text", + "preprocess": [ + "main_content" + ], + "suffix": [ + "3gp", + "3gpp", + "avi", + "flv", + "mkv", + "mov", + "mp4", + "mpeg", + "mpg", + "webm", + "wmv" + ] } } }, @@ -83,7 +94,7 @@ }, "TokenChunker:BrightColorsGlow": { "downstream": [ - "Tokenizer:SharpLensFocus" + "Extractor:AutoExtractDefault" ], "obj": { "component_name": "OneChunker", @@ -115,8 +126,31 @@ } }, "upstream": [ - "TokenChunker:BrightColorsGlow" + "Extractor:AutoExtractDefault" ] + }, + "Extractor:AutoExtractDefault": { + "upstream": [ + "TokenChunker:BrightColorsGlow" + ], + "downstream": [ + "Tokenizer:SharpLensFocus" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "auto_keywords": 0, + "auto_questions": 0, + "llm_id": "" + } + } } }, "globals": { @@ -139,12 +173,16 @@ "targetHandle": "end" }, { - "data": { - "isHovered": false - }, - "id": "xy-edge__TokenChunker:BrightColorsGlowstart-Tokenizer:SharpLensFocusend", + "id": "xy-edge__TokenChunker:BrightColorsGlowstart-Extractor:AutoExtractDefaultend", "source": "TokenChunker:BrightColorsGlow", "sourceHandle": "start", + "target": "Extractor:AutoExtractDefault", + "targetHandle": "end" + }, + { + "id": "xy-edge__Extractor:AutoExtractDefaultstart-Tokenizer:SharpLensFocusend", + "source": "Extractor:AutoExtractDefault", + "sourceHandle": "start", "target": "Tokenizer:SharpLensFocus", "targetHandle": "end" } @@ -278,6 +316,18 @@ "sourcePosition": "right", "targetPosition": "left", "type": "tokenizerNode" + }, + { + "id": "Extractor:AutoExtractDefault", + "type": "contextNode", + "position": { + "x": 796.9952409420641, + "y": 195.39629819663406 + }, + "data": { + "label": "Extractor", + "name": "Extractor_0" + } } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json index ef6bd0ba32..2237191391 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json @@ -49,30 +49,28 @@ "value": "" } }, - "setups": { - "pdf": { - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "pdf" - ], - "vlm": {} - }, - "slides": { - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "pptx", - "ppt" - ] - } + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "slides": { + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pptx", + "ppt" + ] } } }, @@ -95,12 +93,12 @@ } }, "upstream": [ - "PresentationChunker:HappyHillsGlow" + "Extractor:AutoExtractDefault" ] }, "PresentationChunker:HappyHillsGlow": { "downstream": [ - "Tokenizer:TallTreesDance" + "Extractor:AutoExtractDefault" ], "obj": { "component_name": "PresentationChunker", @@ -116,6 +114,29 @@ "upstream": [ "Parser:HipSignsRhyme" ] + }, + "Extractor:AutoExtractDefault": { + "upstream": [ + "PresentationChunker:HappyHillsGlow" + ], + "downstream": [ + "Tokenizer:TallTreesDance" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "auto_keywords": 0, + "auto_questions": 0, + "llm_id": "" + } + } } }, "globals": { @@ -138,12 +159,16 @@ "targetHandle": "end" }, { - "data": { - "isHovered": false - }, - "id": "xy-edge__PresentationChunker:HappyHillsGlowstart-Tokenizer:TallTreesDanceend", + "id": "xy-edge__PresentationChunker:HappyHillsGlowstart-Extractor:AutoExtractDefaultend", "source": "PresentationChunker:HappyHillsGlow", "sourceHandle": "start", + "target": "Extractor:AutoExtractDefault", + "targetHandle": "end" + }, + { + "id": "xy-edge__Extractor:AutoExtractDefaultstart-Tokenizer:TallTreesDanceend", + "source": "Extractor:AutoExtractDefault", + "sourceHandle": "start", "target": "Tokenizer:TallTreesDance", "targetHandle": "end" } @@ -280,6 +305,18 @@ "sourcePosition": "right", "targetPosition": "left", "type": "tokenizerNode" + }, + { + "id": "Extractor:AutoExtractDefault", + "type": "contextNode", + "position": { + "x": 796.9952409420641, + "y": 195.39629819663406 + }, + "data": { + "label": "Extractor", + "name": "Extractor_0" + } } ] }, diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_qa.json b/internal/ingestion/pipeline/template/ingestion_pipeline_qa.json index 5fa2a9e8a2..e44927acb2 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_qa.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_qa.json @@ -49,79 +49,64 @@ "value": "" } }, - "setups": { - "docx": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "docx" - ], - "vlm": {} - }, - "markdown": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "md", - "markdown", - "mdx" - ], - "vlm": {} - }, - "pdf": { - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "pdf" - ], - "vlm": {} - }, - "spreadsheet": { - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} - }, - "text&code": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" - ] - } + "docx": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "docx" + ], + "vlm": {} + }, + "markdown": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "md", + "markdown", + "mdx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "spreadsheet": { + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "xls", + "xlsx", + "csv" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "txt" + ] } } }, @@ -180,17 +165,14 @@ "targetHandle": "end" }, { - "id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:BraveBirdsSingend", + "id": "xy-edge__Parser:HipSignsRhymestart-QAChunker:TidyCloudsThinkend", "source": "Parser:HipSignsRhyme", "sourceHandle": "start", "target": "QAChunker:TidyCloudsThink", "targetHandle": "end" }, { - "data": { - "isHovered": false - }, - "id": "xy-edge__TokenChunker:BraveBirdsSingstart-Tokenizer:ColdCloudsDreamend", + "id": "xy-edge__QAChunker:TidyCloudsThinkstart-Tokenizer:ColdCloudsDreamend", "source": "QAChunker:TidyCloudsThink", "sourceHandle": "start", "target": "Tokenizer:ColdCloudsDream", diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_resume.json b/internal/ingestion/pipeline/template/ingestion_pipeline_resume.json index 874e643644..2cdecf5bc5 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_resume.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_resume.json @@ -46,7 +46,9 @@ "temperatureEnabled": true, "tenant_llm_id": 29, "topPEnabled": true, - "top_p": 0.3 + "top_p": 0.3, + "auto_keywords": 0, + "auto_questions": 0 } }, "upstream": [ @@ -88,52 +90,37 @@ "value": "" } }, - "setups": { - "docx": { - "flatten_media_to_text": true, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "docx" - ], - "vlm": {} - }, - "pdf": { - "flatten_media_to_text": true, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "pdf" - ], - "vlm": {} - }, - "text&code": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" - ] - } + "docx": { + "flatten_media_to_text": true, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "docx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": true, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "txt" + ] } } }, @@ -426,7 +413,7 @@ "top_p": 0.3 }, "label": "Extractor", - "name": "Auto Metadata" + "name": "Extractor_0" }, "dragging": false, "id": "Extractor:ThreeDrinksAct", diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_table.json b/internal/ingestion/pipeline/template/ingestion_pipeline_table.json index e800d1e2bc..41410f47d2 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_table.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_table.json @@ -26,7 +26,7 @@ }, "Parser:HipSignsRhyme": { "downstream": [ - "TokenChunker:FastFoxesJump" + "TableChunker:FastFoxesJump" ], "obj": { "component_name": "Parser", @@ -49,43 +49,31 @@ "value": "" } }, - "setups": { - "spreadsheet": { - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} - }, - "text&code": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" - ] - } + "spreadsheet": { + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "xls", + "xlsx", + "csv" + ], + "vlm": {}, + "column_mode": "auto", + "column_roles": {}, + "column_names": [] + }, + "text&code": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "txt" + ] } } }, @@ -93,7 +81,7 @@ "File" ] }, - "TokenChunker:FastFoxesJump": { + "TableChunker:FastFoxesJump": { "downstream": [ "Tokenizer:DeepLakesShine" ], @@ -127,7 +115,7 @@ } }, "upstream": [ - "TokenChunker:FastFoxesJump" + "TableChunker:FastFoxesJump" ] } }, @@ -144,18 +132,15 @@ "targetHandle": "end" }, { - "id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:FastFoxesJumpend", + "id": "xy-edge__Parser:HipSignsRhymestart-TableChunker:FastFoxesJumpend", "source": "Parser:HipSignsRhyme", "sourceHandle": "start", - "target": "TokenChunker:FastFoxesJump", + "target": "TableChunker:FastFoxesJump", "targetHandle": "end" }, { - "data": { - "isHovered": false - }, - "id": "xy-edge__TokenChunker:FastFoxesJumpstart-Tokenizer:DeepLakesShineend", - "source": "TokenChunker:FastFoxesJump", + "id": "xy-edge__TableChunker:FastFoxesJumpstart-Tokenizer:DeepLakesShineend", + "source": "TableChunker:FastFoxesJump", "sourceHandle": "start", "target": "Tokenizer:DeepLakesShine", "targetHandle": "end" @@ -250,9 +235,9 @@ } }, "label": "TableChunker", - "name": "TableChunker_0" + "name": "Table Chunker_0" }, - "id": "TokenChunker:FastFoxesJump", + "id": "TableChunker:FastFoxesJump", "measured": { "height": 74, "width": 200 diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_tag.json b/internal/ingestion/pipeline/template/ingestion_pipeline_tag.json index 9397841577..88ef93f4dd 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_tag.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_tag.json @@ -49,43 +49,28 @@ "value": "" } }, - "setups": { - "spreadsheet": { - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} - }, - "text&code": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" - ] - } + "spreadsheet": { + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "xls", + "xlsx", + "csv" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "txt" + ] } } }, @@ -151,9 +136,6 @@ "targetHandle": "end" }, { - "data": { - "isHovered": false - }, "id": "xy-edge__TagChunker:NewNoonsGlowstart-Tokenizer:OldOwlsWatchend", "source": "TagChunker:NewNoonsGlow", "sourceHandle": "start", diff --git a/internal/ingestion/task/chunk_process.go b/internal/ingestion/task/chunk_process.go index 4408522dcd..0a8c8db47c 100644 --- a/internal/ingestion/task/chunk_process.go +++ b/internal/ingestion/task/chunk_process.go @@ -18,6 +18,7 @@ package task import ( "fmt" + "strings" "time" "ragflow/internal/common" @@ -171,3 +172,116 @@ func processChunkPositions(ck map[string]any) { } delete(ck, "positions") } + +// AggregateTableDocMetadata collects unique per-column values across all chunks +// for columns with role "metadata" or "both", merges them into document metadata. +// Mirrors Python: rag/utils/table_es_metadata.py:aggregate_table_doc_metadata +func AggregateTableDocMetadata(chunks []map[string]any, parserConfig map[string]interface{}) map[string]any { + mode, roles, tableColumnNames := resolveTableColumnConfig(parserConfig) + if mode == "" { + mode = "auto" + } + if mode != "auto" && mode != "manual" { + return nil + } + if roles == nil { + roles = map[string]interface{}{} + } + var metaCols []string + if len(tableColumnNames) > 0 { + for _, n := range tableColumnNames { + col, _ := n.(string) + if col == "" { + continue + } + role, _ := roles[col].(string) + if role == "" { + role = "both" + } + if role == "metadata" || role == "both" { + metaCols = append(metaCols, col) + } + } + } else { + for col, v := range roles { + role, _ := v.(string) + if role == "metadata" || role == "both" { + metaCols = append(metaCols, col) + } + } + } + if len(metaCols) == 0 { + return nil + } + + acc := make(map[string]map[string]struct{}, len(metaCols)) + for _, col := range metaCols { + acc[col] = make(map[string]struct{}) + } + for _, ck := range chunks { + cd, _ := ck["chunk_data"].(map[string]interface{}) + if cd == nil { + continue + } + for _, col := range metaCols { + val, ok := cd[col] + if !ok { + continue + } + s, _ := val.(string) + if s == "" { + continue + } + acc[col][s] = struct{}{} + } + } + + out := make(map[string]any, len(acc)) + for col, vals := range acc { + if len(vals) == 0 { + continue + } + deduped := make([]string, 0, len(vals)) + for v := range vals { + deduped = append(deduped, v) + } + out[col] = deduped + } + return out +} + +// resolveTableColumnConfig reads table column settings from parser_config. +// Tries root-level flat keys first; falls back to resolving from a Parser +// component entry's spreadsheet config in a component-ID-keyed parser_config. +func resolveTableColumnConfig(parserConfig map[string]interface{}) (mode string, roles map[string]interface{}, names []interface{}) { + mode, _ = parserConfig["table_column_mode"].(string) + roles, _ = parserConfig["table_column_roles"].(map[string]interface{}) + rawNames, _ := parserConfig["table_column_names"].([]interface{}) + if mode != "" || roles != nil || len(rawNames) > 0 { + return mode, roles, rawNames + } + for cid, raw := range parserConfig { + if !strings.HasPrefix(cid, "Parser:") { + continue + } + comp, _ := raw.(map[string]interface{}) + if comp == nil { + continue + } + ss, _ := comp["spreadsheet"].(map[string]interface{}) + if ss == nil { + continue + } + if v, ok := ss["column_mode"].(string); ok { + mode = v + } + if v, ok := ss["column_roles"].(map[string]interface{}); ok { + roles = v + } + if v, ok := ss["column_names"].([]interface{}); ok { + rawNames = v + } + return mode, roles, rawNames + } + return "", nil, nil +} diff --git a/internal/ingestion/task/embedder.go b/internal/ingestion/task/embedder.go index cfcf2d7e5a..cb5447c236 100644 --- a/internal/ingestion/task/embedder.go +++ b/internal/ingestion/task/embedder.go @@ -21,7 +21,6 @@ import ( "strings" "ragflow/internal/dao" - "ragflow/internal/entity" "ragflow/internal/entity/models" componentpkg "ragflow/internal/ingestion/component" "ragflow/internal/service" @@ -55,28 +54,23 @@ func (e *embedder) Encode(texts []string) ([]componentpkg.EmbeddingResult, error } // newEmbedderResolver builds the production embedder resolver used by the -// Tokenizer component. It honors an explicit embedding-model id (from the -// Tokenizer's setups) and falls back to the dataset's configured embd_id when -// none is given. Kept as a constructor over injectable deps so the resolution -// logic stays unit-testable without a live model provider / DB. +// Tokenizer component. It always resolves the embedder from the dataset's +// configured embd_id (looked up by kbID). If the dataset has no embd_id +// configured, it returns nil (no embedding). Kept as a constructor over +// injectable deps so the resolution logic stays unit-testable without a live +// model provider / DB. func newEmbedderResolver( + getKBEmbdID func(kbID string) (string, error), getEmbeddingModel func(tenantID, embdID string) (*models.EmbeddingModel, error), - getKnowledgebaseByID func(kbID string) (*entity.Knowledgebase, error), ) componentpkg.EmbedderResolver { - return func(tenantID, kbID, embeddingModel string) (componentpkg.Embedder, error) { - embdID := strings.TrimSpace(embeddingModel) + return func(tenantID, kbID, _ string) (componentpkg.Embedder, error) { + embdID, err := getKBEmbdID(kbID) + if err != nil { + return nil, fmt.Errorf("embedder: resolve kb embd_id for kb_id=%s: %w", kbID, err) + } + embdID = strings.TrimSpace(embdID) if embdID == "" { - if strings.TrimSpace(kbID) == "" { - return nil, fmt.Errorf("embedding requested but neither embedding_model nor kb_id provided") - } - kb, err := getKnowledgebaseByID(kbID) - if err != nil { - return nil, err - } - if kb == nil || strings.TrimSpace(kb.EmbdID) == "" { - return nil, fmt.Errorf("embedding requested but dataset has no embd_id configured") - } - embdID = kb.EmbdID + return nil, nil } model, err := getEmbeddingModel(tenantID, embdID) if err != nil { @@ -95,7 +89,16 @@ func newEmbedderResolver( // composition root for ingestion runs. func init() { componentpkg.DefaultEmbedderResolver = newEmbedderResolver( + func(kbID string) (string, error) { + kb, err := dao.NewKnowledgebaseDAO().GetByID(kbID) + if err != nil { + return "", err + } + if kb == nil { + return "", nil + } + return kb.EmbdID, nil + }, service.NewModelProviderService().GetEmbeddingModel, - dao.NewKnowledgebaseDAO().GetByID, ) } diff --git a/internal/ingestion/task/embedder_test.go b/internal/ingestion/task/embedder_test.go index 0a21c295bb..619392d4f3 100644 --- a/internal/ingestion/task/embedder_test.go +++ b/internal/ingestion/task/embedder_test.go @@ -17,10 +17,8 @@ package task import ( - "strings" "testing" - "ragflow/internal/entity" "ragflow/internal/entity/models" ) @@ -28,88 +26,45 @@ func makeEmbeddingModelForResolver() *models.EmbeddingModel { return models.NewEmbeddingModel(&stubDriver{}, strPtr("embed"), &models.APIConfig{}, 128) } -func TestEmbedderResolver_ExplicitEmbeddingModelWins(t *testing.T) { +func TestEmbedderResolver_UsesKBEmbdID(t *testing.T) { var gotTenantID, gotEmbdID string resolver := newEmbedderResolver( + func(kbID string) (string, error) { + return "kb-embd-1", nil + }, func(tenantID, embdID string) (*models.EmbeddingModel, error) { gotTenantID, gotEmbdID = tenantID, embdID return makeEmbeddingModelForResolver(), nil }, - func(string) (*entity.Knowledgebase, error) { - t.Fatal("kb lookup should not run when embedding_model is set") - return nil, nil - }, ) - emb, err := resolver("tenant-1", "kb-1", "explicit-embd") + emb, err := resolver("tenant-1", "kb-1", "should-be-ignored") if err != nil { t.Fatalf("resolver: %v", err) } if emb == nil { t.Fatal("expected embedder") } - if gotTenantID != "tenant-1" || gotEmbdID != "explicit-embd" { - t.Fatalf("resolver args = (%q, %q), want (tenant-1, explicit-embd)", gotTenantID, gotEmbdID) + if gotTenantID != "tenant-1" || gotEmbdID != "kb-embd-1" { + t.Fatalf("resolver args = (%q, %q), want (tenant-1, kb-embd-1)", gotTenantID, gotEmbdID) } } -func TestEmbedderResolver_FallsBackToDatasetEmbedding(t *testing.T) { - var gotEmbdID string +func TestEmbedderResolver_EmptyKBEmbdIDReturnsNil(t *testing.T) { resolver := newEmbedderResolver( - func(_ string, embdID string) (*models.EmbeddingModel, error) { - gotEmbdID = embdID - return makeEmbeddingModelForResolver(), nil + func(kbID string) (string, error) { + return "", nil }, - func(kbID string) (*entity.Knowledgebase, error) { - if kbID != "kb-1" { - t.Fatalf("kb lookup id = %q, want kb-1", kbID) - } - return &entity.Knowledgebase{ID: "kb-1", EmbdID: "lookup-embd"}, nil + func(string, string) (*models.EmbeddingModel, error) { + t.Fatal("model resolver should not be called when kb embd_id is empty") + return nil, nil }, ) - if _, err := resolver("tenant-1", "kb-1", ""); err != nil { + emb, err := resolver("tenant-1", "kb-1", "ignored") + if err != nil { t.Fatalf("resolver: %v", err) } - if gotEmbdID != "lookup-embd" { - t.Fatalf("got embd id %q, want lookup-embd", gotEmbdID) - } -} - -func TestEmbedderResolver_MissingDatasetEmbeddingReturnsError(t *testing.T) { - resolver := newEmbedderResolver( - func(string, string) (*models.EmbeddingModel, error) { - t.Fatal("model resolver should not be called") - return nil, nil - }, - func(string) (*entity.Knowledgebase, error) { - return &entity.Knowledgebase{ID: "kb-1", EmbdID: ""}, nil - }, - ) - _, err := resolver("tenant-1", "kb-1", "") - if err == nil { - t.Fatal("expected error when dataset embd_id is missing, got nil") - } - if !strings.Contains(err.Error(), "dataset has no embd_id configured") { - t.Fatalf("err = %v, want dataset has no embd_id configured", err) - } -} - -func TestEmbedderResolver_MissingEmbeddingModelAndKBReturnsError(t *testing.T) { - resolver := newEmbedderResolver( - func(string, string) (*models.EmbeddingModel, error) { - t.Fatal("model resolver should not be called") - return nil, nil - }, - func(string) (*entity.Knowledgebase, error) { - t.Fatal("kb lookup should not be called without a kb_id") - return nil, nil - }, - ) - _, err := resolver("tenant-1", "", "") - if err == nil { - t.Fatal("expected error when neither embedding_model nor kb_id provided") - } - if !strings.Contains(err.Error(), "neither embedding_model nor kb_id") { - t.Fatalf("err = %v, want neither embedding_model nor kb_id", err) + if emb != nil { + t.Fatal("expected nil embedder when kb has no embd_id") } } diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go index 6b0bb6478b..4d12c9979e 100644 --- a/internal/ingestion/task/pipeline_executor.go +++ b/internal/ingestion/task/pipeline_executor.go @@ -102,8 +102,8 @@ func NewPipelineExecutor( ), logCreateFunc: dao.NewPipelineOperationLogDAO().Create, } - svc.loadDSLFunc = svc.defaultLoadDSL - svc.runPipelineFunc = svc.defaultRunPipeline + svc.loadDSLFunc = svc.loadDSLFromCanvas + svc.runPipelineFunc = svc.runPipelineWithDSL return svc, nil } @@ -206,6 +206,18 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map return nil, err } + tableMeta := AggregateTableDocMetadata(chunks, map[string]interface{}(s.taskCtx.Doc.ParserConfig)) + if tableMeta != nil { + if metadata == nil { + metadata = make(map[string]any) + } + for k, v := range tableMeta { + if _, exists := metadata[k]; !exists { + metadata[k] = v + } + } + } + if err := s.indexWriter.Write(ctx, chunks); err != nil { return nil, err } @@ -243,3 +255,71 @@ func (s *PipelineExecutor) recordPipelineLog(docID, dsl, status string) { common.Warn(fmt.Sprintf("failed to record pipeline log: %v", err)) } } + +func (s *PipelineExecutor) loadDSLFromCanvas(ctx context.Context, canvasID string) (string, string, error) { + if s == nil || s.taskCtx == nil { + return "", "", fmt.Errorf("pipeline executor: nil task context") + } + if canvasID == "" { + return "", "", fmt.Errorf("pipeline executor: empty canvas id") + } + canvas, err := dao.NewUserCanvasDAO().GetByID(canvasID) + if err != nil { + return "", "", fmt.Errorf("load canvas %s: %w", canvasID, err) + } + + canvasTitle := "" + if canvas.Title != nil { + canvasTitle = *canvas.Title + } + common.Info(fmt.Sprintf("load canvas %s, name %s", canvasID, canvasTitle)) + + raw, err := json.Marshal(canvas.DSL) + if err != nil { + return "", "", fmt.Errorf("marshal canvas dsl %s: %w", canvasID, err) + } + return string(raw), canvasID, nil +} + +func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (map[string]any, string, error) { + if s == nil || s.taskCtx == nil { + return nil, dsl, fmt.Errorf("pipeline executor: nil task context") + } + + parserConfig := map[string]interface{}(s.taskCtx.Doc.ParserConfig) + common.InjectExtractorLLMID(parserConfig, s.taskCtx.Tenant.LLMID) + patchedDSL, err := pipelinepkg.PatchDSL(dsl, parserConfig) + if err != nil { + return nil, dsl, fmt.Errorf("patch dsl with parser_config: %w", err) + } + + pipelineID := "pipeline_" + s.taskCtx.Doc.ID + if s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.ID != "" { + pipelineID = s.taskCtx.IngestionTask.ID + } + pipe, err := pipelinepkg.NewPipelineFromDSL([]byte(patchedDSL), pipelineID, + pipelinepkg.WithProgressSink(s.progressSink), + pipelinepkg.WithDocumentID(s.taskCtx.Doc.ID)) + if err != nil { + return nil, patchedDSL, fmt.Errorf("compile pipeline dsl: %w", err) + } + inputs := map[string]any{} + if s.taskCtx.Doc.ID != "" { + inputs["doc_id"] = s.taskCtx.Doc.ID + } + if s.taskCtx.File != nil { + inputs["file"] = s.taskCtx.File + } + inputs["tenant_id"] = s.taskCtx.Tenant.ID + inputs["kb_id"] = s.taskCtx.KB.ID + + output, err := pipe.Run(ctx, inputs, map[string]interface{}(s.taskCtx.Doc.ParserConfig)) + if err != nil { + return nil, patchedDSL, err + } + payload, err := pipelinepkg.ExtractPayload(patchedDSL, output) + if err != nil { + return nil, patchedDSL, err + } + return payload, patchedDSL, nil +} diff --git a/internal/ingestion/task/pipeline_executor_defaults.go b/internal/ingestion/task/pipeline_executor_defaults.go deleted file mode 100644 index 519cfe8c7d..0000000000 --- a/internal/ingestion/task/pipeline_executor_defaults.go +++ /dev/null @@ -1,94 +0,0 @@ -// -// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package task - -import ( - "context" - "encoding/json" - "fmt" - - "ragflow/internal/common" - "ragflow/internal/dao" - pipelinepkg "ragflow/internal/ingestion/pipeline" -) - -func (s *PipelineExecutor) defaultLoadDSL(ctx context.Context, canvasID string) (string, string, error) { - if s == nil || s.taskCtx == nil { - return "", "", fmt.Errorf("pipeline executor: nil task context") - } - if canvasID == "" { - return "", "", fmt.Errorf("pipeline executor: empty canvas id") - } - canvas, err := dao.NewUserCanvasDAO().GetByID(canvasID) - if err != nil { - return "", "", fmt.Errorf("load canvas %s: %w", canvasID, err) - } - - canvasTitle := "" - if canvas.Title != nil { - canvasTitle = *canvas.Title - } - common.Info(fmt.Sprintf("load canvas %s, name %s", canvasID, canvasTitle)) - - raw, err := json.Marshal(canvas.DSL) - if err != nil { - return "", "", fmt.Errorf("marshal canvas dsl %s: %w", canvasID, err) - } - return string(raw), canvasID, nil -} - -func (s *PipelineExecutor) defaultRunPipeline(ctx context.Context, dsl string) (map[string]any, string, error) { - if s == nil || s.taskCtx == nil { - return nil, dsl, fmt.Errorf("pipeline executor: nil task context") - } - - // Use doc ID as pipeline ID if available, otherwise a placeholder - pipelineID := "pipeline_" + s.taskCtx.Doc.ID - if s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.ID != "" { - pipelineID = s.taskCtx.IngestionTask.ID - } - opts := []pipelinepkg.PipelineOption{ - pipelinepkg.WithProgressSink(s.progressSink), - pipelinepkg.WithDocumentID(s.taskCtx.Doc.ID), - } - if s.requireResume { - opts = append(opts, pipelinepkg.WithRequireResume()) - } - pipe, err := pipelinepkg.NewPipelineFromDSL([]byte(dsl), pipelineID, opts...) - if err != nil { - return nil, dsl, fmt.Errorf("compile pipeline dsl: %w", err) - } - inputs := map[string]any{} - if s.taskCtx.Doc.ID != "" { - inputs["doc_id"] = s.taskCtx.Doc.ID - } - if s.taskCtx.File != nil { - inputs["file"] = s.taskCtx.File - } - inputs["tenant_id"] = s.taskCtx.Tenant.ID - inputs["kb_id"] = s.taskCtx.KB.ID - - output, err := pipe.Run(ctx, inputs, nil) - if err != nil { - return nil, dsl, err - } - payload, err := pipelinepkg.ExtractPayload(dsl, output) - if err != nil { - return nil, dsl, err - } - return payload, dsl, nil -} diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go index 75475184de..1b82656809 100644 --- a/internal/ingestion/task/pipeline_executor_test.go +++ b/internal/ingestion/task/pipeline_executor_test.go @@ -409,7 +409,7 @@ func TestPipelineExecutor_Execute_PropagatesContext(t *testing.T) { // ============================================================================= // recordingProgressSink captures progress events for asserting the executor -// forwards its sink through defaultRunPipeline into the pipeline. +// forwards its sink through runPipelineWithDSL into the pipeline. type recordingProgressSink struct { mu sync.Mutex total int @@ -436,10 +436,10 @@ func (sinkPassthroughStage) Invoke(_ context.Context, inputs map[string]any) (ma return inputs, nil } -// TestPipelineExecutorDefaultRunPipelineForwardsSink verifies the sink set via -// WithProgressSink is threaded through defaultRunPipeline into the pipeline, +// TestPipelineExecutorRunPipelineWithDSLForwardsSink verifies the sink set via +// WithProgressSink is threaded through runPipelineWithDSL into the pipeline, // which reports the component total and lifecycle events back to the sink. -func TestPipelineExecutorDefaultRunPipelineForwardsSink(t *testing.T) { +func TestPipelineExecutorRunPipelineWithDSLForwardsSink(t *testing.T) { const nameA = "task.SinkPassthroughA" runtime.MustRegister(nameA, runtime.CategoryIngestion, func(_ string, _ map[string]any) (runtime.Component, error) { return sinkPassthroughStage{}, nil }, @@ -451,8 +451,8 @@ func TestPipelineExecutorDefaultRunPipelineForwardsSink(t *testing.T) { dsl := `{"dsl":{"components":{"begin":{"obj":{"component_name":"Begin","params":{}},"downstream":["a"]},"a":{"obj":{"component_name":"` + nameA + `","params":{}},"upstream":["begin"]}},"path":["begin","a"],"graph":{"nodes":[]}}}` - if _, _, err := svc.defaultRunPipeline(context.Background(), dsl); err != nil { - t.Fatalf("defaultRunPipeline: %v", err) + if _, _, err := svc.runPipelineWithDSL(context.Background(), dsl); err != nil { + t.Fatalf("runPipelineWithDSL: %v", err) } sink.mu.Lock() diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go index d4b91a6f7b..aa527fc5ec 100644 --- a/internal/parser/parser/csv_parser.go +++ b/internal/parser/parser/csv_parser.go @@ -126,9 +126,10 @@ func (p *CSVParser) ParseWithResult(filename string, data []byte) ParseResult { case "", "csv": // Continue with the local CSV parser. default: - return ParseResult{ - Err: fmt.Errorf("unsupported CSV parse method: %q", p.ParseMethod), - } + // PDF-specific methods like "DeepDOC" / "PaddleOCR" / "MinerU" + // are meaningless for CSV; treat them as the default CSV path, + // matching Python's behaviour where parse_method is irrelevant + // for CSV processing. } text := string(data) @@ -147,6 +148,7 @@ func (p *CSVParser) ParseWithResult(filename string, data []byte) ParseResult { reader := csv.NewReader(strings.NewReader(text)) reader.LazyQuotes = true reader.TrimLeadingSpace = true + reader.FieldsPerRecord = -1 // Allow variable column counts, matching Python csv.reader behaviour. records, err := reader.ReadAll() if err != nil { diff --git a/internal/parser/parser/pdf_postprocess.go b/internal/parser/parser/pdf_postprocess.go index 5ce3d2a0d7..e116a924e8 100644 --- a/internal/parser/parser/pdf_postprocess.go +++ b/internal/parser/parser/pdf_postprocess.go @@ -30,7 +30,7 @@ func applyPDFPostProcess(result *deepdoctype.ParseResult, opts pdfPostProcessOpt reorderPDFMultiColumn(result, opts.pageWidth, opts.zoom) } if opts.removeTOC { - removePDFTOCByOutlines(result, result.Outlines) + applyRemoveTOC(result) } normalizePDFLayoutTypes(result) if opts.removeHeaderFooter { @@ -83,6 +83,86 @@ func assignPDFDocTypeKeywords(result *deepdoctype.ParseResult, flatten bool) { } } +// applyRemoveTOC mirrors Python parser.py:663-681 three-way dispatch: +// - No outlines → pattern-based remove_toc on all sections +// - First outline on page 1 → outline-based remove_toc_pdf +// - First outline after page 1 → pattern-based on pages before the first outline +func applyRemoveTOC(result *deepdoctype.ParseResult) { + if result == nil { + return + } + outlines := result.Outlines + if len(outlines) == 0 { + removePDFTOC(result) + return + } + firstOutlinePage := outlines[0].PageNumber + if firstOutlinePage <= 1 { + removePDFTOCByOutlines(result, outlines) + return + } + splitAt := len(result.Sections) + for i, s := range result.Sections { + if firstSectionPage(s) >= firstOutlinePage { + splitAt = i + break + } + } + beforeSplit := &deepdoctype.ParseResult{Sections: result.Sections[:splitAt]} + removePDFTOC(beforeSplit) + result.Sections = append(beforeSplit.Sections, result.Sections[splitAt:]...) +} + +func removePDFTOC(result *deepdoctype.ParseResult) { + sections := result.Sections + i := 0 + for i < len(sections) { + text := sectionText(sections[i]) + if !pdfTOCTitlePattern.MatchString(strings.ToLower(strings.TrimSpace(text))) { + i++ + continue + } + sections = append(sections[:i], sections[i+1:]...) + if i >= len(sections) { + break + } + prefix := sectionTextPrefix(sections[i], 3) + for prefix == "" { + sections = append(sections[:i], sections[i+1:]...) + if i >= len(sections) { + break + } + prefix = sectionTextPrefix(sections[i], 3) + } + if i >= len(sections) || prefix == "" { + break + } + sections = append(sections[:i], sections[i+1:]...) + if i >= len(sections) || prefix == "" { + break + } + for j := i; j < len(sections) && j < i+128; j++ { + if !strings.HasPrefix(sectionText(sections[j]), prefix) { + continue + } + sections = append(sections[:i], sections[j:]...) + break + } + } + result.Sections = sections +} + +func sectionText(s deepdoctype.Section) string { + return strings.TrimSpace(s.Text) +} + +func sectionTextPrefix(s deepdoctype.Section, n int) string { + text := sectionText(s) + if len(text) < n { + return text + } + return text[:n] +} func removePDFTOCByOutlines(result *deepdoctype.ParseResult, outlines []deepdoctype.Outline) { if result == nil || len(outlines) == 0 { return diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go index a61d4d30ad..0ffd4f3c7a 100644 --- a/internal/parser/parser/xlsx_parser.go +++ b/internal/parser/parser/xlsx_parser.go @@ -103,9 +103,10 @@ func (p *XLSXParser) ParseWithResult(filename string, data []byte) ParseResult { case "", "excelize": // Continue with the local Excelize parser. default: - return ParseResult{ - Err: fmt.Errorf("unsupported XLSX parse method: %q", p.ParseMethod), - } + // PDF-specific methods like "DeepDOC" / "PaddleOCR" / "MinerU" + // are meaningless for XLSX; treat them as the default excelize path, + // matching Python's behaviour where parse_method is irrelevant + // for spreadsheet processing. } f, err := excelize.OpenReader(bytes.NewReader(data)) diff --git a/internal/service/dataset.go b/internal/service/dataset.go index 7c19621293..4ef5bdf51f 100644 --- a/internal/service/dataset.go +++ b/internal/service/dataset.go @@ -1863,8 +1863,30 @@ func (d *DatasetService) CreateDataset(req *CreateDatasetRequest, tenantID strin parserConfig = applyAutoMetadataConfig(parserConfig, req.AutoMetadataConfig) } - parserConfig = common.GetParserConfig(parserID, parserConfig) - parserConfig["llm_id"] = tenant.LLMID + // Build parser_config. When pipeline_id is set, seed from pipeline DSL defaults. + var parserConfigMap map[string]interface{} + if pipelineID != nil { + if parserConfig == nil { + parserConfig = make(map[string]interface{}) + } + canvas, err := dao.NewUserCanvasDAO().GetByID(*pipelineID) + if err != nil || canvas == nil { + return nil, common.CodeDataError, fmt.Errorf("pipeline %s not found", *pipelineID) + } + pipelineDefaults := common.ExtractPipelineDefaults(canvas.DSL) + if pipelineDefaults != nil { + parserConfigMap = common.DeepMergeMaps(pipelineDefaults, parserConfig) + } else { + parserConfigMap = common.DeepMergeMaps(nil, parserConfig) + } + common.InjectExtractorLLMID(parserConfigMap, tenant.LLMID) + } else { + parserConfigMap = common.GetParserConfig(parserID, parserConfig) + parserConfigMap["llm_id"] = tenant.LLMID + } + if err := validateDatasetParserConfigSize(parserConfigMap); err != nil { + return nil, common.CodeDataError, err + } embdID := tenant.EmbdID tenantEmbdID := ptrStringValue(tenant.TenantEmbdID) @@ -1904,7 +1926,7 @@ func (d *DatasetService) CreateDataset(req *CreateDatasetRequest, tenantID strin CreatedBy: tenantID, ParserID: parserID, PipelineID: pipelineID, - ParserConfig: parserConfig, + ParserConfig: entity.JSONMap(parserConfigMap), Permission: permission, EmbdID: embdID, TenantEmbdID: stringPtrIfNotEmpty(tenantEmbdID), @@ -2222,8 +2244,11 @@ func (d *DatasetService) UpdateDataset(datasetID, tenantID string, req UpdateDat return nil, common.CodeDataError, err } if len(req.ParserConfig) > 0 { - parserConfig := normalizeDatasetUpdateParserConfig(req.ParserConfig) - updates["parser_config"] = entity.JSONMap(common.DeepMergeMaps(kb.ParserConfig, parserConfig)) + merged := common.DeepMergeMaps(kb.ParserConfig, req.ParserConfig) + if err := validateDatasetParserConfigSize(merged); err != nil { + return nil, common.CodeDataError, err + } + updates["parser_config"] = entity.JSONMap(merged) } } @@ -2257,6 +2282,38 @@ func (d *DatasetService) UpdateDataset(datasetID, tenantID string, req UpdateDat } } + // Regenerate parser_config from the new pipeline's DSL defaults when + // the pipeline changes, so that component-level parameters stay in sync. + pipelineChanged := req.PipelineID != nil && (kb.PipelineID == nil || *req.PipelineID != *kb.PipelineID) + if pipelineChanged { + cfgParserID := kb.ParserID + if parserIDProvided { + cfgParserID = parserID + } + parserConfigMap := common.GetParserConfig(cfgParserID, nil) + if upPipelineID, ok := updates["pipeline_id"].(string); ok && upPipelineID != "" { + canvas, err := dao.NewUserCanvasDAO().GetByID(upPipelineID) + if err != nil || canvas == nil { + return nil, common.CodeDataError, fmt.Errorf("pipeline %s not found", upPipelineID) + } + pipelineDefaults := common.ExtractPipelineDefaults(canvas.DSL) + if pipelineDefaults != nil { + parserConfigMap = common.DeepMergeMaps(pipelineDefaults, parserConfigMap) + } + } + if req.ParserConfig != nil && len(req.ParserConfig) > 0 { + parserConfigMap = common.DeepMergeMaps(parserConfigMap, req.ParserConfig) + } + tenant, err := d.tenantDAO.GetByID(tenantID) + if err == nil && tenant != nil { + common.InjectExtractorLLMID(parserConfigMap, tenant.LLMID) + } + if err := validateDatasetParserConfigSize(parserConfigMap); err != nil { + return nil, common.CodeDataError, err + } + updates["parser_config"] = entity.JSONMap(parserConfigMap) + } + if nameValue, ok := updates["name"].(string); ok && strings.ToLower(nameValue) != strings.ToLower(kb.Name) { existing, lookupErr := d.kbDAO.GetByName(nameValue, tenantID) if lookupErr != nil && !dao.IsNotFoundErr(lookupErr) { @@ -2378,57 +2435,6 @@ func normalizeDatasetUpdateExt(ext map[string]interface{}) map[string]interface{ return updates } -func normalizeDatasetUpdateParserConfig(parserConfig map[string]interface{}) map[string]interface{} { - normalized := common.DeepMergeMaps(nil, parserConfig) - parentChild, _ := normalized["parent_child"].(map[string]interface{}) - if parentChild == nil { - parentChild = map[string]interface{}{} - } - - if datasetBoolValue(parentChild["use_parent_child"]) { - childrenDelimiter, ok := parentChild["children_delimiter"] - if !ok { - childrenDelimiter = "\n" - } - normalized["children_delimiter"] = childrenDelimiter - enableChildren, ok := parentChild["use_parent_child"] - if !ok { - enableChildren = true - } - normalized["enable_children"] = enableChildren - } else { - normalized["children_delimiter"] = "" - normalized["enable_children"] = false - normalized["parent_child"] = map[string]interface{}{} - } - - if extFields, ok := normalized["ext"].(map[string]interface{}); ok { - delete(normalized, "ext") - for key, value := range extFields { - normalized[key] = value - } - } - - return normalized -} - -func datasetBoolValue(value interface{}) bool { - switch typedValue := value.(type) { - case bool: - return typedValue - case string: - return typedValue == "1" || strings.EqualFold(typedValue, "true") - case int: - return typedValue != 0 - case int64: - return typedValue != 0 - case float64: - return typedValue != 0 - default: - return false - } -} - // GetMetadataConfig gets the auto-metadata configuration for a dataset. func (d *DatasetService) GetMetadataConfig(datasetID, tenantID string) (map[string]interface{}, common.ErrorCode, error) { kb, err := d.kbDAO.GetByIDAndTenantID(datasetID, tenantID) diff --git a/internal/service/dataset_update_test.go b/internal/service/dataset_update_test.go index 1f67c3a936..aab189338c 100644 --- a/internal/service/dataset_update_test.go +++ b/internal/service/dataset_update_test.go @@ -31,6 +31,7 @@ func TestDatasetServiceUpdateDatasetUpdatesFields(t *testing.T) { db := setupDatasetUpdateTestDB(t) pushServiceDB(t, db) insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original") + insertDatasetUpdateCanvas(t, "abcdef0123456789abcdef0123456789", "tenant-1") name := " Renamed Dataset " description := "updated description" @@ -101,17 +102,11 @@ func TestDatasetServiceUpdateDatasetUpdatesFields(t *testing.T) { if persisted.PipelineID == nil || *persisted.PipelineID != strings.ToLower(pipelineID) { t.Fatalf("expected normalized pipeline id persisted, got %#v", persisted.PipelineID) } - if persisted.ParserConfig["chunk_token_num"] != float64(128) { - t.Fatalf("expected existing parser_config value preserved, got %#v", persisted.ParserConfig) + if pc, ok := persisted.ParserConfig["parent_child"].(map[string]interface{}); !ok || pc["use_parent_child"] != true { + t.Fatalf("expected parent_child preserved as nested, got %#v", persisted.ParserConfig) } - if persisted.ParserConfig["enable_children"] != true { - t.Fatalf("expected enable_children normalized, got %#v", persisted.ParserConfig) - } - if persisted.ParserConfig["children_delimiter"] != "\n" { - t.Fatalf("expected default children delimiter, got %#v", persisted.ParserConfig["children_delimiter"]) - } - if persisted.ParserConfig["delimiter"] != "\n\n" { - t.Fatalf("expected ext parser config fields flattened, got %#v", persisted.ParserConfig) + if pc, ok := persisted.ParserConfig["ext"].(map[string]interface{}); !ok || pc["delimiter"] != "\n\n" { + t.Fatalf("expected ext preserved as nested, got %#v", persisted.ParserConfig) } } @@ -361,6 +356,7 @@ func setupDatasetUpdateTestDB(t *testing.T) *gorm.DB { &entity.TenantModelProvider{}, &entity.TenantModelInstance{}, &entity.TenantModel{}, + &entity.UserCanvas{}, ); err != nil { t.Fatalf("failed to migrate dataset update tables: %v", err) } @@ -396,6 +392,18 @@ func insertDatasetUpdateKB(t *testing.T, id, tenantID, name string) { } } +func insertDatasetUpdateCanvas(t *testing.T, id, userID string) { + t.Helper() + canvas := &entity.UserCanvas{ + ID: id, + UserID: userID, + DSL: entity.JSONMap{}, + } + if err := dao.DB.Create(canvas).Error; err != nil { + t.Fatalf("insert test canvas: %v", err) + } +} + func insertDatasetUpdateConnector(t *testing.T, id, tenantID string) { t.Helper() diff --git a/internal/service/model_service.go b/internal/service/model_service.go index 6ded690a9d..92a8c0d5b3 100644 --- a/internal/service/model_service.go +++ b/internal/service/model_service.go @@ -3087,6 +3087,15 @@ func (m *ModelProviderService) ResolveModelID(tenantID string, modelType entity. return "", err } + // Builtin provider is a local service (TEI), not a tenant-enrolled + // provider. There is no row in tenant_model_provider for it, so skip + // database lookups. Mirrors GetModelConfigFromProviderInstance's + // Builtin short-circuit and Python's resolve_model_id which returns + // None for Builtin TEI embeddings. + if modelType == entity.ModelTypeEmbedding && providerName == "Builtin" { + return "", nil + } + provider, err := m.modelProviderDAO.GetByTenantIDAndProviderName(tenantID, providerName) if err != nil { return "", fmt.Errorf("provider %q lookup failed: %w", providerName, err) diff --git a/rag/flow/tests/dsl_examples/title_chunker.json b/rag/flow/tests/dsl_examples/title_chunker.json index e5a3be9f86..046755703b 100644 --- a/rag/flow/tests/dsl_examples/title_chunker.json +++ b/rag/flow/tests/dsl_examples/title_chunker.json @@ -13,44 +13,42 @@ "obj": { "component_name": "Parser", "params": { - "setups": { - "pdf": { - "parse_method": "deepdoc", - "vlm_name": "", - "lang": "Chinese", - "suffix": [ - "pdf" - ], - "output_format": "json" - }, - "spreadsheet": { - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "output_format": "html" - }, - "word": { - "suffix": [ - "doc", - "docx" - ], - "output_format": "json" - }, - "markdown": { - "suffix": [ - "md", - "markdown" - ], - "output_format": "text" - }, - "text": { - "suffix": ["txt"], - "output_format": "json" - } + "pdf": { + "parse_method": "deepdoc", + "vlm_name": "", + "lang": "Chinese", + "suffix": [ + "pdf" + ], + "output_format": "json" + }, + "spreadsheet": { + "suffix": [ + "xls", + "xlsx", + "csv" + ], + "output_format": "html" + }, + "word": { + "suffix": [ + "doc", + "docx" + ], + "output_format": "json" + }, + "markdown": { + "suffix": [ + "md", + "markdown" + ], + "output_format": "text" + }, + "text": { + "suffix": ["txt"], + "output_format": "json" } - } + } }, "downstream": ["TokenChunker:0"], "upstream": ["File"] diff --git a/test/testcases/restful_api/conftest.py b/test/testcases/restful_api/conftest.py index 77bc6ee620..7530ce1a8a 100644 --- a/test/testcases/restful_api/conftest.py +++ b/test/testcases/restful_api/conftest.py @@ -58,6 +58,7 @@ GO_ONLY_SKIPS = { "test_dataset_update_parser_config_defaults_contract", "test_dataset_update_parser_config_invalid_contract", "test_dataset_update_field_unset_and_unsupported_contract", + "test_dataset_create_parser_config_different_chunk_methods_contract", "test_dataset_create_parser_config_missing_raptor_and_graphrag", "test_dataset_create_embedding_model_contract", "test_dataset_create_embedding_model_format_contract", @@ -315,6 +316,7 @@ def ensure_parsed_document(rest_client, create_document): @pytest.hookimpl(tryfirst=True, hookwrapper=True) def pytest_runtest_protocol(item, nextitem): import time + start = time.perf_counter() yield duration = time.perf_counter() - start