From d549194562bef1ce1c37e41b9fb6fbb64a7dd172 Mon Sep 17 00:00:00 2001 From: qinling0210 <88864212+qinling0210@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:51:40 +0800 Subject: [PATCH] Implement builtin chunk method as ingestion pipeline in GO (#16822) ### Summary Implement builtin chunk mehtod as ingestion pipeline in GO --- agent/templates/ingestion_pipeline_audio.json | 275 +++++ agent/templates/ingestion_pipeline_book.json | 199 +--- agent/templates/ingestion_pipeline_email.json | 335 +++++++ .../templates/ingestion_pipeline_general.json | 122 +-- agent/templates/ingestion_pipeline_laws.json | 943 ++++++++---------- .../templates/ingestion_pipeline_manual.json | 880 +++++++--------- agent/templates/ingestion_pipeline_one.json | 115 +-- agent/templates/ingestion_pipeline_paper.json | 274 +---- .../templates/ingestion_pipeline_picture.json | 294 ++++++ .../ingestion_pipeline_presentation.json | 296 ++++++ agent/templates/ingestion_pipeline_qa.json | 372 +++++++ .../templates/ingestion_pipeline_resume.json | 943 ++++++++---------- agent/templates/ingestion_pipeline_table.json | 308 ++++++ agent/templates/ingestion_pipeline_tag.json | 308 ++++++ go.mod | 1 + go.sum | 4 + internal/agent/dsl/normalize.go | 64 +- internal/dao/canvas_template_seed.go | 9 + internal/engine/elasticsearch/chunk.go | 2 + internal/engine/elasticsearch/chunk_test.go | 32 +- internal/entity/kb.go | 1 - internal/entity/models/types.go | 1 + .../components_handler_test.go | 13 +- .../ingestion/component/chunker/common.go | 10 + internal/ingestion/component/chunker/group.go | 169 ++-- .../ingestion/component/chunker/group_test.go | 74 +- .../ingestion/component/chunker/hierarchy.go | 218 ++-- .../component/chunker/hierarchy_test.go | 106 +- internal/ingestion/component/chunker/image.go | 109 ++ internal/ingestion/component/chunker/one.go | 173 ++++ .../ingestion/component/chunker/one_test.go | 100 ++ .../component/chunker/presentation.go | 108 ++ .../component/chunker/presentation_test.go | 119 +++ internal/ingestion/component/chunker/qa.go | 313 ++++++ .../ingestion/component/chunker/qa_test.go | 169 ++++ .../component/chunker/register_test.go | 40 + internal/ingestion/component/chunker/table.go | 122 +++ .../ingestion/component/chunker/table_test.go | 82 ++ internal/ingestion/component/chunker/tag.go | 225 +++++ .../ingestion/component/chunker/tag_test.go | 135 +++ internal/ingestion/component/chunker/title.go | 287 ++++-- .../ingestion/component/chunker/title_test.go | 152 ++- internal/ingestion/component/chunker/token.go | 276 +++-- .../ingestion/component/chunker/token_test.go | 44 +- .../ingestion/component/dispatch_model.go | 300 ++++++ .../component/docx_vision_dispatch.go | 300 ++++++ internal/ingestion/component/extractor.go | 13 +- .../ingestion/component/extractor_test.go | 12 +- internal/ingestion/component/file.go | 7 +- internal/ingestion/component/file_test.go | 15 - .../component/markdown_vision_dispatch.go | 162 +++ .../ingestion/component/media_dispatch.go | 601 +++++++++++ internal/ingestion/component/parser.go | 64 +- .../ingestion/component/parser_dispatch.go | 61 +- .../component/parser_dispatch_test.go | 90 +- internal/ingestion/component/parser_test.go | 2 +- .../component/pdf_vision_dispatch.go | 565 ++++++----- .../ingestion/component/schema/chunk_types.go | 3 +- .../ingestion/component/schema/chunker.go | 6 +- internal/ingestion/component/schema/parser.go | 5 + internal/ingestion/component/tokenizer.go | 57 +- .../ingestion/component/tokenizer_test.go | 12 - internal/ingestion/pipeline/pipeline_test.go | 3 - .../pipeline/template_integration_test.go | 14 +- internal/parser/parser/audio_parser.go | 97 ++ internal/parser/parser/audio_parser_test.go | 98 ++ internal/parser/parser/csv_parser.go | 251 +++++ internal/parser/parser/docx_parser.go | 199 +++- internal/parser/parser/email_parser.go | 379 +++++++ internal/parser/parser/email_parser_test.go | 157 +++ internal/parser/parser/epub_parser.go | 311 ++++++ internal/parser/parser/factory.go | 4 +- internal/parser/parser/factory_test.go | 1 - internal/parser/parser/json_parser.go | 228 +++++ internal/parser/parser/markdown_parser.go | 315 +++++- .../parser/parser/markdown_parser_test.go | 179 ++++ .../parser/parser/office_parsers_no_cgo.go | 36 +- .../parser/office_parsers_nocgo_test.go | 2 - .../parser/parser/office_parsers_shared.go | 12 + internal/parser/parser/paddleocr.go | 433 ++++++++ internal/parser/parser/parse_result_test.go | 2 +- internal/parser/parser/parser_type.go | 18 +- internal/parser/parser/pdf_parser_mineru.go | 2 +- .../parser/pdf_parser_plaintext_nocgo.go | 34 +- internal/parser/parser/picture_parser.go | 147 +++ internal/parser/parser/picture_parser_test.go | 145 +++ internal/parser/parser/xls_parser.go | 68 +- internal/parser/parser/xls_tcadp.go | 83 ++ internal/parser/parser/xls_tcadp_test.go | 241 +++++ internal/parser/parser/xlsx_parser.go | 78 +- internal/service/chunk/chunk.go | 1 - internal/service/components_service_test.go | 6 +- internal/service/dataset.go | 1 - internal/service/document.go | 1 - internal/service/load_prompt.go | 37 +- internal/tokenizer/tokenizer.go | 8 +- internal/utility/file.go | 24 +- 97 files changed, 11427 insertions(+), 3285 deletions(-) create mode 100644 agent/templates/ingestion_pipeline_audio.json create mode 100644 agent/templates/ingestion_pipeline_email.json create mode 100644 agent/templates/ingestion_pipeline_picture.json create mode 100644 agent/templates/ingestion_pipeline_presentation.json create mode 100644 agent/templates/ingestion_pipeline_qa.json create mode 100644 agent/templates/ingestion_pipeline_table.json create mode 100644 agent/templates/ingestion_pipeline_tag.json create mode 100644 internal/ingestion/component/chunker/image.go create mode 100644 internal/ingestion/component/chunker/one.go create mode 100644 internal/ingestion/component/chunker/one_test.go create mode 100644 internal/ingestion/component/chunker/presentation.go create mode 100644 internal/ingestion/component/chunker/presentation_test.go create mode 100644 internal/ingestion/component/chunker/qa.go create mode 100644 internal/ingestion/component/chunker/qa_test.go create mode 100644 internal/ingestion/component/chunker/register_test.go create mode 100644 internal/ingestion/component/chunker/table.go create mode 100644 internal/ingestion/component/chunker/table_test.go create mode 100644 internal/ingestion/component/chunker/tag.go create mode 100644 internal/ingestion/component/chunker/tag_test.go create mode 100644 internal/ingestion/component/dispatch_model.go create mode 100644 internal/ingestion/component/docx_vision_dispatch.go create mode 100644 internal/ingestion/component/markdown_vision_dispatch.go create mode 100644 internal/ingestion/component/media_dispatch.go create mode 100644 internal/parser/parser/audio_parser.go create mode 100644 internal/parser/parser/audio_parser_test.go create mode 100644 internal/parser/parser/csv_parser.go create mode 100644 internal/parser/parser/email_parser.go create mode 100644 internal/parser/parser/email_parser_test.go create mode 100644 internal/parser/parser/epub_parser.go create mode 100644 internal/parser/parser/json_parser.go create mode 100644 internal/parser/parser/markdown_parser_test.go create mode 100644 internal/parser/parser/office_parsers_shared.go create mode 100644 internal/parser/parser/paddleocr.go create mode 100644 internal/parser/parser/picture_parser.go create mode 100644 internal/parser/parser/picture_parser_test.go create mode 100644 internal/parser/parser/xls_tcadp.go create mode 100644 internal/parser/parser/xls_tcadp_test.go diff --git a/agent/templates/ingestion_pipeline_audio.json b/agent/templates/ingestion_pipeline_audio.json new file mode 100644 index 0000000000..13cfa7be58 --- /dev/null +++ b/agent/templates/ingestion_pipeline_audio.json @@ -0,0 +1,275 @@ +{ + "id": 45, + "title": { + "en": "Audio", + "de": "Audio", + "zh": "音频" + }, + "description": { + "en": "This template transcribes audio files into text using a speech-to-text model. Supports .wav, .mp3, .aac, .flac, .ogg and other common audio formats.", + "de": "Diese Vorlage transkribiert Audiodateien mithilfe eines Speech-to-Text-Modells in Text. Unterstützt .wav, .mp3, .aac, .flac, .ogg und andere gängige Audioformate.", + "zh": "此模板使用语音转文本模型将音频文件转录为文本。支持 .wav、.mp3、.aac、.flac、.ogg 及其他常见音频格式。" + }, + "canvas_type": "Ingestion Pipeline", + "canvas_category": "dataflow_canvas", + "dsl": { + "components": { + "File": { + "downstream": [ + "Parser:SongsFillAir" + ], + "obj": { + "component_name": "File", + "params": {} + }, + "upstream": [] + }, + "Parser:SongsFillAir": { + "downstream": [ + "TokenChunker:BlueSkiesLaugh" + ], + "obj": { + "component_name": "Parser", + "params": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": { + "audio": { + "output_format": "text", + "preprocess": [ + "main_content" + ], + "suffix": [ + "wav", + "mp3", + "aac", + "flac", + "ogg" + ] + } + } + } + }, + "upstream": [ + "File" + ] + }, + "TokenChunker:BlueSkiesLaugh": { + "downstream": [ + "Tokenizer:KindEyesWatch" + ], + "obj": { + "component_name": "OneChunker", + "params": { + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + } + } + }, + "upstream": [ + "Parser:SongsFillAir" + ] + }, + "Tokenizer:KindEyesWatch": { + "downstream": [], + "obj": { + "component_name": "Tokenizer", + "params": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + } + }, + "upstream": [ + "TokenChunker:BlueSkiesLaugh" + ] + } + }, + "globals": { + "sys.history": [] + }, + "graph": { + "edges": [ + { + "id": "xy-edge__Filestart-Parser:SongsFillAirend", + "source": "File", + "sourceHandle": "start", + "target": "Parser:SongsFillAir", + "targetHandle": "end" + }, + { + "id": "xy-edge__Parser:SongsFillAirstart-TokenChunker:BlueSkiesLaughend", + "source": "Parser:SongsFillAir", + "sourceHandle": "start", + "target": "TokenChunker:BlueSkiesLaugh", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__TokenChunker:BlueSkiesLaughstart-Tokenizer:KindEyesWatchend", + "source": "TokenChunker:BlueSkiesLaugh", + "sourceHandle": "start", + "target": "Tokenizer:KindEyesWatch", + "targetHandle": "end" + } + ], + "nodes": [ + { + "data": { + "label": "File", + "name": "File" + }, + "id": "File", + "measured": { + "height": 50, + "width": 200 + }, + "position": { + "x": 50, + "y": 200 + }, + "sourcePosition": "left", + "targetPosition": "right", + "type": "beginNode" + }, + { + "data": { + "form": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": [ + { + "fileFormat": "audio", + "output_format": "text", + "preprocess": [ + "main_content" + ] + } + ] + }, + "label": "Parser", + "name": "Parser_audio_0" + }, + "dragging": false, + "id": "Parser:SongsFillAir", + "measured": { + "height": 57, + "width": 200 + }, + "position": { + "x": 316.99524094206413, + "y": 195.39629819663406 + }, + "selected": true, + "sourcePosition": "right", + "targetPosition": "left", + "type": "parserNode" + }, + { + "data": { + "form": { + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + } + }, + "label": "OneChunker", + "name": "OneChunker_audio_0" + }, + "id": "TokenChunker:BlueSkiesLaugh", + "measured": { + "height": 74, + "width": 200 + }, + "position": { + "x": 616.9952409420641, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "chunkerNode" + }, + { + "data": { + "form": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + }, + "label": "Tokenizer", + "name": "Indexer_audio_0" + }, + "id": "Tokenizer:KindEyesWatch", + "measured": { + "height": 114, + "width": 200 + }, + "position": { + "x": 916.9952409420641, + "y": 195.39629819663406 + }, + "sourcePosition": "right", + "targetPosition": "left", + "type": "tokenizerNode" + } + ] + }, + "history": [], + "messages": [], + "path": [], + "retrieval": [], + "variables": [] + }, + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAGP5JREFUeAHtW2lsXNd1Pve9N/vK4U6KIiWRWi1rceQlm7ckTp3WrVuoSNP+aIImqFMgRYKmAYoGcGC0QNog+dEATYKkaZ2mTqLYaVVbst16gzdFlhd5ky1TIimS4jYkZ1/ednvOuffNUF4ayUtRILrGiG/evOXes37nO9cAF8ev9xDw/2AcO/a5UKIwkk4PXdYtITYYS3d22jV7on/TnqfgPR7/pwI4ePBz8eHQ9qHsurGxaGZgfSyaXScsMWKAsQGEv8EMRbtD4ZgB0pWeD5Wpk7/82NiuG4/AezgseJfHQz/8YRQSx7cObb5mS65703ozlBoyLGO9Ycj1phXuEmBkDdNIGoYJ0vdRAT7eJVkTUkrwnSr+9YQwRDIWT+3H0++pAC7IAo4dvDUeivQmc4MDafBjg6F0ZtAUoVFc0DZhRLZF4uleIaAnEksJtTgPV+VL/OASaZG8TH4vfecVS6mOeUjpe65AAaBcPFmrlWY61+1bD+/heEsBvHzs4N6+wdHrUUHDOJthYYZHwpFYzgyH09IXccsKCcCJ8mR57qhJgcvnxfDi+C+e4kV5vit9p8lrbgkAlHxoGnQd34EC4X/xG1oB/h7+luvJGP4c9V0nJQ0jhMdxywjFBR7jjQkwhHAbjbsf+tLX/ur3Dxzw4ALGm7rAj7/z5bF1G/Y+HomEIr7vaSlpWfmkSV/6LmnVB6fZgKWFWegZ2EAmLUkgnucI3/NQiS4tCa3egnAkDlYkB77bENJ1wfcdfJaL8/cluoYIhyMsASldMgoSJf5LLxVfZLFKPV36AY/JhQC0qHzXsLp6t3V/at8dcODAc/AOBSBWFiqj9VopEgpl8KvBL9Uv49fjOnGCPni4kLnpccj1rBOmIaSwUCEQxitjPHvDMPBWuh//4rGBf6VM4DmTn8kPw8U4aBmu0+DvPrkOv87nVZNASTL0H13jubZgLdDvOA8UHL/Hl0Xo7Vt/Fd54QQI4xwV+8t2/2bv3qqu/PTl+YnPfwGBu844951wmOVaRADy5ND8tmvUqLMycgm17rxbJVFpS5MKFSu0JrEb8l86zDIXWGv0gA7sShtQH5A70CKnULVqCV0MqaQnlPzgH8i0SCrmWcNHaPNep1GvVx2278vmRrVdPwHkMY+2XWDz+O/i5Ev27s7C6AmiaSnv4n+s6UK2UYHV5CU48fxRisSh4dgMKK8s82UDDNGcjmDx+J+3oC0DNXlsEW4AZLEkosQiWGz5BCUhfq+/l83QX3cNHeLWh5WuwRTSSzerSx8KhxN333X5rzwULoFwsLTiOA+lMFmbPTIEwrUD3+FsBmo06BTSolooobQ9N1+aF8xo4yPPcwMbzjm2z0FzHbZ1nC5KtR2qXCobQwg4ODfXM1j2BKyq3oqULMwQGfkz6oPtZVgSPLbBCoe2XXXfzvz1x7/dzv0oArRiwff/f9T045Wd2FEuQSKYla7xchmjE4gnlurppQaJeKcpmvSKqpQKFQtYkal6ZvV5NrVZBAThqIbRMMm3TlHRdoD2pMwROWmQ6e/lYC0YJjLIBuQffod1FewbdTAKixIMC4Gs4WOPiw+EYB8ZUJnfdhu37bsOr/+xXCmD9B2/ZGBnbffLeyXHjuvFpec1VuwSmPLEwc1qOjG4F8jUMPlA78V3pnT0M0XFfNtfdJkwdxWg+tFIVtAEy2U6WB85IG7chVdAzOPkJ7Q6SZQB6hVIEcECqNCBblrF2sGBUtlVp0lTuh5mGMoNhmspo8LpMKnnLi0d+MX/JlTff9lYCYBcIi3zZrVcNI5qAJ54bh0g4JhLJJPr3EgcuisS1lVmwp/9RZOrPgShOg10v80KC+atoz84OdrMpK6WyrFUq6DZNdhf1qrWfYG2BF7CvK5AAGk5ACzsJZRzQSh0qIOuYwcHX1HPlK/HQIrMTAyOb//LFI/9xLfxvFjD+6IGlsY3XzoUS3f2TpTCsrq7I/v4BePH4Mdh71bWU11mbrteBVxcgYqJ/o48j4lszS0pbJlvCC88ehfzSAmbFkDRxUuSXpmWij5Kf4jnTEoaFvoqTDIXD6LP4wYBr4jmTXEWnT9SmxGfQ/XiPCfFkRqqAqJIGi80wCXSyxbB0CH96DsuK7o9GE8mhsZ0/GD9+35Wju25YfFMB0PCrxTuiPSNfOv6aCROnxmHTpg0IXqJoBXkIWwh6DII9PfiKCRSAK0q1EqQ7e3j5hno56LQHu/ddBc2mLTEtESTGwkahQtfzECD5lMMofuB/hP0djiW2XQOvhgDJIyDlspaFRs5Sm8C6kc2EOdTJlmuQAVjStZtCI2s4/fLTsHT2FOS6eyHbtR4zVnIkne39CV583VsKoFZcud9y7C85iR5x/LU52LHzEvYrcoPurg7UeBNsPyMlKsAyELg0KqxR5cRSZWcWBMF4H91SQCgaBdI0AR/SKgUsjRXYTFvpPsiEAYJW65L0HApuJGAPj9GKpI4VVEGw1jHC4u0W4w96BqXuaCIjzkyckc8//QzUa2UYGh6CXVfecO348Qf+YnTX9d94UwGYlcljZm3rfCSR6vuvF0/D732iCdlsFhbnpqGrKwd2oy7qdhJEHGQE51HPn9XzBGjVMlpXiOxko14TGverqo9qAtQ8uROKAi0KJ4s4QPCkLRKUoGApNGYI0qtyCZOzBBqLoO/sHqh1DoKUEDlLWgQMIRJNwsDQJplIZkW5tCpLmL4pmz106E7YfcWHvvrwXd96/Jrf/eKTbxDAzJMHVjYOXna4e/TST89MpCC/iJrv7YXXXjkBl+y5HFOgK+teWjiuRJ8FqFdWCRMSNmmLQEd1yiDhkNI+aV7qIKkQi1IdZwRJKmTIi27i8VNI4+QaoCAxwU5+D7uO9CS5N1/D9YSn86zklOI5hD1sFJoBnb39kMxkoKOzCqgMqFfLspifTw1vufyzONM3CoDd4NjDt4Xnn4ldMmh8cmE+Bzt37mCcjjldUJ1TtjtkDRdMZVBlleIJoT9T5T8VkLmWqyJWqJSKIqjogrxNEJ7/w0V19g3ByuIszx6DlaBAiZGbpUkBj9wFNY/R3OCgaZL7YCHIRRC+x6DAypZh6Ezh84cKMZQSArAmWkNNIKfAx4RdyB0wgX/g61+/KfWVrxwsv0EA85P3Tu7f9huf3nn53puWFs7G7a2b8QUWmlBJGqEYJIe3wZ3/eSUYzpJcMMPiCt9vpXBo1YwCUqkOTKMZTksYCyhFBVFZV3EquqUynVw9kmv4rGmuM6jY5CoZgwBahkNVpXRRu6h1PO8JjypN9H2KooRMVSxo+yG9k7yjAwFWKBIXHgoArRKi8QS+OhTv87LhN7UAGv9w+HDz21fvvCcdie8vF1ahq7sLJsdfgU1bt4MVXYIN1/0BrOSXwSkUGAoH+F9HL7YE8r1KpcjubIBQ0AX9m+BqrnsAllHzAnQAk6DSJKZMEhDHAUPXEHQX+jpyECpMGhQPFDJWVmLx+3XZpRAAuQJXjFJS6pWUUbAUD2Ml6dgximVN1yzYbykAHr78UTwe25/PL0Fffz+ceOkF2LFnnwiFoxBPpGShUMLUEtPL1lYQ4BIcHZ09oqOrN0CBXLzqyQnC7r2Dw4zoOG3hjz6nL8kcAvs7WRYbAPINDvIH5HRoDbLlQopOQR6AY4NiBXzIdvZjbCqiz5clCpQTc9+6jcoNIcpQGWuUYn/sIyiAg28tgBefOn500ydvKkyfmero7e2XhOtLq8syHI2BWatxPidNkSkq85dr0kGL9uKcHMA4RoqYqmihlD651geNamUAj7WgQEO+9o+0alUKiIBMEro4V190OQaJdIeCZb7CEwHsVqHDhKbjlG78wheawbuMNxPAd35x/2J+tfhEIh6TtXoFzTMMpdIqBhU0IcxFCJBkCM/VUBgBWFFBSJeFXBfohI3R3LGbktyFBSZkC+MLXSUyByDaHxEcq4dpUGzI4Pq2t8s1sxbtD2VF1DbOUWozDDIQCaW4dq1vyQqfeOnEz/a979Ibl5cWYXhoHZyZmoLL9l0JKawR5udm0ZoMoNIZjNfLUEJ14qCw8n8Psv9rILo/BMXVPGpEsmkTiyS5ipQy2zkoKuUV9lmFJBnMCAWdLSK+BMNiSp0U8VlleA7xQ5BiKQsxXGarUiUZ24wI6kiNGAML873V8xJAPl84hFNxC4WC1dvXD9WZWUJliO0NiEQiFJEFFjsyZApoMUCG8gUD2WCzvgy+4QCSoeAxMjQI1yOFFeW6AFWEbKALyVSWtUULiMbSwnXq7NcU5bnc5rpHakzA/AEDKiRIweP44TF8pnOcDDzmJYXUFFM0lsRyu0dbnEGcRum8BPC9O+7Of+SjHz6Eb7uJfIect4IcgBWy2PSSyaQsFovQnUtrCStAQmYaG7kZvIEb0FWSZHKyI9dJBKGi+FQgEwxwgFgtrA98R9pIrlKKLK4sBEGQ3UcqdITr8jXqJhqdgKQurQ3NHeAEKLZEEynI5Ho0AeNrh6Gffb7M9Rvl8xIAjQceeOTnN95wzU3jJ1+Fnt4e9vl4LIZVnInEQxgBTxl6u7Ig1nqBNjwzHCd70xVeiBEjk6FaWuoP+6xsFwQguweGNcbXya11vSZW2hyS0J0FRaW1gt45HJOekq9cgYBS0145bwGs2NV/R4taQWSSo3VNjJ+EXXsvYxNuYvVFxoYQWVdsejn4j7NyTIr87QK6PwNubItYWV4K0gSvTPm1SR4uAnAkFN5XBaVhaT7FYrmp2l4FswgCGhdjD5W6CjeoKNwGWVKTJqrLEPQnyBoIKteKs+dvAQcOPFz5+Ac/9EDYEvupiEE4yQYVQq2uFosyhIJooOkGFEagHnfxcRFb+Qk0rb0Qzu6GdLaLqzWKAxQhqPBB02f0hplBM12SfZrNnfzbc7Vve7wAn6+TshMp+Pz8GVVgGdRE8lno5DQk2MH1W7gGaWVmFggRph7VCXIlP31+WSAYE1PT/7Jr2+h+x3GRkEhBpVwhDE7YgAsaYnsC8kZq84+M3iKbpZtFpHME6o0alIrLKizrTJDr7oc80uqBfoyAViJ/9qUIUp/CUYr2RSir3APPrtu4HQI6XWPggEsFnZNV+lPwGr96KC9HOI0q1BYmqhckgMbS2UcyH7hi9tT4ycGenj44MzEOPT09YOYtNqk1QECoSVEjKIyLVy29eCwpY+sSCom0egUGJFIUPJkX0EWk0EoLgI1aIGF/kK+flWijLNAVtMalLZBBWYG7rS7HB99tYkWYh1gu07ggAXzjR/dXR3df/kgmnfpUPBaGxYV52LBpjL0VI7NU9bkmNDQi85uL4C3cAVbfH0LNjYn84ny7KlScnmgRPYbQNJmpuCUiNlrsH+F9QxH/WszEp5CwYomMJGAGurMg/TbfztUUQWteuMOL970G1JZehkbNdi9IADSOHX3627/1m9d/an5+idAV19e29n1KT6DxWWCLtZO3Q7L2TYwPMYhv+VPZxcHOCohKba+GUjp1jn2piU6pCiQFc3XyD2C14gX4SHED3JtkKUrRuiZop3F1idqnUthDuo0E7czeBYab8S9YAN//6eEj1157xWuOa491dvXB0uIcw4x0Ogm1apXkLSh+mdp3I8N/BNXpTogM/DYjv4ruIVAa0z4POma0aLFWbUDFUGu7gC/akVzZhGFoxAiKU6ECrat3iKs+9QwGSRxwXK4A6+A0ShCNhsBfOYqP+ShcsABo1Kru7el05jbs9MDyUl5G43ERQdK0UaeYEtQlKoBZ6X6wdnxGpyih2VwuYnkhqjpTCLJFp7faZYZcA1/lGu9iMSCuaFW+TLCwmdv8V3LWQK27DelhmnaaVRRAFTGLBacf/R50JXCuDXh7Ajh5evLwnp2bv7a6smxmshmUfCwogKBdoChz5D0Buu3lUbsc/VA5iAsBoNGKJ25YaxsUW+wr81Va90Wwb4AWS6CqZ2CEvijgQ91iRJp4LMjXqbVOWcl1ati2r+H1WLRZIWxjPAb1qXsgtAWz1qov3pYAEt0zz4PY/JhlmR9myFpYhjSVnrrNpRGZjsNaHPgqoq0yuS5VlrB8WrWDklxQMQeoUO8YaXeGQHEHql3OUd0hCt03kJ+IScvQcDlMLoAR37OFZ0uB+ofS0gTUC+Mw89SdcNkeE6l3Q+lgzTDOVwC33vqw+8r45J0UeWnJ2P3hOZu6gdru7a0tV4OPFLqYXysf0MAQ2qWsPtvuEzKMVc0/yWCGWnT5uUnMAkkszJAApY9ogCnrILwK+M1V2SjPy+LiKVnKj8PsM4dgKPYSZEaHeb5YIYu3JQAap06d/kUmk5lbWFwSqXQaGo06FXUt7BFoTx2vWajOeW8cOtoFPt26X99EmUBRQOwsxO25dl0iwwtqHxGavYfchtuU5A52rQjV4gIUFs9AfXUCVieegHj5CAzvG8SVp9T+hLdrATR+fNfDM1PT8w9GI2EZJsoJAxe2t1qpSleE7A68kPb2pyCXtSUCsp0F1Lm28DSRQr7N6c5zsUxuCKLHmo2asNG/KepTfnfx2K6XRL2ch+LyLCzPT4BdGofxI3eL+quHYdv7chBGeg5ot4sgJst5+xZA4+ixZ7/R298vVldWgUgR+qi1ybYg1qy39TYdHcTrcZ1mtgA0iyRaeI7JP47utMHKo56ALSmteUysNBjaNlHrlcICUuxTsDz3GjSXX4LpZw9DrHgUdr8vDqmxIdY+WEnVoQLvnDVf8D7Bf/rpfcfff8XeE81mfVuoFoFsOqWpO18FbSMoCYLgGKxN6J6W7p7wX8URtsC83gukXChoiLj8IfO37QYWZFWsAiPYmivLeimPGCMPtVJe1stnwWychlef/G/osebF9j0R6NpBmk/iKpEOt6JM8Qsh35kF0DgzffabWey6VCtVCNpTep+ADBhdJjxU8SM0yAsWpSMfd3iEXqzecuezufvMIvGeH3B5twkuvEHdnYooIGFiYQouLJxGen2S/kq0dahNPwrP3PMzWBdfgO2XhqBrW6cUUeQkDOQiMEz6nuIk8KEmvBMLYAHMLN1z6SWbC6uFclaxbkJbgNrGomCrynFKyx4orGBo+9doVDMaQSpUwpPMEhEN7vMmKBszDvp9oyrrlQI2axfFlks2yulXHgHTWwCoTMIvH30UIo0ZuHQjwMiWEHRsSkuIoc9bWHCZMZ4TdfhDEYRivvfOBfDPBw7NX3/NvjuwcLnFAFN5MKUpQxEPElo7QAiYKoRHmUwEtLZsU9wQoCBtRT7vL1SLt23hYoRHAcgGtuORB0A+YBBmThyCkDMFTz34CDSXTov1XY4Y3gxyZJsB8V7UNHaxwIgISdtnmGwxmK0z40nsW9rvXAA0Hnry6e/t2jzyx8IUMR23qe5mrOOrHQvqQl9tieO6XoW31m9SQUH9xdctMt5oyawPBjrpYKqtVQvY5q5ArrcbKnNHYfL4g3DymWfEQKoht2PVvWmTAb0bLbBSEWDNx7pR8x1od1m0KizbK3XhlB5HcmYIGr54dwTwg3+9+/m//vNPPrjNGPsEcjmoNYO5Uw51zNAEWB+UG6gNrKpdpmWjt8VqvEPkKLFBTUG+72MbmOKFFQ7J2kJZTJ58Tk4+e7dYmXoJelOOvHREyJ5eC5IdMWiEQzC1iGzzagozBFJmLr7Vr6Ewm0TfilAiJ20nI8bevxEiqXMowbcvAJr5p8/mD+ESPkGbJ6LhsAz2AQjaTqIMXtdyvt70pMyf94jrtjjjel3+MNdvJjHaF+X05CSceOGYnHr1OBZfcxLcssgkIjK7YZ9IdGSRpcxCI9cBkWxORtJZtPoENj8zEMIGaDiWQeuP4LOwp2jF8LkROPv8j9E16kjWOu+OAGiEo9GpaqUCS7OvQnrHBwG0v2tepCUrj3t+oBoj6N9YIhOgoe10oloqwXJ+DlYX5+TZmQnM5/NUyAKCLaLeoae/F0a3jkIm0yFzuOBMNiuJmosjoxSNpcCMJBDghdH6wpgeo2h9Jqc7oA8FXS6vPewJ59AtSiIer7x7AujqzhbKxVUswqriift/hD7ry3qDGxvEwgnunnPNqksAtf9HUViqiSIsZnepwyPEhg39cvPoEDJExAQJ3gtgqNY6979L5bIolYr8fKr8hCZE6LlK+Lx/GJftqu9ADRkfSVxXjoxhPICt2Bucc941AVTLzVpPTxfW3Q0sTAQ0fU2Q8mYI7oWC0gLwZmpJJi4j7AZBHIRgryBe7Hi0kQoX5rgt4Kg32kq9sUIauiGq4otKoYpfI7aBg5BiGn0K/igNtLYGot+Xnl+B3NTjjxVWxJF3TQBLM5XT1ZpdqFRrHdTiDodMlHZMtvkK9T9JcItLUTiq/tesn0KGunfZ4hA0ZlI1MrSSidoGo0Cj5t/U5lR9Sug2pWz1Q1u/k6UkkpEmAre7Pv4nf3tOFBTwDseXP79/d39P92fxQVEpiJjQqxbMTikmmDZv+gaf4C2ubJ4IynzCaLz7X+0ZI/QYUOLBpkn6rvt+ajOlghyW3o5DzzGDa00WIvUrWfZInxE13Kg37NlyYeXBr37z5y+fwzNcHBfHxXFxXBwXx8Xxaz3+Bwejx8HM3R61AAAAAElFTkSuQmCC", + "parser_ids": [ + "audio" + ] +} diff --git a/agent/templates/ingestion_pipeline_book.json b/agent/templates/ingestion_pipeline_book.json index 9ff36d0a67..973611b204 100644 --- a/agent/templates/ingestion_pipeline_book.json +++ b/agent/templates/ingestion_pipeline_book.json @@ -1,14 +1,14 @@ { "id": 29, "title": { - "en": "Book", - "de": "Buch", - "zh": "书籍" + "en": "Book", + "de": "Buch", + "zh": "书籍" }, "description": { - "en": "This template segments parsed files by book structure. Best for books, long-form manuscripts, literary works, and other documents with defined chapters and sections.", - "de": "Diese Vorlage segmentiert die geparste Datei anhand der Struktur eines Buches. Sie eignet sich für Dokumente mit klar definierten Kapiteln und Abschnitten, wie Bücher, längere Manuskripte, literarische Werke und andere kapitelbasierte Texte.", - "zh": "此模板将解析后的文件按书籍结构进行切片，适用于具有清晰章节层级的文档类型，如书籍、长篇手稿、文学作品及其他按章节组织的文本。" + "en": "This template segments parsed files by book structure. Best for books, long-form manuscripts, literary works, and other documents with defined chapters and sections.", + "de": "Diese Vorlage segmentiert die geparste Datei anhand der Struktur eines Buches. Sie eignet sich für Dokumente mit klar definierten Kapiteln und Abschnitten, wie Bücher, längere Manuskripte, literarische Werke und andere kapitelbasierte Texte.", + "zh": "此模板将解析后的文件按书籍结构进行切片，适用于具有清晰章节层级的文档类型，如书籍、长篇手稿、文学作品及其他按章节组织的文本。" }, "canvas_type": "Ingestion Pipeline", "canvas_category": "dataflow_canvas", @@ -70,26 +70,6 @@ ], "vlm": {} }, - "email": { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "output_format": "text", - "preprocess": [ - "main_content" - ], - "suffix": [ - "eml", - "msg" - ] - }, "html": { "output_format": "json", "preprocess": [ @@ -100,32 +80,6 @@ "html" ] }, - "image": { - "output_format": "text", - "parse_method": "ocr", - "preprocess": [ - "main_content" - ], - "suffix": [ - "jpg", - "jpeg", - "png", - "gif" - ] - }, - "markdown": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "md", - "markdown", - "mdx" - ], - "vlm": {} - }, "pdf": { "flatten_media_to_text": false, "output_format": "json", @@ -139,31 +93,6 @@ ], "vlm": {} }, - "slides": { - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "pptx", - "ppt" - ] - }, - "spreadsheet": { - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} - }, "text&code": { "output_format": "json", "preprocess": [ @@ -210,25 +139,25 @@ "^####[^#]" ], [ - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+(\u5206?\u7f16|\u90e8\u5206)", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u6761", - "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]" + "第[零一二三四五六七八九十百0-9]+(分?编|部分)", + "第[零一二三四五六七八九十百0-9]+章", + "第[零一二三四五六七八九十百0-9]+节", + "第[零一二三四五六七八九十百0-9]+条", + "[\\(（][零一二三四五六七八九十百]+[\\)）]" ], [ - "\u7b2c[0-9]+\u7ae0", - "\u7b2c[0-9]+\u8282", - "[0-9]{1,2}[\\. \u3001]", + "第[0-9]+章", + "第[0-9]+节", + "[0-9]{1,2}[\\. 、]", "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])", "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}" ], [ - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282", - "[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[ \u3001]", - "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]", - "[\\(\uff08][0-9]{,2}[\\)\uff09]" + "第[零一二三四五六七八九十百0-9]+章", + "第[零一二三四五六七八九十百0-9]+节", + "[零一二三四五六七八九十百]+[ 、]", + "[\\(（][零一二三四五六七八九十百]+[\\)）]", + "[\\(（][0-9]{,2}[\\)）]" ], [ "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)", @@ -342,51 +271,6 @@ "remove_toc": true, "vlm": {} }, - { - "fileFormat": "spreadsheet", - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "vlm": {} - }, - { - "fileFormat": "image", - "output_format": "text", - "parse_method": "ocr", - "preprocess": [ - "main_content" - ] - }, - { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "fileFormat": "email", - "output_format": "text", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "markdown", - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "remove_toc": true, - "vlm": {} - }, { "fileFormat": "text&code", "output_format": "json", @@ -418,14 +302,6 @@ ], "remove_toc": true, "vlm": {} - }, - { - "fileFormat": "slides", - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ] } ] }, @@ -479,32 +355,32 @@ { "levels": [ { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+(\u5206?\u7f16|\u90e8\u5206)" + "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)" }, { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0" + "expression": "第[零一二三四五六七八九十百0-9]+章" }, { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282" + "expression": "第[零一二三四五六七八九十百0-9]+节" }, { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u6761" + "expression": "第[零一二三四五六七八九十百0-9]+条" }, { - "expression": "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]" + "expression": "[\\(（][零一二三四五六七八九十百]+[\\)）]" } ] }, { "levels": [ { - "expression": "\u7b2c[0-9]+\u7ae0" + "expression": "第[0-9]+章" }, { - "expression": "\u7b2c[0-9]+\u8282" + "expression": "第[0-9]+节" }, { - "expression": "[0-9]{1,2}[\\. \u3001]" + "expression": "[0-9]{1,2}[\\. 、]" }, { "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])" @@ -517,19 +393,19 @@ { "levels": [ { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0" + "expression": "第[零一二三四五六七八九十百0-9]+章" }, { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282" + "expression": "第[零一二三四五六七八九十百0-9]+节" }, { - "expression": "[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[ \u3001]" + "expression": "[零一二三四五六七八九十百]+[ 、]" }, { - "expression": "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]" + "expression": "[\\(（][零一二三四五六七八九十百]+[\\)）]" }, { - "expression": "[\\(\uff08][0-9]{,2}[\\)\uff09]" + "expression": "[\\(（][0-9]{,2}[\\)）]" } ] }, @@ -549,7 +425,8 @@ } ] } - ] + ], + "setups": [] }, "label": "TitleChunker", "name": "Title Chunker_0" @@ -577,7 +454,8 @@ "search_method": [ "embedding", "full_text" - ] + ], + "setups": [] }, "label": "Tokenizer", "name": "Indexer_0" @@ -603,5 +481,8 @@ "retrieval": [], "variables": [] }, - "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAHaRJREFUeAFlW1msZFd13ecONbyq9169oSe73W7bGLttBrcHcEgwkDApiIAU8pOgSCiKFCJFCn/5iZS//PKTj0SKkKIEPpKPIIGUL0eWwEhgbDxgt216cNPz8OZX870na+19zr23Oq9dfvWq6p579rT22nufcoKf11577XTms++5NH2qdDIoJjOZzWbSynNxSSIuTSTNMn5U0iQV772UvhSPf4lL9DdeEPwhfNr84Wedc/rclfy8k4JX8DW85+eFJFjf471iPpeyLKXd6UiSZ3od/+Yaej3+9kUpZYEVnP3N/XCPRVnonrgf3scu8TLHa4J7cB3+ZGkqSZL891Sm3zl79uwlp8JL6zXcZOAyE24+nkiBm6RYOOu0JcVmsKwU4Qa6NG8iJhyvwTZUKG4iDUooE1cpwAUF6FvcPDc9t3vo5vA5Kt3jed7tSAbl6+d4Xwrsw/14n9nclI1rdT/O9uOCYp2uh/+SoEDcJ66RwZC8d5KkO9PSn02SIv0urh9wA1y8hCbjj6O2stRuqgZOxNW2rZ7rq/hAog8I7sT+LsNnuPngHPpW8AxaQ+L7wUISP8MbNrxJvciH16n0sLYawFbU/xLvGzvE+/o/R6vr36oQPqQc5Ln7XpKm6dfiBRTeh5vgdclaLRVefFwuLurtUfpqx7ZhXwmgm6RySlMKN6JKSFwVChKsyteiUsS5hbBZCKG4l/hauD5GoIR1ovX1KR2FXqHhYgZUb0AoZaV7KqFmgoo1VmK80O0dN2uOhRv4Skh1/yC8arislRPMUm2wjNYNipAQKvYIAkYLRyWIKaGKfQnruaD8aGFXe0dKHPFRSb7ag+orPE9crWjzAj9IKKyCDbRSBDdM1fWzsEBZCauCFBavcYNJdFld0MIk8VIrNSomgJnGpxrSa6johsuG2wbvqqzMz7pFZDUr+8ozoxIpYBk8ij9JUE7TI+P1UREJBSdwzQMC02IpXF8aN/E+IG9hm9fPFUUFXgZQZh1/D2q7hhIioruGJFzbhFyQEdclYfMh9plx8DwRV8W9GWFRKAnXWHg6BWcXMMkU6mpcwr9MU9q8qCyat9sADFcJZq5vXuBC/EWBo4tX8riGFI0bxh9VnNTr1Aq+J3cGkbh2EhShXsj7wqy+MMEVWyrh7X5JWeMGPac081ceurAvvKDJvYRlPdMeLR9SRwSb+XymHpoGJI3pxDVijO5daTUIQ2UkYWNMhwRDCpQFt43YEa0mDc/RqyKvCCDnAmZEpSluiGGQ6rgML0j8bHQry25MfWUxDzghVfrOCgMDXZTAxxcn4AGT4UhuvvuK3Ll8TsZ721g/lfWTD8v9Z56R9RMPVimRoVC5ra/Tnv0KeVlqq6jHEBAJvnge7z8ZjWVr665cvXZFWp0lefbpZ2oniQRLre8WPK7WSQDZgIEScCDmXiNg6gtqUMfQw4IZY19RFISHnx1D8Cvn3pBrb7wom8tOHn/29+Tmay/JzUsX5dbd83L+Fy/KiceekWe/8k0AZUv1q8I4c7MqxqRG9jJ4DX/zfuqOI1GBf/nKz+XOrauyt78vx088IO9e+K08ffasnH3qKXhdakAZrKsZyTUAOGKPhPeivGlSG4X70XgBm00z826GIo0AGpKR9ZHmqnvPvFx951V5+b/+WT7zlS/JfZ/6uiTjLekf/VPpvfQDaSFgxjORd869Jj/7zz351Df+WrrIGM7RfZ1RWoYNwTLgyJysDU/Hk7F8cP43cv78e7K/dyDXb9+U61cuySeefV7+8Ot/Iku9ZVXM89vbcvPWbWNxLvKHiPhSh0PwLh8VE7IZP1omIguUPGQTJUPqieYJuEiyna1b4N5LASim8sqP/0MG3UTyxFxa+sfF7V2V9Uefktmt87J78aI88ehxOff+BXnrJz+W577wDcsOuWl3PplCwB25e/eOXLt2Ve5u3ZH3z78rt27dlI3N4xoCW7sH8q0//xaum8jPf/q/8tbrv5Lf/czvQ1kzObKxKQd7h4bexmKq1BjBLgrFFK2oKDUmqf8DJIlZnnGPfy6my0oXxswQgJId7G7B7Q/15pOd67LWnsnxjRW58+6vZHDytOTrp3Qj+fpJSWa7srp7R9340UeOya/f/Ikc/s4XUDQBPKelXLhwUV5//RV5781XpYtscuLUafngymV56MMfld7Khly6fFm+/MUvy+tvvSXrm5uyvrEuLXjNG6/+XC68+4488qHHZDQeS6edB3AL7D+GVVFG8A5sNLHfwbJVyBNbIvipIsuKGHFNlwRVMuV/8dOf/oddWGl1eVXefvnH8uBmrgt02hkKoT4Kk66Uu1clXRqgQoOn7F9VxO0sdWR/a1fy1fvl2vmL8uYvfib/+E/flY996FH5JNz6I2efk+3t27K2fkKeOfsJefLxJ1QBn//MH8jGkU1589dvyxOPPS4ra2tSTA7k7ddfl9W1Tbl85Yp8+PHHFZDV0IGp+pDwmN9jZZdoUZPUwoXPlkEpyiX0JVcBs71fBpDG9Tt3r8ndm5el0+8CmLYEBYI4pIsUWirHu1IM93AFSNJ8opkmO/msgQdusL6+gsrxQObbu/KRp5+X7/zl38jjTz4l/ZVVuXP1srwKoY6trcjWtUuyc/saMCSR2XQi9x07LsePHZP3338fVV9bNo7eJ0ePHZEffP9fpQfFZlke2KBUeTsKskB4IskKmBdJWIqwyZL4vosQEDwAQms6cfo7mRzsM7co+SlLq8tTAECr05LO8kCy/pqk7SUp7/wGiplK7qAcJSde2kBFulcOND+4c1tOrA3kcH9XpqORbN25AeuuS7+/jKovkb2dLXnk9IMogadSTmfykTNnpLe8jLR3TW7vHACIE3jHZ+VthENkNwpskemV1oNgqM5nU5kAVEe4zxQKLbBegj23ey3pr/VkcGSAx6p0l9paz1RAGJSANKHW52uZloqAnP3dXfj1itbkfVxIJHetLshRR/z2RdbPMrt+TiYHe9Lq9sRRKX5fOsOZDAGkBzfwWp4qws9W+nLl+hX5OLzhyH0nZXdnWy5fuiQvvPB5abc7WDOX9tKSrK6vqaynHzgp49GTygU+/vznJGcPAMJrTyI0P8hWZxB8OptIFw2TDvbYQXi2kb4TzURV8g2CQiEw4gTNHWWPwe1rTzCMyVj0kKWdf+OX8sjHnpO7516U7uqq9FfXZHL3iuT0gO4A1meT5Ka01o5J5rCpJAd3WJVB3pf2o48JEEwOtu+oC1744Lycu3hZTj7xrLx/8beyhjj/4pe+KhmAkY2OHPdk6mWDZYaswdBN05a0u8p0ZAhQJmubwsppmsMb21B6SwbYVw62mmhOT2uLxgwZWGEERFaIxladhUkZ06jXkOKnlJm+94tX5Ppvzsuf/d3fy84wlY2dHWnf/6R0j90nwyvndOMdgFnr4efFASeICzsfnJMe8GDy2nlZ/dwL0jl1UpAwZfvmNeT8A3l8lpjQScw9iVqKG2f+ZiahK08REoz3yeEQVp+rq0/Hh3BpJ0tLfRiio/VJ2mqrQFHoiu/fUz5z/bJRD+QIjQk9SVt3VosYLpiSsvHevty8cAEXzOVH3/sX+dpffVt+/T//Jg/7l2V0+5T0HvgoImNdpns3xd18HZbLZbZzR2b9R6V3A/GM9lXn5APaaiJybzz4kHzi6Dflk5Iq8xJgRhnQXHkCawtsiDopGL/8G0IfHmzj76GMD3fw/kwGG0clXVlRL8lF6opTZKFJEmyvIeprilB9xnqZ8wZXiJcYSALj1uToqVNy/sJ5ufbby5J+/9/lj//i2/LuT38k6fkLcuIA3pARFDsyQxyORnNZeuBpKa8OZYY09+DXv9ZogFjstTpd3NNiuPRzFZDan8PatHCOPN8HAHb7x6QF69744D25/M7bcuz+U+KBD5OJxTsBrwQ1Zy7vkmtwHVfXGZrSAq0lG+X981YeXvcaWjn7mYmrCi9zGB8oe8Is1JbHXvi0bD78kL6RLvXk5ZdelEfOPIe0hYtB2ufzkcwcNNnqy2xYyo1z12WwvCIPf/WPxOE3N6ENTaCx8itYdTqdo6ga6kY78JL2UldWN1cQy0vYVK6pLsstnvur63JwuC8n8DeqM7l1547sDeEdSQfXHAmuzl6Fk+lwiFDKrLkZzE3rG2nyqvQsS5Sai3phEholZZUSNQ0yJEiFCUosHo6ePq1IO5xMxE2ncvn9d2HxmcZTmmSyyVy92UGaWZZTZz4GBN/EezMZoVI0FE60re2tMwkrt5AGewpa2lxl/KPBKs0mRnDrDpTY7a/KCOA3mRayDkLEz7aQZgso9vAQr2NtVolqN3X3UoWIjdWKCYYqL2KCujnWGSFbiVjl6qgw9j+hlOz4qUfEKne4C1yXTWF6CMGkYOOQG8HrHViuRVYIDc9AV5nvZxNs7PBAekBnMsNOv1XFOi2URJAKZbIPzZUJrp9D2e0O3LRA8zVry1Gky4M9FF4rR2T77l1gzkQypMUjx9tqzTb2wIyxhPRpXSk2S0JVpxzGh9gvG1mhVoAbWRaIswzGh+e9NzZPWDrAhltAXU0ZiF+t7KjL1AYNBaq6CSx0eLArh9u3cH0pjz71jAIdNemKUis/onuMzYqENMBKFd3KDTRTUxJ3vn70pNxEddjtb2grfjoEUBJD8Ei6bMTMNBP4QIqpwA68zLm61cawillBO0oBB+gltLo1c0stgjQQ8Fq21OsHQTMwqR4LKb2YLeQpND6B+xWBfW3fvSG3Lr8DYIfVV8AN3DMqJL3G6Knx87ghX7W+6g7YGOzNu4kswaPoYTmEopWWN45YdYf/j3HfXq+rLn443IdnYlLE3gAHHNlcN94GkGq7jrQ9tQZuEadMvqxwI+JABpI250bLUDqHYijLAXqaP2ER3ihl/68wRGVJSRffunNVDuGeh+wMzYayBGJSzkhUhtJJV9QK+wcHYGjdulwVa4Ts4voEOJNqmsypadDjVeDL3Ko4bgYW7/ZWISQnUCWE78Hdx2B9M+nSrbHxKXBJlSXW19eGp/Yx0qqBmtrYq8rxzdY6Q3k2tdZ4WSmHCsitA0x6ygVcSRrqtUdINjZEfj7cu6ubYMYAvutNmF52b9+QzoMratmlfp87kyEIzXA4rIoS3qaL97hZtqJyWIth0tFqz/h94lMV7gg6QuQAHRiCbTnKQc8jRjHLEFh91Qj1AMmWFTfYT+xqF4WFYKtKh6UNebK0JgmNn4RDSB1/uZplucATXepQsKwg1rqWtkBLEzw8y0Esur91OwwrBBabyPUbN+Cyh4rgdDtaWZmft7kfUwpDi4XPFCmzKGoL0d1PAJAPd7fVM9Rdnc0Fh/CiFhUWen/RsrNQFE1wb6vvy+o97xf5Qkyb9ICkosjECB9me67Bq52EuAFhGaxBCQMFrk4PmaC1pAoinZ2AspZza3kxzXWB0G14yQQMj+/zeuII05uxNVF+zyyQhBSWMaenqVqzt3YE8T+ClyRokfUQAiYgNcwQmKEQYgebIzwdmobhahQ0DnlU6Y1MYJZOwvSoDlG3OMSp62vt+OaZWryN1Le0OlAKzFF0C9WcLVIqlR0jK5BzsMBhCT3GRtuoIMuAI3RJEh9qieHGEriPkGjBnZOYw8NjCYruL/XVg2gUdppyfIb0mR6mFbyrJ1G0pI7pvTVb1TDeMkQ1zWp4RGSJsa+oHmGIGVsNUnmAjpBTVm0txPcy4m/JhotaumUSBvTgAzvmmvhzOp7qbN+GFKIze1LdFTRImL/zyN68/38uy9eoqLQ3gLAjpcv0Cgo2xd823bXOMpuZaWbWZCbh5ZlmAgtxeqFlhqTCAQ2DMO+sxNTxehwxeFkYeKgXZKaEFjfPzYhliyxt6+Kkl4f720GbiR6ooDUYZ1TEAHXG6uqgMRMoq1kA0+EUVm1aiLn61CMflm10jPPUuEIbayY6qxhrbaDKVdArQjiEgYqTamijHoPw8VXrK+JAHphkbCEhCzgXZ+dOmjO9QLTDBdhIpwd039V29XwKlliCqeHfASpDH0Y13X4PjBEpch4tbCN3bnwIgWmVFPmYXeg+wDXTQYwVTdqJgusvofIkMaW70lqM/TYzR2lF1QhZhhkjTqliie0CO6zAr6wxIXoAP8OKlYwyltJJrSG/iAHBC3wAC1JhYoJOdRAWZRg+lrMCDHE/DC706tCDtDXpwgf7B+qWyyvL6hF8PgWPOAB3YMosYC0qiu7dAUdgt9dDYHqfeh7baHiQDTKdsdhieFH4PLbj5/PAASxzRFe/95hNFo7exAq2UoCv2sZugcwkIS22gMpttMJ0QkuNp3lAE0Hq2qqySB3j9sO47kNwovnu7g6o9L6680S7PbYZtt+U3CiYovNz9IQ2YXkfhgsptg903Bop88rCTKfO1QUSX4suXpa1RzcVII09JrKQFu7l7qKaJNrytEi727fKDjulN3id9pYy3NvRBeOpjxncltbd293DgOQuGqI7umkC1nQy184w8YLVI1OZBNCMwHT0/tOys7WlXmeVJEKgmFUjNi1pEwPuJAxOtPdQSmiVWR3QVEARZphpltYZTz0gWDyewmhqSxXFvJpY4ZKjhe1dEmIus0EntH6wc1eHJz4MJHYh+BhW9lpTpHVF6K0xQo+aK4e3vr0yNfYGQp2/unFMGSDDogtMUQ6B/1NxsUr1pVRT7Mj96zyfBLaaVyBY4wCrQ3td02BUvY//7mk3RWaoYQBOwM6QtpPzjpappc7/JoYDYriRs6VOOgs3Z6zyeaYbL1QxsSokUYp01sfDS2yg9NAfYPlNPpDY3HKsjdKZZo9Ee4rTRsVpjQ4qj0Lz0SzKFo/fiAJx7A4nek5noaV1r/CuopBE5lZAYFvAFMNOzQjtcs3PqR1/46OD4oiCE8npjh2QqB5SKtvZS3h4PZRlzE7z/XSqSiJhakEJxWys1/VQsDFFsmnKTVKpXJf7obBtkCoKb2yRTZy59R7L+nicYULgA+wThnGa6c83g36xilItqcvz3FAbBKVbu1LWlug/u6gWoycR7WMokR6TD/SXjf2xj8+gO9xnmIzA80fqFdy8hJYWN3cMc8kZGiLkE4fIFKJVqo3W4/Gc+XSmIEihJ5Np1XAhRzAgdNI8e1ON1EItoM9jHVDx0dhPCGPneMSMrmhU2Op0ZVFQgMYj4noMILQDC07dd7CxBga4oi7ODbJKPNjfR3WJpgqez1gMsYXmC1UI+3jRhXn7ARo1Q7BMWpc9xTjaJiWmv5u3zMMasetUR3Vcy7rCrlLAQpdIlApbw8I65pGWmvASwoN+ooNIuh17BnBRthTolvPSgHCK9lih3SAXj2fpjTgVYupjZkgDDpAY1T8Rd2wsF4XpLC1jzp/oWC2CWBJKX55eIXcgs1PBilI9ix7GMlhBlhkGCtICKhz+XOgP5PHEaBIAoWwIf2861BrArJshT+eIZUVdzQi5QRquPcDo3E5wKGjoBngTYkEXlaTW+HidG+1gw8QBBUimRE6COCgBcNJjOLEarB9FQ2bfKjkAIitFCmSoz5J6bK1vCD2bGYaMETZUchFosmJU4xheHQappUGepuBJjDK6iPgqlqPW9DctqsVRrjigDCCkSKWhcOURZggVlLBAgaCrKKcZx7T+YA0DS2aSlh3H2d/bkz08VOjp7B7PcBhwnsCgZE8/y/cZrhyGEuDIBi3OvWYbOwBlhR09wtJ6suD6CwqgQfXwVlEuxEh8NF3Gig07eWX0tFMxLy91jT1GJoiHEgyCDFxix2YI1x3hsQNiNERY0OpZYlmCvEALpXBSlVeuHbsfXjOC4tZRkfZUOFaGbLToabbCCiJeU1F374NwYj1C4lbIXIsKMK/KpHGeJgJN80N1FFj8pSBA7AmQFKnqUNiM0DjVzaFzQyuy7VViBxNsdgq3p1takyXVtXO4v7XFM7VmRo/wkYnUXkjmiVigScUOqRLo4N4KgGWI8ZmSKJ0ka7md2Cwi4EXEmfjTbJwQBxLnw+lLaZ7q9FU4uEaNoBlBO0Utu6n4Cuy08MGgZApCVIsiOttjJedCQ4Q/U6Q9up/GuhMFyOpEaOkDPRb9fM5hCMZr7CvQwqU2U0Xxx06HoArlfBKhRbbK7q+xXGvaxJFwkxBRWexUjQ9HkoRqqNYQhS7tFGb8TkAVlQQ9Mj0SFeBAKPiAAx1zL8/8vmVdVwkzfcRuprMDq9NVEVrhFSGN2RBFByYclpbhHJC3Cc7J0x9GrbGnwhI82U7nPIBtcZ4TILGiyyeyWNEu0HlvBy/JGSYYuR3uHCLFjrQ+SRIXy8ZQ/ftwFk8Cd2+0ljQw6QEZW2VL5nLKB8KhJnx2iIovekwHlJgbjEduY6WmgId8rlxdzyHPNXRMERZuLHuZHVYBhIcA18FgIKvoLDFr5ElS7c039haTVxTeGqojFGWHSMcHEB6hM7MOUpQrK1zM+SF/8/UimKBUiLOjZq5unDIM6AFaealLWsubH+H0iIzN+nepAlY8Fapj8dQ21l7u6exRT5iFLg0rRJ1Vuhp7OIApg6tp8USFzaeB4hIYgP6pnXCl35FVzjGY1RMu8KA0HKKy+YGrBiLRo7PIhzXNeVcRoOj5dlg6hECF8Jm1ycEEPfv2Wg9wHbo5CqMRBia9ZfUQncBCKA5SGPdsd9PydL9W1lIw6610NMYra4qdWnc6uYEXrW4AWw6UjutgZF6EapG9AjwvEM/8atDM2nHqlXkWPMpXHh5JLgkbs5nJElE/urjE9BVOXdBtI5Hw4csNeoNcByVl9B6WxyQfjGXkbgme0kUhk4YvXk1hcRZEa3DnjfV1dIj7oMvLeuLM7tvs61vzk8B34sHHjOvD03JkhnZvHWEHLlKSWULI0rytg/0QX7RQYqtMqz1z7MSHw9tlnKyEEBCpNWOaimfu6iKp5gLhs84YobXIdd6Bm7bsibBvtysrcr9+jhSVrI+ZxhhZGXRt/2LGcQ3OoVhBPuIQwwdDCDyQzVNnai90uTVpsjx0dl29f66RhG+kBFmZUzJrGmpFyTPIIVWQB9jF6sJBE7E0FjvdX/f4GlWjTlbzrs710pKBnekJEi6+v3VLjj90RkNKO0fVCS0WPmnjixalWk+/ZhaEZ8NyMpqEM0OljsV5eTvv6m6SUJjFQ5Fp+DKULDR0w/7jfQM1N2WTWsSv0vC8Y2laLSPIBatYpyipNuYNnnWvdhJTlGFlCQR0yOfwhtF4h0nCQAi5nQRHGt5VtU0Tsw7H8PNQAnvl8KWmWCu8OEgtFAMkjLXjCc/YzIxKiF+VkaaNtFli2FQ6CynX8JQkeEjmG2WvxHiQ2hP06E0j3ZRxCXRu2WLmhksUJdwcraQ1O8Ngfxejrk31lAiqHL0XGoPhyy+OVLQVWmwMK18ff6W1eBaY13vrEbiybnB61/gSlEjIHMFYZIV5LIN9ALtElWCd44BnvGcS48bVM7bYI4zfzpRQErvgVj7EIqdG7PGzJVbOEw2JeTlEEdTVdNRBzi2SCKhm1TR1oWWVVkAZv3cYLSM65LBBieFOI/YaP77aiVRhEDmHfZ0mhEWj1kmdC9a1NbLK5RdXltrQsedmjKD6GgobmdLWgqW/cdQIDgoXuv8yhpwGueGoWhhT6RcZysYBRgqdpqFhqma0UFSiMtdY1z0EAGs2bF1DtGr3rv7eQF3ihNAOoO6qs4bWNqcCdiDToEL50G/PEjtpxRG5D51jbYwQA4LrsUMkPP3ZMp6uXHyVfcNu6Bj7Ki+XLtw1CBu/A+QaCBElid8NSCowdtV3kKqPBs+ovsvqazPGEJFAjnxIX83iLvYsOOX8Ff76bJ3j4uZcYzGpqG78JqZZhRo1JqfTV1SIPGil4RE4g1TgagImMayigOKlWYvoidZwrQ8XmaC1m0t04wVpZDGMYv3fUEos3CKeoVD7YVKkxbegmp1qgQbqa+ir4GGRxM4FaFgkIX45KYYbz3g+DwpQF0tqBYbjuZo9XMCWeB+iuwscumq8BKXp7yQJ4dD4LmKjxPUNMKsKgbBGcz1XBYz96MdTt+OS1t8mZ86cuQSGdRYW/mEkPorEYsJX7K80aqpVYrhhWTFHY0PaZvLREnWsukZTJXx7r0LvUmp3XvgGmqvHdHHmp3zFRUNJdbihiWC6pvcLFLgCCqffJ9rBXV4qfHr2zJmHLv0fmt93XiHDmvoAAAAASUVORK5CYII=" + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAHaRJREFUeAFlW1msZFd13ecONbyq9169oSe73W7bGLttBrcHcEgwkDApiIAU8pOgSCiKFCJFCn/5iZS//PKTj0SKkKIEPpKPIIGUL0eWwEhgbDxgt216cNPz8OZX870na+19zr23Oq9dfvWq6p579rT22nufcoKf11577XTms++5NH2qdDIoJjOZzWbSynNxSSIuTSTNMn5U0iQV772UvhSPf4lL9DdeEPwhfNr84Wedc/rclfy8k4JX8DW85+eFJFjf471iPpeyLKXd6UiSZ3od/+Yaej3+9kUpZYEVnP3N/XCPRVnonrgf3scu8TLHa4J7cB3+ZGkqSZL891Sm3zl79uwlp8JL6zXcZOAyE24+nkiBm6RYOOu0JcVmsKwU4Qa6NG8iJhyvwTZUKG4iDUooE1cpwAUF6FvcPDc9t3vo5vA5Kt3jed7tSAbl6+d4Xwrsw/14n9nclI1rdT/O9uOCYp2uh/+SoEDcJ66RwZC8d5KkO9PSn02SIv0urh9wA1y8hCbjj6O2stRuqgZOxNW2rZ7rq/hAog8I7sT+LsNnuPngHPpW8AxaQ+L7wUISP8MbNrxJvciH16n0sLYawFbU/xLvGzvE+/o/R6vr36oQPqQc5Ln7XpKm6dfiBRTeh5vgdclaLRVefFwuLurtUfpqx7ZhXwmgm6RySlMKN6JKSFwVChKsyteiUsS5hbBZCKG4l/hauD5GoIR1ovX1KR2FXqHhYgZUb0AoZaV7KqFmgoo1VmK80O0dN2uOhRv4Skh1/yC8arislRPMUm2wjNYNipAQKvYIAkYLRyWIKaGKfQnruaD8aGFXe0dKHPFRSb7ag+orPE9crWjzAj9IKKyCDbRSBDdM1fWzsEBZCauCFBavcYNJdFld0MIk8VIrNSomgJnGpxrSa6johsuG2wbvqqzMz7pFZDUr+8ozoxIpYBk8ij9JUE7TI+P1UREJBSdwzQMC02IpXF8aN/E+IG9hm9fPFUUFXgZQZh1/D2q7hhIioruGJFzbhFyQEdclYfMh9plx8DwRV8W9GWFRKAnXWHg6BWcXMMkU6mpcwr9MU9q8qCyat9sADFcJZq5vXuBC/EWBo4tX8riGFI0bxh9VnNTr1Aq+J3cGkbh2EhShXsj7wqy+MMEVWyrh7X5JWeMGPac081ceurAvvKDJvYRlPdMeLR9SRwSb+XymHpoGJI3pxDVijO5daTUIQ2UkYWNMhwRDCpQFt43YEa0mDc/RqyKvCCDnAmZEpSluiGGQ6rgML0j8bHQry25MfWUxDzghVfrOCgMDXZTAxxcn4AGT4UhuvvuK3Ll8TsZ721g/lfWTD8v9Z56R9RMPVimRoVC5ra/Tnv0KeVlqq6jHEBAJvnge7z8ZjWVr665cvXZFWp0lefbpZ2oniQRLre8WPK7WSQDZgIEScCDmXiNg6gtqUMfQw4IZY19RFISHnx1D8Cvn3pBrb7wom8tOHn/29+Tmay/JzUsX5dbd83L+Fy/KiceekWe/8k0AZUv1q8I4c7MqxqRG9jJ4DX/zfuqOI1GBf/nKz+XOrauyt78vx088IO9e+K08ffasnH3qKXhdakAZrKsZyTUAOGKPhPeivGlSG4X70XgBm00z826GIo0AGpKR9ZHmqnvPvFx951V5+b/+WT7zlS/JfZ/6uiTjLekf/VPpvfQDaSFgxjORd869Jj/7zz351Df+WrrIGM7RfZ1RWoYNwTLgyJysDU/Hk7F8cP43cv78e7K/dyDXb9+U61cuySeefV7+8Ot/Iku9ZVXM89vbcvPWbWNxLvKHiPhSh0PwLh8VE7IZP1omIguUPGQTJUPqieYJuEiyna1b4N5LASim8sqP/0MG3UTyxFxa+sfF7V2V9Uefktmt87J78aI88ehxOff+BXnrJz+W577wDcsOuWl3PplCwB25e/eOXLt2Ve5u3ZH3z78rt27dlI3N4xoCW7sH8q0//xaum8jPf/q/8tbrv5Lf/czvQ1kzObKxKQd7h4bexmKq1BjBLgrFFK2oKDUmqf8DJIlZnnGPfy6my0oXxswQgJId7G7B7Q/15pOd67LWnsnxjRW58+6vZHDytOTrp3Qj+fpJSWa7srp7R9340UeOya/f/Ikc/s4XUDQBPKelXLhwUV5//RV5781XpYtscuLUafngymV56MMfld7Khly6fFm+/MUvy+tvvSXrm5uyvrEuLXjNG6/+XC68+4488qHHZDQeS6edB3AL7D+GVVFG8A5sNLHfwbJVyBNbIvipIsuKGHFNlwRVMuV/8dOf/oddWGl1eVXefvnH8uBmrgt02hkKoT4Kk66Uu1clXRqgQoOn7F9VxO0sdWR/a1fy1fvl2vmL8uYvfib/+E/flY996FH5JNz6I2efk+3t27K2fkKeOfsJefLxJ1QBn//MH8jGkU1589dvyxOPPS4ra2tSTA7k7ddfl9W1Tbl85Yp8+PHHFZDV0IGp+pDwmN9jZZdoUZPUwoXPlkEpyiX0JVcBs71fBpDG9Tt3r8ndm5el0+8CmLYEBYI4pIsUWirHu1IM93AFSNJ8opkmO/msgQdusL6+gsrxQObbu/KRp5+X7/zl38jjTz4l/ZVVuXP1srwKoY6trcjWtUuyc/saMCSR2XQi9x07LsePHZP3338fVV9bNo7eJ0ePHZEffP9fpQfFZlke2KBUeTsKskB4IskKmBdJWIqwyZL4vosQEDwAQms6cfo7mRzsM7co+SlLq8tTAECr05LO8kCy/pqk7SUp7/wGiplK7qAcJSde2kBFulcOND+4c1tOrA3kcH9XpqORbN25AeuuS7+/jKovkb2dLXnk9IMogadSTmfykTNnpLe8jLR3TW7vHACIE3jHZ+VthENkNwpskemV1oNgqM5nU5kAVEe4zxQKLbBegj23ey3pr/VkcGSAx6p0l9paz1RAGJSANKHW52uZloqAnP3dXfj1itbkfVxIJHetLshRR/z2RdbPMrt+TiYHe9Lq9sRRKX5fOsOZDAGkBzfwWp4qws9W+nLl+hX5OLzhyH0nZXdnWy5fuiQvvPB5abc7WDOX9tKSrK6vqaynHzgp49GTygU+/vznJGcPAMJrTyI0P8hWZxB8OptIFw2TDvbYQXi2kb4TzURV8g2CQiEw4gTNHWWPwe1rTzCMyVj0kKWdf+OX8sjHnpO7516U7uqq9FfXZHL3iuT0gO4A1meT5Ka01o5J5rCpJAd3WJVB3pf2o48JEEwOtu+oC1744Lycu3hZTj7xrLx/8beyhjj/4pe+KhmAkY2OHPdk6mWDZYaswdBN05a0u8p0ZAhQJmubwsppmsMb21B6SwbYVw62mmhOT2uLxgwZWGEERFaIxladhUkZ06jXkOKnlJm+94tX5Ppvzsuf/d3fy84wlY2dHWnf/6R0j90nwyvndOMdgFnr4efFASeICzsfnJMe8GDy2nlZ/dwL0jl1UpAwZfvmNeT8A3l8lpjQScw9iVqKG2f+ZiahK08REoz3yeEQVp+rq0/Hh3BpJ0tLfRiio/VJ2mqrQFHoiu/fUz5z/bJRD+QIjQk9SVt3VosYLpiSsvHevty8cAEXzOVH3/sX+dpffVt+/T//Jg/7l2V0+5T0HvgoImNdpns3xd18HZbLZbZzR2b9R6V3A/GM9lXn5APaaiJybzz4kHzi6Dflk5Iq8xJgRhnQXHkCawtsiDopGL/8G0IfHmzj76GMD3fw/kwGG0clXVlRL8lF6opTZKFJEmyvIeprilB9xnqZ8wZXiJcYSALj1uToqVNy/sJ5ufbby5J+/9/lj//i2/LuT38k6fkLcuIA3pARFDsyQxyORnNZeuBpKa8OZYY09+DXv9ZogFjstTpd3NNiuPRzFZDan8PatHCOPN8HAHb7x6QF69744D25/M7bcuz+U+KBD5OJxTsBrwQ1Zy7vkmtwHVfXGZrSAq0lG+X981YeXvcaWjn7mYmrCi9zGB8oe8Is1JbHXvi0bD78kL6RLvXk5ZdelEfOPIe0hYtB2ufzkcwcNNnqy2xYyo1z12WwvCIPf/WPxOE3N6ENTaCx8itYdTqdo6ga6kY78JL2UldWN1cQy0vYVK6pLsstnvur63JwuC8n8DeqM7l1547sDeEdSQfXHAmuzl6Fk+lwiFDKrLkZzE3rG2nyqvQsS5Sai3phEholZZUSNQ0yJEiFCUosHo6ePq1IO5xMxE2ncvn9d2HxmcZTmmSyyVy92UGaWZZTZz4GBN/EezMZoVI0FE60re2tMwkrt5AGewpa2lxl/KPBKs0mRnDrDpTY7a/KCOA3mRayDkLEz7aQZgso9vAQr2NtVolqN3X3UoWIjdWKCYYqL2KCujnWGSFbiVjl6qgw9j+hlOz4qUfEKne4C1yXTWF6CMGkYOOQG8HrHViuRVYIDc9AV5nvZxNs7PBAekBnMsNOv1XFOi2URJAKZbIPzZUJrp9D2e0O3LRA8zVry1Gky4M9FF4rR2T77l1gzkQypMUjx9tqzTb2wIyxhPRpXSk2S0JVpxzGh9gvG1mhVoAbWRaIswzGh+e9NzZPWDrAhltAXU0ZiF+t7KjL1AYNBaq6CSx0eLArh9u3cH0pjz71jAIdNemKUis/onuMzYqENMBKFd3KDTRTUxJ3vn70pNxEddjtb2grfjoEUBJD8Ei6bMTMNBP4QIqpwA68zLm61cawillBO0oBB+gltLo1c0stgjQQ8Fq21OsHQTMwqR4LKb2YLeQpND6B+xWBfW3fvSG3Lr8DYIfVV8AN3DMqJL3G6Knx87ghX7W+6g7YGOzNu4kswaPoYTmEopWWN45YdYf/j3HfXq+rLn443IdnYlLE3gAHHNlcN94GkGq7jrQ9tQZuEadMvqxwI+JABpI250bLUDqHYijLAXqaP2ER3ihl/68wRGVJSRffunNVDuGeh+wMzYayBGJSzkhUhtJJV9QK+wcHYGjdulwVa4Ts4voEOJNqmsypadDjVeDL3Ko4bgYW7/ZWISQnUCWE78Hdx2B9M+nSrbHxKXBJlSXW19eGp/Yx0qqBmtrYq8rxzdY6Q3k2tdZ4WSmHCsitA0x6ygVcSRrqtUdINjZEfj7cu6ubYMYAvutNmF52b9+QzoMratmlfp87kyEIzXA4rIoS3qaL97hZtqJyWIth0tFqz/h94lMV7gg6QuQAHRiCbTnKQc8jRjHLEFh91Qj1AMmWFTfYT+xqF4WFYKtKh6UNebK0JgmNn4RDSB1/uZplucATXepQsKwg1rqWtkBLEzw8y0Esur91OwwrBBabyPUbN+Cyh4rgdDtaWZmft7kfUwpDi4XPFCmzKGoL0d1PAJAPd7fVM9Rdnc0Fh/CiFhUWen/RsrNQFE1wb6vvy+o97xf5Qkyb9ICkosjECB9me67Bq52EuAFhGaxBCQMFrk4PmaC1pAoinZ2AspZza3kxzXWB0G14yQQMj+/zeuII05uxNVF+zyyQhBSWMaenqVqzt3YE8T+ClyRokfUQAiYgNcwQmKEQYgebIzwdmobhahQ0DnlU6Y1MYJZOwvSoDlG3OMSp62vt+OaZWryN1Le0OlAKzFF0C9WcLVIqlR0jK5BzsMBhCT3GRtuoIMuAI3RJEh9qieHGEriPkGjBnZOYw8NjCYruL/XVg2gUdppyfIb0mR6mFbyrJ1G0pI7pvTVb1TDeMkQ1zWp4RGSJsa+oHmGIGVsNUnmAjpBTVm0txPcy4m/JhotaumUSBvTgAzvmmvhzOp7qbN+GFKIze1LdFTRImL/zyN68/38uy9eoqLQ3gLAjpcv0Cgo2xd823bXOMpuZaWbWZCbh5ZlmAgtxeqFlhqTCAQ2DMO+sxNTxehwxeFkYeKgXZKaEFjfPzYhliyxt6+Kkl4f720GbiR6ooDUYZ1TEAHXG6uqgMRMoq1kA0+EUVm1aiLn61CMflm10jPPUuEIbayY6qxhrbaDKVdArQjiEgYqTamijHoPw8VXrK+JAHphkbCEhCzgXZ+dOmjO9QLTDBdhIpwd039V29XwKlliCqeHfASpDH0Y13X4PjBEpch4tbCN3bnwIgWmVFPmYXeg+wDXTQYwVTdqJgusvofIkMaW70lqM/TYzR2lF1QhZhhkjTqliie0CO6zAr6wxIXoAP8OKlYwyltJJrSG/iAHBC3wAC1JhYoJOdRAWZRg+lrMCDHE/DC706tCDtDXpwgf7B+qWyyvL6hF8PgWPOAB3YMosYC0qiu7dAUdgt9dDYHqfeh7baHiQDTKdsdhieFH4PLbj5/PAASxzRFe/95hNFo7exAq2UoCv2sZugcwkIS22gMpttMJ0QkuNp3lAE0Hq2qqySB3j9sO47kNwovnu7g6o9L6680S7PbYZtt+U3CiYovNz9IQ2YXkfhgsptg903Bop88rCTKfO1QUSX4suXpa1RzcVII09JrKQFu7l7qKaJNrytEi727fKDjulN3id9pYy3NvRBeOpjxncltbd293DgOQuGqI7umkC1nQy184w8YLVI1OZBNCMwHT0/tOys7WlXmeVJEKgmFUjNi1pEwPuJAxOtPdQSmiVWR3QVEARZphpltYZTz0gWDyewmhqSxXFvJpY4ZKjhe1dEmIus0EntH6wc1eHJz4MJHYh+BhW9lpTpHVF6K0xQo+aK4e3vr0yNfYGQp2/unFMGSDDogtMUQ6B/1NxsUr1pVRT7Mj96zyfBLaaVyBY4wCrQ3td02BUvY//7mk3RWaoYQBOwM6QtpPzjpappc7/JoYDYriRs6VOOgs3Z6zyeaYbL1QxsSokUYp01sfDS2yg9NAfYPlNPpDY3HKsjdKZZo9Ee4rTRsVpjQ4qj0Lz0SzKFo/fiAJx7A4nek5noaV1r/CuopBE5lZAYFvAFMNOzQjtcs3PqR1/46OD4oiCE8npjh2QqB5SKtvZS3h4PZRlzE7z/XSqSiJhakEJxWys1/VQsDFFsmnKTVKpXJf7obBtkCoKb2yRTZy59R7L+nicYULgA+wThnGa6c83g36xilItqcvz3FAbBKVbu1LWlug/u6gWoycR7WMokR6TD/SXjf2xj8+gO9xnmIzA80fqFdy8hJYWN3cMc8kZGiLkE4fIFKJVqo3W4/Gc+XSmIEihJ5Np1XAhRzAgdNI8e1ON1EItoM9jHVDx0dhPCGPneMSMrmhU2Op0ZVFQgMYj4noMILQDC07dd7CxBga4oi7ODbJKPNjfR3WJpgqez1gMsYXmC1UI+3jRhXn7ARo1Q7BMWpc9xTjaJiWmv5u3zMMasetUR3Vcy7rCrlLAQpdIlApbw8I65pGWmvASwoN+ooNIuh17BnBRthTolvPSgHCK9lih3SAXj2fpjTgVYupjZkgDDpAY1T8Rd2wsF4XpLC1jzp/oWC2CWBJKX55eIXcgs1PBilI9ix7GMlhBlhkGCtICKhz+XOgP5PHEaBIAoWwIf2861BrArJshT+eIZUVdzQi5QRquPcDo3E5wKGjoBngTYkEXlaTW+HidG+1gw8QBBUimRE6COCgBcNJjOLEarB9FQ2bfKjkAIitFCmSoz5J6bK1vCD2bGYaMETZUchFosmJU4xheHQappUGepuBJjDK6iPgqlqPW9DctqsVRrjigDCCkSKWhcOURZggVlLBAgaCrKKcZx7T+YA0DS2aSlh3H2d/bkz08VOjp7B7PcBhwnsCgZE8/y/cZrhyGEuDIBi3OvWYbOwBlhR09wtJ6suD6CwqgQfXwVlEuxEh8NF3Gig07eWX0tFMxLy91jT1GJoiHEgyCDFxix2YI1x3hsQNiNERY0OpZYlmCvEALpXBSlVeuHbsfXjOC4tZRkfZUOFaGbLToabbCCiJeU1F374NwYj1C4lbIXIsKMK/KpHGeJgJN80N1FFj8pSBA7AmQFKnqUNiM0DjVzaFzQyuy7VViBxNsdgq3p1takyXVtXO4v7XFM7VmRo/wkYnUXkjmiVigScUOqRLo4N4KgGWI8ZmSKJ0ka7md2Cwi4EXEmfjTbJwQBxLnw+lLaZ7q9FU4uEaNoBlBO0Utu6n4Cuy08MGgZApCVIsiOttjJedCQ4Q/U6Q9up/GuhMFyOpEaOkDPRb9fM5hCMZr7CvQwqU2U0Xxx06HoArlfBKhRbbK7q+xXGvaxJFwkxBRWexUjQ9HkoRqqNYQhS7tFGb8TkAVlQQ9Mj0SFeBAKPiAAx1zL8/8vmVdVwkzfcRuprMDq9NVEVrhFSGN2RBFByYclpbhHJC3Cc7J0x9GrbGnwhI82U7nPIBtcZ4TILGiyyeyWNEu0HlvBy/JGSYYuR3uHCLFjrQ+SRIXy8ZQ/ftwFk8Cd2+0ljQw6QEZW2VL5nLKB8KhJnx2iIovekwHlJgbjEduY6WmgId8rlxdzyHPNXRMERZuLHuZHVYBhIcA18FgIKvoLDFr5ElS7c039haTVxTeGqojFGWHSMcHEB6hM7MOUpQrK1zM+SF/8/UimKBUiLOjZq5unDIM6AFaealLWsubH+H0iIzN+nepAlY8Fapj8dQ21l7u6exRT5iFLg0rRJ1Vuhp7OIApg6tp8USFzaeB4hIYgP6pnXCl35FVzjGY1RMu8KA0HKKy+YGrBiLRo7PIhzXNeVcRoOj5dlg6hECF8Jm1ycEEPfv2Wg9wHbo5CqMRBia9ZfUQncBCKA5SGPdsd9PydL9W1lIw6610NMYra4qdWnc6uYEXrW4AWw6UjutgZF6EapG9AjwvEM/8atDM2nHqlXkWPMpXHh5JLgkbs5nJElE/urjE9BVOXdBtI5Hw4csNeoNcByVl9B6WxyQfjGXkbgme0kUhk4YvXk1hcRZEa3DnjfV1dIj7oMvLeuLM7tvs61vzk8B34sHHjOvD03JkhnZvHWEHLlKSWULI0rytg/0QX7RQYqtMqz1z7MSHw9tlnKyEEBCpNWOaimfu6iKp5gLhs84YobXIdd6Bm7bsibBvtysrcr9+jhSVrI+ZxhhZGXRt/2LGcQ3OoVhBPuIQwwdDCDyQzVNnai90uTVpsjx0dl29f66RhG+kBFmZUzJrGmpFyTPIIVWQB9jF6sJBE7E0FjvdX/f4GlWjTlbzrs710pKBnekJEi6+v3VLjj90RkNKO0fVCS0WPmnjixalWk+/ZhaEZ8NyMpqEM0OljsV5eTvv6m6SUJjFQ5Fp+DKULDR0w/7jfQM1N2WTWsSv0vC8Y2laLSPIBatYpyipNuYNnnWvdhJTlGFlCQR0yOfwhtF4h0nCQAi5nQRHGt5VtU0Tsw7H8PNQAnvl8KWmWCu8OEgtFAMkjLXjCc/YzIxKiF+VkaaNtFli2FQ6CynX8JQkeEjmG2WvxHiQ2hP06E0j3ZRxCXRu2WLmhksUJdwcraQ1O8Ngfxejrk31lAiqHL0XGoPhyy+OVLQVWmwMK18ff6W1eBaY13vrEbiybnB61/gSlEjIHMFYZIV5LIN9ALtElWCd44BnvGcS48bVM7bYI4zfzpRQErvgVj7EIqdG7PGzJVbOEw2JeTlEEdTVdNRBzi2SCKhm1TR1oWWVVkAZv3cYLSM65LBBieFOI/YaP77aiVRhEDmHfZ0mhEWj1kmdC9a1NbLK5RdXltrQsedmjKD6GgobmdLWgqW/cdQIDgoXuv8yhpwGueGoWhhT6RcZysYBRgqdpqFhqma0UFSiMtdY1z0EAGs2bF1DtGr3rv7eQF3ihNAOoO6qs4bWNqcCdiDToEL50G/PEjtpxRG5D51jbYwQA4LrsUMkPP3ZMp6uXHyVfcNu6Bj7Ki+XLtw1CBu/A+QaCBElid8NSCowdtV3kKqPBs+ovsvqazPGEJFAjnxIX83iLvYsOOX8Ff76bJ3j4uZcYzGpqG78JqZZhRo1JqfTV1SIPGil4RE4g1TgagImMayigOKlWYvoidZwrQ8XmaC1m0t04wVpZDGMYv3fUEos3CKeoVD7YVKkxbegmp1qgQbqa+ir4GGRxM4FaFgkIX45KYYbz3g+DwpQF0tqBYbjuZo9XMCWeB+iuwscumq8BKXp7yQJ4dD4LmKjxPUNMKsKgbBGcz1XBYz96MdTt+OS1t8mZ86cuQSGdRYW/mEkPorEYsJX7K80aqpVYrhhWTFHY0PaZvLREnWsukZTJXx7r0LvUmp3XvgGmqvHdHHmp3zFRUNJdbihiWC6pvcLFLgCCqffJ9rBXV4qfHr2zJmHLv0fmt93XiHDmvoAAAAASUVORK5CYII=", + "parser_ids": [ + "book" + ] } diff --git a/agent/templates/ingestion_pipeline_email.json b/agent/templates/ingestion_pipeline_email.json new file mode 100644 index 0000000000..ddf2d348fc --- /dev/null +++ b/agent/templates/ingestion_pipeline_email.json @@ -0,0 +1,335 @@ +{ + "id": 44, + "title": { + "en": "Email", + "de": "E-Mail", + "zh": "邮件" + }, + "description": { + "en": "This template parses email files, extracting headers, body content, and attachments. Best for .eml and .msg files.", + "de": "Diese Vorlage analysiert E-Mail-Dateien und extrahiert Kopfzeilen, Nachrichtentext und Anhänge. Geeignet für .eml- und .msg-Dateien.", + "zh": "此模板解析电子邮件文件，提取标题、正文内容和附件。适用于 .eml 和 .msg 文件。" + }, + "canvas_type": "Ingestion Pipeline", + "canvas_category": "dataflow_canvas", + "dsl": { + "components": { + "File": { + "downstream": [ + "Parser:BirdsFlutterHigh" + ], + "obj": { + "component_name": "File", + "params": {} + }, + "upstream": [] + }, + "Parser:BirdsFlutterHigh": { + "downstream": [ + "TokenChunker:WarmBreadSmells" + ], + "obj": { + "component_name": "Parser", + "params": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": { + "email": { + "fields": [ + "from", + "to", + "cc", + "bcc", + "date", + "subject", + "body", + "attachments" + ], + "output_format": "text", + "preprocess": [ + "main_content" + ], + "suffix": [ + "eml", + "msg" + ] + } + } + } + }, + "upstream": [ + "File" + ] + }, + "TokenChunker:WarmBreadSmells": { + "downstream": [ + "Tokenizer:NiceWordsSpoken" + ], + "obj": { + "component_name": "TokenChunker", + "params": { + "children_delimiters": [], + "chunk_token_size": 512, + "delimiter_mode": "token_size", + "delimiters": [ + "\n", + "!", + "?", + "。", + "；", + "！", + "？" + ], + "image_context_size": 0, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "overlapped_percent": 0, + "table_context_size": 0 + } + }, + "upstream": [ + "Parser:BirdsFlutterHigh" + ] + }, + "Tokenizer:NiceWordsSpoken": { + "downstream": [], + "obj": { + "component_name": "Tokenizer", + "params": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + } + }, + "upstream": [ + "TokenChunker:WarmBreadSmells" + ] + } + }, + "globals": { + "sys.history": [] + }, + "graph": { + "edges": [ + { + "id": "xy-edge__Filestart-Parser:BirdsFlutterHighend", + "source": "File", + "sourceHandle": "start", + "target": "Parser:BirdsFlutterHigh", + "targetHandle": "end" + }, + { + "id": "xy-edge__Parser:BirdsFlutterHighstart-TokenChunker:WarmBreadSmellsend", + "source": "Parser:BirdsFlutterHigh", + "sourceHandle": "start", + "target": "TokenChunker:WarmBreadSmells", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__TokenChunker:WarmBreadSmellsstart-Tokenizer:NiceWordsSpokenend", + "source": "TokenChunker:WarmBreadSmells", + "sourceHandle": "start", + "target": "Tokenizer:NiceWordsSpoken", + "targetHandle": "end" + } + ], + "nodes": [ + { + "data": { + "label": "File", + "name": "File" + }, + "id": "File", + "measured": { + "height": 50, + "width": 200 + }, + "position": { + "x": 50, + "y": 200 + }, + "sourcePosition": "left", + "targetPosition": "right", + "type": "beginNode" + }, + { + "data": { + "form": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": [ + { + "fields": [ + "from", + "to", + "cc", + "bcc", + "date", + "subject", + "body", + "attachments" + ], + "fileFormat": "email", + "output_format": "text", + "preprocess": [ + "main_content" + ] + } + ] + }, + "label": "Parser", + "name": "Parser_email_0" + }, + "dragging": false, + "id": "Parser:BirdsFlutterHigh", + "measured": { + "height": 57, + "width": 200 + }, + "position": { + "x": 316.99524094206413, + "y": 195.39629819663406 + }, + "selected": true, + "sourcePosition": "right", + "targetPosition": "left", + "type": "parserNode" + }, + { + "data": { + "form": { + "children_delimiters": [], + "chunk_token_size": 512, + "delimiter_mode": "token_size", + "delimiters": [ + { + "value": "\n" + }, + { + "value": "!" + }, + { + "value": "?" + }, + { + "value": "。" + }, + { + "value": "；" + }, + { + "value": "！" + }, + { + "value": "？" + } + ], + "image_table_context_window": 0, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "overlapped_percent": 0 + }, + "label": "TokenChunker", + "name": "Token Chunker_email_0" + }, + "id": "TokenChunker:WarmBreadSmells", + "measured": { + "height": 74, + "width": 200 + }, + "position": { + "x": 616.9952409420641, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "chunkerNode" + }, + { + "data": { + "form": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + }, + "label": "Tokenizer", + "name": "Indexer_email_0" + }, + "id": "Tokenizer:NiceWordsSpoken", + "measured": { + "height": 114, + "width": 200 + }, + "position": { + "x": 916.9952409420641, + "y": 195.39629819663406 + }, + "sourcePosition": "right", + "targetPosition": "left", + "type": "tokenizerNode" + } + ] + }, + "history": [], + "messages": [], + "path": [], + "retrieval": [], + "variables": [] + }, + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAGP5JREFUeAHtW2lsXNd1Pve9N/vK4U6KIiWRWi1rceQlm7ckTp3WrVuoSNP+aIImqFMgRYKmAYoGcGC0QNog+dEATYKkaZ2mTqLYaVVbst16gzdFlhd5ky1TIimS4jYkZ1/ednvOuffNUF4ayUtRILrGiG/evOXes37nO9cAF8ev9xDw/2AcO/a5UKIwkk4PXdYtITYYS3d22jV7on/TnqfgPR7/pwI4ePBz8eHQ9qHsurGxaGZgfSyaXScsMWKAsQGEv8EMRbtD4ZgB0pWeD5Wpk7/82NiuG4/AezgseJfHQz/8YRQSx7cObb5mS65703ozlBoyLGO9Ycj1phXuEmBkDdNIGoYJ0vdRAT7eJVkTUkrwnSr+9YQwRDIWT+3H0++pAC7IAo4dvDUeivQmc4MDafBjg6F0ZtAUoVFc0DZhRLZF4uleIaAnEksJtTgPV+VL/OASaZG8TH4vfecVS6mOeUjpe65AAaBcPFmrlWY61+1bD+/heEsBvHzs4N6+wdHrUUHDOJthYYZHwpFYzgyH09IXccsKCcCJ8mR57qhJgcvnxfDi+C+e4kV5vit9p8lrbgkAlHxoGnQd34EC4X/xG1oB/h7+luvJGP4c9V0nJQ0jhMdxywjFBR7jjQkwhHAbjbsf+tLX/ur3Dxzw4ALGm7rAj7/z5bF1G/Y+HomEIr7vaSlpWfmkSV/6LmnVB6fZgKWFWegZ2EAmLUkgnucI3/NQiS4tCa3egnAkDlYkB77bENJ1wfcdfJaL8/cluoYIhyMsASldMgoSJf5LLxVfZLFKPV36AY/JhQC0qHzXsLp6t3V/at8dcODAc/AOBSBWFiqj9VopEgpl8KvBL9Uv49fjOnGCPni4kLnpccj1rBOmIaSwUCEQxitjPHvDMPBWuh//4rGBf6VM4DmTn8kPw8U4aBmu0+DvPrkOv87nVZNASTL0H13jubZgLdDvOA8UHL/Hl0Xo7Vt/Fd54QQI4xwV+8t2/2bv3qqu/PTl+YnPfwGBu844951wmOVaRADy5ND8tmvUqLMycgm17rxbJVFpS5MKFSu0JrEb8l86zDIXWGv0gA7sShtQH5A70CKnULVqCV0MqaQnlPzgH8i0SCrmWcNHaPNep1GvVx2278vmRrVdPwHkMY+2XWDz+O/i5Ev27s7C6AmiaSnv4n+s6UK2UYHV5CU48fxRisSh4dgMKK8s82UDDNGcjmDx+J+3oC0DNXlsEW4AZLEkosQiWGz5BCUhfq+/l83QX3cNHeLWh5WuwRTSSzerSx8KhxN333X5rzwULoFwsLTiOA+lMFmbPTIEwrUD3+FsBmo06BTSolooobQ9N1+aF8xo4yPPcwMbzjm2z0FzHbZ1nC5KtR2qXCobQwg4ODfXM1j2BKyq3oqULMwQGfkz6oPtZVgSPLbBCoe2XXXfzvz1x7/dzv0oArRiwff/f9T045Wd2FEuQSKYla7xchmjE4gnlurppQaJeKcpmvSKqpQKFQtYkal6ZvV5NrVZBAThqIbRMMm3TlHRdoD2pMwROWmQ6e/lYC0YJjLIBuQffod1FewbdTAKixIMC4Gs4WOPiw+EYB8ZUJnfdhu37bsOr/+xXCmD9B2/ZGBnbffLeyXHjuvFpec1VuwSmPLEwc1qOjG4F8jUMPlA78V3pnT0M0XFfNtfdJkwdxWg+tFIVtAEy2U6WB85IG7chVdAzOPkJ7Q6SZQB6hVIEcECqNCBblrF2sGBUtlVp0lTuh5mGMoNhmspo8LpMKnnLi0d+MX/JlTff9lYCYBcIi3zZrVcNI5qAJ54bh0g4JhLJJPr3EgcuisS1lVmwp/9RZOrPgShOg10v80KC+atoz84OdrMpK6WyrFUq6DZNdhf1qrWfYG2BF7CvK5AAGk5ACzsJZRzQSh0qIOuYwcHX1HPlK/HQIrMTAyOb//LFI/9xLfxvFjD+6IGlsY3XzoUS3f2TpTCsrq7I/v4BePH4Mdh71bWU11mbrteBVxcgYqJ/o48j4lszS0pbJlvCC88ehfzSAmbFkDRxUuSXpmWij5Kf4jnTEoaFvoqTDIXD6LP4wYBr4jmTXEWnT9SmxGfQ/XiPCfFkRqqAqJIGi80wCXSyxbB0CH96DsuK7o9GE8mhsZ0/GD9+35Wju25YfFMB0PCrxTuiPSNfOv6aCROnxmHTpg0IXqJoBXkIWwh6DII9PfiKCRSAK0q1EqQ7e3j5hno56LQHu/ddBc2mLTEtESTGwkahQtfzECD5lMMofuB/hP0djiW2XQOvhgDJIyDlspaFRs5Sm8C6kc2EOdTJlmuQAVjStZtCI2s4/fLTsHT2FOS6eyHbtR4zVnIkne39CV583VsKoFZcud9y7C85iR5x/LU52LHzEvYrcoPurg7UeBNsPyMlKsAyELg0KqxR5cRSZWcWBMF4H91SQCgaBdI0AR/SKgUsjRXYTFvpPsiEAYJW65L0HApuJGAPj9GKpI4VVEGw1jHC4u0W4w96BqXuaCIjzkyckc8//QzUa2UYGh6CXVfecO348Qf+YnTX9d94UwGYlcljZm3rfCSR6vuvF0/D732iCdlsFhbnpqGrKwd2oy7qdhJEHGQE51HPn9XzBGjVMlpXiOxko14TGverqo9qAtQ8uROKAi0KJ4s4QPCkLRKUoGApNGYI0qtyCZOzBBqLoO/sHqh1DoKUEDlLWgQMIRJNwsDQJplIZkW5tCpLmL4pmz106E7YfcWHvvrwXd96/Jrf/eKTbxDAzJMHVjYOXna4e/TST89MpCC/iJrv7YXXXjkBl+y5HFOgK+teWjiuRJ8FqFdWCRMSNmmLQEd1yiDhkNI+aV7qIKkQi1IdZwRJKmTIi27i8VNI4+QaoCAxwU5+D7uO9CS5N1/D9YSn86zklOI5hD1sFJoBnb39kMxkoKOzCqgMqFfLspifTw1vufyzONM3CoDd4NjDt4Xnn4ldMmh8cmE+Bzt37mCcjjldUJ1TtjtkDRdMZVBlleIJoT9T5T8VkLmWqyJWqJSKIqjogrxNEJ7/w0V19g3ByuIszx6DlaBAiZGbpUkBj9wFNY/R3OCgaZL7YCHIRRC+x6DAypZh6Ezh84cKMZQSArAmWkNNIKfAx4RdyB0wgX/g61+/KfWVrxwsv0EA85P3Tu7f9huf3nn53puWFs7G7a2b8QUWmlBJGqEYJIe3wZ3/eSUYzpJcMMPiCt9vpXBo1YwCUqkOTKMZTksYCyhFBVFZV3EquqUynVw9kmv4rGmuM6jY5CoZgwBahkNVpXRRu6h1PO8JjypN9H2KooRMVSxo+yG9k7yjAwFWKBIXHgoArRKi8QS+OhTv87LhN7UAGv9w+HDz21fvvCcdie8vF1ahq7sLJsdfgU1bt4MVXYIN1/0BrOSXwSkUGAoH+F9HL7YE8r1KpcjubIBQ0AX9m+BqrnsAllHzAnQAk6DSJKZMEhDHAUPXEHQX+jpyECpMGhQPFDJWVmLx+3XZpRAAuQJXjFJS6pWUUbAUD2Ml6dgximVN1yzYbykAHr78UTwe25/PL0Fffz+ceOkF2LFnnwiFoxBPpGShUMLUEtPL1lYQ4BIcHZ09oqOrN0CBXLzqyQnC7r2Dw4zoOG3hjz6nL8kcAvs7WRYbAPINDvIH5HRoDbLlQopOQR6AY4NiBXzIdvZjbCqiz5clCpQTc9+6jcoNIcpQGWuUYn/sIyiAg28tgBefOn500ydvKkyfmero7e2XhOtLq8syHI2BWatxPidNkSkq85dr0kGL9uKcHMA4RoqYqmihlD651geNamUAj7WgQEO+9o+0alUKiIBMEro4V190OQaJdIeCZb7CEwHsVqHDhKbjlG78wheawbuMNxPAd35x/2J+tfhEIh6TtXoFzTMMpdIqBhU0IcxFCJBkCM/VUBgBWFFBSJeFXBfohI3R3LGbktyFBSZkC+MLXSUyByDaHxEcq4dpUGzI4Pq2t8s1sxbtD2VF1DbOUWozDDIQCaW4dq1vyQqfeOnEz/a979Ibl5cWYXhoHZyZmoLL9l0JKawR5udm0ZoMoNIZjNfLUEJ14qCw8n8Psv9rILo/BMXVPGpEsmkTiyS5ipQy2zkoKuUV9lmFJBnMCAWdLSK+BMNiSp0U8VlleA7xQ5BiKQsxXGarUiUZ24wI6kiNGAML873V8xJAPl84hFNxC4WC1dvXD9WZWUJliO0NiEQiFJEFFjsyZApoMUCG8gUD2WCzvgy+4QCSoeAxMjQI1yOFFeW6AFWEbKALyVSWtUULiMbSwnXq7NcU5bnc5rpHakzA/AEDKiRIweP44TF8pnOcDDzmJYXUFFM0lsRyu0dbnEGcRum8BPC9O+7Of+SjHz6Eb7uJfIect4IcgBWy2PSSyaQsFovQnUtrCStAQmYaG7kZvIEb0FWSZHKyI9dJBKGi+FQgEwxwgFgtrA98R9pIrlKKLK4sBEGQ3UcqdITr8jXqJhqdgKQurQ3NHeAEKLZEEynI5Ho0AeNrh6Gffb7M9Rvl8xIAjQceeOTnN95wzU3jJ1+Fnt4e9vl4LIZVnInEQxgBTxl6u7Ig1nqBNjwzHCd70xVeiBEjk6FaWuoP+6xsFwQguweGNcbXya11vSZW2hyS0J0FRaW1gt45HJOekq9cgYBS0145bwGs2NV/R4taQWSSo3VNjJ+EXXsvYxNuYvVFxoYQWVdsejn4j7NyTIr87QK6PwNubItYWV4K0gSvTPm1SR4uAnAkFN5XBaVhaT7FYrmp2l4FswgCGhdjD5W6CjeoKNwGWVKTJqrLEPQnyBoIKteKs+dvAQcOPFz5+Ac/9EDYEvupiEE4yQYVQq2uFosyhIJooOkGFEagHnfxcRFb+Qk0rb0Qzu6GdLaLqzWKAxQhqPBB02f0hplBM12SfZrNnfzbc7Vve7wAn6+TshMp+Pz8GVVgGdRE8lno5DQk2MH1W7gGaWVmFggRph7VCXIlP31+WSAYE1PT/7Jr2+h+x3GRkEhBpVwhDE7YgAsaYnsC8kZq84+M3iKbpZtFpHME6o0alIrLKizrTJDr7oc80uqBfoyAViJ/9qUIUp/CUYr2RSir3APPrtu4HQI6XWPggEsFnZNV+lPwGr96KC9HOI0q1BYmqhckgMbS2UcyH7hi9tT4ycGenj44MzEOPT09YOYtNqk1QECoSVEjKIyLVy29eCwpY+sSCom0egUGJFIUPJkX0EWk0EoLgI1aIGF/kK+flWijLNAVtMalLZBBWYG7rS7HB99tYkWYh1gu07ggAXzjR/dXR3df/kgmnfpUPBaGxYV52LBpjL0VI7NU9bkmNDQi85uL4C3cAVbfH0LNjYn84ny7KlScnmgRPYbQNJmpuCUiNlrsH+F9QxH/WszEp5CwYomMJGAGurMg/TbfztUUQWteuMOL970G1JZehkbNdi9IADSOHX3627/1m9d/an5+idAV19e29n1KT6DxWWCLtZO3Q7L2TYwPMYhv+VPZxcHOCohKba+GUjp1jn2piU6pCiQFc3XyD2C14gX4SHED3JtkKUrRuiZop3F1idqnUthDuo0E7czeBYab8S9YAN//6eEj1157xWuOa491dvXB0uIcw4x0Ogm1apXkLSh+mdp3I8N/BNXpTogM/DYjv4ruIVAa0z4POma0aLFWbUDFUGu7gC/akVzZhGFoxAiKU6ECrat3iKs+9QwGSRxwXK4A6+A0ShCNhsBfOYqP+ShcsABo1Kru7el05jbs9MDyUl5G43ERQdK0UaeYEtQlKoBZ6X6wdnxGpyih2VwuYnkhqjpTCLJFp7faZYZcA1/lGu9iMSCuaFW+TLCwmdv8V3LWQK27DelhmnaaVRRAFTGLBacf/R50JXCuDXh7Ajh5evLwnp2bv7a6smxmshmUfCwogKBdoChz5D0Buu3lUbsc/VA5iAsBoNGKJ25YaxsUW+wr81Va90Wwb4AWS6CqZ2CEvijgQ91iRJp4LMjXqbVOWcl1ati2r+H1WLRZIWxjPAb1qXsgtAWz1qov3pYAEt0zz4PY/JhlmR9myFpYhjSVnrrNpRGZjsNaHPgqoq0yuS5VlrB8WrWDklxQMQeoUO8YaXeGQHEHql3OUd0hCt03kJ+IScvQcDlMLoAR37OFZ0uB+ofS0gTUC+Mw89SdcNkeE6l3Q+lgzTDOVwC33vqw+8r45J0UeWnJ2P3hOZu6gdru7a0tV4OPFLqYXysf0MAQ2qWsPtvuEzKMVc0/yWCGWnT5uUnMAkkszJAApY9ogCnrILwK+M1V2SjPy+LiKVnKj8PsM4dgKPYSZEaHeb5YIYu3JQAap06d/kUmk5lbWFwSqXQaGo06FXUt7BFoTx2vWajOeW8cOtoFPt26X99EmUBRQOwsxO25dl0iwwtqHxGavYfchtuU5A52rQjV4gIUFs9AfXUCVieegHj5CAzvG8SVp9T+hLdrATR+fNfDM1PT8w9GI2EZJsoJAxe2t1qpSleE7A68kPb2pyCXtSUCsp0F1Lm28DSRQr7N6c5zsUxuCKLHmo2asNG/KepTfnfx2K6XRL2ch+LyLCzPT4BdGofxI3eL+quHYdv7chBGeg5ot4sgJst5+xZA4+ixZ7/R298vVldWgUgR+qi1ybYg1qy39TYdHcTrcZ1mtgA0iyRaeI7JP47utMHKo56ALSmteUysNBjaNlHrlcICUuxTsDz3GjSXX4LpZw9DrHgUdr8vDqmxIdY+WEnVoQLvnDVf8D7Bf/rpfcfff8XeE81mfVuoFoFsOqWpO18FbSMoCYLgGKxN6J6W7p7wX8URtsC83gukXChoiLj8IfO37QYWZFWsAiPYmivLeimPGCMPtVJe1stnwWychlef/G/osebF9j0R6NpBmk/iKpEOt6JM8Qsh35kF0DgzffabWey6VCtVCNpTep+ADBhdJjxU8SM0yAsWpSMfd3iEXqzecuezufvMIvGeH3B5twkuvEHdnYooIGFiYQouLJxGen2S/kq0dahNPwrP3PMzWBdfgO2XhqBrW6cUUeQkDOQiMEz6nuIk8KEmvBMLYAHMLN1z6SWbC6uFclaxbkJbgNrGomCrynFKyx4orGBo+9doVDMaQSpUwpPMEhEN7vMmKBszDvp9oyrrlQI2axfFlks2yulXHgHTWwCoTMIvH30UIo0ZuHQjwMiWEHRsSkuIoc9bWHCZMZ4TdfhDEYRivvfOBfDPBw7NX3/NvjuwcLnFAFN5MKUpQxEPElo7QAiYKoRHmUwEtLZsU9wQoCBtRT7vL1SLt23hYoRHAcgGtuORB0A+YBBmThyCkDMFTz34CDSXTov1XY4Y3gxyZJsB8V7UNHaxwIgISdtnmGwxmK0z40nsW9rvXAA0Hnry6e/t2jzyx8IUMR23qe5mrOOrHQvqQl9tieO6XoW31m9SQUH9xdctMt5oyawPBjrpYKqtVQvY5q5ArrcbKnNHYfL4g3DymWfEQKoht2PVvWmTAb0bLbBSEWDNx7pR8x1od1m0KizbK3XhlB5HcmYIGr54dwTwg3+9+/m//vNPPrjNGPsEcjmoNYO5Uw51zNAEWB+UG6gNrKpdpmWjt8VqvEPkKLFBTUG+72MbmOKFFQ7J2kJZTJ58Tk4+e7dYmXoJelOOvHREyJ5eC5IdMWiEQzC1iGzzagozBFJmLr7Vr6Ewm0TfilAiJ20nI8bevxEiqXMowbcvAJr5p8/mD+ESPkGbJ6LhsAz2AQjaTqIMXtdyvt70pMyf94jrtjjjel3+MNdvJjHaF+X05CSceOGYnHr1OBZfcxLcssgkIjK7YZ9IdGSRpcxCI9cBkWxORtJZtPoENj8zEMIGaDiWQeuP4LOwp2jF8LkROPv8j9E16kjWOu+OAGiEo9GpaqUCS7OvQnrHBwG0v2tepCUrj3t+oBoj6N9YIhOgoe10oloqwXJ+DlYX5+TZmQnM5/NUyAKCLaLeoae/F0a3jkIm0yFzuOBMNiuJmosjoxSNpcCMJBDghdH6wpgeo2h9Jqc7oA8FXS6vPewJ59AtSiIer7x7AujqzhbKxVUswqriift/hD7ry3qDGxvEwgnunnPNqksAtf9HUViqiSIsZnepwyPEhg39cvPoEDJExAQJ3gtgqNY6979L5bIolYr8fKr8hCZE6LlK+Lx/GJftqu9ADRkfSVxXjoxhPICt2Bucc941AVTLzVpPTxfW3Q0sTAQ0fU2Q8mYI7oWC0gLwZmpJJi4j7AZBHIRgryBe7Hi0kQoX5rgt4Kg32kq9sUIauiGq4otKoYpfI7aBg5BiGn0K/igNtLYGot+Xnl+B3NTjjxVWxJF3TQBLM5XT1ZpdqFRrHdTiDodMlHZMtvkK9T9JcItLUTiq/tesn0KGunfZ4hA0ZlI1MrSSidoGo0Cj5t/U5lR9Sug2pWz1Q1u/k6UkkpEmAre7Pv4nf3tOFBTwDseXP79/d39P92fxQVEpiJjQqxbMTikmmDZv+gaf4C2ubJ4IynzCaLz7X+0ZI/QYUOLBpkn6rvt+ajOlghyW3o5DzzGDa00WIvUrWfZInxE13Kg37NlyYeXBr37z5y+fwzNcHBfHxXFxXBwXx8Xxaz3+Bwejx8HM3R61AAAAAElFTkSuQmCC", + "parser_ids": [ + "email" + ] +} diff --git a/agent/templates/ingestion_pipeline_general.json b/agent/templates/ingestion_pipeline_general.json index d49b70d542..1851aa45c4 100644 --- a/agent/templates/ingestion_pipeline_general.json +++ b/agent/templates/ingestion_pipeline_general.json @@ -1,14 +1,14 @@ { "id": 33, "title": { - "en": "General", - "de": "Allgemein", - "zh": "通用" + "en": "General", + "de": "Allgemein", + "zh": "通用" }, "description": { - "en": "This general-purpose template segments parsed files by token count. Ideal for unstructured documents that lack a fixed layout.", - "de": "Diese Vorlage verwendet eine allgemeine Segmentierungsstrategie und teilt die geparste Datei anhand der Token-Anzahl auf. Sie eignet sich für verschiedenste Dokumenttypen, wenn kein spezifisches Strukturmuster erforderlich ist.", - "zh": "此模板采用通用切分逻辑，按照 token 数量对解析后的文件进行切片，适用于不依赖特定结构模式的各类文档。" + "en": "This general-purpose template segments parsed files by token count. Ideal for unstructured documents that lack a fixed layout.", + "de": "Diese Vorlage verwendet eine allgemeine Segmentierungsstrategie und teilt die geparste Datei anhand der Token-Anzahl auf. Sie eignet sich für verschiedenste Dokumenttypen, wenn kein spezifisches Strukturmuster erforderlich ist.", + "zh": "此模板采用通用切分逻辑，按照 token 数量对解析后的文件进行切片，适用于不依赖特定结构模式的各类文档。" }, "canvas_type": "Ingestion Pipeline", "canvas_category": "dataflow_canvas", @@ -70,26 +70,6 @@ ], "vlm": {} }, - "email": { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "output_format": "text", - "preprocess": [ - "main_content" - ], - "suffix": [ - "eml", - "msg" - ] - }, "html": { "output_format": "json", "preprocess": [ @@ -100,19 +80,6 @@ "html" ] }, - "image": { - "output_format": "text", - "parse_method": "ocr", - "preprocess": [ - "main_content" - ], - "suffix": [ - "jpg", - "jpeg", - "png", - "gif" - ] - }, "markdown": { "flatten_media_to_text": false, "output_format": "json", @@ -138,17 +105,6 @@ ], "vlm": {} }, - "slides": { - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "pptx", - "ppt" - ] - }, "spreadsheet": { "flatten_media_to_text": false, "output_format": "html", @@ -202,7 +158,15 @@ "children_delimiters": [], "chunk_token_size": 512, "delimiter_mode": "token_size", - "delimiters": [], + "delimiters": [ + "\n", + "!", + "?", + "。", + "；", + "！", + "？" + ], "image_context_size": 0, "outputs": { "chunks": { @@ -328,31 +292,6 @@ ], "vlm": {} }, - { - "fileFormat": "image", - "output_format": "text", - "parse_method": "ocr", - "preprocess": [ - "main_content" - ] - }, - { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "fileFormat": "email", - "output_format": "text", - "preprocess": [ - "main_content" - ] - }, { "fileFormat": "markdown", "flatten_media_to_text": false, @@ -391,14 +330,6 @@ "main_content" ], "vlm": {} - }, - { - "fileFormat": "slides", - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ] } ] }, @@ -429,6 +360,24 @@ "delimiters": [ { "value": "\n" + }, + { + "value": "!" + }, + { + "value": "?" + }, + { + "value": "。" + }, + { + "value": "；" + }, + { + "value": "！" + }, + { + "value": "？" } ], "image_table_context_window": 0, @@ -492,5 +441,8 @@ "retrieval": [], "variables": [] }, - "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAGP5JREFUeAHtW2lsXNd1Pve9N/vK4U6KIiWRWi1rceQlm7ckTp3WrVuoSNP+aIImqFMgRYKmAYoGcGC0QNog+dEATYKkaZ2mTqLYaVVbst16gzdFlhd5ky1TIimS4jYkZ1/ednvOuffNUF4ayUtRILrGiG/evOXes37nO9cAF8ev9xDw/2AcO/a5UKIwkk4PXdYtITYYS3d22jV7on/TnqfgPR7/pwI4ePBz8eHQ9qHsurGxaGZgfSyaXScsMWKAsQGEv8EMRbtD4ZgB0pWeD5Wpk7/82NiuG4/AezgseJfHQz/8YRQSx7cObb5mS65703ozlBoyLGO9Ycj1phXuEmBkDdNIGoYJ0vdRAT7eJVkTUkrwnSr+9YQwRDIWT+3H0++pAC7IAo4dvDUeivQmc4MDafBjg6F0ZtAUoVFc0DZhRLZF4uleIaAnEksJtTgPV+VL/OASaZG8TH4vfecVS6mOeUjpe65AAaBcPFmrlWY61+1bD+/heEsBvHzs4N6+wdHrUUHDOJthYYZHwpFYzgyH09IXccsKCcCJ8mR57qhJgcvnxfDi+C+e4kV5vit9p8lrbgkAlHxoGnQd34EC4X/xG1oB/h7+luvJGP4c9V0nJQ0jhMdxywjFBR7jjQkwhHAbjbsf+tLX/ur3Dxzw4ALGm7rAj7/z5bF1G/Y+HomEIr7vaSlpWfmkSV/6LmnVB6fZgKWFWegZ2EAmLUkgnucI3/NQiS4tCa3egnAkDlYkB77bENJ1wfcdfJaL8/cluoYIhyMsASldMgoSJf5LLxVfZLFKPV36AY/JhQC0qHzXsLp6t3V/at8dcODAc/AOBSBWFiqj9VopEgpl8KvBL9Uv49fjOnGCPni4kLnpccj1rBOmIaSwUCEQxitjPHvDMPBWuh//4rGBf6VM4DmTn8kPw8U4aBmu0+DvPrkOv87nVZNASTL0H13jubZgLdDvOA8UHL/Hl0Xo7Vt/Fd54QQI4xwV+8t2/2bv3qqu/PTl+YnPfwGBu844951wmOVaRADy5ND8tmvUqLMycgm17rxbJVFpS5MKFSu0JrEb8l86zDIXWGv0gA7sShtQH5A70CKnULVqCV0MqaQnlPzgH8i0SCrmWcNHaPNep1GvVx2278vmRrVdPwHkMY+2XWDz+O/i5Ev27s7C6AmiaSnv4n+s6UK2UYHV5CU48fxRisSh4dgMKK8s82UDDNGcjmDx+J+3oC0DNXlsEW4AZLEkosQiWGz5BCUhfq+/l83QX3cNHeLWh5WuwRTSSzerSx8KhxN333X5rzwULoFwsLTiOA+lMFmbPTIEwrUD3+FsBmo06BTSolooobQ9N1+aF8xo4yPPcwMbzjm2z0FzHbZ1nC5KtR2qXCobQwg4ODfXM1j2BKyq3oqULMwQGfkz6oPtZVgSPLbBCoe2XXXfzvz1x7/dzv0oArRiwff/f9T045Wd2FEuQSKYla7xchmjE4gnlurppQaJeKcpmvSKqpQKFQtYkal6ZvV5NrVZBAThqIbRMMm3TlHRdoD2pMwROWmQ6e/lYC0YJjLIBuQffod1FewbdTAKixIMC4Gs4WOPiw+EYB8ZUJnfdhu37bsOr/+xXCmD9B2/ZGBnbffLeyXHjuvFpec1VuwSmPLEwc1qOjG4F8jUMPlA78V3pnT0M0XFfNtfdJkwdxWg+tFIVtAEy2U6WB85IG7chVdAzOPkJ7Q6SZQB6hVIEcECqNCBblrF2sGBUtlVp0lTuh5mGMoNhmspo8LpMKnnLi0d+MX/JlTff9lYCYBcIi3zZrVcNI5qAJ54bh0g4JhLJJPr3EgcuisS1lVmwp/9RZOrPgShOg10v80KC+atoz84OdrMpK6WyrFUq6DZNdhf1qrWfYG2BF7CvK5AAGk5ACzsJZRzQSh0qIOuYwcHX1HPlK/HQIrMTAyOb//LFI/9xLfxvFjD+6IGlsY3XzoUS3f2TpTCsrq7I/v4BePH4Mdh71bWU11mbrteBVxcgYqJ/o48j4lszS0pbJlvCC88ehfzSAmbFkDRxUuSXpmWij5Kf4jnTEoaFvoqTDIXD6LP4wYBr4jmTXEWnT9SmxGfQ/XiPCfFkRqqAqJIGi80wCXSyxbB0CH96DsuK7o9GE8mhsZ0/GD9+35Wju25YfFMB0PCrxTuiPSNfOv6aCROnxmHTpg0IXqJoBXkIWwh6DII9PfiKCRSAK0q1EqQ7e3j5hno56LQHu/ddBc2mLTEtESTGwkahQtfzECD5lMMofuB/hP0djiW2XQOvhgDJIyDlspaFRs5Sm8C6kc2EOdTJlmuQAVjStZtCI2s4/fLTsHT2FOS6eyHbtR4zVnIkne39CV583VsKoFZcud9y7C85iR5x/LU52LHzEvYrcoPurg7UeBNsPyMlKsAyELg0KqxR5cRSZWcWBMF4H91SQCgaBdI0AR/SKgUsjRXYTFvpPsiEAYJW65L0HApuJGAPj9GKpI4VVEGw1jHC4u0W4w96BqXuaCIjzkyckc8//QzUa2UYGh6CXVfecO348Qf+YnTX9d94UwGYlcljZm3rfCSR6vuvF0/D732iCdlsFhbnpqGrKwd2oy7qdhJEHGQE51HPn9XzBGjVMlpXiOxko14TGverqo9qAtQ8uROKAi0KJ4s4QPCkLRKUoGApNGYI0qtyCZOzBBqLoO/sHqh1DoKUEDlLWgQMIRJNwsDQJplIZkW5tCpLmL4pmz106E7YfcWHvvrwXd96/Jrf/eKTbxDAzJMHVjYOXna4e/TST89MpCC/iJrv7YXXXjkBl+y5HFOgK+teWjiuRJ8FqFdWCRMSNmmLQEd1yiDhkNI+aV7qIKkQi1IdZwRJKmTIi27i8VNI4+QaoCAxwU5+D7uO9CS5N1/D9YSn86zklOI5hD1sFJoBnb39kMxkoKOzCqgMqFfLspifTw1vufyzONM3CoDd4NjDt4Xnn4ldMmh8cmE+Bzt37mCcjjldUJ1TtjtkDRdMZVBlleIJoT9T5T8VkLmWqyJWqJSKIqjogrxNEJ7/w0V19g3ByuIszx6DlaBAiZGbpUkBj9wFNY/R3OCgaZL7YCHIRRC+x6DAypZh6Ezh84cKMZQSArAmWkNNIKfAx4RdyB0wgX/g61+/KfWVrxwsv0EA85P3Tu7f9huf3nn53puWFs7G7a2b8QUWmlBJGqEYJIe3wZ3/eSUYzpJcMMPiCt9vpXBo1YwCUqkOTKMZTksYCyhFBVFZV3EquqUynVw9kmv4rGmuM6jY5CoZgwBahkNVpXRRu6h1PO8JjypN9H2KooRMVSxo+yG9k7yjAwFWKBIXHgoArRKi8QS+OhTv87LhN7UAGv9w+HDz21fvvCcdie8vF1ahq7sLJsdfgU1bt4MVXYIN1/0BrOSXwSkUGAoH+F9HL7YE8r1KpcjubIBQ0AX9m+BqrnsAllHzAnQAk6DSJKZMEhDHAUPXEHQX+jpyECpMGhQPFDJWVmLx+3XZpRAAuQJXjFJS6pWUUbAUD2Ml6dgximVN1yzYbykAHr78UTwe25/PL0Fffz+ceOkF2LFnnwiFoxBPpGShUMLUEtPL1lYQ4BIcHZ09oqOrN0CBXLzqyQnC7r2Dw4zoOG3hjz6nL8kcAvs7WRYbAPINDvIH5HRoDbLlQopOQR6AY4NiBXzIdvZjbCqiz5clCpQTc9+6jcoNIcpQGWuUYn/sIyiAg28tgBefOn500ydvKkyfmero7e2XhOtLq8syHI2BWatxPidNkSkq85dr0kGL9uKcHMA4RoqYqmihlD651geNamUAj7WgQEO+9o+0alUKiIBMEro4V190OQaJdIeCZb7CEwHsVqHDhKbjlG78wheawbuMNxPAd35x/2J+tfhEIh6TtXoFzTMMpdIqBhU0IcxFCJBkCM/VUBgBWFFBSJeFXBfohI3R3LGbktyFBSZkC+MLXSUyByDaHxEcq4dpUGzI4Pq2t8s1sxbtD2VF1DbOUWozDDIQCaW4dq1vyQqfeOnEz/a979Ibl5cWYXhoHZyZmoLL9l0JKawR5udm0ZoMoNIZjNfLUEJ14qCw8n8Psv9rILo/BMXVPGpEsmkTiyS5ipQy2zkoKuUV9lmFJBnMCAWdLSK+BMNiSp0U8VlleA7xQ5BiKQsxXGarUiUZ24wI6kiNGAML873V8xJAPl84hFNxC4WC1dvXD9WZWUJliO0NiEQiFJEFFjsyZApoMUCG8gUD2WCzvgy+4QCSoeAxMjQI1yOFFeW6AFWEbKALyVSWtUULiMbSwnXq7NcU5bnc5rpHakzA/AEDKiRIweP44TF8pnOcDDzmJYXUFFM0lsRyu0dbnEGcRum8BPC9O+7Of+SjHz6Eb7uJfIect4IcgBWy2PSSyaQsFovQnUtrCStAQmYaG7kZvIEb0FWSZHKyI9dJBKGi+FQgEwxwgFgtrA98R9pIrlKKLK4sBEGQ3UcqdITr8jXqJhqdgKQurQ3NHeAEKLZEEynI5Ho0AeNrh6Gffb7M9Rvl8xIAjQceeOTnN95wzU3jJ1+Fnt4e9vl4LIZVnInEQxgBTxl6u7Ig1nqBNjwzHCd70xVeiBEjk6FaWuoP+6xsFwQguweGNcbXya11vSZW2hyS0J0FRaW1gt45HJOekq9cgYBS0145bwGs2NV/R4taQWSSo3VNjJ+EXXsvYxNuYvVFxoYQWVdsejn4j7NyTIr87QK6PwNubItYWV4K0gSvTPm1SR4uAnAkFN5XBaVhaT7FYrmp2l4FswgCGhdjD5W6CjeoKNwGWVKTJqrLEPQnyBoIKteKs+dvAQcOPFz5+Ac/9EDYEvupiEE4yQYVQq2uFosyhIJooOkGFEagHnfxcRFb+Qk0rb0Qzu6GdLaLqzWKAxQhqPBB02f0hplBM12SfZrNnfzbc7Vve7wAn6+TshMp+Pz8GVVgGdRE8lno5DQk2MH1W7gGaWVmFggRph7VCXIlP31+WSAYE1PT/7Jr2+h+x3GRkEhBpVwhDE7YgAsaYnsC8kZq84+M3iKbpZtFpHME6o0alIrLKizrTJDr7oc80uqBfoyAViJ/9qUIUp/CUYr2RSir3APPrtu4HQI6XWPggEsFnZNV+lPwGr96KC9HOI0q1BYmqhckgMbS2UcyH7hi9tT4ycGenj44MzEOPT09YOYtNqk1QECoSVEjKIyLVy29eCwpY+sSCom0egUGJFIUPJkX0EWk0EoLgI1aIGF/kK+flWijLNAVtMalLZBBWYG7rS7HB99tYkWYh1gu07ggAXzjR/dXR3df/kgmnfpUPBaGxYV52LBpjL0VI7NU9bkmNDQi85uL4C3cAVbfH0LNjYn84ny7KlScnmgRPYbQNJmpuCUiNlrsH+F9QxH/WszEp5CwYomMJGAGurMg/TbfztUUQWteuMOL970G1JZehkbNdi9IADSOHX3627/1m9d/an5+idAV19e29n1KT6DxWWCLtZO3Q7L2TYwPMYhv+VPZxcHOCohKba+GUjp1jn2piU6pCiQFc3XyD2C14gX4SHED3JtkKUrRuiZop3F1idqnUthDuo0E7czeBYab8S9YAN//6eEj1157xWuOa491dvXB0uIcw4x0Ogm1apXkLSh+mdp3I8N/BNXpTogM/DYjv4ruIVAa0z4POma0aLFWbUDFUGu7gC/akVzZhGFoxAiKU6ECrat3iKs+9QwGSRxwXK4A6+A0ShCNhsBfOYqP+ShcsABo1Kru7el05jbs9MDyUl5G43ERQdK0UaeYEtQlKoBZ6X6wdnxGpyih2VwuYnkhqjpTCLJFp7faZYZcA1/lGu9iMSCuaFW+TLCwmdv8V3LWQK27DelhmnaaVRRAFTGLBacf/R50JXCuDXh7Ajh5evLwnp2bv7a6smxmshmUfCwogKBdoChz5D0Buu3lUbsc/VA5iAsBoNGKJ25YaxsUW+wr81Va90Wwb4AWS6CqZ2CEvijgQ91iRJp4LMjXqbVOWcl1ati2r+H1WLRZIWxjPAb1qXsgtAWz1qov3pYAEt0zz4PY/JhlmR9myFpYhjSVnrrNpRGZjsNaHPgqoq0yuS5VlrB8WrWDklxQMQeoUO8YaXeGQHEHql3OUd0hCt03kJ+IScvQcDlMLoAR37OFZ0uB+ofS0gTUC+Mw89SdcNkeE6l3Q+lgzTDOVwC33vqw+8r45J0UeWnJ2P3hOZu6gdru7a0tV4OPFLqYXysf0MAQ2qWsPtvuEzKMVc0/yWCGWnT5uUnMAkkszJAApY9ogCnrILwK+M1V2SjPy+LiKVnKj8PsM4dgKPYSZEaHeb5YIYu3JQAap06d/kUmk5lbWFwSqXQaGo06FXUt7BFoTx2vWajOeW8cOtoFPt26X99EmUBRQOwsxO25dl0iwwtqHxGavYfchtuU5A52rQjV4gIUFs9AfXUCVieegHj5CAzvG8SVp9T+hLdrATR+fNfDM1PT8w9GI2EZJsoJAxe2t1qpSleE7A68kPb2pyCXtSUCsp0F1Lm28DSRQr7N6c5zsUxuCKLHmo2asNG/KepTfnfx2K6XRL2ch+LyLCzPT4BdGofxI3eL+quHYdv7chBGeg5ot4sgJst5+xZA4+ixZ7/R298vVldWgUgR+qi1ybYg1qy39TYdHcTrcZ1mtgA0iyRaeI7JP47utMHKo56ALSmteUysNBjaNlHrlcICUuxTsDz3GjSXX4LpZw9DrHgUdr8vDqmxIdY+WEnVoQLvnDVf8D7Bf/rpfcfff8XeE81mfVuoFoFsOqWpO18FbSMoCYLgGKxN6J6W7p7wX8URtsC83gukXChoiLj8IfO37QYWZFWsAiPYmivLeimPGCMPtVJe1stnwWychlef/G/osebF9j0R6NpBmk/iKpEOt6JM8Qsh35kF0DgzffabWey6VCtVCNpTep+ADBhdJjxU8SM0yAsWpSMfd3iEXqzecuezufvMIvGeH3B5twkuvEHdnYooIGFiYQouLJxGen2S/kq0dahNPwrP3PMzWBdfgO2XhqBrW6cUUeQkDOQiMEz6nuIk8KEmvBMLYAHMLN1z6SWbC6uFclaxbkJbgNrGomCrynFKyx4orGBo+9doVDMaQSpUwpPMEhEN7vMmKBszDvp9oyrrlQI2axfFlks2yulXHgHTWwCoTMIvH30UIo0ZuHQjwMiWEHRsSkuIoc9bWHCZMZ4TdfhDEYRivvfOBfDPBw7NX3/NvjuwcLnFAFN5MKUpQxEPElo7QAiYKoRHmUwEtLZsU9wQoCBtRT7vL1SLt23hYoRHAcgGtuORB0A+YBBmThyCkDMFTz34CDSXTov1XY4Y3gxyZJsB8V7UNHaxwIgISdtnmGwxmK0z40nsW9rvXAA0Hnry6e/t2jzyx8IUMR23qe5mrOOrHQvqQl9tieO6XoW31m9SQUH9xdctMt5oyawPBjrpYKqtVQvY5q5ArrcbKnNHYfL4g3DymWfEQKoht2PVvWmTAb0bLbBSEWDNx7pR8x1od1m0KizbK3XhlB5HcmYIGr54dwTwg3+9+/m//vNPPrjNGPsEcjmoNYO5Uw51zNAEWB+UG6gNrKpdpmWjt8VqvEPkKLFBTUG+72MbmOKFFQ7J2kJZTJ58Tk4+e7dYmXoJelOOvHREyJ5eC5IdMWiEQzC1iGzzagozBFJmLr7Vr6Ewm0TfilAiJ20nI8bevxEiqXMowbcvAJr5p8/mD+ESPkGbJ6LhsAz2AQjaTqIMXtdyvt70pMyf94jrtjjjel3+MNdvJjHaF+X05CSceOGYnHr1OBZfcxLcssgkIjK7YZ9IdGSRpcxCI9cBkWxORtJZtPoENj8zEMIGaDiWQeuP4LOwp2jF8LkROPv8j9E16kjWOu+OAGiEo9GpaqUCS7OvQnrHBwG0v2tepCUrj3t+oBoj6N9YIhOgoe10oloqwXJ+DlYX5+TZmQnM5/NUyAKCLaLeoae/F0a3jkIm0yFzuOBMNiuJmosjoxSNpcCMJBDghdH6wpgeo2h9Jqc7oA8FXS6vPewJ59AtSiIer7x7AujqzhbKxVUswqriift/hD7ry3qDGxvEwgnunnPNqksAtf9HUViqiSIsZnepwyPEhg39cvPoEDJExAQJ3gtgqNY6979L5bIolYr8fKr8hCZE6LlK+Lx/GJftqu9ADRkfSVxXjoxhPICt2Bucc941AVTLzVpPTxfW3Q0sTAQ0fU2Q8mYI7oWC0gLwZmpJJi4j7AZBHIRgryBe7Hi0kQoX5rgt4Kg32kq9sUIauiGq4otKoYpfI7aBg5BiGn0K/igNtLYGot+Xnl+B3NTjjxVWxJF3TQBLM5XT1ZpdqFRrHdTiDodMlHZMtvkK9T9JcItLUTiq/tesn0KGunfZ4hA0ZlI1MrSSidoGo0Cj5t/U5lR9Sug2pWz1Q1u/k6UkkpEmAre7Pv4nf3tOFBTwDseXP79/d39P92fxQVEpiJjQqxbMTikmmDZv+gaf4C2ubJ4IynzCaLz7X+0ZI/QYUOLBpkn6rvt+ajOlghyW3o5DzzGDa00WIvUrWfZInxE13Kg37NlyYeXBr37z5y+fwzNcHBfHxXFxXBwXx8Xxaz3+Bwejx8HM3R61AAAAAElFTkSuQmCC" + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAGP5JREFUeAHtW2lsXNd1Pve9N/vK4U6KIiWRWi1rceQlm7ckTp3WrVuoSNP+aIImqFMgRYKmAYoGcGC0QNog+dEATYKkaZ2mTqLYaVVbst16gzdFlhd5ky1TIimS4jYkZ1/ednvOuffNUF4ayUtRILrGiG/evOXes37nO9cAF8ev9xDw/2AcO/a5UKIwkk4PXdYtITYYS3d22jV7on/TnqfgPR7/pwI4ePBz8eHQ9qHsurGxaGZgfSyaXScsMWKAsQGEv8EMRbtD4ZgB0pWeD5Wpk7/82NiuG4/AezgseJfHQz/8YRQSx7cObb5mS65703ozlBoyLGO9Ycj1phXuEmBkDdNIGoYJ0vdRAT7eJVkTUkrwnSr+9YQwRDIWT+3H0++pAC7IAo4dvDUeivQmc4MDafBjg6F0ZtAUoVFc0DZhRLZF4uleIaAnEksJtTgPV+VL/OASaZG8TH4vfecVS6mOeUjpe65AAaBcPFmrlWY61+1bD+/heEsBvHzs4N6+wdHrUUHDOJthYYZHwpFYzgyH09IXccsKCcCJ8mR57qhJgcvnxfDi+C+e4kV5vit9p8lrbgkAlHxoGnQd34EC4X/xG1oB/h7+luvJGP4c9V0nJQ0jhMdxywjFBR7jjQkwhHAbjbsf+tLX/ur3Dxzw4ALGm7rAj7/z5bF1G/Y+HomEIr7vaSlpWfmkSV/6LmnVB6fZgKWFWegZ2EAmLUkgnucI3/NQiS4tCa3egnAkDlYkB77bENJ1wfcdfJaL8/cluoYIhyMsASldMgoSJf5LLxVfZLFKPV36AY/JhQC0qHzXsLp6t3V/at8dcODAc/AOBSBWFiqj9VopEgpl8KvBL9Uv49fjOnGCPni4kLnpccj1rBOmIaSwUCEQxitjPHvDMPBWuh//4rGBf6VM4DmTn8kPw8U4aBmu0+DvPrkOv87nVZNASTL0H13jubZgLdDvOA8UHL/Hl0Xo7Vt/Fd54QQI4xwV+8t2/2bv3qqu/PTl+YnPfwGBu844951wmOVaRADy5ND8tmvUqLMycgm17rxbJVFpS5MKFSu0JrEb8l86zDIXWGv0gA7sShtQH5A70CKnULVqCV0MqaQnlPzgH8i0SCrmWcNHaPNep1GvVx2278vmRrVdPwHkMY+2XWDz+O/i5Ev27s7C6AmiaSnv4n+s6UK2UYHV5CU48fxRisSh4dgMKK8s82UDDNGcjmDx+J+3oC0DNXlsEW4AZLEkosQiWGz5BCUhfq+/l83QX3cNHeLWh5WuwRTSSzerSx8KhxN333X5rzwULoFwsLTiOA+lMFmbPTIEwrUD3+FsBmo06BTSolooobQ9N1+aF8xo4yPPcwMbzjm2z0FzHbZ1nC5KtR2qXCobQwg4ODfXM1j2BKyq3oqULMwQGfkz6oPtZVgSPLbBCoe2XXXfzvz1x7/dzv0oArRiwff/f9T045Wd2FEuQSKYla7xchmjE4gnlurppQaJeKcpmvSKqpQKFQtYkal6ZvV5NrVZBAThqIbRMMm3TlHRdoD2pMwROWmQ6e/lYC0YJjLIBuQffod1FewbdTAKixIMC4Gs4WOPiw+EYB8ZUJnfdhu37bsOr/+xXCmD9B2/ZGBnbffLeyXHjuvFpec1VuwSmPLEwc1qOjG4F8jUMPlA78V3pnT0M0XFfNtfdJkwdxWg+tFIVtAEy2U6WB85IG7chVdAzOPkJ7Q6SZQB6hVIEcECqNCBblrF2sGBUtlVp0lTuh5mGMoNhmspo8LpMKnnLi0d+MX/JlTff9lYCYBcIi3zZrVcNI5qAJ54bh0g4JhLJJPr3EgcuisS1lVmwp/9RZOrPgShOg10v80KC+atoz84OdrMpK6WyrFUq6DZNdhf1qrWfYG2BF7CvK5AAGk5ACzsJZRzQSh0qIOuYwcHX1HPlK/HQIrMTAyOb//LFI/9xLfxvFjD+6IGlsY3XzoUS3f2TpTCsrq7I/v4BePH4Mdh71bWU11mbrteBVxcgYqJ/o48j4lszS0pbJlvCC88ehfzSAmbFkDRxUuSXpmWij5Kf4jnTEoaFvoqTDIXD6LP4wYBr4jmTXEWnT9SmxGfQ/XiPCfFkRqqAqJIGi80wCXSyxbB0CH96DsuK7o9GE8mhsZ0/GD9+35Wju25YfFMB0PCrxTuiPSNfOv6aCROnxmHTpg0IXqJoBXkIWwh6DII9PfiKCRSAK0q1EqQ7e3j5hno56LQHu/ddBc2mLTEtESTGwkahQtfzECD5lMMofuB/hP0djiW2XQOvhgDJIyDlspaFRs5Sm8C6kc2EOdTJlmuQAVjStZtCI2s4/fLTsHT2FOS6eyHbtR4zVnIkne39CV583VsKoFZcud9y7C85iR5x/LU52LHzEvYrcoPurg7UeBNsPyMlKsAyELg0KqxR5cRSZWcWBMF4H91SQCgaBdI0AR/SKgUsjRXYTFvpPsiEAYJW65L0HApuJGAPj9GKpI4VVEGw1jHC4u0W4w96BqXuaCIjzkyckc8//QzUa2UYGh6CXVfecO348Qf+YnTX9d94UwGYlcljZm3rfCSR6vuvF0/D732iCdlsFhbnpqGrKwd2oy7qdhJEHGQE51HPn9XzBGjVMlpXiOxko14TGverqo9qAtQ8uROKAi0KJ4s4QPCkLRKUoGApNGYI0qtyCZOzBBqLoO/sHqh1DoKUEDlLWgQMIRJNwsDQJplIZkW5tCpLmL4pmz106E7YfcWHvvrwXd96/Jrf/eKTbxDAzJMHVjYOXna4e/TST89MpCC/iJrv7YXXXjkBl+y5HFOgK+teWjiuRJ8FqFdWCRMSNmmLQEd1yiDhkNI+aV7qIKkQi1IdZwRJKmTIi27i8VNI4+QaoCAxwU5+D7uO9CS5N1/D9YSn86zklOI5hD1sFJoBnb39kMxkoKOzCqgMqFfLspifTw1vufyzONM3CoDd4NjDt4Xnn4ldMmh8cmE+Bzt37mCcjjldUJ1TtjtkDRdMZVBlleIJoT9T5T8VkLmWqyJWqJSKIqjogrxNEJ7/w0V19g3ByuIszx6DlaBAiZGbpUkBj9wFNY/R3OCgaZL7YCHIRRC+x6DAypZh6Ezh84cKMZQSArAmWkNNIKfAx4RdyB0wgX/g61+/KfWVrxwsv0EA85P3Tu7f9huf3nn53puWFs7G7a2b8QUWmlBJGqEYJIe3wZ3/eSUYzpJcMMPiCt9vpXBo1YwCUqkOTKMZTksYCyhFBVFZV3EquqUynVw9kmv4rGmuM6jY5CoZgwBahkNVpXRRu6h1PO8JjypN9H2KooRMVSxo+yG9k7yjAwFWKBIXHgoArRKi8QS+OhTv87LhN7UAGv9w+HDz21fvvCcdie8vF1ahq7sLJsdfgU1bt4MVXYIN1/0BrOSXwSkUGAoH+F9HL7YE8r1KpcjubIBQ0AX9m+BqrnsAllHzAnQAk6DSJKZMEhDHAUPXEHQX+jpyECpMGhQPFDJWVmLx+3XZpRAAuQJXjFJS6pWUUbAUD2Ml6dgximVN1yzYbykAHr78UTwe25/PL0Fffz+ceOkF2LFnnwiFoxBPpGShUMLUEtPL1lYQ4BIcHZ09oqOrN0CBXLzqyQnC7r2Dw4zoOG3hjz6nL8kcAvs7WRYbAPINDvIH5HRoDbLlQopOQR6AY4NiBXzIdvZjbCqiz5clCpQTc9+6jcoNIcpQGWuUYn/sIyiAg28tgBefOn500ydvKkyfmero7e2XhOtLq8syHI2BWatxPidNkSkq85dr0kGL9uKcHMA4RoqYqmihlD651geNamUAj7WgQEO+9o+0alUKiIBMEro4V190OQaJdIeCZb7CEwHsVqHDhKbjlG78wheawbuMNxPAd35x/2J+tfhEIh6TtXoFzTMMpdIqBhU0IcxFCJBkCM/VUBgBWFFBSJeFXBfohI3R3LGbktyFBSZkC+MLXSUyByDaHxEcq4dpUGzI4Pq2t8s1sxbtD2VF1DbOUWozDDIQCaW4dq1vyQqfeOnEz/a979Ibl5cWYXhoHZyZmoLL9l0JKawR5udm0ZoMoNIZjNfLUEJ14qCw8n8Psv9rILo/BMXVPGpEsmkTiyS5ipQy2zkoKuUV9lmFJBnMCAWdLSK+BMNiSp0U8VlleA7xQ5BiKQsxXGarUiUZ24wI6kiNGAML873V8xJAPl84hFNxC4WC1dvXD9WZWUJliO0NiEQiFJEFFjsyZApoMUCG8gUD2WCzvgy+4QCSoeAxMjQI1yOFFeW6AFWEbKALyVSWtUULiMbSwnXq7NcU5bnc5rpHakzA/AEDKiRIweP44TF8pnOcDDzmJYXUFFM0lsRyu0dbnEGcRum8BPC9O+7Of+SjHz6Eb7uJfIect4IcgBWy2PSSyaQsFovQnUtrCStAQmYaG7kZvIEb0FWSZHKyI9dJBKGi+FQgEwxwgFgtrA98R9pIrlKKLK4sBEGQ3UcqdITr8jXqJhqdgKQurQ3NHeAEKLZEEynI5Ho0AeNrh6Gffb7M9Rvl8xIAjQceeOTnN95wzU3jJ1+Fnt4e9vl4LIZVnInEQxgBTxl6u7Ig1nqBNjwzHCd70xVeiBEjk6FaWuoP+6xsFwQguweGNcbXya11vSZW2hyS0J0FRaW1gt45HJOekq9cgYBS0145bwGs2NV/R4taQWSSo3VNjJ+EXXsvYxNuYvVFxoYQWVdsejn4j7NyTIr87QK6PwNubItYWV4K0gSvTPm1SR4uAnAkFN5XBaVhaT7FYrmp2l4FswgCGhdjD5W6CjeoKNwGWVKTJqrLEPQnyBoIKteKs+dvAQcOPFz5+Ac/9EDYEvupiEE4yQYVQq2uFosyhIJooOkGFEagHnfxcRFb+Qk0rb0Qzu6GdLaLqzWKAxQhqPBB02f0hplBM12SfZrNnfzbc7Vve7wAn6+TshMp+Pz8GVVgGdRE8lno5DQk2MH1W7gGaWVmFggRph7VCXIlP31+WSAYE1PT/7Jr2+h+x3GRkEhBpVwhDE7YgAsaYnsC8kZq84+M3iKbpZtFpHME6o0alIrLKizrTJDr7oc80uqBfoyAViJ/9qUIUp/CUYr2RSir3APPrtu4HQI6XWPggEsFnZNV+lPwGr96KC9HOI0q1BYmqhckgMbS2UcyH7hi9tT4ycGenj44MzEOPT09YOYtNqk1QECoSVEjKIyLVy29eCwpY+sSCom0egUGJFIUPJkX0EWk0EoLgI1aIGF/kK+flWijLNAVtMalLZBBWYG7rS7HB99tYkWYh1gu07ggAXzjR/dXR3df/kgmnfpUPBaGxYV52LBpjL0VI7NU9bkmNDQi85uL4C3cAVbfH0LNjYn84ny7KlScnmgRPYbQNJmpuCUiNlrsH+F9QxH/WszEp5CwYomMJGAGurMg/TbfztUUQWteuMOL970G1JZehkbNdi9IADSOHX3627/1m9d/an5+idAV19e29n1KT6DxWWCLtZO3Q7L2TYwPMYhv+VPZxcHOCohKba+GUjp1jn2piU6pCiQFc3XyD2C14gX4SHED3JtkKUrRuiZop3F1idqnUthDuo0E7czeBYab8S9YAN//6eEj1157xWuOa491dvXB0uIcw4x0Ogm1apXkLSh+mdp3I8N/BNXpTogM/DYjv4ruIVAa0z4POma0aLFWbUDFUGu7gC/akVzZhGFoxAiKU6ECrat3iKs+9QwGSRxwXK4A6+A0ShCNhsBfOYqP+ShcsABo1Kru7el05jbs9MDyUl5G43ERQdK0UaeYEtQlKoBZ6X6wdnxGpyih2VwuYnkhqjpTCLJFp7faZYZcA1/lGu9iMSCuaFW+TLCwmdv8V3LWQK27DelhmnaaVRRAFTGLBacf/R50JXCuDXh7Ajh5evLwnp2bv7a6smxmshmUfCwogKBdoChz5D0Buu3lUbsc/VA5iAsBoNGKJ25YaxsUW+wr81Va90Wwb4AWS6CqZ2CEvijgQ91iRJp4LMjXqbVOWcl1ati2r+H1WLRZIWxjPAb1qXsgtAWz1qov3pYAEt0zz4PY/JhlmR9myFpYhjSVnrrNpRGZjsNaHPgqoq0yuS5VlrB8WrWDklxQMQeoUO8YaXeGQHEHql3OUd0hCt03kJ+IScvQcDlMLoAR37OFZ0uB+ofS0gTUC+Mw89SdcNkeE6l3Q+lgzTDOVwC33vqw+8r45J0UeWnJ2P3hOZu6gdru7a0tV4OPFLqYXysf0MAQ2qWsPtvuEzKMVc0/yWCGWnT5uUnMAkkszJAApY9ogCnrILwK+M1V2SjPy+LiKVnKj8PsM4dgKPYSZEaHeb5YIYu3JQAap06d/kUmk5lbWFwSqXQaGo06FXUt7BFoTx2vWajOeW8cOtoFPt26X99EmUBRQOwsxO25dl0iwwtqHxGavYfchtuU5A52rQjV4gIUFs9AfXUCVieegHj5CAzvG8SVp9T+hLdrATR+fNfDM1PT8w9GI2EZJsoJAxe2t1qpSleE7A68kPb2pyCXtSUCsp0F1Lm28DSRQr7N6c5zsUxuCKLHmo2asNG/KepTfnfx2K6XRL2ch+LyLCzPT4BdGofxI3eL+quHYdv7chBGeg5ot4sgJst5+xZA4+ixZ7/R298vVldWgUgR+qi1ybYg1qy39TYdHcTrcZ1mtgA0iyRaeI7JP47utMHKo56ALSmteUysNBjaNlHrlcICUuxTsDz3GjSXX4LpZw9DrHgUdr8vDqmxIdY+WEnVoQLvnDVf8D7Bf/rpfcfff8XeE81mfVuoFoFsOqWpO18FbSMoCYLgGKxN6J6W7p7wX8URtsC83gukXChoiLj8IfO37QYWZFWsAiPYmivLeimPGCMPtVJe1stnwWychlef/G/osebF9j0R6NpBmk/iKpEOt6JM8Qsh35kF0DgzffabWey6VCtVCNpTep+ADBhdJjxU8SM0yAsWpSMfd3iEXqzecuezufvMIvGeH3B5twkuvEHdnYooIGFiYQouLJxGen2S/kq0dahNPwrP3PMzWBdfgO2XhqBrW6cUUeQkDOQiMEz6nuIk8KEmvBMLYAHMLN1z6SWbC6uFclaxbkJbgNrGomCrynFKyx4orGBo+9doVDMaQSpUwpPMEhEN7vMmKBszDvp9oyrrlQI2axfFlks2yulXHgHTWwCoTMIvH30UIo0ZuHQjwMiWEHRsSkuIoc9bWHCZMZ4TdfhDEYRivvfOBfDPBw7NX3/NvjuwcLnFAFN5MKUpQxEPElo7QAiYKoRHmUwEtLZsU9wQoCBtRT7vL1SLt23hYoRHAcgGtuORB0A+YBBmThyCkDMFTz34CDSXTov1XY4Y3gxyZJsB8V7UNHaxwIgISdtnmGwxmK0z40nsW9rvXAA0Hnry6e/t2jzyx8IUMR23qe5mrOOrHQvqQl9tieO6XoW31m9SQUH9xdctMt5oyawPBjrpYKqtVQvY5q5ArrcbKnNHYfL4g3DymWfEQKoht2PVvWmTAb0bLbBSEWDNx7pR8x1od1m0KizbK3XhlB5HcmYIGr54dwTwg3+9+/m//vNPPrjNGPsEcjmoNYO5Uw51zNAEWB+UG6gNrKpdpmWjt8VqvEPkKLFBTUG+72MbmOKFFQ7J2kJZTJ58Tk4+e7dYmXoJelOOvHREyJ5eC5IdMWiEQzC1iGzzagozBFJmLr7Vr6Ewm0TfilAiJ20nI8bevxEiqXMowbcvAJr5p8/mD+ESPkGbJ6LhsAz2AQjaTqIMXtdyvt70pMyf94jrtjjjel3+MNdvJjHaF+X05CSceOGYnHr1OBZfcxLcssgkIjK7YZ9IdGSRpcxCI9cBkWxORtJZtPoENj8zEMIGaDiWQeuP4LOwp2jF8LkROPv8j9E16kjWOu+OAGiEo9GpaqUCS7OvQnrHBwG0v2tepCUrj3t+oBoj6N9YIhOgoe10oloqwXJ+DlYX5+TZmQnM5/NUyAKCLaLeoae/F0a3jkIm0yFzuOBMNiuJmosjoxSNpcCMJBDghdH6wpgeo2h9Jqc7oA8FXS6vPewJ59AtSiIer7x7AujqzhbKxVUswqriift/hD7ry3qDGxvEwgnunnPNqksAtf9HUViqiSIsZnepwyPEhg39cvPoEDJExAQJ3gtgqNY6979L5bIolYr8fKr8hCZE6LlK+Lx/GJftqu9ADRkfSVxXjoxhPICt2Bucc941AVTLzVpPTxfW3Q0sTAQ0fU2Q8mYI7oWC0gLwZmpJJi4j7AZBHIRgryBe7Hi0kQoX5rgt4Kg32kq9sUIauiGq4otKoYpfI7aBg5BiGn0K/igNtLYGot+Xnl+B3NTjjxVWxJF3TQBLM5XT1ZpdqFRrHdTiDodMlHZMtvkK9T9JcItLUTiq/tesn0KGunfZ4hA0ZlI1MrSSidoGo0Cj5t/U5lR9Sug2pWz1Q1u/k6UkkpEmAre7Pv4nf3tOFBTwDseXP79/d39P92fxQVEpiJjQqxbMTikmmDZv+gaf4C2ubJ4IynzCaLz7X+0ZI/QYUOLBpkn6rvt+ajOlghyW3o5DzzGDa00WIvUrWfZInxE13Kg37NlyYeXBr37z5y+fwzNcHBfHxXFxXBwXx8Xxaz3+Bwejx8HM3R61AAAAAElFTkSuQmCC", + "parser_ids": [ + "naive" + ] } diff --git a/agent/templates/ingestion_pipeline_laws.json b/agent/templates/ingestion_pipeline_laws.json index cd408eac66..d9e1a5f4ef 100644 --- a/agent/templates/ingestion_pipeline_laws.json +++ b/agent/templates/ingestion_pipeline_laws.json @@ -6,31 +6,247 @@ "zh": "法律条文" }, "description": { - "en": "This template segments parsed files by legal provision hierarchy. Best for documents with clearly defined articles, such as laws, regulations, judicial interpretations, and compliance policies.", - "de": "Diese Vorlage segmentiert die geparste Datei anhand der Struktur von Gesetzesbestimmungen. Sie eignet sich für Dokumente mit klar definierten Artikel- und Paragraphenhierarchien, wie Gesetze, Verordnungen, richterliche Auslegungen, Compliance-Richtlinien und andere Rechtstexte.", - "zh": "此模板将解析后的文件按法律条文结构进行切片，适用于具有清晰条、款、项层级的文档类型，如法律、法规、司法解释、合规制度及其他法律文本。" + "en": "This template segments parsed files by legal provision hierarchy. Best for documents with clearly defined articles, such as laws, regulations, judicial interpretations, and compliance policies.", + "de": "Diese Vorlage segmentiert die geparste Datei anhand der Struktur von Gesetzesbestimmungen. Sie eignet sich für Dokumente mit klar definierten Artikel- und Paragraphenhierarchien, wie Gesetze, Verordnungen, richterliche Auslegungen, Compliance-Richtlinien und andere Rechtstexte.", + "zh": "此模板将解析后的文件按法律条文结构进行切片，适用于具有清晰条、款、项层级的文档类型，如法律、法规、司法解释、合规制度及其他法律文本。" }, "canvas_type": "Ingestion Pipeline", "canvas_category": "dataflow_canvas", - "dsl": { - "components": { - "File": { - "downstream": [ - "Parser:HipSignsRhyme" - ], - "obj": { - "component_name": "File", - "params": {} - }, - "upstream": [] + "dsl": { + "components": { + "File": { + "downstream": [ + "Parser:HipSignsRhyme" + ], + "obj": { + "component_name": "File", + "params": {} }, - "Parser:HipSignsRhyme": { - "downstream": [ - "TitleChunker:SpicyKeysKick" - ], - "obj": { - "component_name": "Parser", - "params": { + "upstream": [] + }, + "Parser:HipSignsRhyme": { + "downstream": [ + "TitleChunker:SpicyKeysKick" + ], + "obj": { + "component_name": "Parser", + "params": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": { + "doc": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "doc" + ] + }, + "docx": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "docx" + ], + "vlm": {} + }, + "html": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "htm", + "html" + ] + }, + "markdown": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "md", + "markdown", + "mdx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": "main_content", + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "txt", + "py", + "js", + "java", + "c", + "cpp", + "h", + "php", + "go", + "ts", + "sh", + "cs", + "kt", + "sql" + ] + } + } + } + }, + "upstream": [ + "File" + ] + }, + "TitleChunker:SpicyKeysKick": { + "downstream": [ + "Tokenizer:PublicJobsTake" + ], + "obj": { + "component_name": "TitleChunker", + "params": { + "hierarchy": 2, + "include_heading_content": false, + "levels": [ + [ + "^#[^#]", + "^##[^#]", + "^###[^#]", + "^####[^#]" + ], + [ + "第[零一二三四五六七八九十百0-9]+(分?编|部分)", + "第[零一二三四五六七八九十百0-9]+章", + "第[零一二三四五六七八九十百0-9]+节", + "第[零一二三四五六七八九十百0-9]+条", + "[\\(（][零一二三四五六七八九十百]+[\\)）]" + ], + [ + "第[0-9]+章", + "第[0-9]+节", + "[0-9]{1,2}[\\. 、]", + "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])", + "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}" + ], + [ + "第[零一二三四五六七八九十百0-9]+章", + "第[零一二三四五六七八九十百0-9]+节", + "[零一二三四五六七八九十百]+[ 、]", + "[\\(（][零一二三四五六七八九十百]+[\\)）]", + "[\\(（][0-9]{,2}[\\)）]" + ], + [ + "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)", + "Chapter (I+V?|VI*|XI|IX|X)", + "Section [0-9]+", + "Article [0-9]+" + ] + ], + "method": "hierarchy" + } + }, + "upstream": [ + "Parser:HipSignsRhyme" + ] + }, + "Tokenizer:PublicJobsTake": { + "downstream": [], + "obj": { + "component_name": "Tokenizer", + "params": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + } + }, + "upstream": [ + "TitleChunker:SpicyKeysKick" + ] + } + }, + "globals": { + "sys.history": [] + }, + "graph": { + "edges": [ + { + "data": { + "isHovered": false + }, + "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", + "source": "File", + "sourceHandle": "start", + "target": "Parser:HipSignsRhyme", + "targetHandle": "end" + }, + { + "id": "xy-edge__Parser:HipSignsRhymestart-TitleChunker:SpicyKeysKickend", + "source": "Parser:HipSignsRhyme", + "sourceHandle": "start", + "target": "TitleChunker:SpicyKeysKick", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__TitleChunker:SpicyKeysKickstart-Tokenizer:PublicJobsTakeend", + "source": "TitleChunker:SpicyKeysKick", + "sourceHandle": "start", + "target": "Tokenizer:PublicJobsTake", + "targetHandle": "end" + } + ], + "nodes": [ + { + "data": { + "label": "File", + "name": "File" + }, + "id": "File", + "measured": { + "height": 50, + "width": 200 + }, + "position": { + "x": 50, + "y": 200 + }, + "sourcePosition": "left", + "targetPosition": "right", + "type": "beginNode" + }, + { + "data": { + "form": { "outputs": { "html": { "type": "string", @@ -49,186 +265,185 @@ "value": "" } }, - "setups": { - "doc": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "doc" - ] - }, - "docx": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "docx" - ], - "vlm": {} - }, - "email": { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "output_format": "text", - "preprocess": "main_content", - "suffix": [ - "eml", - "msg" - ] - }, - "html": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "htm", - "html" - ] - }, - "image": { - "output_format": "text", - "parse_method": "ocr", - "preprocess": "main_content", - "suffix": [ - "jpg", - "jpeg", - "png", - "gif" - ], - "system_prompt": "" - }, - "markdown": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "md", - "markdown", - "mdx" - ], - "vlm": {} - }, - "pdf": { + "setups": [ + { + "fileFormat": "pdf", "flatten_media_to_text": false, "output_format": "json", "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pdf" - ], - "vlm": {} + "preprocess": "main_content" }, - "slides": { + { + "fileFormat": "markdown", + "flatten_media_to_text": false, "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pptx", - "ppt" + "preprocess": "main_content" + }, + { + "fileFormat": "text&code", + "output_format": "json", + "preprocess": "main_content" + }, + { + "fileFormat": "html", + "output_format": "json", + "preprocess": "main_content" + }, + { + "fileFormat": "doc", + "output_format": "json", + "preprocess": "main_content" + }, + { + "fileFormat": "docx", + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": "main_content" + } + ] + }, + "label": "Parser", + "name": "Parser_0" + }, + "dragging": false, + "id": "Parser:HipSignsRhyme", + "measured": { + "height": 57, + "width": 200 + }, + "position": { + "x": 316.99524094206413, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "parserNode" + }, + { + "data": { + "form": { + "hierarchy": "2", + "include_heading_content": false, + "method": "hierarchy", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "rules": [ + { + "levels": [ + { + "expression": "^#[^#]" + }, + { + "expression": "^##[^#]" + }, + { + "expression": "^###[^#]" + }, + { + "expression": "^####[^#]" + } ] }, - "spreadsheet": { - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} + { + "levels": [ + { + "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)" + }, + { + "expression": "第[零一二三四五六七八九十百0-9]+章" + }, + { + "expression": "第[零一二三四五六七八九十百0-9]+节" + }, + { + "expression": "第[零一二三四五六七八九十百0-9]+条" + }, + { + "expression": "[\\(（][零一二三四五六七八九十百]+[\\)）]" + } + ] }, - "text&code": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" + { + "levels": [ + { + "expression": "第[0-9]+章" + }, + { + "expression": "第[0-9]+节" + }, + { + "expression": "[0-9]{1,2}[\\. 、]" + }, + { + "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])" + }, + { + "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}" + } + ] + }, + { + "levels": [ + { + "expression": "第[零一二三四五六七八九十百0-9]+章" + }, + { + "expression": "第[零一二三四五六七八九十百0-9]+节" + }, + { + "expression": "[零一二三四五六七八九十百]+[ 、]" + }, + { + "expression": "[\\(（][零一二三四五六七八九十百]+[\\)）]" + }, + { + "expression": "[\\(（][0-9]{,2}[\\)）]" + } + ] + }, + { + "levels": [ + { + "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)" + }, + { + "expression": "Chapter (I+V?|VI*|XI|IX|X)" + }, + { + "expression": "Section [0-9]+" + }, + { + "expression": "Article [0-9]+" + } ] } - } - } + ] + }, + "label": "TitleChunker", + "name": "Title Chunker_0" }, - "upstream": [ - "File" - ] - }, - "TitleChunker:SpicyKeysKick": { - "downstream": [ - "Tokenizer:PublicJobsTake" - ], - "obj": { - "component_name": "TitleChunker", - "params": { - "hierarchy": 2, - "include_heading_content": false, - "levels": [ - [ - "^#[^#]", - "^##[^#]", - "^###[^#]", - "^####[^#]" - ], - [ - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+(\u5206?\u7f16|\u90e8\u5206)", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u6761", - "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]" - ], - [ - "\u7b2c[0-9]+\u7ae0", - "\u7b2c[0-9]+\u8282", - "[0-9]{1,2}[\\. \u3001]", - "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])", - "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}" - ], - [ - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282", - "[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[ \u3001]", - "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]", - "[\\(\uff08][0-9]{,2}[\\)\uff09]" - ], - [ - "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)", - "Chapter (I+V?|VI*|XI|IX|X)", - "Section [0-9]+", - "Article [0-9]+" - ] - ], - "method": "hierarchy" - } + "id": "TitleChunker:SpicyKeysKick", + "measured": { + "height": 74, + "width": 200 }, - "upstream": [ - "Parser:HipSignsRhyme" - ] + "position": { + "x": 616.9952409420641, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "chunkerNode" }, - "Tokenizer:PublicJobsTake": { - "downstream": [], - "obj": { - "component_name": "Tokenizer", - "params": { + { + "data": { + "form": { "fields": "text", "filename_embd_weight": 0.1, "outputs": {}, @@ -236,332 +451,34 @@ "embedding", "full_text" ] - } + }, + "label": "Tokenizer", + "name": "Indexer_0" }, - "upstream": [ - "TitleChunker:SpicyKeysKick" - ] + "id": "Tokenizer:PublicJobsTake", + "measured": { + "height": 114, + "width": 200 + }, + "position": { + "x": 916.9952409420641, + "y": 195.39629819663406 + }, + "selected": true, + "sourcePosition": "right", + "targetPosition": "left", + "type": "tokenizerNode" } - }, - "globals": { - "sys.history": [] - }, - "graph": { - "edges": [ - { - "data": { - "isHovered": false - }, - "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", - "source": "File", - "sourceHandle": "start", - "target": "Parser:HipSignsRhyme", - "targetHandle": "end" - }, - { - "id": "xy-edge__Parser:HipSignsRhymestart-TitleChunker:SpicyKeysKickend", - "source": "Parser:HipSignsRhyme", - "sourceHandle": "start", - "target": "TitleChunker:SpicyKeysKick", - "targetHandle": "end" - }, - { - "data": { - "isHovered": false - }, - "id": "xy-edge__TitleChunker:SpicyKeysKickstart-Tokenizer:PublicJobsTakeend", - "source": "TitleChunker:SpicyKeysKick", - "sourceHandle": "start", - "target": "Tokenizer:PublicJobsTake", - "targetHandle": "end" - } - ], - "nodes": [ - { - "data": { - "label": "File", - "name": "File" - }, - "id": "File", - "measured": { - "height": 50, - "width": 200 - }, - "position": { - "x": 50, - "y": 200 - }, - "sourcePosition": "left", - "targetPosition": "right", - "type": "beginNode" - }, - { - "data": { - "form": { - "outputs": { - "html": { - "type": "string", - "value": "" - }, - "json": { - "type": "Array", - "value": [] - }, - "markdown": { - "type": "string", - "value": "" - }, - "text": { - "type": "string", - "value": "" - } - }, - "setups": [ - { - "fileFormat": "pdf", - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content" - }, - { - "fileFormat": "spreadsheet", - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": "main_content" - }, - { - "fileFormat": "image", - "output_format": "text", - "parse_method": "ocr", - "preprocess": "main_content", - "system_prompt": "" - }, - { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "fileFormat": "email", - "output_format": "text", - "preprocess": "main_content" - }, - { - "fileFormat": "markdown", - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "text&code", - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "html", - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "doc", - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "docx", - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "slides", - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content" - } - ] - }, - "label": "Parser", - "name": "Parser_0" - }, - "dragging": false, - "id": "Parser:HipSignsRhyme", - "measured": { - "height": 57, - "width": 200 - }, - "position": { - "x": 316.99524094206413, - "y": 195.39629819663406 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "parserNode" - }, - { - "data": { - "form": { - "hierarchy": "2", - "include_heading_content": false, - "method": "hierarchy", - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "rules": [ - { - "levels": [ - { - "expression": "^#[^#]" - }, - { - "expression": "^##[^#]" - }, - { - "expression": "^###[^#]" - }, - { - "expression": "^####[^#]" - } - ] - }, - { - "levels": [ - { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+(\u5206?\u7f16|\u90e8\u5206)" - }, - { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0" - }, - { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282" - }, - { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u6761" - }, - { - "expression": "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]" - } - ] - }, - { - "levels": [ - { - "expression": "\u7b2c[0-9]+\u7ae0" - }, - { - "expression": "\u7b2c[0-9]+\u8282" - }, - { - "expression": "[0-9]{1,2}[\\. \u3001]" - }, - { - "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])" - }, - { - "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}" - } - ] - }, - { - "levels": [ - { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0" - }, - { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282" - }, - { - "expression": "[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[ \u3001]" - }, - { - "expression": "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]" - }, - { - "expression": "[\\(\uff08][0-9]{,2}[\\)\uff09]" - } - ] - }, - { - "levels": [ - { - "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)" - }, - { - "expression": "Chapter (I+V?|VI*|XI|IX|X)" - }, - { - "expression": "Section [0-9]+" - }, - { - "expression": "Article [0-9]+" - } - ] - } - ] - }, - "label": "TitleChunker", - "name": "Title Chunker_0" - }, - "id": "TitleChunker:SpicyKeysKick", - "measured": { - "height": 74, - "width": 200 - }, - "position": { - "x": 616.9952409420641, - "y": 195.39629819663406 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "chunkerNode" - }, - { - "data": { - "form": { - "fields": "text", - "filename_embd_weight": 0.1, - "outputs": {}, - "search_method": [ - "embedding", - "full_text" - ] - }, - "label": "Tokenizer", - "name": "Indexer_0" - }, - "id": "Tokenizer:PublicJobsTake", - "measured": { - "height": 114, - "width": 200 - }, - "position": { - "x": 916.9952409420641, - "y": 195.39629819663406 - }, - "selected": true, - "sourcePosition": "right", - "targetPosition": "left", - "type": "tokenizerNode" - } - ] - }, - "history": [], - "messages": [], - "path": [], - "retrieval": [], - "variables": [] + ] }, - "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAFOdJREFUeAHtWnmQXWWVP+cub3+9pNfXnaQ7e0JIIEJAlokpC4JREqHcRmRQxEFnYUaUcpYaLawpl5qB0lGHKreRUUeDE8ZRXGqAYMQIRkI2ZAnZl+50d3p9/fZ773fmfMu973WEcgIZ/MP+ktfv3ve+d+93tt/5nfNdgNkxO2bH7PgDHgivwdix4wfZFQtWvd6KpZK2YwP4AQQo7+3wt340z1Z/HQj4M9u2SZ4FPDc85l+gPCc+DzBAm2zCIEBS3wf8JfAx6LmB/J0lyBKnP//5r+69++67Bfy+xujwyS8JERCRIPNHmGOqv/+fhghf+kA0nAv5Ci8s5JA/CIJADA0c/8zLrc2C12DEk5lebPQ1ooazc3BC4sn8YrnU5VD/FqPrEP7W1dgd2Guq75kcPnbX+OnDb2ZPmCHza6IAfRPRsCQ4l0HRkZYaEPGsCVqhjR/zHP5Y/1R4lXkW2v+cndP90O23vPmmxt868P8wvvflzzYvWbrkRieRaCrkJ2E6P7Eslc1QgwjhcSRCw+lZ3834MPyOpNDS3uwNSlgpK5nLovQS5SvqTqQVJt8srBUKGyeHjp8uTAztnbvi8rHzroC7717vXH3thgcsy90gGKAsXoVXKaolc4SCjEy1VGqwmBHLth0pzEv5hzSxUZ72ISU8GEtD+E0UFCbGMPKOcApVyzc1d/TcFEsmxg7u+OEbzrsC+qDfsRC7+CV1roTVSyXwvBqNjJwBqQPUi1cWNEJBZ0cnJJPJUGZlwnqANw5jaVB6RXODl5hB4aG6hlIGaZXYltMmqHzreVdAf1+/jtFGe6nbI8XjCezp6TEuGxq1vnZOdw1XwkaXnyEXNoQQ4ktGTDhVGQAMGkT31f+BHTJ53kHwzESTNq5cAN8BkBoWJFO4RZZlKWH52LzbRvizwa3+wdZtz8OS278LX/nJb4zywoHU+IOXOJY2hxAQFWaoVAIyJPH8g+DcEPX1jawQrczNa7UqQAheOkZB4yKny1gcfrH3BB9bUCjXYOeLQ7DruUGKxR1Ex6WyJ/AHO4/Tw7uOUKlYwZULO+GKC3qwszlF+XIVNly6ABxWqhJV33GGi5FyBWE+kkoIrPOugMREnm8qNJLJm5E6VlaTwo+MjISGVZ+FGGBZNvT29sDAeBE+/cAuePuVi+jaS+bD+zasxEKpBp994CnwfY+SMcCPv/Myam1K0ZmJAux8foj+actTcMuGCyAQQikgVCvUJYdIJxDGkaU+Pe8K6Mnl+EbCksuw2M8sux6k8Xic5s2bp0+lG2OjgfTy3nrVErj72ztp+bxWuHrVvEiEe/50Hbz9kw/BZ997BSyeO0f9YH5XE41PlWGkUIUbrlgIMdeO4LEOgOH1BdQxUXqgUDc8rxhwz/q25UcevO+ufd/7WteJfbtBeYL0AJmYNSjUY/dlMv2PdhyABS0x3H94WKG2sSUKPj4xUULPF1Fqk988vu84XtHfClt//kLDZemsuzQApQo9HQZyZefFAz66pqUvaVn3xN34jXPnt1tjx3fhw1seoOs/cQ91dbUgmaD0fZ9K5bKGhwaxZRikUilwnRj86tkBuGRZDoqVGhTZ9TOpuJpXLNfIY3tNFasGMYDTagAx9rXujiwcHZiYKScYFkAhDigEUCGn0El+EhC+ag9494Lspumi2FmuBW/Llzz7Wz/YDV/4r2dh25ECPv3UXnkXDgU9l0kQllkB5XKFKpUK8TuUS+ocfM+DY4MT0NOe5bCx4bq1C+FHTxyKTJkv1jhgHShVfMMkCX6++yhctDwH0itWLuqip58fiNZliJAqGqIgQzyLLwSvTgGb56f/uuiJB/PVoPNUwYeHXsjDnuEq3HzDavrbG5ZSrjbCQnucbgI1343Z0NXZCZ2d7ShfXV2d0N2tX0n2gEd3HYMNly9SDv6Gi/vg18+cDEMASlVfCVXxAwpD4IHHnud585VbX7V6Hj66hzMI1IlVnRxrKFCgy0Bp8jRICHzFCnh9e2pTviLuLXiBe7TgwROnKzC3IwV3bVoMc0QF5y+fD2LgWdiz9QtAA/8tzW+MISR5MeCkY1GupVbz4ZmDp2HVok4dp+w2q5d2w5GT4+p+xaqvROHQUEuX3tLX3QxN6YS60hIGRgoETU1XzAopygVkwA/roaCnMFa+Igy4vDO2WgTi62WfrAK733A5wOsWZKG/2YHde06hzWjc5zu46e9vgfaOKaChfVB99lMQW/5RRp2kzPMhM4mwYO+Lw3A1W12zSEvRxLUrcvDIU0do0bw29AKhzFb1lAVp++5jcMt1q4wcktoiXNDfhvsODdG6NX0my0AkdFhBhEPzEM6+cO7DCXz7E9WA2id5MadKPq1ujxP5gg4Pl2HfmQrNvTQN7/xQjjpzJcKmZWDlVoBLe8E/+CUyyIS6JxDqgeCbP90PiVgMHv31UTg5XsTHnj4OA2eKsOvQGShXPI5z7iFxRmEghSp7y6N7TuLBUxNq/tGhPHL4kMvF1NafHagXSaRBMLK4Pp8pDJzjuKA10cPX3igLmsFyAP1pF12+9GDRx4oI4LYbs/CO9QTx6mkQo/sR53D8Z1dwV6ICOPgLrO6vQHz1x2lGU0Q2tLit9aGv/QpRUmImUNueH1WLnccpUcoidFmL8r41z8fD4xV47307IIzwW+/bgQyysKY3o6yvboCak4clla4D6tRcku9z9oAmS6TjlhUbrwWQchDmZhyYrAo8VfRh8xUOvemCALx8GbzJSQwmB4DGn2PkGgRoWQOibTnt3vYQ/OJLd0G1Uqq7Jr8+fduV+LZLc5Thxh63sUDUPFq/rI2++7FrIRF3lJiyy2VxeGTTcfr6HetpbV8zEM8Tsk9YZTK0pgu++uE3GoyRAxvYJpksEN5TIhCdOxX2xmqHnC7n38sBvb8raUOFQbnE8bn5UgdvWAnoFTntWT44iQpZMabFOAy28xsYnJyEQz89CI9vH4P9B/4N773mVupbvsJcFWFOcwK+eMd6JjoEpwbHYU5rBpuyCWXCGiu7MF2Caxc3g8eC+hwOFyxsh+9/8nqSXOHMmTzkGBDjMachzHWYoakFTTkIZzHPc1fA06yD9cOlDyUcZ0veFzmv4H2wr9W+8t0XWeSzJ3i2kAqgssNFj1Xk2LXhyZ9Nw87/eQwef3EKhSdocUcaoFyAqcGTkGptAzeeMswYwXUtWNDXrkUwxnrxxCjcv/2gMuJ/PH4YLl7cRWtX9ihx0kkXM31tFMU2UVhbqSPTetDoL+kv1S/MLdNXlgW2y1627z8qO9r9KSuzaal9VVBkltNsQa1E0iVRBJ5sv8DWHw/D478swTMjVbikOwnXXt6NiUwGhrbdC4cHTsOxI+N06Z99Cldv2MTzrZC+4U92HqV7H9wH+bIPNd9XTi10RwM/8IXtzPstSMdd+MA1S+nmjSvNymgmEzTXCrkAKn3MKKVffTUohPXi5IhXGE7amU5hQ7rFwWpZQIXTxJd/VsRHdldhgPFhZWsMutIOHDyRhxV9FqcSDpWWVjgyehwfu/MO+NYzG5VDomojIay7MIdr+uZE9gNTU4fdJDDZPMM8QEstoLHAaKz8JKhq5kENNRKfsUZftQJOVPxtDw84X+fU9FerSgJlsZdpceD7ewP44S6PTjNH6GPBU44Fg3kPepuATo1M4ws/3gtPHJyA0ZIHiXQWahzb8QTjsqUFzaTjkE4nlDOHTc2wg1CvJBvHbzVTVGvQ1OI64HVqDNuF6hfnoxjCdMy5+JFheWHOAJz1pnjdP97r46lSAO1xdlV21/GqUIKMs0IOcR1/fMqHgPRnl6xbB2PjE5BK8zn3BUTU6ZWCWmbRaKqasLjBEN0RDeybLqG6Jpfe3F+Mg2kaqkpU1iLaAfRsgfTqQuDGRZnOJsQPD5f8dZxV6TsnCDYHAp+d9uBAPqAc58m2uA0FJkwZF2GClXCGOz2TtYAc1aRGaO/uhptuuw2qVQ9st6YFU/1LZfoI3DCqAaVBRXQSCh/m97CpLN9ZASpeOH2a/mHYLtAdqVfVErtzaVtvrMXeWypW205NA5xkUiSz9YOnuO/HWJZkl7+oKw0n8zVgoGbfICp6hBXm6zbPc225Dgv++PYPQneuk5qbstja1o4SlmiGMZFMCw0jgaXnANbbrY2KwgaSTVH7nUwbEHWXSmlMzX9FCljfAZnTtcrHLko0te0b8HCUyYBkYVLy5uamr33iH/5ic6FU7tz6jS3sd1X0eBUV7udVmS/EmLQnLBWdsG7zW2EBc4F8oYyxVJFDwlJVkFyaJDwiou8YdXYa2+BhqDR2ekIlpFJJxpAU1TcQdGoUQFGBzBuH5+4BNy1I9M1JJe7ndL7+kQOTlI5ZUOaFZOMWxWz8yzvvvP07Sxb3vZFX2vldN4mnuFLkz5XLJVhBST52UinafPOfwNqrr8ZsNq0Wmkwk0LFtdgA3onEA4cZP1ADS7zrXm1BpxIC6pFbYYidT/RGZblD4sVANkXNSwI292aVdaeuHFYJlx6Y8kPS3MCUoyZQ4jjg5kfC+2doKToyLGmZrcNlVl9L+F46gw66eclEqCOZfuAre9b5bYO78eZhtaoZMUxODXwbb2lp50S6g7WAof90B6ow9TGVY73A1bLAghD2PkP024kg9PoRJnOcAgjcvTm3uTtn3F3xqPTxRg7FKANMMbjUhamWBn2uJu/cPD0Mxk+5tcl1HNQE3XnMl02CXThwbRBnjF122lno4T/IGCdevMWkArDH1dfg6E1MMJGgrXICGXn8dAsl0DnQ3+ewsGCogxEoZAslkQn8p6pVnnQaQ7CL/bgW8g5fq9qdun5eNf26iIuJHJj2V0karAUiwm+M6XzlY8P5uQvX7+Rxk58pVwnCXFq9Z9zpa/ue3cpyXKC89hsEynkhAjNOUy57icJvLjbl6c1M1UHXHK9o7VAYXGHo8znAEojqekdkS1TOE7PyErSAL6vtsmiBrtYrfvS+A7sLMF1d2pz54eKxqSSLDlR+MsfCMZ5R27C09hdpHDp71I2l1tAJiTwCfgcZ2Y1yoCGjisGxqcVn4BDhuDBPJFKhXIqnmSA4gX0Ymg/pWvYCPXFoqygp3iMIImEkDVQgIjZCR8wA0VqDy9y+rAI73to4m+4FlufQbnxkswOlpn9jyKIX3WLtMbnaJrtqt2w81PONiXMBhywvfQpetLOmt3PWVFvcqAZbZ5z1RBdeTj3Egyj5HueqRpoCWXj02lO2RU5MkDqalJ9SuA4RcqSEcDGpKYGU2mdRYaQAwpM/1PszLKOD6XHZ5W5Pz7bUr2l/31KFxGJr2UZIYJjBQ4zzu2tYeRuz7FlTj//JHS6zemIXJkIJ862P/GOvs7+8hBjP5WIz0tnSKLW5ZyBjI77xBwn+4dEVZvmaamyDd3oFzFq+G5v4LKZ3rVwzJVIe6kad6G2ZzIVx5HfhCmABoSB86lKTwAepQCVsBwvgWgf9SIbCxJ3nVolziG1e+rnfxQztO4Gi+Kt2epPAV2UdH61TVw/dw1ttyYtpfNVYGbOZUeFkPuzJ7sCM3Z8aPKmvJfXJbChtIgW3e47O59++gLHljXMnJ56Vsj8Nk5ABT4Sdg4mkX2y+/GVa86QPGWgbrzAMxoDscYOhvOCWyfDQVNPsjEwLhLpDEkkgJ+mIzPeC63qa3LJzjbl2xoDXxvW1HaarsQZ7Z6XiVXTeQLSQamgisDfFabYClSUhyM83fD3JLvMxenHGkY0rZtUPbqC3hmHNWCAuNigYr4ZVH6GLHsjTW9x5/mNrXbIramKZBWvdwReYCeZdQ6IgHmCnqOpwFKJNNGzWRwgMd9mFG0B3pSAGb52Xf35aAf+1qiSUefnqIpjnWxznVTbJgRe788jrHq771lv40LEtnk9tKfpCrCfnEG2uR11Lg96rmmmojRHqoimolvFIMC8/fcXvfkoowuyW2maTm8R979AQM73kEOi9cBzaDpVQO1km86XRopUTPVkToHlEhdS1Ne00AENV7BSogRB0EN+bS70nZ8FWOTXzySB64lIdxlk6mO455thRWfQ+uH/W93S212OaaFfQwE5S0ltqTDqxojTMTdNjlCW259w+6ry/5vhTUdfU7NzFIHTvqhYwlYPG5LedxWFhKGUV8YetnoDQ5Tpfc+H6FfJwZTMxjxPFZi2R6A8oDBOlyN/IEoxTZRRYBv4QP9a6w0MgRKoB3cN8mjXCIze0p4QN2fW5qSO5u4XiR4K0Tvv+kmmsJUiUqL6jMgFjkL5nEc8FjKZeWrNc2VtfnqPiCrT2Cw0Fb27F1f0Z+Zxkrh1tocvEXdx6FOasPoxuLEzNElteWABQVQfIhC6KwdIiosmXewx4ClYolxQmELxUQKGWAPJfPK5Ephnh3Rz57wNYGHKkwWWG3r0nt8H4m6+Gmiaq/oxErVGVFEZ6iTGK2xAiD8gHodUUK0MGM6jtLt6Yc0m1p9Z1SAilFqPDkt/zkGAwePcBxnIV4Mq3IlXqISscXaQWYh6cawkQBh8EV+Y89EqeHjlDP3HUQ+FWWnVdHNVZETQaBVsBoSWypCvEuaQSJ9HL3LWVb08Ua3dHR4v+SW5QZlo93Z8AKPEvKobO23GrjhbPxmedb0poS+PQaQXkC2qaVZXABQlNpR9TDElrocNNbuvrpk0fh+Z3buVhKUToZU6RK4oUUXCvM1ulRRojyAjuix5ZCVDUX5Y5spqMHLSZagwf3Y7VaAL9ShMmRARg7M/Jc6HR2LmnfwP3Hv2E9LuGiJc/B/J9cUO3n5fuoynm+JHtNNfB7fIFX82qX8uU7GdQ72uMOxuTi6v0IlXQtQKMM87CEIXIIYaagGe6v87dKctC1eCEsX70cspkkJRIxZFqtcp8GxSgPSg2EFZO8tqZGqPFBLkE/NyuXzvuLUxOUm9sbpJpa8hNjo788fOC5j8woKS7hDdwX0uk2yyr68Xh7tUV2UpjZ+b6wUvzii8m+vc3A6PBnruuLzFRF5JIuMcGVj0EHcNZQXmkrZw8AzLsOikA/HG0ehLbr5S5KJ+1ozxS6cxlPRqnLktkxJpccx3KDzK6yPK4vTcNxFOM2AgrXYecETwEhxzn/5+rT8QMRJERL1uYzO6jFnGBuIhvUHL9a9q38FTC3BrNjdsyO2TE7ZsfsmB2zY3bMjtnxhzj+F+LeQhipiiQ4AAAAAElFTkSuQmCC" + "history": [], + "messages": [], + "path": [], + "retrieval": [], + "variables": [] + }, + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAFOdJREFUeAHtWnmQXWWVP+cub3+9pNfXnaQ7e0JIIEJAlokpC4JREqHcRmRQxEFnYUaUcpYaLawpl5qB0lGHKreRUUeDE8ZRXGqAYMQIRkI2ZAnZl+50d3p9/fZ773fmfMu973WEcgIZ/MP+ktfv3ve+d+93tt/5nfNdgNkxO2bH7PgDHgivwdix4wfZFQtWvd6KpZK2YwP4AQQo7+3wt340z1Z/HQj4M9u2SZ4FPDc85l+gPCc+DzBAm2zCIEBS3wf8JfAx6LmB/J0lyBKnP//5r+69++67Bfy+xujwyS8JERCRIPNHmGOqv/+fhghf+kA0nAv5Ci8s5JA/CIJADA0c/8zLrc2C12DEk5lebPQ1ooazc3BC4sn8YrnU5VD/FqPrEP7W1dgd2Guq75kcPnbX+OnDb2ZPmCHza6IAfRPRsCQ4l0HRkZYaEPGsCVqhjR/zHP5Y/1R4lXkW2v+cndP90O23vPmmxt868P8wvvflzzYvWbrkRieRaCrkJ2E6P7Eslc1QgwjhcSRCw+lZ3834MPyOpNDS3uwNSlgpK5nLovQS5SvqTqQVJt8srBUKGyeHjp8uTAztnbvi8rHzroC7717vXH3thgcsy90gGKAsXoVXKaolc4SCjEy1VGqwmBHLth0pzEv5hzSxUZ72ISU8GEtD+E0UFCbGMPKOcApVyzc1d/TcFEsmxg7u+OEbzrsC+qDfsRC7+CV1roTVSyXwvBqNjJwBqQPUi1cWNEJBZ0cnJJPJUGZlwnqANw5jaVB6RXODl5hB4aG6hlIGaZXYltMmqHzreVdAf1+/jtFGe6nbI8XjCezp6TEuGxq1vnZOdw1XwkaXnyEXNoQQ4ktGTDhVGQAMGkT31f+BHTJ53kHwzESTNq5cAN8BkBoWJFO4RZZlKWH52LzbRvizwa3+wdZtz8OS278LX/nJb4zywoHU+IOXOJY2hxAQFWaoVAIyJPH8g+DcEPX1jawQrczNa7UqQAheOkZB4yKny1gcfrH3BB9bUCjXYOeLQ7DruUGKxR1Ex6WyJ/AHO4/Tw7uOUKlYwZULO+GKC3qwszlF+XIVNly6ABxWqhJV33GGi5FyBWE+kkoIrPOugMREnm8qNJLJm5E6VlaTwo+MjISGVZ+FGGBZNvT29sDAeBE+/cAuePuVi+jaS+bD+zasxEKpBp994CnwfY+SMcCPv/Myam1K0ZmJAux8foj+actTcMuGCyAQQikgVCvUJYdIJxDGkaU+Pe8K6Mnl+EbCksuw2M8sux6k8Xic5s2bp0+lG2OjgfTy3nrVErj72ztp+bxWuHrVvEiEe/50Hbz9kw/BZ997BSyeO0f9YH5XE41PlWGkUIUbrlgIMdeO4LEOgOH1BdQxUXqgUDc8rxhwz/q25UcevO+ufd/7WteJfbtBeYL0AJmYNSjUY/dlMv2PdhyABS0x3H94WKG2sSUKPj4xUULPF1Fqk988vu84XtHfClt//kLDZemsuzQApQo9HQZyZefFAz66pqUvaVn3xN34jXPnt1tjx3fhw1seoOs/cQ91dbUgmaD0fZ9K5bKGhwaxZRikUilwnRj86tkBuGRZDoqVGhTZ9TOpuJpXLNfIY3tNFasGMYDTagAx9rXujiwcHZiYKScYFkAhDigEUCGn0El+EhC+ag9494Lspumi2FmuBW/Llzz7Wz/YDV/4r2dh25ECPv3UXnkXDgU9l0kQllkB5XKFKpUK8TuUS+ocfM+DY4MT0NOe5bCx4bq1C+FHTxyKTJkv1jhgHShVfMMkCX6++yhctDwH0itWLuqip58fiNZliJAqGqIgQzyLLwSvTgGb56f/uuiJB/PVoPNUwYeHXsjDnuEq3HzDavrbG5ZSrjbCQnucbgI1343Z0NXZCZ2d7ShfXV2d0N2tX0n2gEd3HYMNly9SDv6Gi/vg18+cDEMASlVfCVXxAwpD4IHHnud585VbX7V6Hj66hzMI1IlVnRxrKFCgy0Bp8jRICHzFCnh9e2pTviLuLXiBe7TgwROnKzC3IwV3bVoMc0QF5y+fD2LgWdiz9QtAA/8tzW+MISR5MeCkY1GupVbz4ZmDp2HVok4dp+w2q5d2w5GT4+p+xaqvROHQUEuX3tLX3QxN6YS60hIGRgoETU1XzAopygVkwA/roaCnMFa+Igy4vDO2WgTi62WfrAK733A5wOsWZKG/2YHde06hzWjc5zu46e9vgfaOKaChfVB99lMQW/5RRp2kzPMhM4mwYO+Lw3A1W12zSEvRxLUrcvDIU0do0bw29AKhzFb1lAVp++5jcMt1q4wcktoiXNDfhvsODdG6NX0my0AkdFhBhEPzEM6+cO7DCXz7E9WA2id5MadKPq1ujxP5gg4Pl2HfmQrNvTQN7/xQjjpzJcKmZWDlVoBLe8E/+CUyyIS6JxDqgeCbP90PiVgMHv31UTg5XsTHnj4OA2eKsOvQGShXPI5z7iFxRmEghSp7y6N7TuLBUxNq/tGhPHL4kMvF1NafHagXSaRBMLK4Pp8pDJzjuKA10cPX3igLmsFyAP1pF12+9GDRx4oI4LYbs/CO9QTx6mkQo/sR53D8Z1dwV6ICOPgLrO6vQHz1x2lGU0Q2tLit9aGv/QpRUmImUNueH1WLnccpUcoidFmL8r41z8fD4xV47307IIzwW+/bgQyysKY3o6yvboCak4clla4D6tRcku9z9oAmS6TjlhUbrwWQchDmZhyYrAo8VfRh8xUOvemCALx8GbzJSQwmB4DGn2PkGgRoWQOibTnt3vYQ/OJLd0G1Uqq7Jr8+fduV+LZLc5Thxh63sUDUPFq/rI2++7FrIRF3lJiyy2VxeGTTcfr6HetpbV8zEM8Tsk9YZTK0pgu++uE3GoyRAxvYJpksEN5TIhCdOxX2xmqHnC7n38sBvb8raUOFQbnE8bn5UgdvWAnoFTntWT44iQpZMabFOAy28xsYnJyEQz89CI9vH4P9B/4N773mVupbvsJcFWFOcwK+eMd6JjoEpwbHYU5rBpuyCWXCGiu7MF2Caxc3g8eC+hwOFyxsh+9/8nqSXOHMmTzkGBDjMachzHWYoakFTTkIZzHPc1fA06yD9cOlDyUcZ0veFzmv4H2wr9W+8t0XWeSzJ3i2kAqgssNFj1Xk2LXhyZ9Nw87/eQwef3EKhSdocUcaoFyAqcGTkGptAzeeMswYwXUtWNDXrkUwxnrxxCjcv/2gMuJ/PH4YLl7cRWtX9ihx0kkXM31tFMU2UVhbqSPTetDoL+kv1S/MLdNXlgW2y1627z8qO9r9KSuzaal9VVBkltNsQa1E0iVRBJ5sv8DWHw/D478swTMjVbikOwnXXt6NiUwGhrbdC4cHTsOxI+N06Z99Cldv2MTzrZC+4U92HqV7H9wH+bIPNd9XTi10RwM/8IXtzPstSMdd+MA1S+nmjSvNymgmEzTXCrkAKn3MKKVffTUohPXi5IhXGE7amU5hQ7rFwWpZQIXTxJd/VsRHdldhgPFhZWsMutIOHDyRhxV9FqcSDpWWVjgyehwfu/MO+NYzG5VDomojIay7MIdr+uZE9gNTU4fdJDDZPMM8QEstoLHAaKz8JKhq5kENNRKfsUZftQJOVPxtDw84X+fU9FerSgJlsZdpceD7ewP44S6PTjNH6GPBU44Fg3kPepuATo1M4ws/3gtPHJyA0ZIHiXQWahzb8QTjsqUFzaTjkE4nlDOHTc2wg1CvJBvHbzVTVGvQ1OI64HVqDNuF6hfnoxjCdMy5+JFheWHOAJz1pnjdP97r46lSAO1xdlV21/GqUIKMs0IOcR1/fMqHgPRnl6xbB2PjE5BK8zn3BUTU6ZWCWmbRaKqasLjBEN0RDeybLqG6Jpfe3F+Mg2kaqkpU1iLaAfRsgfTqQuDGRZnOJsQPD5f8dZxV6TsnCDYHAp+d9uBAPqAc58m2uA0FJkwZF2GClXCGOz2TtYAc1aRGaO/uhptuuw2qVQ9st6YFU/1LZfoI3DCqAaVBRXQSCh/m97CpLN9ZASpeOH2a/mHYLtAdqVfVErtzaVtvrMXeWypW205NA5xkUiSz9YOnuO/HWJZkl7+oKw0n8zVgoGbfICp6hBXm6zbPc225Dgv++PYPQneuk5qbstja1o4SlmiGMZFMCw0jgaXnANbbrY2KwgaSTVH7nUwbEHWXSmlMzX9FCljfAZnTtcrHLko0te0b8HCUyYBkYVLy5uamr33iH/5ic6FU7tz6jS3sd1X0eBUV7udVmS/EmLQnLBWdsG7zW2EBc4F8oYyxVJFDwlJVkFyaJDwiou8YdXYa2+BhqDR2ekIlpFJJxpAU1TcQdGoUQFGBzBuH5+4BNy1I9M1JJe7ndL7+kQOTlI5ZUOaFZOMWxWz8yzvvvP07Sxb3vZFX2vldN4mnuFLkz5XLJVhBST52UinafPOfwNqrr8ZsNq0Wmkwk0LFtdgA3onEA4cZP1ADS7zrXm1BpxIC6pFbYYidT/RGZblD4sVANkXNSwI292aVdaeuHFYJlx6Y8kPS3MCUoyZQ4jjg5kfC+2doKToyLGmZrcNlVl9L+F46gw66eclEqCOZfuAre9b5bYO78eZhtaoZMUxODXwbb2lp50S6g7WAof90B6ow9TGVY73A1bLAghD2PkP024kg9PoRJnOcAgjcvTm3uTtn3F3xqPTxRg7FKANMMbjUhamWBn2uJu/cPD0Mxk+5tcl1HNQE3XnMl02CXThwbRBnjF122lno4T/IGCdevMWkArDH1dfg6E1MMJGgrXICGXn8dAsl0DnQ3+ewsGCogxEoZAslkQn8p6pVnnQaQ7CL/bgW8g5fq9qdun5eNf26iIuJHJj2V0karAUiwm+M6XzlY8P5uQvX7+Rxk58pVwnCXFq9Z9zpa/ue3cpyXKC89hsEynkhAjNOUy57icJvLjbl6c1M1UHXHK9o7VAYXGHo8znAEojqekdkS1TOE7PyErSAL6vtsmiBrtYrfvS+A7sLMF1d2pz54eKxqSSLDlR+MsfCMZ5R27C09hdpHDp71I2l1tAJiTwCfgcZ2Y1yoCGjisGxqcVn4BDhuDBPJFKhXIqnmSA4gX0Ymg/pWvYCPXFoqygp3iMIImEkDVQgIjZCR8wA0VqDy9y+rAI73to4m+4FlufQbnxkswOlpn9jyKIX3WLtMbnaJrtqt2w81PONiXMBhywvfQpetLOmt3PWVFvcqAZbZ5z1RBdeTj3Egyj5HueqRpoCWXj02lO2RU5MkDqalJ9SuA4RcqSEcDGpKYGU2mdRYaQAwpM/1PszLKOD6XHZ5W5Pz7bUr2l/31KFxGJr2UZIYJjBQ4zzu2tYeRuz7FlTj//JHS6zemIXJkIJ862P/GOvs7+8hBjP5WIz0tnSKLW5ZyBjI77xBwn+4dEVZvmaamyDd3oFzFq+G5v4LKZ3rVwzJVIe6kad6G2ZzIVx5HfhCmABoSB86lKTwAepQCVsBwvgWgf9SIbCxJ3nVolziG1e+rnfxQztO4Gi+Kt2epPAV2UdH61TVw/dw1ttyYtpfNVYGbOZUeFkPuzJ7sCM3Z8aPKmvJfXJbChtIgW3e47O59++gLHljXMnJ56Vsj8Nk5ABT4Sdg4mkX2y+/GVa86QPGWgbrzAMxoDscYOhvOCWyfDQVNPsjEwLhLpDEkkgJ+mIzPeC63qa3LJzjbl2xoDXxvW1HaarsQZ7Z6XiVXTeQLSQamgisDfFabYClSUhyM83fD3JLvMxenHGkY0rZtUPbqC3hmHNWCAuNigYr4ZVH6GLHsjTW9x5/mNrXbIramKZBWvdwReYCeZdQ6IgHmCnqOpwFKJNNGzWRwgMd9mFG0B3pSAGb52Xf35aAf+1qiSUefnqIpjnWxznVTbJgRe788jrHq771lv40LEtnk9tKfpCrCfnEG2uR11Lg96rmmmojRHqoimolvFIMC8/fcXvfkoowuyW2maTm8R979AQM73kEOi9cBzaDpVQO1km86XRopUTPVkToHlEhdS1Ne00AENV7BSogRB0EN+bS70nZ8FWOTXzySB64lIdxlk6mO455thRWfQ+uH/W93S212OaaFfQwE5S0ltqTDqxojTMTdNjlCW259w+6ry/5vhTUdfU7NzFIHTvqhYwlYPG5LedxWFhKGUV8YetnoDQ5Tpfc+H6FfJwZTMxjxPFZi2R6A8oDBOlyN/IEoxTZRRYBv4QP9a6w0MgRKoB3cN8mjXCIze0p4QN2fW5qSO5u4XiR4K0Tvv+kmmsJUiUqL6jMgFjkL5nEc8FjKZeWrNc2VtfnqPiCrT2Cw0Fb27F1f0Z+Zxkrh1tocvEXdx6FOasPoxuLEzNElteWABQVQfIhC6KwdIiosmXewx4ClYolxQmELxUQKGWAPJfPK5Ephnh3Rz57wNYGHKkwWWG3r0nt8H4m6+Gmiaq/oxErVGVFEZ6iTGK2xAiD8gHodUUK0MGM6jtLt6Yc0m1p9Z1SAilFqPDkt/zkGAwePcBxnIV4Mq3IlXqISscXaQWYh6cawkQBh8EV+Y89EqeHjlDP3HUQ+FWWnVdHNVZETQaBVsBoSWypCvEuaQSJ9HL3LWVb08Ua3dHR4v+SW5QZlo93Z8AKPEvKobO23GrjhbPxmedb0poS+PQaQXkC2qaVZXABQlNpR9TDElrocNNbuvrpk0fh+Z3buVhKUToZU6RK4oUUXCvM1ulRRojyAjuix5ZCVDUX5Y5spqMHLSZagwf3Y7VaAL9ShMmRARg7M/Jc6HR2LmnfwP3Hv2E9LuGiJc/B/J9cUO3n5fuoynm+JHtNNfB7fIFX82qX8uU7GdQ72uMOxuTi6v0IlXQtQKMM87CEIXIIYaagGe6v87dKctC1eCEsX70cspkkJRIxZFqtcp8GxSgPSg2EFZO8tqZGqPFBLkE/NyuXzvuLUxOUm9sbpJpa8hNjo788fOC5j8woKS7hDdwX0uk2yyr68Xh7tUV2UpjZ+b6wUvzii8m+vc3A6PBnruuLzFRF5JIuMcGVj0EHcNZQXmkrZw8AzLsOikA/HG0ehLbr5S5KJ+1ozxS6cxlPRqnLktkxJpccx3KDzK6yPK4vTcNxFOM2AgrXYecETwEhxzn/5+rT8QMRJERL1uYzO6jFnGBuIhvUHL9a9q38FTC3BrNjdsyO2TE7ZsfsmB2zY3bMjtnxhzj+F+LeQhipiiQ4AAAAAElFTkSuQmCC", + "parser_ids": [ + "laws" + ] } diff --git a/agent/templates/ingestion_pipeline_manual.json b/agent/templates/ingestion_pipeline_manual.json index d54ffb858a..56a5ec9dbe 100644 --- a/agent/templates/ingestion_pipeline_manual.json +++ b/agent/templates/ingestion_pipeline_manual.json @@ -1,36 +1,207 @@ { "id": 35, "title": { - "en": "Manual", - "de": "Handbuch", - "zh": "手册" + "en": "Manual", + "de": "Handbuch", + "zh": "手册" }, "description": { - "en": "This template segments parsed files by manual structure. Best for technical documents with clearly defined sections and operational guidance, such as product manuals, user guides, and installation instructions.", - "de": "Diese Vorlage segmentiert die geparste Datei anhand der Struktur eines Handbuchs. Sie eignet sich für Dokumente mit klar definierten Abschnitten und Handlungsanweisungen, wie Produkthandbücher, Benutzerhandbücher, Installationsanleitungen und technische Dokumentationen.", - "zh": "此模板将解析后的文件按手册结构进行切片，适用于具有清晰章节层级和操作说明的文档类型，如产品手册、用户指南、安装说明和技术文档。" + "en": "This template segments parsed files by manual structure. Best for technical documents with clearly defined sections and operational guidance, such as product manuals, user guides, and installation instructions.", + "de": "Diese Vorlage segmentiert die geparste Datei anhand der Struktur eines Handbuchs. Sie eignet sich für Dokumente mit klar definierten Abschnitten und Handlungsanweisungen, wie Produkthandbücher, Benutzerhandbücher, Installationsanleitungen und technische Dokumentationen.", + "zh": "此模板将解析后的文件按手册结构进行切片，适用于具有清晰章节层级和操作说明的文档类型，如产品手册、用户指南、安装说明和技术文档。" }, "canvas_type": "Ingestion Pipeline", "canvas_category": "dataflow_canvas", - "dsl": { - "components": { - "File": { - "downstream": [ - "Parser:HipSignsRhyme" - ], - "obj": { - "component_name": "File", - "params": {} - }, - "upstream": [] + "dsl": { + "components": { + "File": { + "downstream": [ + "Parser:HipSignsRhyme" + ], + "obj": { + "component_name": "File", + "params": {} }, - "Parser:HipSignsRhyme": { - "downstream": [ - "TitleChunker:NineInsectsFind" - ], - "obj": { - "component_name": "Parser", - "params": { + "upstream": [] + }, + "Parser:HipSignsRhyme": { + "downstream": [ + "TitleChunker:NineInsectsFind" + ], + "obj": { + "component_name": "Parser", + "params": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": { + "doc": { + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "doc" + ] + }, + "docx": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": "main_content", + "suffix": [ + "docx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": "main_content", + "suffix": [ + "pdf" + ], + "vlm": {} + } + } + } + }, + "upstream": [ + "File" + ] + }, + "TitleChunker:NineInsectsFind": { + "downstream": [ + "Tokenizer:FunnyBalloonsGrin" + ], + "obj": { + "component_name": "TitleChunker", + "params": { + "hierarchy": 0, + "include_heading_content": false, + "levels": [ + [ + "^#[^#]", + "^##[^#]", + "^###[^#]", + "^####[^#]" + ], + [ + "第[零一二三四五六七八九十百0-9]+(分?编|部分)", + "第[零一二三四五六七八九十百0-9]+章", + "第[零一二三四五六七八九十百0-9]+节", + "第[零一二三四五六七八九十百0-9]+条", + "[\\(（][零一二三四五六七八九十百]+[\\)）]" + ], + [ + "第[0-9]+章", + "第[0-9]+节", + "[0-9]{1,2}[\\. 、]", + "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])", + "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}" + ], + [ + "第[零一二三四五六七八九十百0-9]+章", + "第[零一二三四五六七八九十百0-9]+节", + "[零一二三四五六七八九十百]+[ 、]", + "[\\(（][零一二三四五六七八九十百]+[\\)）]", + "[\\(（][0-9]{,2}[\\)）]" + ], + [ + "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)", + "Chapter (I+V?|VI*|XI|IX|X)", + "Section [0-9]+", + "Article [0-9]+" + ] + ], + "method": "group" + } + }, + "upstream": [ + "Parser:HipSignsRhyme" + ] + }, + "Tokenizer:FunnyBalloonsGrin": { + "downstream": [], + "obj": { + "component_name": "Tokenizer", + "params": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + } + }, + "upstream": [ + "TitleChunker:NineInsectsFind" + ] + } + }, + "globals": { + "sys.history": [] + }, + "graph": { + "edges": [ + { + "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", + "source": "File", + "sourceHandle": "start", + "target": "Parser:HipSignsRhyme", + "targetHandle": "end" + }, + { + "id": "xy-edge__Parser:HipSignsRhymestart-TitleChunker:NineInsectsFindend", + "source": "Parser:HipSignsRhyme", + "sourceHandle": "start", + "target": "TitleChunker:NineInsectsFind", + "targetHandle": "end" + }, + { + "id": "xy-edge__TitleChunker:NineInsectsFindstart-Tokenizer:FunnyBalloonsGrinend", + "source": "TitleChunker:NineInsectsFind", + "sourceHandle": "start", + "target": "Tokenizer:FunnyBalloonsGrin", + "targetHandle": "end" + } + ], + "nodes": [ + { + "data": { + "label": "File", + "name": "File" + }, + "id": "File", + "measured": { + "height": 50, + "width": 200 + }, + "position": { + "x": 50, + "y": 200 + }, + "sourcePosition": "left", + "targetPosition": "right", + "type": "beginNode" + }, + { + "data": { + "form": { "outputs": { "html": { "type": "string", @@ -49,186 +220,169 @@ "value": "" } }, - "setups": { - "doc": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "doc" - ] - }, - "docx": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "docx" - ], - "vlm": {} - }, - "email": { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "output_format": "text", - "preprocess": "main_content", - "suffix": [ - "eml", - "msg" - ] - }, - "html": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "htm", - "html" - ] - }, - "image": { - "output_format": "text", - "parse_method": "ocr", - "preprocess": "main_content", - "suffix": [ - "jpg", - "jpeg", - "png", - "gif" - ], - "system_prompt": "" - }, - "markdown": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "md", - "markdown", - "mdx" - ], - "vlm": {} - }, - "pdf": { + "setups": [ + { + "fileFormat": "pdf", "flatten_media_to_text": false, "output_format": "json", "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pdf" - ], - "vlm": {} + "preprocess": "main_content" }, - "slides": { + { + "fileFormat": "doc", "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pptx", - "ppt" + "preprocess": "main_content" + }, + { + "fileFormat": "docx", + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": "main_content" + } + ] + }, + "label": "Parser", + "name": "Parser_0" + }, + "dragging": false, + "id": "Parser:HipSignsRhyme", + "measured": { + "height": 57, + "width": 200 + }, + "position": { + "x": 316.99524094206413, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "parserNode" + }, + { + "data": { + "form": { + "hierarchy": "0", + "include_heading_content": false, + "method": "group", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "rules": [ + { + "levels": [ + { + "expression": "^#[^#]" + }, + { + "expression": "^##[^#]" + }, + { + "expression": "^###[^#]" + }, + { + "expression": "^####[^#]" + } ] }, - "spreadsheet": { - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} + { + "levels": [ + { + "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)" + }, + { + "expression": "第[零一二三四五六七八九十百0-9]+章" + }, + { + "expression": "第[零一二三四五六七八九十百0-9]+节" + }, + { + "expression": "第[零一二三四五六七八九十百0-9]+条" + }, + { + "expression": "[\\(（][零一二三四五六七八九十百]+[\\)）]" + } + ] }, - "text&code": { - "output_format": "json", - "preprocess": "main_content", - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" + { + "levels": [ + { + "expression": "第[0-9]+章" + }, + { + "expression": "第[0-9]+节" + }, + { + "expression": "[0-9]{1,2}[\\. 、]" + }, + { + "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])" + }, + { + "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}" + } + ] + }, + { + "levels": [ + { + "expression": "第[零一二三四五六七八九十百0-9]+章" + }, + { + "expression": "第[零一二三四五六七八九十百0-9]+节" + }, + { + "expression": "[零一二三四五六七八九十百]+[ 、]" + }, + { + "expression": "[\\(（][零一二三四五六七八九十百]+[\\)）]" + }, + { + "expression": "[\\(（][0-9]{,2}[\\)）]" + } + ] + }, + { + "levels": [ + { + "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)" + }, + { + "expression": "Chapter (I+V?|VI*|XI|IX|X)" + }, + { + "expression": "Section [0-9]+" + }, + { + "expression": "Article [0-9]+" + } ] } - } - } + ] + }, + "label": "TitleChunker", + "name": "Title Chunker_0" }, - "upstream": [ - "File" - ] - }, - "TitleChunker:NineInsectsFind": { - "downstream": [ - "Tokenizer:FunnyBalloonsGrin" - ], - "obj": { - "component_name": "TitleChunker", - "params": { - "hierarchy": 0, - "include_heading_content": false, - "levels": [ - [ - "^#[^#]", - "^##[^#]", - "^###[^#]", - "^####[^#]" - ], - [ - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+(\u5206?\u7f16|\u90e8\u5206)", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u6761", - "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]" - ], - [ - "\u7b2c[0-9]+\u7ae0", - "\u7b2c[0-9]+\u8282", - "[0-9]{1,2}[\\. \u3001]", - "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])", - "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}" - ], - [ - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282", - "[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[ \u3001]", - "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]", - "[\\(\uff08][0-9]{,2}[\\)\uff09]" - ], - [ - "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)", - "Chapter (I+V?|VI*|XI|IX|X)", - "Section [0-9]+", - "Article [0-9]+" - ] - ], - "method": "group" - } + "id": "TitleChunker:NineInsectsFind", + "measured": { + "height": 74, + "width": 200 }, - "upstream": [ - "Parser:HipSignsRhyme" - ] + "position": { + "x": 616.9952409420641, + "y": 195.39629819663406 + }, + "selected": true, + "sourcePosition": "right", + "targetPosition": "left", + "type": "chunkerNode" }, - "Tokenizer:FunnyBalloonsGrin": { - "downstream": [], - "obj": { - "component_name": "Tokenizer", - "params": { + { + "data": { + "form": { "fields": "text", "filename_embd_weight": 0.1, "outputs": {}, @@ -236,325 +390,33 @@ "embedding", "full_text" ] - } + }, + "label": "Tokenizer", + "name": "Indexer_0" }, - "upstream": [ - "TitleChunker:NineInsectsFind" - ] + "id": "Tokenizer:FunnyBalloonsGrin", + "measured": { + "height": 114, + "width": 200 + }, + "position": { + "x": 916.9952409420641, + "y": 195.39629819663406 + }, + "sourcePosition": "right", + "targetPosition": "left", + "type": "tokenizerNode" } - }, - "globals": { - "sys.history": [] - }, - "graph": { - "edges": [ - { - "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", - "source": "File", - "sourceHandle": "start", - "target": "Parser:HipSignsRhyme", - "targetHandle": "end" - }, - { - "id": "xy-edge__Parser:HipSignsRhymestart-TitleChunker:NineInsectsFindend", - "source": "Parser:HipSignsRhyme", - "sourceHandle": "start", - "target": "TitleChunker:NineInsectsFind", - "targetHandle": "end" - }, - { - "id": "xy-edge__TitleChunker:NineInsectsFindstart-Tokenizer:FunnyBalloonsGrinend", - "source": "TitleChunker:NineInsectsFind", - "sourceHandle": "start", - "target": "Tokenizer:FunnyBalloonsGrin", - "targetHandle": "end" - } - ], - "nodes": [ - { - "data": { - "label": "File", - "name": "File" - }, - "id": "File", - "measured": { - "height": 50, - "width": 200 - }, - "position": { - "x": 50, - "y": 200 - }, - "sourcePosition": "left", - "targetPosition": "right", - "type": "beginNode" - }, - { - "data": { - "form": { - "outputs": { - "html": { - "type": "string", - "value": "" - }, - "json": { - "type": "Array", - "value": [] - }, - "markdown": { - "type": "string", - "value": "" - }, - "text": { - "type": "string", - "value": "" - } - }, - "setups": [ - { - "fileFormat": "pdf", - "flatten_media_to_text": false, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content" - }, - { - "fileFormat": "spreadsheet", - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": "main_content" - }, - { - "fileFormat": "image", - "output_format": "text", - "parse_method": "ocr", - "preprocess": "main_content", - "system_prompt": "" - }, - { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "fileFormat": "email", - "output_format": "text", - "preprocess": "main_content" - }, - { - "fileFormat": "markdown", - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "text&code", - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "html", - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "doc", - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "docx", - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": "main_content" - }, - { - "fileFormat": "slides", - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content" - } - ] - }, - "label": "Parser", - "name": "Parser_0" - }, - "dragging": false, - "id": "Parser:HipSignsRhyme", - "measured": { - "height": 57, - "width": 200 - }, - "position": { - "x": 316.99524094206413, - "y": 195.39629819663406 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "parserNode" - }, - { - "data": { - "form": { - "hierarchy": "0", - "include_heading_content": false, - "method": "group", - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "rules": [ - { - "levels": [ - { - "expression": "^#[^#]" - }, - { - "expression": "^##[^#]" - }, - { - "expression": "^###[^#]" - }, - { - "expression": "^####[^#]" - } - ] - }, - { - "levels": [ - { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+(\u5206?\u7f16|\u90e8\u5206)" - }, - { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0" - }, - { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282" - }, - { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u6761" - }, - { - "expression": "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]" - } - ] - }, - { - "levels": [ - { - "expression": "\u7b2c[0-9]+\u7ae0" - }, - { - "expression": "\u7b2c[0-9]+\u8282" - }, - { - "expression": "[0-9]{1,2}[\\. \u3001]" - }, - { - "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])" - }, - { - "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}" - } - ] - }, - { - "levels": [ - { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0" - }, - { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282" - }, - { - "expression": "[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[ \u3001]" - }, - { - "expression": "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]" - }, - { - "expression": "[\\(\uff08][0-9]{,2}[\\)\uff09]" - } - ] - }, - { - "levels": [ - { - "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)" - }, - { - "expression": "Chapter (I+V?|VI*|XI|IX|X)" - }, - { - "expression": "Section [0-9]+" - }, - { - "expression": "Article [0-9]+" - } - ] - } - ] - }, - "label": "TitleChunker", - "name": "Title Chunker_0" - }, - "id": "TitleChunker:NineInsectsFind", - "measured": { - "height": 74, - "width": 200 - }, - "position": { - "x": 616.9952409420641, - "y": 195.39629819663406 - }, - "selected": true, - "sourcePosition": "right", - "targetPosition": "left", - "type": "chunkerNode" - }, - { - "data": { - "form": { - "fields": "text", - "filename_embd_weight": 0.1, - "outputs": {}, - "search_method": [ - "embedding", - "full_text" - ] - }, - "label": "Tokenizer", - "name": "Indexer_0" - }, - "id": "Tokenizer:FunnyBalloonsGrin", - "measured": { - "height": 114, - "width": 200 - }, - "position": { - "x": 916.9952409420641, - "y": 195.39629819663406 - }, - "sourcePosition": "right", - "targetPosition": "left", - "type": "tokenizerNode" - } - ] - }, - "history": [], - "messages": [], - "path": [], - "retrieval": [], - "variables": [] + ] }, - "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAHkZJREFUeAHtW3d4XNWVP+dNb5JG1erFckOyXOQeYwMGU2IcdoHEC9jkSwjLhhKDQ0J2U5zkY5fNQhLCLglLCgECLBDANsbGFBvbGHfJapZkadS7Zkaj6fXuufe9NyMSXL/N7h+bm1jT3sy7p//O71wA/rr+fy+8mItf3Xqrvj+tSFNZORsZ02p0OgP6fHrU6fxYlpEBQYNJisXMGo0miJqgRvy2Vq+XYnqfRgpb0CxJqNHGpKgmJiWYUatJhDRAX9do9VoGMZ3JaNHrNVodg4QhFIrptAYjanUoJRKg09GVCUAN3VgDCZRAQmOMNsHicQ0Ak+hmqNVo+yZHO/f4NSWTixYtil6ITBekgL3vv3qd3aK/W6fVFuhNVqMlLZM2gtoEgAYYI7FoQwCSRiNJDEEXjyV09MsIDBAZaBAT2gRtnF7w6+me/CvKBuiJJHF5gF/OnyP9n9EfujRBnyJ/W2yUMfl68RTF4p/S24zuT6phCdIH66fLWpvbG+5YteqLY+eT7bwK2Lp1q7Tuiqrm8SHHrEgMcfmV68FiszNJowV5L5jcFYqnXAdcdpA3yFWg3kwIxfi7n767fJ2sG/5NkL/H5J9gQjO0ElzL/CYMFV0giczkq/nXGYg//Jb+yfFTe/cf/dz69esD55JPOteHWx54oHawf6RuzOmZ5feHoKW1i8ysk2/D4pBIxIGJfzHaXZw2StuR36Pt8EfaMkvwR+SP/B+KbSZkcfhzvmEmhARZBVwA/koIwmSzCym5nwtlUogo1wgtMFV4YQAJxTPyohn0po1r61IVoK2cOXtXS3NTu1ZiTr61aIz7OYLX68VJjwdTBkQmu4Mwrmxg/lRSHENCYX1hckm2tuIuyD1Gvl6xM39NN2OgOj7Anziq/JH6HoUJ9ypIOZrYTyAYsWwbLzry9dc7Wn74Vv21cAkKiHt9QWfV3Pk1/aNhg8FeDPMWXw6Dwy544Xe/hW/ctxk6znRAyptRsYbE/0FSE2oYqAphclBPDQOmis9kWyufq7EASYHZFMFT4k55ZEquYBgn7+sZ9pX0uxOzA1G48VIUwJ559um1/QPdvzx27Bju2P4uvPXWW/Du2zvw2PF6CAT9TKvVCD9UvBhVRWzf2wT/8crHLE5e3tHrZD/49z3cskAJCn/2wsds/wmHeP3yznr4zR+PCrd5ZVcjOPqc8MTvDgh97D3aBR8d6xFCv/B2I7x7qFMoc9/xbtzNn09Vgcg7kNRm0iDcO0gRCZaQLkUB0NnZ2aeJR3etWDQrUZ6rg9wME9yx6Xb20yd/Bk8+9SQWFRfTTVTLi+gW+3jzgxZ47LnD2No1Ak+9dBB+va0OySXhWEMP2763BZ54/hPaFMCuQx3w2O8PseYzw2zP0W7WP+xl//VemwiDk62j0NA5zjp6x+G1PU3wxItHIBaLQ33bCBynz1LCc6dJyKkFlBRKvoSikmjlcInH8ZIU8PhPfvqIwZZzksqvLT0jTcSuhrJ/eno65E2bxqhkT0nnmEwDHm8Allfns3cOtIHL5YNMq4F5PH5SxEm8b8MydPQ7oaG1T1j3rhtrcPPj76BnMpA0Iv/LkyllWXh1TzNctWQ6aLQa/PBwO71F4vLkqiwR/4BqIVLelEOJhxyvGppzyHhWBdTW1uqstrQfdzran7VnpA8bTGZKgBJMeDzw0Uf74dTJeohGImrQKb4nZ6JAJMFuWj0TX9hZB2tXzgGrxUC5YwJONPfDwVPdrLI0F7bvaxep/6qlM6E0PwNOnBkHk0FLQAnhcH0PtFE45GYYccfBDujsc2KmzQDP726FOClmYjIATW2D4PYEVIHFb6leIDTI8w6vQLzm8Cp1lnVW5QwNDSUWLV42Pysr59ZJXzizsGwmls2oIkvo4aMP3oOXX3oFV6xcDhkZdpHclCQo9NFI1r3txkXYSSFw95eWQ4djhKpkFBZVlcA/3XMNVFfk4KET7VCYl8aW1JTgVYvLoXfACTdfM5fNKbXjK3saWE1FNi6rLoI0ix5+cM+VcM2SMmhs64Oa6TkwMDIBzR3DUFM5DdJtRkjlWyUR0kPANwnbGj1AVsQcbeD4wTd+s/Oz5DwfEMK1a29YsuCyom1u53heNEoWu2YNO3PGIe5z3wNfZ7a0dLmaUdxKkpz1CcuBCtpkJCeBnOtTO2Uq4JFfy9conqRAG/kCppZYNhUKMnXrTP4hScUFjMkgcWy4Dzb+ZDcY7faQSa95+JVHbvmPzxJQex4FsP7+7rqNNy2L9zpC4A0Bu/baa+DG9SaeC6gKyF/nmw0G/Oh2OSnzMyUuVZgrwxu5xMsVIxWvKENgBdZKyUTKVO3JiFr9AVCTrYr+JJEA+aU5uUVAvUNy4/F4BKePvdlQjP0/ev1l166zCXhOBXzzm49c7XJNfk/SGvJtVjMLRkPipjqdPlnKUSk9JrOZ6fQGGboqYEaBwqBgZjlE6PNoOII8kRnpOzIE4NfK2IElIW2ClKeREaHIaZBUmgx3VeQj4CLy5KzCaP5o0sTZt646Njd7tuXlv/1C7ndmrR564qIUUFlZaZg+ffq2/fuf23v95TNcJos1k7lDpNk4dnf3gNFowPyCfAW2yBvT6+SyIyNAZKiCuaQCAOpPnoSAxwkV5UUQ9FlQ0hkhM7cgmcWd3Z+QJxwB72gdQvr9rGTmoqTgAEnQJMAWpCCw/AhyAPHFgy7NGkcTjOp0cWv52eQ8axXo6OiITE76HbNmzZnR3uO0mLPKoGbx5ayppZ09/dQv4NvffJiNDA6pGRiVzSU3yQScZclAjYZD4rGsvIyS4wDUn2qCiHdMSBONhMS3RjvfoUbrdTTq9mLnacYy80qpHEbl78rCKZUWmXI3EV+MJVsD9X1eKpPOJ3qRi1UA/7FnnvnFylDI+2vX2Eh0z+49uHPbdujuaAejwQQzZ85AvVEva5xNgaM4FY+pFmHQ2dEG7aebwWqzsnXrPw/+UAKCkRiE/W423N9Dl1D7HtuJxvhBcA0yqFq9BbhDjbT/G4z2vAFTIB5ve9VGINk4MUgZAtTeAFO7ONs6Zw5wOBweyeH4xS2//tfN7c1+q8sbxRvXr2O3btjAuBG0VBJxipyyheSYlNUgp4poLIod7WfYnBlF0NbcBDk5uTBjOkeRCSpXXgyEo8yfZQNNrBkgGsCc7BHwRd/EiR435GV+Ar3Dm0DNBUpGle/Gpmhd/jzpF3IOkjdHiVm6JAVs3rw5w+XyPkqch91qs6E36BWZXORfjUaRHNTbKsmKkVtJHIKgWr54gHr9EUKFbsjOzsZJ5yBYDBKEIxHo6HJAVfVs7Dj6ClQvtQOEo6D390Cm5jkAQxwGTkQQCkvFrZgqlIJ3VeWnbK0mQUjmHb54xTqbjOdEgmWl01t6unsu12i1MZ7leUvLk08gGGDRaPQs3WoqVtVF5ZItXrII3vnwBDa1nIG+/kFoON0JB442wpzZxZDwfAyVc/aD+72jEBweglDMAEOOMHyyMwxNndezwsvWKL2/4unKXdSVwgCgKhyTuZf2HD9HDjirB9hsNl7WQkXFJQNuf7zUkDYNp8/Jh3HnBGx/44/YPzCEm7dsZgUFRUptF5hE7u8VrSSbdHpSWFTA1ly1mh34+BD09Q6gBiKwojoLYmM9UFzeCLETDWDMNUNPaxQ+eFti9tKlWDj/arjyho2g1Zmm7CzpArIaFA4FYIrWZZyFFHkQ9ACEhsbOXLQC9u3bF8vLK7mOFPjwqaYOXUa6lcqcBt7dvYdyQxeMjrk4AaR0AEr0JwMTVYoHFXADBqMRliythcrppegcH4PWllMQGXoRiku8EDrqAFuFHQ4cisCOvXmw6eEfweyaJcxkzSCn0yZDW7Gz0vILnJFkhVgKKYrHCIXh8x8taC4M7P1Kj8vSejY5zweF+TK+9J+PdXafac13+aL40Lf/kU16/WC1WjEtLZ3QoE7hPHj7KXeFkuD+lNKuxiqqO0+gd7wfxrr+jZWXNaNzVz0YyyxwrNUIr38wHe746p3QPehl5TPnwIpVV3Jui316m6kak6QQVe4QZAvw7bhHe/HwW78KzfI/2RZMN32v+i7vDrgYD+CrpqbGUlOz8BsSau06Ij80UpxpiBMrKChIEPedhKugWEINBQXKsqkbU7g+mHQ6mM+5FYsz23BshwPSazLgow/j8MtdmVBdO4P99g870UAAsKf9FPb3ONiNt94OZrPlU/tiClROoQE2lTEWS6/XsczcfIPRwebp0/WcEvtMBZwLB0h33Lbp6EDf4Ff1JmM8LSMdDHo9ctoeFco2GY/sz4hHVDE9qNgfE9DXsgu0kfsgK1YHrr2dJHw6HD8QgzO+G+D+R77LTjY78PDxRmjpHMSOfjdrOnkEnvrn78NA/4Dym0zmxVgyzYBa9PFTUIxzlzog/l5kJBbBiy+DZWVl+gx7ZoUt3f47gu03xxKSleo+6+7pgQPEBxQQDF573bUU2+YplUeGvyngLmN/3pd3HH8WDP5fQXDMhRJ1VcZZdjh9NAjt3vVs3ab7wJ6Tj1u2GOHpp/4dzjh6WDAcJRYpCiuWFbHWxjqIhEKsrKJC7XdVT08qRoXD6l64AnRGm8wpXwoj1N3dHXrjzVf/ITvL/oUPDrbkMFMBK5uzAFtPt4HHOQa//90LODIyKiceNSb/3BPER8P9faifeBxOHdTjntctkFVlhqZPAtDsWQ/XbNwMRRWz0GpLZzXz57EHtzzE5syswIDfB3HUw+zqhZBjm8Tje34JaqVRPY6xFPxl7NN35byFibiKQAA+xSD96TpnDti9e/dz+fn5r37jq7c6mo59mBtHLdy+6ctUx/vg1g1fArvdLl+In4a+mOwRRYkCvUHPjh9AWLxgCDRLAQ68EoIh7Zdg1c33QEHpdJ7pRZwQ+cJmzQH8yt1/D79+5lmah+nhDy+9BgZNAsxGHd5CZQdlruFPFqYaIQWE8sENdZMQ1xKVJxjIS1AA/+XLV1y+pqy0yBwLTuAk9SRVVVUwf8FCkXB4BQA1xyu9ukqPyw2K7BBa6vhM874L2z54nknxABYuWger1m6AnIJyAazUtpngJWYRTK5dtAjS09LY/gOf4Mm6BtBaLLDxa19T5wUqBQlJAKpuFnEKKcwdJc6k9CJIRLsDl6SAf/rH7/22saFlAyFBncliAi/vyjjJSJqVUaGKdlENphQcnzIAs6alwcJVN7CKuUtBq6EuL2camq0ZnxqjgUKI8B/KzskBm9UKRcVFsG7dWrBnZrKCwpLkfCFl6VT4JdUglwMZLBOGGDKv+b2/6ZknL1oBFRUV6cXF5Xe+8eZbj5jNVz+o1xvyJAwLeV1uN6OujjgBo9gIL/v8/THnpOgE8nPTwTfp4RAUJCUnSuQtWbnThIa4Q/q8k7KvyKMsUVmMVO5i0QjG4zER1FlZWeIfV2bQ71WJEz4DVadOQmSVajNZbWplkJVFCshbvrF9xld+3nfRCuCdYFdXx2tXrL7quyeaB6z5uSVQXWQjPqANtr3xJtFfLvjhj38I2bm5ov7HaDR4zw//yI41D8CZdx5BTo3FYjGFLwS5i1HgoqROe1A0TqCQYqBAeEG6SJKkZDbe7wtIKQQHjIsvinSASucpKHANY0kUqkJziRspD86xzhkCf3jp+S8vXbriBpM2+NzoYI+FrIilpQPM0dEJJosFk+mXNqIlyvyhO1djQ3s/0xt0zOMexYhCZPAd85E3yBMczCRPmHQ7SdAYV4YQlksT8E6kYK0CHyQVCSqlTuQMcjl7XlGqL57S+ykvBGcQiwTBlG7IP5eM54XCVAXMj3//XkdrS1NeKK6Hh771beYPRMBiNaHVmsZ4W4zKbvnGhDDqO6hYLgmLhaCy7ZmC4RXCE5MWFySbXD5k32bJ3kexMB+JS/KcUTyXUxGmaiJw4cPYeHg3K5qz/JO8aeWfg0vxgE2bNtUQFf4EanS5hAJZKMgED5Cdk0U9toZJSuORBPudOyA+Xkel7vswNNAH4XBQociZIDhl7pp31RIolDnmFZWBa7gfEkp8KIQ5qoSoCnhU3ltBPEzVo0zAIfGKJaL08bG71zlAICrMuIdpIJEztXG6YAVwUrR24dJ9//Xqa0dNxsUBi8Vi9pJA8rZFyyupXJTK9iaGD6I09B6wmgeQQgR0Br1qelDHVPL3UySpz+MGg9kKZvImrjDadbLMqVZFZa6gMxg4wsNQwEvRLTPFVKEo+WUIFjka8EAwMCk8zXG6DnKn5fAzNfbNm2/LpUtHLkoBNP9L0I8HiwqKPG5vKBrnJ10IsDgcPXD06BEsoRK1+oorwUCb4kuAs0U/IAN8F8CcDvqEVwARTI7HQXgDNzE/BiPcWD39wa+hmRjxD8Jqyf5RWF6Neyq/BIxQACuTrCF+6IJ+IBKYEPkkFqXv0h7dY4OUgEOYlpHNYrGwdc2qKwqffPLli1PAiRMnohUV719nsWS8eLiuI61qzgxYSITI2Pg4eN0ueHr7LlZVXQ0FhUUyT89dW2dSRpWMeb0TAs5KyY5REgmPv8wtKIWJ0YHUcRk5Zcgnh9SEoeQU/lWj2Yrp2dMI1ySEUgwCgAEGJseZ3kylj4SnygExYpeDAR+caW3Eylmzmd5oomoUM2RmF5bR9Sc/S87zJsGioiLT9zZ/2dF6ujnPFwL8u42boLGplWVlZsDatWtoIGKVOUJKYpLKQ8u5QfivnMVBLmuoZvNkCyJnNUy2skxWX0KNWQVhyhcrCU+saCgAIwNdRBfOoEYpgCQ4I4IV+rvPUCgFcPGK1eRVenCN9mCY2R6cPrv25/x7W+++25wzb5507733+s7pAXwtX77cVD2n5q70NGtaho2mw6T42bNmMOIJxGkurU6X6gFIsFjrmxB37AL9NT8Dty9I1gjIypGVgmqFUEGMAoRALXeq8SWl2xalk6XGZExuatBoMpOwHuAeFg76CCT5xPORwV50OkfZgiUreJImj9RR4vZBiMG8H3/r729ZtLCmVmu2XXngwBHuDV8/nwKkG65b9/7BQ0dr9MbLtHriAiAUEW5JuYEp3pw0JE+MMXcvxgfrGIsEkJ+Z0vKTZBpJxLnFamUc4ESjEZCmJEFZOCnlBMpHhDxFVo+GA0qhYZCWmSumzN6JceHqEfotjidoLgnOsSHo7e5g82qXyE0a7XOgvxfGRt1gTdfccdMtX7yzf2AQ3/vw0Ommxsafqzs/Jx+QO61gkdvt/hez0XSPVqfPZYmwgGt+n5+ZzCZih6QkD8fNp1v+DTCsfFA4slUb5EdTZHqM/mck3kCcLVPQoerLEqbOECULGwoqWyhWLwaeMt8Xj4XAP+mmKuAjKO0RpXJywgVuas+7HK0wb+FiyMorgP7+Qezv7YVp1D9MK18AnZ3d2q6u1tj77+466HaNbNqz/1gSGp8zB9x775bvx2PR72RlGA0F0+yUpY0wc9ZM9u7OneCd9OCD39xCU9k8NgXuyqww/R0Z7AM/x/uqn3DYixr1WjnpcWAkeiopeXhQUhsZ8hxJ/klmScugXGMBjhTDQb9w966uTsgvIGG7HVwBWDl7DvMGoqQ4A6O8gGMuPx48cIAN9nbw00n8l+871ux4hpO9U2U8bxJcs2ZNzbIFsw46Orus4UgcCwumsda2dqLNrfDYT/4F8vLyRfZG+YSYsBYvfwr4EHGsDDJVgI5SspHBJIfDr1OhXKrFI9/hGZ4sHw7QBIks7/d6WXfnGTATFB8Z6GNuDzVm6Tlos0+DubXL4VTDaUaWlnQswAwUqdnT8iEzI43VLF55f03t6j87I3DOJMjBUHZ23qw5lcWIwXGY8MXhwYceAI/HRxYxYaadd2o8OTMFs8rMCz9AqaJd/hZiQlGMOClKX1DKHECSO0gwpb8Vx1qYMmRKQDwaJGTXBRPD7QTBAT1+aobiEXZo3yGWnltMA9ur0WLL4ifX2K5HHwMbTdxzLRqWlpmPy1d+jlVUzkQiWAm6W/6WbndRCsAvb/rq24ePHLmC4lFjMZvBR0iNkiGUlBRzClzEoNyVgRA+ReCDQg7LxztVWM9rNcpnd2S0IGcCedqDyuEncbKU2uEEAZuIF9564SkIdb0LN21MYF6ejtW1zoah8ZXw+dvux9ExJ9uxcx9MugYh3UztMAYgJ6sALl+9CguKiinvmCiHGCCNqDGz1Xw5n3ZxfHNBCqBSZ87Oyb2SJkCP2DMyHhwymqmrCik9DvKT0UxlciTlzILS6jKAJBmSnBmkxkcJ8WE8HuFljSWoMvBkySFwLBZk4YATw54e6OtuZvs/boAPDw9BZVkV2Ou17AsbjLg8F2H8vSj7z2d+Ax7XGOTaDcymT0Budh4sXn4tlFVMJ6LWxHQ6Q5K0MVnTUCcldN9+6K4FX7z9xNELUkBDQ4O/p7fnuSVLln6n7nRfhtGUDnkFVtY3MAKNp+rE0Za5VTNphC2yNcoujizFA4lTseKVRqMAHy4o9xSyboJFeCnDuvpmGBkaY4FgkDf5YNJ4YWHxKcguiKDTW8OuvWoZZNit4A+G8MNnX4faQie0vf8BzVAXMyv9bmlJES5ethzLK2cwoyWDabRCcNF+85IbpmErQwMbH+7EuTVVN9EuLkwBfL3wwm/vX1Jbe5iFnT+j2mpNS0+HE8dPsOGBXjh67ATO/+etLIs6Q17amHxYXW7VKAfEqV4nYlHC52EIBQI0pwuRdf0QiQRYR9cwtrT3wcSkj/m8QbCYyU0JaFmM9HV9ELIum8sW1WpxPNqGdc0lrKGhH3p7ByEjgWw8n2E0GAejPUKjtnlQUFzCwr5RaK0fIo6Sww6ahZNPcePzc83hcEygRltWOTS1tR36UxnPqYD+/v4gxe1Ljz78lUfr6+otrmHENffcA4dpMrzxzo3M55skIUMcc4BGaXG5lROJqMDlHMTEKImF/OOMU+jHTvVCfXM/WC1WAlM6ys4WyLEbcOZ0DausGIeVnwO0G4KA7iZ01wdhVXoEfvTOJHgCRkEBlaclYCJTghCRPLULCiErI0xDj24k3WIsFiFcoh7KitOjlukMRuQlNOCLodvpGiKaoPuiFKAug8nIYmRVo9FC+SmKJYXZLBoO4UDvAIvGIoTOYgpQiQiho+Gw7AEkvGt8BCw6mgW63HCi1QL2rCxMsxqhIDsMSxcHcOUKBnnE+6O7D+LDk3C8GWHHITscb7ewdaXDaMYYBGmXWTYjlKcFcTJzOtjTikjBfnCPh0Wcx2MJ0Q4n+COH0NS46nVaoO4S9E4/+AJhVl5VEdiw/q6mi1bA0NBQ2OVyOy1mU24whhgKh1gkEub348dXqUc3E+Tlc0NKkcEAeNwx5I8+9whkSSfhzqVD0DWsRU0Zwu2fn2RdATObUWnE8hxyS3STtUdhrD0C7x7Ww77mPDjdpwN/hNA0KbrNpBH3mGbWs7xMM1qyclnMbEEb8QyBAMcHjCdjUM8ncpqeeyI/xC3mAnotC8cRIzHNW81NrVs+S74L8YB4Y1P7zcT5/SLO4ld1d7RrQmRh8nNyuygLBYLgI2Q2MUHNSTAYp9dhz+SkMTfTJh0cs8PRDjPbsGYM19xWw7Mk1LoIhY6fhlBfFOo7JfjjoXQ41JoHvqiG+SjRhSNBZaYQY71BG5nSDuk2Ez9SB3HJCFFmomt1EJP09NpEQxM9JWIDV1RMa9AHtJIUoKLjCoUCrc4xX7PZojsTZ9oPNt3/+NBnCXdeJKgu/p/OxD0Dc8KRyBqirm0Ea4ME8Mh30ZVIaCcIsrj1poS3t2co0dExcCwajdr5CP2GK5ew2kXVYI4+DStKHMh8NAztkeCn23Ohud9I8R2HEGVqjhE4IEizmoBPoovzcwTeKC4shKLSApafl+0yW8zDVqvZScYYI3ZonJDIIFWUYWq8zgQhNpBmNLmG/YFgbe2NwbNRYJesgAtdVdOnUyqLHiRHgOKCHFhzxTJqSUN4uqWF/euNdfjiPgNUFiTg0XdyuNCMH1/hbssPVJPwjKZIuKCmis2aOQNKSgtbsuyZb9Ck+815q26uv1ChLmZdUBK8mBUIBPrtdiO1qhL0DY7C69ve470dxOIxfPE9Ewz76ZaUHPnMgPcQWRk2AVb4/LD6sstw3vy5wfKK0o8zM2zPdI01br/+6q9F4C+4/sc9gK9lNZXv+IOR6wMhufePxhkjfgIW5wQw3xZhCQKRR8ZKIEpKyKEx2GWzZ8GC+dXOrGz7TzPt9rd3H2xqopBLwP/C+h/3AL6oEt5pNht+RRb+m0AoijotYppZD9kWD6suY/haaxFvq0Pza6qGZlRWfJKWbvrNqus37v1LuPj51l/EA/jinaQxEf08kxJrcrIz8yk7S0SrefLy8oaqqi/ry87KOtnSd6pu69bnQvDX9df11/V/tf4b8hrFeF52qJEAAAAASUVORK5CYII=" + "history": [], + "messages": [], + "path": [], + "retrieval": [], + "variables": [] + }, + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAHkZJREFUeAHtW3d4XNWVP+dNb5JG1erFckOyXOQeYwMGU2IcdoHEC9jkSwjLhhKDQ0J2U5zkY5fNQhLCLglLCgECLBDANsbGFBvbGHfJapZkadS7Zkaj6fXuufe9NyMSXL/N7h+bm1jT3sy7p//O71wA/rr+fy+8mItf3Xqrvj+tSFNZORsZ02p0OgP6fHrU6fxYlpEBQYNJisXMGo0miJqgRvy2Vq+XYnqfRgpb0CxJqNHGpKgmJiWYUatJhDRAX9do9VoGMZ3JaNHrNVodg4QhFIrptAYjanUoJRKg09GVCUAN3VgDCZRAQmOMNsHicQ0Ak+hmqNVo+yZHO/f4NSWTixYtil6ITBekgL3vv3qd3aK/W6fVFuhNVqMlLZM2gtoEgAYYI7FoQwCSRiNJDEEXjyV09MsIDBAZaBAT2gRtnF7w6+me/CvKBuiJJHF5gF/OnyP9n9EfujRBnyJ/W2yUMfl68RTF4p/S24zuT6phCdIH66fLWpvbG+5YteqLY+eT7bwK2Lp1q7Tuiqrm8SHHrEgMcfmV68FiszNJowV5L5jcFYqnXAdcdpA3yFWg3kwIxfi7n767fJ2sG/5NkL/H5J9gQjO0ElzL/CYMFV0giczkq/nXGYg//Jb+yfFTe/cf/dz69esD55JPOteHWx54oHawf6RuzOmZ5feHoKW1i8ysk2/D4pBIxIGJfzHaXZw2StuR36Pt8EfaMkvwR+SP/B+KbSZkcfhzvmEmhARZBVwA/koIwmSzCym5nwtlUogo1wgtMFV4YQAJxTPyohn0po1r61IVoK2cOXtXS3NTu1ZiTr61aIz7OYLX68VJjwdTBkQmu4Mwrmxg/lRSHENCYX1hckm2tuIuyD1Gvl6xM39NN2OgOj7Anziq/JH6HoUJ9ypIOZrYTyAYsWwbLzry9dc7Wn74Vv21cAkKiHt9QWfV3Pk1/aNhg8FeDPMWXw6Dwy544Xe/hW/ctxk6znRAyptRsYbE/0FSE2oYqAphclBPDQOmis9kWyufq7EASYHZFMFT4k55ZEquYBgn7+sZ9pX0uxOzA1G48VIUwJ559um1/QPdvzx27Bju2P4uvPXWW/Du2zvw2PF6CAT9TKvVCD9UvBhVRWzf2wT/8crHLE5e3tHrZD/49z3cskAJCn/2wsds/wmHeP3yznr4zR+PCrd5ZVcjOPqc8MTvDgh97D3aBR8d6xFCv/B2I7x7qFMoc9/xbtzNn09Vgcg7kNRm0iDcO0gRCZaQLkUB0NnZ2aeJR3etWDQrUZ6rg9wME9yx6Xb20yd/Bk8+9SQWFRfTTVTLi+gW+3jzgxZ47LnD2No1Ak+9dBB+va0OySXhWEMP2763BZ54/hPaFMCuQx3w2O8PseYzw2zP0W7WP+xl//VemwiDk62j0NA5zjp6x+G1PU3wxItHIBaLQ33bCBynz1LCc6dJyKkFlBRKvoSikmjlcInH8ZIU8PhPfvqIwZZzksqvLT0jTcSuhrJ/eno65E2bxqhkT0nnmEwDHm8Allfns3cOtIHL5YNMq4F5PH5SxEm8b8MydPQ7oaG1T1j3rhtrcPPj76BnMpA0Iv/LkyllWXh1TzNctWQ6aLQa/PBwO71F4vLkqiwR/4BqIVLelEOJhxyvGppzyHhWBdTW1uqstrQfdzran7VnpA8bTGZKgBJMeDzw0Uf74dTJeohGImrQKb4nZ6JAJMFuWj0TX9hZB2tXzgGrxUC5YwJONPfDwVPdrLI0F7bvaxep/6qlM6E0PwNOnBkHk0FLQAnhcH0PtFE45GYYccfBDujsc2KmzQDP726FOClmYjIATW2D4PYEVIHFb6leIDTI8w6vQLzm8Cp1lnVW5QwNDSUWLV42Pysr59ZJXzizsGwmls2oIkvo4aMP3oOXX3oFV6xcDhkZdpHclCQo9NFI1r3txkXYSSFw95eWQ4djhKpkFBZVlcA/3XMNVFfk4KET7VCYl8aW1JTgVYvLoXfACTdfM5fNKbXjK3saWE1FNi6rLoI0ix5+cM+VcM2SMmhs64Oa6TkwMDIBzR3DUFM5DdJtRkjlWyUR0kPANwnbGj1AVsQcbeD4wTd+s/Oz5DwfEMK1a29YsuCyom1u53heNEoWu2YNO3PGIe5z3wNfZ7a0dLmaUdxKkpz1CcuBCtpkJCeBnOtTO2Uq4JFfy9conqRAG/kCppZYNhUKMnXrTP4hScUFjMkgcWy4Dzb+ZDcY7faQSa95+JVHbvmPzxJQex4FsP7+7rqNNy2L9zpC4A0Bu/baa+DG9SaeC6gKyF/nmw0G/Oh2OSnzMyUuVZgrwxu5xMsVIxWvKENgBdZKyUTKVO3JiFr9AVCTrYr+JJEA+aU5uUVAvUNy4/F4BKePvdlQjP0/ev1l166zCXhOBXzzm49c7XJNfk/SGvJtVjMLRkPipjqdPlnKUSk9JrOZ6fQGGboqYEaBwqBgZjlE6PNoOII8kRnpOzIE4NfK2IElIW2ClKeREaHIaZBUmgx3VeQj4CLy5KzCaP5o0sTZt646Njd7tuXlv/1C7ndmrR564qIUUFlZaZg+ffq2/fuf23v95TNcJos1k7lDpNk4dnf3gNFowPyCfAW2yBvT6+SyIyNAZKiCuaQCAOpPnoSAxwkV5UUQ9FlQ0hkhM7cgmcWd3Z+QJxwB72gdQvr9rGTmoqTgAEnQJMAWpCCw/AhyAPHFgy7NGkcTjOp0cWv52eQ8axXo6OiITE76HbNmzZnR3uO0mLPKoGbx5ayppZ09/dQv4NvffJiNDA6pGRiVzSU3yQScZclAjYZD4rGsvIyS4wDUn2qCiHdMSBONhMS3RjvfoUbrdTTq9mLnacYy80qpHEbl78rCKZUWmXI3EV+MJVsD9X1eKpPOJ3qRi1UA/7FnnvnFylDI+2vX2Eh0z+49uHPbdujuaAejwQQzZ85AvVEva5xNgaM4FY+pFmHQ2dEG7aebwWqzsnXrPw/+UAKCkRiE/W423N9Dl1D7HtuJxvhBcA0yqFq9BbhDjbT/G4z2vAFTIB5ve9VGINk4MUgZAtTeAFO7ONs6Zw5wOBweyeH4xS2//tfN7c1+q8sbxRvXr2O3btjAuBG0VBJxipyyheSYlNUgp4poLIod7WfYnBlF0NbcBDk5uTBjOkeRCSpXXgyEo8yfZQNNrBkgGsCc7BHwRd/EiR435GV+Ar3Dm0DNBUpGle/Gpmhd/jzpF3IOkjdHiVm6JAVs3rw5w+XyPkqch91qs6E36BWZXORfjUaRHNTbKsmKkVtJHIKgWr54gHr9EUKFbsjOzsZJ5yBYDBKEIxHo6HJAVfVs7Dj6ClQvtQOEo6D390Cm5jkAQxwGTkQQCkvFrZgqlIJ3VeWnbK0mQUjmHb54xTqbjOdEgmWl01t6unsu12i1MZ7leUvLk08gGGDRaPQs3WoqVtVF5ZItXrII3vnwBDa1nIG+/kFoON0JB442wpzZxZDwfAyVc/aD+72jEBweglDMAEOOMHyyMwxNndezwsvWKL2/4unKXdSVwgCgKhyTuZf2HD9HDjirB9hsNl7WQkXFJQNuf7zUkDYNp8/Jh3HnBGx/44/YPzCEm7dsZgUFRUptF5hE7u8VrSSbdHpSWFTA1ly1mh34+BD09Q6gBiKwojoLYmM9UFzeCLETDWDMNUNPaxQ+eFti9tKlWDj/arjyho2g1Zmm7CzpArIaFA4FYIrWZZyFFHkQ9ACEhsbOXLQC9u3bF8vLK7mOFPjwqaYOXUa6lcqcBt7dvYdyQxeMjrk4AaR0AEr0JwMTVYoHFXADBqMRliythcrppegcH4PWllMQGXoRiku8EDrqAFuFHQ4cisCOvXmw6eEfweyaJcxkzSCn0yZDW7Gz0vILnJFkhVgKKYrHCIXh8x8taC4M7P1Kj8vSejY5zweF+TK+9J+PdXafac13+aL40Lf/kU16/WC1WjEtLZ3QoE7hPHj7KXeFkuD+lNKuxiqqO0+gd7wfxrr+jZWXNaNzVz0YyyxwrNUIr38wHe746p3QPehl5TPnwIpVV3Jui316m6kak6QQVe4QZAvw7bhHe/HwW78KzfI/2RZMN32v+i7vDrgYD+CrpqbGUlOz8BsSau06Ij80UpxpiBMrKChIEPedhKugWEINBQXKsqkbU7g+mHQ6mM+5FYsz23BshwPSazLgow/j8MtdmVBdO4P99g870UAAsKf9FPb3ONiNt94OZrPlU/tiClROoQE2lTEWS6/XsczcfIPRwebp0/WcEvtMBZwLB0h33Lbp6EDf4Ff1JmM8LSMdDHo9ctoeFco2GY/sz4hHVDE9qNgfE9DXsgu0kfsgK1YHrr2dJHw6HD8QgzO+G+D+R77LTjY78PDxRmjpHMSOfjdrOnkEnvrn78NA/4Dym0zmxVgyzYBa9PFTUIxzlzog/l5kJBbBiy+DZWVl+gx7ZoUt3f47gu03xxKSleo+6+7pgQPEBxQQDF573bUU2+YplUeGvyngLmN/3pd3HH8WDP5fQXDMhRJ1VcZZdjh9NAjt3vVs3ab7wJ6Tj1u2GOHpp/4dzjh6WDAcJRYpCiuWFbHWxjqIhEKsrKJC7XdVT08qRoXD6l64AnRGm8wpXwoj1N3dHXrjzVf/ITvL/oUPDrbkMFMBK5uzAFtPt4HHOQa//90LODIyKiceNSb/3BPER8P9faifeBxOHdTjntctkFVlhqZPAtDsWQ/XbNwMRRWz0GpLZzXz57EHtzzE5syswIDfB3HUw+zqhZBjm8Tje34JaqVRPY6xFPxl7NN35byFibiKQAA+xSD96TpnDti9e/dz+fn5r37jq7c6mo59mBtHLdy+6ctUx/vg1g1fArvdLl+In4a+mOwRRYkCvUHPjh9AWLxgCDRLAQ68EoIh7Zdg1c33QEHpdJ7pRZwQ+cJmzQH8yt1/D79+5lmah+nhDy+9BgZNAsxGHd5CZQdlruFPFqYaIQWE8sENdZMQ1xKVJxjIS1AA/+XLV1y+pqy0yBwLTuAk9SRVVVUwf8FCkXB4BQA1xyu9ukqPyw2K7BBa6vhM874L2z54nknxABYuWger1m6AnIJyAazUtpngJWYRTK5dtAjS09LY/gOf4Mm6BtBaLLDxa19T5wUqBQlJAKpuFnEKKcwdJc6k9CJIRLsDl6SAf/rH7/22saFlAyFBncliAi/vyjjJSJqVUaGKdlENphQcnzIAs6alwcJVN7CKuUtBq6EuL2camq0ZnxqjgUKI8B/KzskBm9UKRcVFsG7dWrBnZrKCwpLkfCFl6VT4JdUglwMZLBOGGDKv+b2/6ZknL1oBFRUV6cXF5Xe+8eZbj5jNVz+o1xvyJAwLeV1uN6OujjgBo9gIL/v8/THnpOgE8nPTwTfp4RAUJCUnSuQtWbnThIa4Q/q8k7KvyKMsUVmMVO5i0QjG4zER1FlZWeIfV2bQ71WJEz4DVadOQmSVajNZbWplkJVFCshbvrF9xld+3nfRCuCdYFdXx2tXrL7quyeaB6z5uSVQXWQjPqANtr3xJtFfLvjhj38I2bm5ov7HaDR4zw//yI41D8CZdx5BTo3FYjGFLwS5i1HgoqROe1A0TqCQYqBAeEG6SJKkZDbe7wtIKQQHjIsvinSASucpKHANY0kUqkJziRspD86xzhkCf3jp+S8vXbriBpM2+NzoYI+FrIilpQPM0dEJJosFk+mXNqIlyvyhO1djQ3s/0xt0zOMexYhCZPAd85E3yBMczCRPmHQ7SdAYV4YQlksT8E6kYK0CHyQVCSqlTuQMcjl7XlGqL57S+ykvBGcQiwTBlG7IP5eM54XCVAXMj3//XkdrS1NeKK6Hh771beYPRMBiNaHVmsZ4W4zKbvnGhDDqO6hYLgmLhaCy7ZmC4RXCE5MWFySbXD5k32bJ3kexMB+JS/KcUTyXUxGmaiJw4cPYeHg3K5qz/JO8aeWfg0vxgE2bNtUQFf4EanS5hAJZKMgED5Cdk0U9toZJSuORBPudOyA+Xkel7vswNNAH4XBQociZIDhl7pp31RIolDnmFZWBa7gfEkp8KIQ5qoSoCnhU3ltBPEzVo0zAIfGKJaL08bG71zlAICrMuIdpIJEztXG6YAVwUrR24dJ9//Xqa0dNxsUBi8Vi9pJA8rZFyyupXJTK9iaGD6I09B6wmgeQQgR0Br1qelDHVPL3UySpz+MGg9kKZvImrjDadbLMqVZFZa6gMxg4wsNQwEvRLTPFVKEo+WUIFjka8EAwMCk8zXG6DnKn5fAzNfbNm2/LpUtHLkoBNP9L0I8HiwqKPG5vKBrnJ10IsDgcPXD06BEsoRK1+oorwUCb4kuAs0U/IAN8F8CcDvqEVwARTI7HQXgDNzE/BiPcWD39wa+hmRjxD8Jqyf5RWF6Neyq/BIxQACuTrCF+6IJ+IBKYEPkkFqXv0h7dY4OUgEOYlpHNYrGwdc2qKwqffPLli1PAiRMnohUV719nsWS8eLiuI61qzgxYSITI2Pg4eN0ueHr7LlZVXQ0FhUUyT89dW2dSRpWMeb0TAs5KyY5REgmPv8wtKIWJ0YHUcRk5Zcgnh9SEoeQU/lWj2Yrp2dMI1ySEUgwCgAEGJseZ3kylj4SnygExYpeDAR+caW3Eylmzmd5oomoUM2RmF5bR9Sc/S87zJsGioiLT9zZ/2dF6ujnPFwL8u42boLGplWVlZsDatWtoIGKVOUJKYpLKQ8u5QfivnMVBLmuoZvNkCyJnNUy2skxWX0KNWQVhyhcrCU+saCgAIwNdRBfOoEYpgCQ4I4IV+rvPUCgFcPGK1eRVenCN9mCY2R6cPrv25/x7W+++25wzb5507733+s7pAXwtX77cVD2n5q70NGtaho2mw6T42bNmMOIJxGkurU6X6gFIsFjrmxB37AL9NT8Dty9I1gjIypGVgmqFUEGMAoRALXeq8SWl2xalk6XGZExuatBoMpOwHuAeFg76CCT5xPORwV50OkfZgiUreJImj9RR4vZBiMG8H3/r729ZtLCmVmu2XXngwBHuDV8/nwKkG65b9/7BQ0dr9MbLtHriAiAUEW5JuYEp3pw0JE+MMXcvxgfrGIsEkJ+Z0vKTZBpJxLnFamUc4ESjEZCmJEFZOCnlBMpHhDxFVo+GA0qhYZCWmSumzN6JceHqEfotjidoLgnOsSHo7e5g82qXyE0a7XOgvxfGRt1gTdfccdMtX7yzf2AQ3/vw0Ommxsafqzs/Jx+QO61gkdvt/hez0XSPVqfPZYmwgGt+n5+ZzCZih6QkD8fNp1v+DTCsfFA4slUb5EdTZHqM/mck3kCcLVPQoerLEqbOECULGwoqWyhWLwaeMt8Xj4XAP+mmKuAjKO0RpXJywgVuas+7HK0wb+FiyMorgP7+Qezv7YVp1D9MK18AnZ3d2q6u1tj77+466HaNbNqz/1gSGp8zB9x775bvx2PR72RlGA0F0+yUpY0wc9ZM9u7OneCd9OCD39xCU9k8NgXuyqww/R0Z7AM/x/uqn3DYixr1WjnpcWAkeiopeXhQUhsZ8hxJ/klmScugXGMBjhTDQb9w966uTsgvIGG7HVwBWDl7DvMGoqQ4A6O8gGMuPx48cIAN9nbw00n8l+871ux4hpO9U2U8bxJcs2ZNzbIFsw46Orus4UgcCwumsda2dqLNrfDYT/4F8vLyRfZG+YSYsBYvfwr4EHGsDDJVgI5SspHBJIfDr1OhXKrFI9/hGZ4sHw7QBIks7/d6WXfnGTATFB8Z6GNuDzVm6Tlos0+DubXL4VTDaUaWlnQswAwUqdnT8iEzI43VLF55f03t6j87I3DOJMjBUHZ23qw5lcWIwXGY8MXhwYceAI/HRxYxYaadd2o8OTMFs8rMCz9AqaJd/hZiQlGMOClKX1DKHECSO0gwpb8Vx1qYMmRKQDwaJGTXBRPD7QTBAT1+aobiEXZo3yGWnltMA9ur0WLL4ifX2K5HHwMbTdxzLRqWlpmPy1d+jlVUzkQiWAm6W/6WbndRCsAvb/rq24ePHLmC4lFjMZvBR0iNkiGUlBRzClzEoNyVgRA+ReCDQg7LxztVWM9rNcpnd2S0IGcCedqDyuEncbKU2uEEAZuIF9564SkIdb0LN21MYF6ejtW1zoah8ZXw+dvux9ExJ9uxcx9MugYh3UztMAYgJ6sALl+9CguKiinvmCiHGCCNqDGz1Xw5n3ZxfHNBCqBSZ87Oyb2SJkCP2DMyHhwymqmrCik9DvKT0UxlciTlzILS6jKAJBmSnBmkxkcJ8WE8HuFljSWoMvBkySFwLBZk4YATw54e6OtuZvs/boAPDw9BZVkV2Ou17AsbjLg8F2H8vSj7z2d+Ax7XGOTaDcymT0Budh4sXn4tlFVMJ6LWxHQ6Q5K0MVnTUCcldN9+6K4FX7z9xNELUkBDQ4O/p7fnuSVLln6n7nRfhtGUDnkFVtY3MAKNp+rE0Za5VTNphC2yNcoujizFA4lTseKVRqMAHy4o9xSyboJFeCnDuvpmGBkaY4FgkDf5YNJ4YWHxKcguiKDTW8OuvWoZZNit4A+G8MNnX4faQie0vf8BzVAXMyv9bmlJES5ethzLK2cwoyWDabRCcNF+85IbpmErQwMbH+7EuTVVN9EuLkwBfL3wwm/vX1Jbe5iFnT+j2mpNS0+HE8dPsOGBXjh67ATO/+etLIs6Q17amHxYXW7VKAfEqV4nYlHC52EIBQI0pwuRdf0QiQRYR9cwtrT3wcSkj/m8QbCYyU0JaFmM9HV9ELIum8sW1WpxPNqGdc0lrKGhH3p7ByEjgWw8n2E0GAejPUKjtnlQUFzCwr5RaK0fIo6Sww6ahZNPcePzc83hcEygRltWOTS1tR36UxnPqYD+/v4gxe1Ljz78lUfr6+otrmHENffcA4dpMrzxzo3M55skIUMcc4BGaXG5lROJqMDlHMTEKImF/OOMU+jHTvVCfXM/WC1WAlM6ys4WyLEbcOZ0DausGIeVnwO0G4KA7iZ01wdhVXoEfvTOJHgCRkEBlaclYCJTghCRPLULCiErI0xDj24k3WIsFiFcoh7KitOjlukMRuQlNOCLodvpGiKaoPuiFKAug8nIYmRVo9FC+SmKJYXZLBoO4UDvAIvGIoTOYgpQiQiho+Gw7AEkvGt8BCw6mgW63HCi1QL2rCxMsxqhIDsMSxcHcOUKBnnE+6O7D+LDk3C8GWHHITscb7ewdaXDaMYYBGmXWTYjlKcFcTJzOtjTikjBfnCPh0Wcx2MJ0Q4n+COH0NS46nVaoO4S9E4/+AJhVl5VEdiw/q6mi1bA0NBQ2OVyOy1mU24whhgKh1gkEub348dXqUc3E+Tlc0NKkcEAeNwx5I8+9whkSSfhzqVD0DWsRU0Zwu2fn2RdATObUWnE8hxyS3STtUdhrD0C7x7Ww77mPDjdpwN/hNA0KbrNpBH3mGbWs7xMM1qyclnMbEEb8QyBAMcHjCdjUM8ncpqeeyI/xC3mAnotC8cRIzHNW81NrVs+S74L8YB4Y1P7zcT5/SLO4ld1d7RrQmRh8nNyuygLBYLgI2Q2MUHNSTAYp9dhz+SkMTfTJh0cs8PRDjPbsGYM19xWw7Mk1LoIhY6fhlBfFOo7JfjjoXQ41JoHvqiG+SjRhSNBZaYQY71BG5nSDuk2Ez9SB3HJCFFmomt1EJP09NpEQxM9JWIDV1RMa9AHtJIUoKLjCoUCrc4xX7PZojsTZ9oPNt3/+NBnCXdeJKgu/p/OxD0Dc8KRyBqirm0Ea4ME8Mh30ZVIaCcIsrj1poS3t2co0dExcCwajdr5CP2GK5ew2kXVYI4+DStKHMh8NAztkeCn23Ohud9I8R2HEGVqjhE4IEizmoBPoovzcwTeKC4shKLSApafl+0yW8zDVqvZScYYI3ZonJDIIFWUYWq8zgQhNpBmNLmG/YFgbe2NwbNRYJesgAtdVdOnUyqLHiRHgOKCHFhzxTJqSUN4uqWF/euNdfjiPgNUFiTg0XdyuNCMH1/hbssPVJPwjKZIuKCmis2aOQNKSgtbsuyZb9Ck+815q26uv1ChLmZdUBK8mBUIBPrtdiO1qhL0DY7C69ve470dxOIxfPE9Ewz76ZaUHPnMgPcQWRk2AVb4/LD6sstw3vy5wfKK0o8zM2zPdI01br/+6q9F4C+4/sc9gK9lNZXv+IOR6wMhufePxhkjfgIW5wQw3xZhCQKRR8ZKIEpKyKEx2GWzZ8GC+dXOrGz7TzPt9rd3H2xqopBLwP/C+h/3AL6oEt5pNht+RRb+m0AoijotYppZD9kWD6suY/haaxFvq0Pza6qGZlRWfJKWbvrNqus37v1LuPj51l/EA/jinaQxEf08kxJrcrIz8yk7S0SrefLy8oaqqi/ry87KOtnSd6pu69bnQvDX9df11/V/tf4b8hrFeF52qJEAAAAASUVORK5CYII=", + "parser_ids": [ + "manual" + ] } diff --git a/agent/templates/ingestion_pipeline_one.json b/agent/templates/ingestion_pipeline_one.json index cf74a4fa23..f3509ca0a7 100644 --- a/agent/templates/ingestion_pipeline_one.json +++ b/agent/templates/ingestion_pipeline_one.json @@ -26,7 +26,7 @@ }, "Parser:HipSignsRhyme": { "downstream": [ - "TokenChunker:DryDrinksVisit" + "OneChunker:DryDrinksVisit" ], "obj": { "component_name": "Parser", @@ -66,24 +66,6 @@ ], "vlm": {} }, - "email": { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "output_format": "text", - "preprocess": "main_content", - "suffix": [ - "eml", - "msg" - ] - }, "html": { "output_format": "json", "preprocess": "main_content", @@ -92,18 +74,6 @@ "html" ] }, - "image": { - "output_format": "text", - "parse_method": "ocr", - "preprocess": "main_content", - "suffix": [ - "jpg", - "jpeg", - "png", - "gif" - ], - "system_prompt": "" - }, "markdown": { "flatten_media_to_text": false, "output_format": "json", @@ -125,15 +95,6 @@ ], "vlm": {} }, - "slides": { - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content", - "suffix": [ - "pptx", - "ppt" - ] - }, "spreadsheet": { "flatten_media_to_text": false, "output_format": "html", @@ -173,26 +134,19 @@ "File" ] }, - "TokenChunker:DryDrinksVisit": { + "OneChunker:DryDrinksVisit": { "downstream": [ "Tokenizer:FrankWeeksListen" ], "obj": { - "component_name": "TokenChunker", + "component_name": "OneChunker", "params": { - "children_delimiters": [], - "chunk_token_size": 512, - "delimiter_mode": "one", - "delimiters": [], - "image_context_size": 0, "outputs": { "chunks": { "type": "Array", "value": [] } - }, - "overlapped_percent": 0, - "table_context_size": 0 + } } }, "upstream": [ @@ -214,7 +168,7 @@ } }, "upstream": [ - "TokenChunker:DryDrinksVisit" + "OneChunker:DryDrinksVisit" ] } }, @@ -231,15 +185,15 @@ "targetHandle": "end" }, { - "id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:DryDrinksVisitend", + "id": "xy-edge__Parser:HipSignsRhymestart-OneChunker:DryDrinksVisitend", "source": "Parser:HipSignsRhyme", "sourceHandle": "start", - "target": "TokenChunker:DryDrinksVisit", + "target": "OneChunker:DryDrinksVisit", "targetHandle": "end" }, { - "id": "xy-edge__TokenChunker:DryDrinksVisitstart-Tokenizer:FrankWeeksListenend", - "source": "TokenChunker:DryDrinksVisit", + "id": "xy-edge__OneChunker:DryDrinksVisitstart-Tokenizer:FrankWeeksListenend", + "source": "OneChunker:DryDrinksVisit", "sourceHandle": "start", "target": "Tokenizer:FrankWeeksListen", "targetHandle": "end" @@ -300,28 +254,6 @@ "parse_method": "DeepDOC", "preprocess": "main_content" }, - { - "fileFormat": "image", - "output_format": "text", - "parse_method": "ocr", - "preprocess": "main_content", - "system_prompt": "" - }, - { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "fileFormat": "email", - "output_format": "text", - "preprocess": "main_content" - }, { "fileFormat": "markdown", "flatten_media_to_text": false, @@ -348,12 +280,6 @@ "flatten_media_to_text": false, "output_format": "json", "preprocess": "main_content" - }, - { - "fileFormat": "slides", - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": "main_content" } ] }, @@ -378,27 +304,17 @@ { "data": { "form": { - "children_delimiters": [], - "chunk_token_size": 512, - "delimiter_mode": "one", - "delimiters": [ - { - "value": "\n" - } - ], - "image_table_context_window": 0, "outputs": { "chunks": { "type": "Array", "value": [] } - }, - "overlapped_percent": 0 + } }, - "label": "TokenChunker", - "name": "Token Chunker_0" + "label": "OneChunker", + "name": "One Chunker_0" }, - "id": "TokenChunker:DryDrinksVisit", + "id": "OneChunker:DryDrinksVisit", "measured": { "height": 74, "width": 200 @@ -447,5 +363,8 @@ "retrieval": [], "variables": [] }, - "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAEz5JREFUeAHtW1msXddZ/tfa+5yzz3TnGw9x7ExNmlCnGdskSFWL0og+UKEAqYraIl4oEhKPCBAPfgAeGB6qqg+0RKhKqkpU0FZCJLQJhTSm0DZRA3Vd4iG277XvbJ975mHv9fMPa+1zLCFzXR/e7or33cNZe/jn7///FYD9sT/2x/6Y8kAAg0gbTGyI4z1tJ06csHpMe388vhY2MPD/PPb8gq985ffnH/u5j35u+dBdv2qNKVhrDX+f4SfwH6aODywf8XON/kfnRIu8CAHDTCNsoQv+mg509Ic3uoX3WUaHKWbpgHYDHKWdoRt21nopvPTlr/3ln5w48S8p3OLYMwNe+/oXnn3k6Y+9Uk5iJguZAUIHUWiYBDnmYdFfAKY+f4G1OPFK5YMnO38JojAAXabMcCNw2RCyUQ9Ggw4M+00YdBrQbW5jsnT8te3NtU+9/+ef24RbGHavEwuluGJt5AlTsZv8dpU274UaNP6qTMMxvQZUrXWuzjP5TGWclc2YiK7ExM8C2KgEUaEMUVyhfQI2LgBm/WcOHT36re+SZsItjD0zAHBCgoEIYYSFYOcmEGsCMzxxqhX+vjDG8+S6saCfw/RHxBtigCVtM7RFRWGCjUuG9iYuVowb9WFxqf7Q8Wc+/cp3Xvq9I/Azjr0zIOPvtajabceSZT2Q6+K/jFFBqkcIvAvc8POUWKMaLw9g5locm43Mo18iZGYwE2xUIA0oITGBtQFHwx5NjWFmLnn84Wc+842fvvHVw/AzjD0zgLTRqcOyEGyfKR0Nh9Dv9Uy/34MBbbzvd7smHA/7fdpoPxjgcNDnOcZlmT7DRuhNwKuD1xojZkBvI1NgLYhiukbmRxszwtI5+wUwRXaSZnYheWz57ve9+urf/8Ei3OSI9zrRxJix8xMWGK/xdD4gopgwYyYkLxJldWApWjEN/h3VTcrzSuUK5M4if4kPDqh+lDWBnKFGFBtjxMRb8QsmS4ciBI0hCAuLlfc+/tQn/vGHrz/4y49/6NNrsMexZwbQdzj5RnV0hoMAH9Rn5kEiopqD+gHjHVwQKhp/m8WxZ3RKIAg+UObJgbCYjphwMRdSBPIHLmUNkGPL5zTSUQqRdRCc7+xi/Yk73fEXz7z+t598z4ee39oLXXs2AecyhxKn1RfoxyqRAQLkTlDImLhZmAIh6AaXz6I2ATEEj4meY/k93i8Q0ciER3ERLTlFlv72lRWiIFG/wrgBh6QJpY8sv+feF098+MN7Eu6eNcCRBrBqCsF0ApFK4eK507B71TNbtFF4gcFfWPEVoHjBqmYY6yMe+GBqvF+JIlF5NqHDx+6Dan1GJilTrERIDo/WFrCY1M2Zt17HU99tiuKVkgrW5hdhZnkRbjty+7Of/es//NLC58q//buff3kwJQZkKn7+HPpA9PabVGoE2DIf5Y1qvkpX7dhaMCHEj7XEGwIpivVgKQAqvZGIlLAodBvnQMyAJB8VyxCPBlCuzePSoWPY2NyAbquBjZ1ruHnpohn0OlAk33jvY0/8xrOf+uQb8PmXX5gKA3DkEDzKVWyrMOjgoSNgDh8N5qDR3toQ4gKp+pAQ9z0u9o/2quBxBk4CRj5nyStGYFahQ1NIZrGKHB4Lplybw0GnDYNui6JNG9JBl7Y27Fx8Fw7cn/zpX/3WY1/97Bff7MItM8A5HINXVkolcnd3h8Jgl62U2TJBlAmRwdOuKFAkbtT0lZ1BM+Q3FPMSr2/QeMBlrCYOWUbmQYAkHQ0JGhOhwxTSjK2fgFNcNPXFAxjHFjL6LUv7zLDlR5/7zK/AF9988dYZEKHIDH2qA94fsP1aLz0LYzV3ISPwUldzAAE44FGjzQGl9SGSNT3CkGSpufhnyHsdvatIILEAhWICaVIDVx9BNuwTwQMsxpEhBqAj4jmHSCl/iOrJo3TnrTPAZak6QYV0gQ1Qq81ApTozVmSriNDrPnlteQWZscMQQvNIAcFyrGqXku7zCVYyzwD5yd+uabU6IOTEicIjZhQlR8SulBkBwoB0YNISIcZWd+ZGdN1UFAD9MHDsqSMrFKyvrUC72aDf0Xt/SWXQqzvWZ+eJCQVoNrb1szFXDXmcjSjOs+54RvgwodhCsJSCKGWPcIW0y0nKxC8RodD38N/lA4chqc0bZgpmIxj1d2G3M7RTYUAUmVTpxwlPbuDg4WNgj9ylIU71H02eDitIYIU4SHN0GAwOQZUAJxKrwGKZF34PVwP3lBmor0GOEOKXQy2BNM0RWpS8oUCn6zAVBrg0dfLRiLkP4P+u7WxBp93Mr3gbh/nFA2Q2GXZau6IxPqyNDSAgoOBHrCZaiMFveKcgDLXKeLEKwxUDhdkCLi3OLRzMnatEDSMfKqGT/cNUGJCRWYXIpeUtkQDW6nOQEK5n0iNrg4SgWKoIoksqVQRv3TARHcIwwVwUHaFXZ/ULiAK59XeDnvnGgxAxD8w9bIjOzBanYJwSqVE6LQaYkcfB+g2BiGKxRMQWJXWV/MBrgI/rBkoJqGL4kuAYLfh5kmHlUhcGQwiP4H2uqvz4awLUmpwd6i7qJ4QJ9E1pNsQb0XUTuQCnAm6ygOWH+gR10Kig3kykeWqrGD43pA4+vqGvEKK/hB5Mov+H46mSXqD6n/xZ+TfgBHNgIoSOuls3ZMCeNWDU7YSqlqiYCpV2w6uAO18GnP04YOVuuHz+rK8QjQsixtcPFpYPQ+Pqpsk/GIMPAA+NLOYJUjCBkF0ChtAid4rFoGciCC6DQqEEC7cdAY3DaihZe2c6DCgVi3kYNHlNgE5X/gbi3b8wrvNTtPd/ifA5V6f0Wz1ZygLyDwRezEJ8WAGUTw+d4Hxvy9eXwSful2NkGGzMmKkadKKgdRSpYpj0U6wFw/bl6TBg4DK8rn6r9S/ApedgsL0G0dJvUr4wMOsr51Eh6xgVBrvXUpeeMDaozS5SJrnpA+pE+gsAE4HAO0rxeygWwg8kHFKdXYK5pUM5KAtVZY0lLK8CmMHlDKbBgGIhyhAn8xeP9ObugujJPwdfIoN73vtwgLaBAcFPeQfpT7z6Lh643eQhEQLSM2AmvY0G+nF+hJKR5m7CF1JomgvAiPZcwCpg1r3kYBoMGJI/dVyv5yiMIlYpYPELs51/grh+3NjqMUJ8O6IeAQ4HFOeJkBNrfPdkQj1M7h1zJhtzvRow2AnXxN6TSl1QYM4/b0dMPD8oHXYJg7emowGUbmWMtpTZGtqZ++7CC6bY/mPMtn8Jsge+YNrNXZEKeqAfkiFPSUB5njLI6wjeHar8r7sX8owR1CuCzKgwxqiF31E1RIgXDeBb+q1tmJ0tTscH0OMgpIOoyFOZXnsKR2uk9nd8VAoX/X6HEpI0Bzz8kcVSGVqUNpsczOYEKdlWS2yi1h5g8XEpqcKBI/eEuUESQWs8o13IzzAnnvekZWlvHaixNh0TsENGtAGX+0onhapo6UEwS19XKErbHXfe5wWsoCii0hl/7+JthyRF5tsDXNbjEDH0Dm/TRivLY+cWbD08Ws0hxA2nhLtUiee9LQN236HawWg6GhCVEzeOUSIuNXP2t40fo63fS56yalq7V6VElif1nghfGvNVBAPXo2ErEsf8ieAVyGLIqUoErcu1ur5b1M/bGYMzKpJQBgiWNicM6CPGZdPbPon/F9TbOxLMYul4myBE4QM5wEt/B/Hqxwy8+zvgYR+EhqngBarmXn/uu0vg+wfaO5BzqwT7tMCgzyn9hipd2lTVScpuSISTaRLeNxnVAThhGbVI9QmcDVaheeFf6cbetHyAOmLIK58qRSzfA+nF+wEOfQC4StfttDEVHwCeyEncHqRvcj9ofUAJyhLcIzOqUCiaxYNHQB2u4zyfoRNwvYkiEt1J2IQ6yLylXAQZdakGcA2iZMlsvv0SFtwqMbwOU2FAoRBjDrMC/GSpLT8C9sBrKmWS9uLyoRCZQxbnQx5fsZibsZTNfDMRg6g1pwjlNskuWdLi9zkXGUkFSDQgGxGjtfqTDTtUI2wSDutRHSCBbPcHsPHmN+H+R2NovwvTYYAQlHd9nAlqO87i6BOpbHR1e4M+bBRS2gk3YDwO8gWTUBdFzOM8T6rPzuHM7ILGd7L1Ub+Fg36LbdBXycjK05SjPFlDKlkoAXUDxUXSphY0Vv8DTPP7WC2fMYWZKhVF7HQY4IoFtn8XIjd/ty4SUZSmJq0lMIehhSKlraA2/g9ORoHrsD8/qlAoUPM0VdyTjcz2xgocOHacfkt9tCfzIliO2ZDm9TDrtcygsw2tqxeguXUeynAJzr/xunnq17jDHkHMVaFpMKCYpaqH4tCtmyxqsKMLpHCBRMtjANrqVoSX1wMQvfbkGSGETJhVn4nnel6WMaEjUuuBLBcQ1QcyB2qKIjm8tN/GYe8a9Fo70G1cgfa1FSL+Iqy+/QO488EmlJdLMOzGvMbATIUBJdYALU9JnpmXAMdJbw5IxtUADDB/XA+Qy258R17oIBqzzPDyGOd0XVA26vvYTtInZiC1xKktjqN+E/rta4T0NrCzuy4MSMxlWD31I1NKL+LRDxTpqQUjnaVCaTpRoBs+GMZlcc1eQg1fSBrn7oE1CKH/C4qgJmqivoCi9Dt2bkT4kFvfJqX217DX1kVWo65xQw5vLeh3r0KvcxW7DSK8uWEGrXWsxatm7fRP0DWu4EO/XoS4TGmxKYFkg9GNSbw5J4g5qjShjJNzISce8mKGn6mk+8p/SI0Usko2oXpAYY3Vngup1Pkx6bCHw6EwgHBNA4dEeHt307DEO7vb0GtuUDN4E8rZJTh98h0ou2vw9PMFSOaskoW6zCb2rfSpMCCHb5ORDXySo5QZj/byRAZDkxBykwfNKBXJcSuMGcuZJjU3JLQNB6Tm1N4adJpQnV+A1s5F6DQ2oNXYwO7uGvTaO1CNNiFrXYKT3z5rjtw2gic+HkP9IEuenJ5JOJ7oe2xpOj4gJ1zFqcRpXhYguixmmCjjoa/i5zoiTs5pSqsLIbXOyHbPKz4ofJKP61Ho68CgR02NnRWYXZyFzQunWPrQaW5SXr4JC+Ut/NHJ82bt7A48eC/gB3/RwvxRNnki3taICRWKjlSNNgPWhCkyQNYwqnTFp9vgvzGvZyjtDiZyAd9Ry0tVqvrS1qJ2ExNPqp8J8X1pevY6DdNvb5Hj78L2pbdg/eIpsv8NqMVbZmN1Hf7htTVYTFI4fp+BJ58FM3cHBxbqKUYkdctRiByfqdLbSsSTKZqAh21a2LA+/oMve6EuVfEzfVRT3Qg9RU+8hAlKmKSr4zTkIRM/ZCdHku80LlPf/xzcdfeiOf3DV7EaUfOlsQXfem0d21t9uOsAwGPvA3j4SYT6kn+dLdKeHJ8p0nGF9jUCLX20UTYdDah4ukSqjMpwEmGFMpfT4qXP+cXMXR48hHQnCE/jPUkfR8O+ycjpDYh48u7s3LDXXIHblgxcePub2N1Zg+/8+wasnOvAoTmAJx4AeD8Rfw9l3eV5emuBy8FWvT2n3rZqIKabC8vUOr9AGpBMhwFZqSBQ10cC1LKTN4E8z5e1T8IQN67g+WqFE/uXJXKM59jmqYjKKj8adCieb9G0EZbK9A5ixr+9/Aqe/+9V2L7ShsNE+C88AubBBwCPHiEXV6c0qGKgRQ5v1KQ2ebNMxFZY9emZXXrPKhTMFXKa61A59vR0mqOsAdqD1CakyyAvd4VSl3OhCaJOD31HPJNFD3IvxfmM1d4MZRHDQHSlSLW9waAJp773KvzX917Hy+9e5L6imakZuP2OBJMkgl6lCG9vJXCqUYI4KkOpWoEKtear9Rms1mtQospTtV6HArXiyhVq2S8ehdLKN6ATL0+HAYMBJTgMU7noQLDU+IXQWgGm5qfv8fuYHzCQJnK+RkB6SiaQmhFpQePaDqxdOgNnf3wSzp3+T2qw9jCplk29Pov3ffBJKpnXcH5hDupE3AxtlWoFk3KZW3EYF4pULksgprYcb5Flu7fSF+B+oCVTiGqHoLP6bVK9WjwVBmSUfu6sn8NKrcqtcg1pXBAZpZCh9k0zxyBmxAsneVWo6fe70G3vQruxha3GJuxQYtNutYDX+YJUeYqmlJTw/ocegmolMTOzVayUy6ZULmMpqUChWDJxIaGMrsgLJGUNMYuT8CJ0h3QwJJgsSVJLlE4CnpHlctQueQsdda3S4nI0FQa0mruY7Ky4q5vnodFoSEFC4zoqiMkYw2eGYjmOe/bcreekzHJuT1KdgZn5WYrMHLW0SSKLYgmyxxFbDEK318X+oCPZsbWqSjrPY2eD3s064/+PBRJIJi+LI/U8MRWwy5QKLB44jGfemVJnaHtte5ck/ebs/OxHYqr9BfTG32VdADUIRfYPTjG/hj9ZX+mbO8CLDbWMJ80L32Cw2gDW9Y6SNIqG+Wx7HHpBayahxmB9YUo7DQR5Il1HEFtp3ZmzlzbaZ85e+dqN6DJwE+PP/ugT9wy6vefpWxKWLOmiZHPi+NkKSKpakbW+WKk7q+uqqWWW+YUS8oufY/VY/jqIMLS1VN5+1Yf8I71mTss/ysjzZ+tc30iliZaL4ZxN4eCfT7zwk+/D/tgf+2N/7I/9sT/2x/8y/gd6Ig36oI1njAAAAABJRU5ErkJggg==" + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAEz5JREFUeAHtW1msXddZ/tfa+5yzz3TnGw9x7ExNmlCnGdskSFWL0og+UKEAqYraIl4oEhKPCBAPfgAeGB6qqg+0RKhKqkpU0FZCJLQJhTSm0DZRA3Vd4iG277XvbJ975mHv9fMPa+1zLCFzXR/e7or33cNZe/jn7///FYD9sT/2x/6Y8kAAg0gbTGyI4z1tJ06csHpMe388vhY2MPD/PPb8gq985ffnH/u5j35u+dBdv2qNKVhrDX+f4SfwH6aODywf8XON/kfnRIu8CAHDTCNsoQv+mg509Ic3uoX3WUaHKWbpgHYDHKWdoRt21nopvPTlr/3ln5w48S8p3OLYMwNe+/oXnn3k6Y+9Uk5iJguZAUIHUWiYBDnmYdFfAKY+f4G1OPFK5YMnO38JojAAXabMcCNw2RCyUQ9Ggw4M+00YdBrQbW5jsnT8te3NtU+9/+ef24RbGHavEwuluGJt5AlTsZv8dpU274UaNP6qTMMxvQZUrXWuzjP5TGWclc2YiK7ExM8C2KgEUaEMUVyhfQI2LgBm/WcOHT36re+SZsItjD0zAHBCgoEIYYSFYOcmEGsCMzxxqhX+vjDG8+S6saCfw/RHxBtigCVtM7RFRWGCjUuG9iYuVowb9WFxqf7Q8Wc+/cp3Xvq9I/Azjr0zIOPvtajabceSZT2Q6+K/jFFBqkcIvAvc8POUWKMaLw9g5locm43Mo18iZGYwE2xUIA0oITGBtQFHwx5NjWFmLnn84Wc+842fvvHVw/AzjD0zgLTRqcOyEGyfKR0Nh9Dv9Uy/34MBbbzvd7smHA/7fdpoPxjgcNDnOcZlmT7DRuhNwKuD1xojZkBvI1NgLYhiukbmRxszwtI5+wUwRXaSZnYheWz57ve9+urf/8Ei3OSI9zrRxJix8xMWGK/xdD4gopgwYyYkLxJldWApWjEN/h3VTcrzSuUK5M4if4kPDqh+lDWBnKFGFBtjxMRb8QsmS4ciBI0hCAuLlfc+/tQn/vGHrz/4y49/6NNrsMexZwbQdzj5RnV0hoMAH9Rn5kEiopqD+gHjHVwQKhp/m8WxZ3RKIAg+UObJgbCYjphwMRdSBPIHLmUNkGPL5zTSUQqRdRCc7+xi/Yk73fEXz7z+t598z4ee39oLXXs2AecyhxKn1RfoxyqRAQLkTlDImLhZmAIh6AaXz6I2ATEEj4meY/k93i8Q0ciER3ERLTlFlv72lRWiIFG/wrgBh6QJpY8sv+feF098+MN7Eu6eNcCRBrBqCsF0ApFK4eK507B71TNbtFF4gcFfWPEVoHjBqmYY6yMe+GBqvF+JIlF5NqHDx+6Dan1GJilTrERIDo/WFrCY1M2Zt17HU99tiuKVkgrW5hdhZnkRbjty+7Of/es//NLC58q//buff3kwJQZkKn7+HPpA9PabVGoE2DIf5Y1qvkpX7dhaMCHEj7XEGwIpivVgKQAqvZGIlLAodBvnQMyAJB8VyxCPBlCuzePSoWPY2NyAbquBjZ1ruHnpohn0OlAk33jvY0/8xrOf+uQb8PmXX5gKA3DkEDzKVWyrMOjgoSNgDh8N5qDR3toQ4gKp+pAQ9z0u9o/2quBxBk4CRj5nyStGYFahQ1NIZrGKHB4Lplybw0GnDYNui6JNG9JBl7Y27Fx8Fw7cn/zpX/3WY1/97Bff7MItM8A5HINXVkolcnd3h8Jgl62U2TJBlAmRwdOuKFAkbtT0lZ1BM+Q3FPMSr2/QeMBlrCYOWUbmQYAkHQ0JGhOhwxTSjK2fgFNcNPXFAxjHFjL6LUv7zLDlR5/7zK/AF9988dYZEKHIDH2qA94fsP1aLz0LYzV3ISPwUldzAAE44FGjzQGl9SGSNT3CkGSpufhnyHsdvatIILEAhWICaVIDVx9BNuwTwQMsxpEhBqAj4jmHSCl/iOrJo3TnrTPAZak6QYV0gQ1Qq81ApTozVmSriNDrPnlteQWZscMQQvNIAcFyrGqXku7zCVYyzwD5yd+uabU6IOTEicIjZhQlR8SulBkBwoB0YNISIcZWd+ZGdN1UFAD9MHDsqSMrFKyvrUC72aDf0Xt/SWXQqzvWZ+eJCQVoNrb1szFXDXmcjSjOs+54RvgwodhCsJSCKGWPcIW0y0nKxC8RodD38N/lA4chqc0bZgpmIxj1d2G3M7RTYUAUmVTpxwlPbuDg4WNgj9ylIU71H02eDitIYIU4SHN0GAwOQZUAJxKrwGKZF34PVwP3lBmor0GOEOKXQy2BNM0RWpS8oUCn6zAVBrg0dfLRiLkP4P+u7WxBp93Mr3gbh/nFA2Q2GXZau6IxPqyNDSAgoOBHrCZaiMFveKcgDLXKeLEKwxUDhdkCLi3OLRzMnatEDSMfKqGT/cNUGJCRWYXIpeUtkQDW6nOQEK5n0iNrg4SgWKoIoksqVQRv3TARHcIwwVwUHaFXZ/ULiAK59XeDnvnGgxAxD8w9bIjOzBanYJwSqVE6LQaYkcfB+g2BiGKxRMQWJXWV/MBrgI/rBkoJqGL4kuAYLfh5kmHlUhcGQwiP4H2uqvz4awLUmpwd6i7qJ4QJ9E1pNsQb0XUTuQCnAm6ygOWH+gR10Kig3kykeWqrGD43pA4+vqGvEKK/hB5Mov+H46mSXqD6n/xZ+TfgBHNgIoSOuls3ZMCeNWDU7YSqlqiYCpV2w6uAO18GnP04YOVuuHz+rK8QjQsixtcPFpYPQ+Pqpsk/GIMPAA+NLOYJUjCBkF0ChtAid4rFoGciCC6DQqEEC7cdAY3DaihZe2c6DCgVi3kYNHlNgE5X/gbi3b8wrvNTtPd/ifA5V6f0Wz1ZygLyDwRezEJ8WAGUTw+d4Hxvy9eXwSful2NkGGzMmKkadKKgdRSpYpj0U6wFw/bl6TBg4DK8rn6r9S/ApedgsL0G0dJvUr4wMOsr51Eh6xgVBrvXUpeeMDaozS5SJrnpA+pE+gsAE4HAO0rxeygWwg8kHFKdXYK5pUM5KAtVZY0lLK8CmMHlDKbBgGIhyhAn8xeP9ObugujJPwdfIoN73vtwgLaBAcFPeQfpT7z6Lh643eQhEQLSM2AmvY0G+nF+hJKR5m7CF1JomgvAiPZcwCpg1r3kYBoMGJI/dVyv5yiMIlYpYPELs51/grh+3NjqMUJ8O6IeAQ4HFOeJkBNrfPdkQj1M7h1zJhtzvRow2AnXxN6TSl1QYM4/b0dMPD8oHXYJg7emowGUbmWMtpTZGtqZ++7CC6bY/mPMtn8Jsge+YNrNXZEKeqAfkiFPSUB5njLI6wjeHar8r7sX8owR1CuCzKgwxqiF31E1RIgXDeBb+q1tmJ0tTscH0OMgpIOoyFOZXnsKR2uk9nd8VAoX/X6HEpI0Bzz8kcVSGVqUNpsczOYEKdlWS2yi1h5g8XEpqcKBI/eEuUESQWs8o13IzzAnnvekZWlvHaixNh0TsENGtAGX+0onhapo6UEwS19XKErbHXfe5wWsoCii0hl/7+JthyRF5tsDXNbjEDH0Dm/TRivLY+cWbD08Ws0hxA2nhLtUiee9LQN236HawWg6GhCVEzeOUSIuNXP2t40fo63fS56yalq7V6VElif1nghfGvNVBAPXo2ErEsf8ieAVyGLIqUoErcu1ur5b1M/bGYMzKpJQBgiWNicM6CPGZdPbPon/F9TbOxLMYul4myBE4QM5wEt/B/Hqxwy8+zvgYR+EhqngBarmXn/uu0vg+wfaO5BzqwT7tMCgzyn9hipd2lTVScpuSISTaRLeNxnVAThhGbVI9QmcDVaheeFf6cbetHyAOmLIK58qRSzfA+nF+wEOfQC4StfttDEVHwCeyEncHqRvcj9ofUAJyhLcIzOqUCiaxYNHQB2u4zyfoRNwvYkiEt1J2IQ6yLylXAQZdakGcA2iZMlsvv0SFtwqMbwOU2FAoRBjDrMC/GSpLT8C9sBrKmWS9uLyoRCZQxbnQx5fsZibsZTNfDMRg6g1pwjlNskuWdLi9zkXGUkFSDQgGxGjtfqTDTtUI2wSDutRHSCBbPcHsPHmN+H+R2NovwvTYYAQlHd9nAlqO87i6BOpbHR1e4M+bBRS2gk3YDwO8gWTUBdFzOM8T6rPzuHM7ILGd7L1Ub+Fg36LbdBXycjK05SjPFlDKlkoAXUDxUXSphY0Vv8DTPP7WC2fMYWZKhVF7HQY4IoFtn8XIjd/ty4SUZSmJq0lMIehhSKlraA2/g9ORoHrsD8/qlAoUPM0VdyTjcz2xgocOHacfkt9tCfzIliO2ZDm9TDrtcygsw2tqxeguXUeynAJzr/xunnq17jDHkHMVaFpMKCYpaqH4tCtmyxqsKMLpHCBRMtjANrqVoSX1wMQvfbkGSGETJhVn4nnel6WMaEjUuuBLBcQ1QcyB2qKIjm8tN/GYe8a9Fo70G1cgfa1FSL+Iqy+/QO488EmlJdLMOzGvMbATIUBJdYALU9JnpmXAMdJbw5IxtUADDB/XA+Qy258R17oIBqzzPDyGOd0XVA26vvYTtInZiC1xKktjqN+E/rta4T0NrCzuy4MSMxlWD31I1NKL+LRDxTpqQUjnaVCaTpRoBs+GMZlcc1eQg1fSBrn7oE1CKH/C4qgJmqivoCi9Dt2bkT4kFvfJqX217DX1kVWo65xQw5vLeh3r0KvcxW7DSK8uWEGrXWsxatm7fRP0DWu4EO/XoS4TGmxKYFkg9GNSbw5J4g5qjShjJNzISce8mKGn6mk+8p/SI0Usko2oXpAYY3Vngup1Pkx6bCHw6EwgHBNA4dEeHt307DEO7vb0GtuUDN4E8rZJTh98h0ou2vw9PMFSOaskoW6zCb2rfSpMCCHb5ORDXySo5QZj/byRAZDkxBykwfNKBXJcSuMGcuZJjU3JLQNB6Tm1N4adJpQnV+A1s5F6DQ2oNXYwO7uGvTaO1CNNiFrXYKT3z5rjtw2gic+HkP9IEuenJ5JOJ7oe2xpOj4gJ1zFqcRpXhYguixmmCjjoa/i5zoiTs5pSqsLIbXOyHbPKz4ofJKP61Ho68CgR02NnRWYXZyFzQunWPrQaW5SXr4JC+Ut/NHJ82bt7A48eC/gB3/RwvxRNnki3taICRWKjlSNNgPWhCkyQNYwqnTFp9vgvzGvZyjtDiZyAd9Ry0tVqvrS1qJ2ExNPqp8J8X1pevY6DdNvb5Hj78L2pbdg/eIpsv8NqMVbZmN1Hf7htTVYTFI4fp+BJ58FM3cHBxbqKUYkdctRiByfqdLbSsSTKZqAh21a2LA+/oMve6EuVfEzfVRT3Qg9RU+8hAlKmKSr4zTkIRM/ZCdHku80LlPf/xzcdfeiOf3DV7EaUfOlsQXfem0d21t9uOsAwGPvA3j4SYT6kn+dLdKeHJ8p0nGF9jUCLX20UTYdDah4ukSqjMpwEmGFMpfT4qXP+cXMXR48hHQnCE/jPUkfR8O+ycjpDYh48u7s3LDXXIHblgxcePub2N1Zg+/8+wasnOvAoTmAJx4AeD8Rfw9l3eV5emuBy8FWvT2n3rZqIKabC8vUOr9AGpBMhwFZqSBQ10cC1LKTN4E8z5e1T8IQN67g+WqFE/uXJXKM59jmqYjKKj8adCieb9G0EZbK9A5ixr+9/Aqe/+9V2L7ShsNE+C88AubBBwCPHiEXV6c0qGKgRQ5v1KQ2ebNMxFZY9emZXXrPKhTMFXKa61A59vR0mqOsAdqD1CakyyAvd4VSl3OhCaJOD31HPJNFD3IvxfmM1d4MZRHDQHSlSLW9waAJp773KvzX917Hy+9e5L6imakZuP2OBJMkgl6lCG9vJXCqUYI4KkOpWoEKtear9Rms1mtQospTtV6HArXiyhVq2S8ehdLKN6ATL0+HAYMBJTgMU7noQLDU+IXQWgGm5qfv8fuYHzCQJnK+RkB6SiaQmhFpQePaDqxdOgNnf3wSzp3+T2qw9jCplk29Pov3ffBJKpnXcH5hDupE3AxtlWoFk3KZW3EYF4pULksgprYcb5Flu7fSF+B+oCVTiGqHoLP6bVK9WjwVBmSUfu6sn8NKrcqtcg1pXBAZpZCh9k0zxyBmxAsneVWo6fe70G3vQruxha3GJuxQYtNutYDX+YJUeYqmlJTw/ocegmolMTOzVayUy6ZULmMpqUChWDJxIaGMrsgLJGUNMYuT8CJ0h3QwJJgsSVJLlE4CnpHlctQueQsdda3S4nI0FQa0mruY7Ky4q5vnodFoSEFC4zoqiMkYw2eGYjmOe/bcreekzHJuT1KdgZn5WYrMHLW0SSKLYgmyxxFbDEK318X+oCPZsbWqSjrPY2eD3s064/+PBRJIJi+LI/U8MRWwy5QKLB44jGfemVJnaHtte5ck/ebs/OxHYqr9BfTG32VdADUIRfYPTjG/hj9ZX+mbO8CLDbWMJ80L32Cw2gDW9Y6SNIqG+Wx7HHpBayahxmB9YUo7DQR5Il1HEFtp3ZmzlzbaZ85e+dqN6DJwE+PP/ugT9wy6vefpWxKWLOmiZHPi+NkKSKpakbW+WKk7q+uqqWWW+YUS8oufY/VY/jqIMLS1VN5+1Yf8I71mTss/ysjzZ+tc30iliZaL4ZxN4eCfT7zwk+/D/tgf+2N/7I/9sT/2x/8y/gd6Ig36oI1njAAAAABJRU5ErkJggg==", + "parser_ids": [ + "one" + ] } diff --git a/agent/templates/ingestion_pipeline_paper.json b/agent/templates/ingestion_pipeline_paper.json index d53ff65ea1..e3f655896e 100644 --- a/agent/templates/ingestion_pipeline_paper.json +++ b/agent/templates/ingestion_pipeline_paper.json @@ -1,14 +1,14 @@ { "id": 32, "title": { - "en": "Paper", - "de": "Wissenschaftliche Arbeit", - "zh": "论文" + "en": "Paper", + "de": "Wissenschaftliche Arbeit", + "zh": "论文" }, "description": { - "en": "This template segments parsed files by paper structure. Best for documents with clearly defined sections, such as scholarly works, conference articles, and research studies.", - "de": "Diese Vorlage segmentiert die geparste Datei anhand der Struktur einer wissenschaftlichen Arbeit. Sie eignet sich für Dokumente mit klar definierten Abschnitten, wie wissenschaftliche Aufsätze, Forschungsartikel, Konferenzbeiträge und technische Studien.", - "zh": "此模板将解析后的文件按论文结构进行切片，适用于具有清晰章节和学术结构的文档类型，如学术论文、研究文章、会议论文和技术研究报告。" + "en": "This template segments parsed files by paper structure. Best for documents with clearly defined sections, such as scholarly works, conference articles, and research studies.", + "de": "Diese Vorlage segmentiert die geparste Datei anhand der Struktur einer wissenschaftlichen Arbeit. Sie eignet sich für Dokumente mit klar definierten Abschnitten, wie wissenschaftliche Aufsätze, Forschungsartikel, Konferenzbeiträge und technische Studien.", + "zh": "此模板将解析后的文件按论文结构进行切片，适用于具有清晰章节和学术结构的文档类型，如学术论文、研究文章、会议论文和技术研究报告。" }, "canvas_type": "Ingestion Pipeline", "canvas_category": "dataflow_canvas", @@ -50,82 +50,6 @@ } }, "setups": { - "doc": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "doc" - ] - }, - "docx": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "docx" - ], - "vlm": {} - }, - "email": { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "output_format": "text", - "preprocess": [ - "main_content" - ], - "suffix": [ - "eml", - "msg" - ] - }, - "html": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "htm", - "html" - ] - }, - "image": { - "output_format": "text", - "parse_method": "ocr", - "preprocess": [ - "main_content" - ], - "suffix": [ - "jpg", - "jpeg", - "png", - "gif" - ] - }, - "markdown": { - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "md", - "markdown", - "mdx" - ], - "vlm": {} - }, "pdf": { "enable_multi_column": true, "flatten_media_to_text": false, @@ -138,53 +62,6 @@ "pdf" ], "vlm": {} - }, - "slides": { - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "pptx", - "ppt" - ] - }, - "spreadsheet": { - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} - }, - "text&code": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" - ] } } } @@ -210,25 +87,25 @@ "^####[^#]" ], [ - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+(\u5206?\u7f16|\u90e8\u5206)", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u6761", - "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]" + "第[零一二三四五六七八九十百0-9]+(分?编|部分)", + "第[零一二三四五六七八九十百0-9]+章", + "第[零一二三四五六七八九十百0-9]+节", + "第[零一二三四五六七八九十百0-9]+条", + "[\\(（][零一二三四五六七八九十百]+[\\)）]" ], [ - "\u7b2c[0-9]+\u7ae0", - "\u7b2c[0-9]+\u8282", - "[0-9]{1,2}[\\. \u3001]", + "第[0-9]+章", + "第[0-9]+节", + "[0-9]{1,2}[\\. 、]", "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])", "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}" ], [ - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0", - "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282", - "[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[ \u3001]", - "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]", - "[\\(\uff08][0-9]{,2}[\\)\uff09]" + "第[零一二三四五六七八九十百0-9]+章", + "第[零一二三四五六七八九十百0-9]+节", + "[零一二三四五六七八九十百]+[ 、]", + "[\\(（][零一二三四五六七八九十百]+[\\)）]", + "[\\(（][0-9]{,2}[\\)）]" ], [ "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)", @@ -347,88 +224,6 @@ "main_content" ], "vlm": {} - }, - { - "fileFormat": "spreadsheet", - "flatten_media_to_text": false, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "vlm": {} - }, - { - "fileFormat": "image", - "output_format": "text", - "parse_method": "ocr", - "preprocess": [ - "main_content" - ] - }, - { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "fileFormat": "email", - "output_format": "text", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "markdown", - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "vlm": {} - }, - { - "fileFormat": "text&code", - "output_format": "json", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "html", - "output_format": "json", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "doc", - "output_format": "json", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "docx", - "flatten_media_to_text": false, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "vlm": {} - }, - { - "fileFormat": "slides", - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ] } ] }, @@ -482,32 +277,32 @@ { "levels": [ { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+(\u5206?\u7f16|\u90e8\u5206)" + "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)" }, { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0" + "expression": "第[零一二三四五六七八九十百0-9]+章" }, { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282" + "expression": "第[零一二三四五六七八九十百0-9]+节" }, { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u6761" + "expression": "第[零一二三四五六七八九十百0-9]+条" }, { - "expression": "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]" + "expression": "[\\(（][零一二三四五六七八九十百]+[\\)）]" } ] }, { "levels": [ { - "expression": "\u7b2c[0-9]+\u7ae0" + "expression": "第[0-9]+章" }, { - "expression": "\u7b2c[0-9]+\u8282" + "expression": "第[0-9]+节" }, { - "expression": "[0-9]{1,2}[\\. \u3001]" + "expression": "[0-9]{1,2}[\\. 、]" }, { "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])" @@ -520,19 +315,19 @@ { "levels": [ { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u7ae0" + "expression": "第[零一二三四五六七八九十百0-9]+章" }, { - "expression": "\u7b2c[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e0-9]+\u8282" + "expression": "第[零一二三四五六七八九十百0-9]+节" }, { - "expression": "[\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[ \u3001]" + "expression": "[零一二三四五六七八九十百]+[ 、]" }, { - "expression": "[\\(\uff08][\u96f6\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e]+[\\)\uff09]" + "expression": "[\\(（][零一二三四五六七八九十百]+[\\)）]" }, { - "expression": "[\\(\uff08][0-9]{,2}[\\)\uff09]" + "expression": "[\\(（][0-9]{,2}[\\)）]" } ] }, @@ -607,5 +402,8 @@ "retrieval": [], "variables": [] }, - "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAHnhJREFUeAHte2mQZeV53vud/e5L9+1tunt2mH2GGWAMwlhGirBkBMgYR7EtRVIcyXiRUnZUpT8KI5flyKlSyv6RWI7LihMrVbFRcKLYoAULjCQEgmGZjZmenpneu2/f7ruvZ/vyvN+5PWCLgp5RXGWXOcWd7nv73HO+d3ve532+A9Hbx9vH28c/5UPQP7BD8prwz9mfpMSVRvpf5XruHltqo3GLhh1HS6VvP/ZY6tf+TSZoN13pu88nMtlviW0PLNN1Hn/vDpAyusepTw/H45NH4+N7j6cCQUky9RE9UxgSpA+HpG3X4qkJqdujQtPz0nML3XYn+cpj3xLWE18Tk+RKg4SwbSHjw/DP/R8iOvwefEJCk+EXpJR/Ye74wPNC+e/ajut2wKM/m7tnx1b3gNQp7yRE2nH0nG44zeS+X3xCT40MkxXLGaY5EHpezicakQEVNF0fkoaZ152UIXRTCE2DETrhXKnWAktI0ykIfBH0utJtt+jyU0+S/uQjdOimuDCSgdQS8OiARb09/5Zo6BBcYEicvEKh9utWonBR5A+euhY7NuWAP3+Q9O3v/427B/LJncXllZFyaW3g3DMvPPDPP3psoLBnizBsR2qWA/dbVJ7ukmcWhIbYSCmEVGESpOGnphukWXGs2SLDTpBuOgRH4HsaseH4A2la/3csDR6CTwJZOvkdyk//vnBq81LIED6T5N/2L6Vx22dxUTgA18YXpWyuTVPY/q7mnfyUGPrV5mZsM97sj+f/+GOphnR+frXufYaMxKRn5SgzGhd2fpxCPUmluZdofGKOqFwX5DawOE04yY9QfOwwbLCwfoONlprpCN0wI2PZIbxgGEs6LCGdPyDO3qha+n+/WjySUhM7qPhInZJNjSxbSviPKH2RjHfAbgojB+L0MNR3a5q5m4rB7+CMH90BQW7yP6/Or/28L2wUYIqGtu2XyzMztHX7MHmBQcW/eYza9hpSFvdHnnueIGPPPMUmjCiKuqYiiqghkB7SAebpUaRJw5sg4NTAlzmCMFzyrxtl/FpFW5k8NX2bvGqT4o5OpgmTZxbIrrfITKdxphadbyCjREidUnMQH1yi63XA6T/6peFuPH13udre02q7wied4uk8tVtd0fMCCmGAE09R1cuKdndNBj6JXo+o3UG0VtdkQjMRfAcOUI5AypoK3JVTVKpjwSoLtOjVr0QZRb+PB/xGOULaqQy5sRT1OmXEGw7xkQm1KnXm5sg6eFhdi2+gmXERdurku60R2uTxhg5wtuz+4uUL8/+iXFkX2Wya1pbXRM91JV4U+j5sMCmVzVF8x130l1PjZFoJaSdTlCsUaO+hO4QwYoi0xQ6QyiLBac61rV01ntNWbEBQVAGRE6QqEvW27wqUuSVoYIw67qz6wLHxb9MjvzyD8470r4LbmJYImwFp2WyertcBL/3Xj+xuNDuTmmGKRrNN2VyaZBhI1/WpVq4pQAsRhngmR7e9714AmUmWZQvTARCiOBnouPZVfQuxYbjkHIcjqNvzqMypHHMol03S+YvLcmJ8kHzPF3/61e/IXdtH6flXpgVKWn7iQ++h4UIKkbWlmRgSsQKuT4EIYyaaiU/B+mzfc7pylEob+NuyExna5PF3HQAQG/vobKl+hw03ez6WjMV7WJzrBtIPmpRK2Bw9YZgwGPVsWJYw2HA44suPPkf3vfsIff27F5AVJu2YGKRHv35S7t+9haq1lnzimXPi3n92jE5PLVJxrU7//fceoj/72rNicDDF96Cfee9xMTSYptHhHC2uVGB8WkRQoAszN060JUaNPT8jczccoNrJr5OJgCg8uZozeCFbglDP0iaPv+sAafqtK922S4WRLNm2qaBYRzSXV1ZofHwMv/MNNcmRZwfoqG+QF9HoeDBsRfa8F8USjNs6VqAXTs3IeMyk+ZWK+JVfvIvmYNR7fnwfnZleRgZYtFysUKlSpysLq2Lvri109sK8XCtnKBmPicViRdBGG0APHbr5OJV2H6F33H6bgo/OsVvJRCsVCkM2gDPk7ENLxh+u0wHUrJbnTLQ4DQgeQ5o2W21yAe+VUpkmxsdkPJ0WnudJJxbvt7MIxM5MLTGCi2deukzbJwqUS8fpiWfOinveeUQ++s0XZKfbE6OFLI0OZWn3xKAsDKTEhUtL8qFfuEt1Pu4Wl2eL1Gr3xKF923Bvi6uAIZHbqRw8cpsoaGb/fpJihRQRvd545QBVA4YVOjKC2bdkhj9EhL73mzdtNQ+/60oDbWeluE6e61Gr2RTttid/6t73ynR+UEOxy4EhBbRq4RqQ/fxMSU6ODYguzu/0Am5sNDaSI0OPUF+RvqvoH4F8GErV/RlTQlDFAAAbeN5VPOTzQ7TP0HdVirNH+GPcU5EfvmZyeLvUFA0Icc9QyG6N5OJzf6Td8OFf3owDfigDam5jZSTo1nxXz8QcizqISBwA5wG8fD9k0JPdrtuPvgodv+SuyQG0ybaEFcKUgYAxcmG2qoyHIaACoTo3BGmIfkf5GMAQECSLmaShC8uOkR1PAHgMZpKCs1CLSg7G9jtDxJj7Rd/vIzKkKJYieit7Dp3AByeuwwFDvenQ8Lvnfd857jggFtQA4oIHxB1qdz1kQhf3k4rgyD5d5Ts3ag2JUlGlY5oWJVNpGGgAIA0J3OAX40dEfNRSo/IRihcoeqgiHAQu+BG4ho9hr+shOyKHoROpDOG0YexhZ/IRzw2BVjvUD7ZU14tqY1PHDzngWIXCKUc/2yzVj4+MjRCjcyKRoG6nJ0rFoozjdxuZwTfiXhV1ayHyg2hlvi9kVLm8WOGjHNxeV8WJo91pNftYLdgQ9Hc4RWWCoRbORiHiSABDlRW3Ux29ncm/UCmAA98LQ+YjHtilL0KuH3VFvjJ+N+MUGnbkhutxAO0n6dZL844FzwLpTSy8UW9zuGSjWkcg2DBSkQ6DUAUT9sseDF0trqiM4OwwLQtdxFE8weRWifemFVPf89xeP15SDYCIMkrEV1ggA1d63ZArHK0W/d62lTPVddkxhg0HWVLHzz5TVPAQLV6qNOKio4fx9gRdhwNwlM+ceil/7H7ldcvSqdcNZbvTIzvhwtSwX358GwVMqiZt4AQbHKh0DdlIMMceOGA05VowRNU7uocHRmnCOE3jqdhQhnGKh8ypGS9UmqgrgyB50ZQMju91wbdlk4kVlmaoUuDssWLOa8wRsQ81A/+e2JQH3tABwuiuCJfHWp+SyTjSuKZAzDJ11Zdi8Tg1ajX8TfVh1Xo4GPFkUqU0Rh+UgydcDAgcXV6aj8GHjelySYAR+q6LrLCl7msc0T6DNImnRgV8akJCQqDuQ2RFgOsxCzWgHQReh1FflYiDGeXq4KjmiFBodtqkR85uatR/QweUyqGZ7nQoNF3Uf5yq5ZpkyDMBhpz0oMnoDh1KpZIUcVCFA2iXdRVVAyBoILLpXF5y1NHaEGmLXKQ+Rz96wcG+K1St97sJnxtLZsmCYb7XY6Il9KgMOOJqYuTPopSXipb3R2ceihVnULzJyQ/QJo8fcsAjZ0nEbM1cX10TwrNpYnJEpVws4XDvkZ1GQ+QLw9QFoAltmCIhR60DHDwmapUyUDxUnFxTRhkAzZgcHB5VABmLJSiZzoLFOVHU++1UtVTOpghT8CN5NSHVh1GIaeN3Gd1UqhYoolE6SgLcRrP2UmE/znqErtkBD+L1NaDb2vISDQ9sQbonlUGq7HGjOlJ/iAmLH6jEV5nIP/HKpDPqZaKfRykc/Yco83LhIOc1C5g/RAvuT74bIzFdne5oQxSAtVxSCvGxlhA3RUlwgYBveGjTpohlcpz+XBqCTGewc+VPR/H9ebpWBzxVImHvCGoeSE0i7og6RAf4QxpAbxd162EN3VaLGR4IC6djoEqAF20YGvndBub2dfIDT43OaG8yNbxTXG3NqH8+2ecyACawITCOdUCcj26A63FLzAwMi3a9rOYQBbma0U951WUAI5EUJkxFh6XKCDUNsjKX0cS+n/0Jot/9yjU74J0Fkt+LhyW9K7CwABGv80gs6pWa5ExwWzXRrpXlwOg4DAjI3IAa3Nlt16hTh0IkLIJ8CZsdOE+j2uwVFUHFGtXoKlEWcWSKTcz+MEoTmpvq/SqT0RE8twPAs7nfi5CdKV0FfP2uq4xHK1QDGXeRqBz4L/yPT9bk3Uel/N3/EWkl1+AAlE0oH4p3cN31udnlwe07J+GAlOx1u4rExeOmTKZTitS4ridNx1LmqzDocdn0Y2iJMShG6PksjIZK5FJNjVthG9njdttwrqv4AGMJmKNknqMrsRTaAkZt07T7X7IUG+SL8HckDxCKDHEZoMsEAFE4yG23yY7ZUvbXIpzU/tcX1KYdwEX3jUTPNbrm0nJxdXDXnp2UyaTF8tKqhFZPo2PDIgQzg2lUWS2K5NbJaC8DruYaR++nTrdDoiYiIoTomJjRLYeNskR2IKZ6OzPEoJ/6TH0xdAmm0qLTUlE1LVuxQkORH4t06AtQkVWOBx6+60UZwU4K0E083BN8oA+HsMJyDtETO9MwqXZNDuAjOFv0e9nh2UTSOrS8uEoOUNzv+WoSzQ0OyvJ6FeNuEnXc6Yu40U0ZEsYnt6EdAgeQMYoahzxAGVReWxX1aiVSkBBdTn8TUrqjDDQED0k9ZEYQER8Bx2AAQ6sEBqRyBeBBRTkzosqmtOJpZSfPGPi+5O8zzqg6QOYJwxkMRj5+B9Fn/uqaHdCEbe2ev2hmdGpCFuuB0FiOST1kgB1LiLmpKTk2drOawgSDVCiVEzj/mLhAJEHZJKI+hZTldE2lMyBCEQdoc6TbTWK+AAyQUI+YKst4MqNaKp/P0eUogw+IZrXELVjw+2R+WLZrJWVsxAQjYSYSadJ9TsaeiRsyfeDduN5jb4YDb+iAB4EDn/+APh1i7JV6B3UfF/l8VmJOgkDSkazF27EYlZZXJR3EsCK9viwVzQgNZAjv7vDczrXMshlHPgV5W7UwOEHVc3+07SF9O+2Gau8gWfwdjrLgdprMFZjyRnyHFP0RsWRasWUSr1V4xI0CcbXu0U2knfvp6cd3fYZoundNDuCv/5bvnZRAfWZ0c/NLcuvECJVW1hSIMQ6xMtxFm+p5khxd6y8CijK4fjIFCRtGIZKChQ6eC0CLecK/iuQWukCUASBE+ElRs4sySVB/j0A5Vci/BWYyana8FcFUm3EEzmQc4IxMDhT64yZoc3ZicjL/iRuJPn3qmhzAh6sbCxCjO/B9bHF+SezetVV6ACymswMDOVqDRJYZyIny0hyNTY5TX6pg5GeVGLgRVywQ0ZOcptH8FA1Gffqsejq3RsaJDakrGmoiiRekH+UHgOu1GQ8EY4Km1GbmUdHcwC+eNA0FkIwdAe6rR14yNFsWtgEH6NodYFBz3TZHq4hYjEHMD8AvgMRNMMEte3bTlYsX6NitR9F+GtSnqP0iEEoMCQOMvMAD6TbJw4YKtzI7t3VDt1DIr4wHdjFoRvjQg9GBgo7s8BYC32Cxgx0IKp4hJ5EG8PYEy/Q8KwRuoDoCYEJtxTFIcitVaKwcCp6SmvgknP4H/Q2XzTuAzJm2F+woJXV9tNnoyNnZOcoP5uXaakXw+BnixonsAF08fUZuO3CEESoiudyWoOq45SuK9ZGZxM6STV3UfXVpQWUQG896AeNDBimrOgLKAFnDMpiEQ0RtragIEbNEnikscAvDjCkDZeipLTbsPPclNjhC9iApuNAPu7jeSJSPHJLc5I3uk7+2D2s7e00O2PcIBad/rjenDYpDuWxCrBfX5eSxg2J5cRk01JKM5uX1stRCjoAPA0RUx6odYpHdEmpwF1XbnOEuD0QUxwYKI4Db6/SpcFeWlua5NQqeBMEbpGMmRAJbYV6vhwRyBQMmmCDup4lAYZKlSorLTLVALRJKiKivqoXcetRYrMBDgIHuuu/XJf2nhzZE0o3h7U0dgKEofL7TOR2EwT0DeUS+xD0A21LYMOEtsnQqTpW1dVEYHZUzr75Cuw8fodeNZNheO46fGG9FQ5RLJUS1Lhg8Y5DUnFhCxlMZwcSI18NjdK/XQbgcsQpMgXHSiSconkhJu5CI2tzGxftaYNgfzq7CZMSh+zKd7AcjVC9jYPL9nf993+evTF0cWKkl/vXXHmqeIXr1D97UAeythyqVl9mZLI21Wx2qVqqSU7eF3wcGB2j24kV5wwfuFaef+Z7cdfgoC93U9zBqv0Hu2qvkjNwq80PDaqlMfZlTNOsV0aiWFdsDUGI3KkX5VFYpv3wvZAs7p19QHNCgP/cztQ772oeQqp0qLIhYJc8Nlm3JWDYrhJoMffUzbJ4fsw4cfGbi9g+OLH7x88WXXln87FuWAB/g/adbzRYAMAQdTirDWSleXVqmm44eoNJ6XRQXiyiDHrUaUIgQ3Y3+rMUGyB69RUUAc4Nqg1wGPF5nUfcu2iTTYQbHFnZ6G+U1heZsuG05cqOcONae5yoSxS+M4UoZDtAhNtQjpV1CbwzUvqUverV1zCKQ59aukJz9X6RvB0EauXM8OPVlmfVnTzz8TKN6QmzCASm3soBl9HBdrN3h+pf7D+yi6QszdOe77iS308byfBrfsVPMn3pW7r393fwsAG0wlKC5gE7WFbncDbIMUGOq22k1lI7HgGeDM7BDWDEW/X0P1hk8Zn+YOH01TqusAMtzwBYd3vqDNgh1WapNE9V/VE5oASk6wpJ6ryW7yy8LY/qL0jg4Kah3kvwXnqTiC1hAauJ/Ep2Tm8qAfYVSZ94Qp3GXm6FuyjjosIsROBm3qNHqUSbpUAnZcPDoTfKFr/432v1j75YGa4DRXgEZuZ24gZCsI6ShUlkYXwUbu7GXi2VwV8Cus3Q7Le5lSilixI+lsoLZYq8FfGi3wCRdJbRwpPnZBNU2uy20QZA8dJ1Qh4ossDUe9qT3wp9RwvkB6Qe2QKc7TZW/XqPGWoxS93wmPpQZ/BR97mP/nvqA+Kby+f5zEEeO7j4eS6aODuRzvEC0Gl/pIPFUjnKZuFiYWwA1zVA6btLchbM0fuNBwe2Loq2svkYREaAOAtBq1KlRrcp6rSw6mAc4hQF2wkmkVLSZ7/cwEbq9NsAwhr3InNIY+Rrq825LsMaIkRnZ4ChVWanQnZrsFtF6n/qCSI3OCH3SJG/mLK09VqbVVZsqO++XO47fQfrIriMe2f/nt7/ygzW28U13UE4wQrfbl7hvO5i1eTCp1xpCAeJ6mbLYHzRNDcNJRYzvupFWZi+K0tzFPttjRhuqcbdcWpb1yjpAKlQCSDKTFQ5+cgtDdKleZYpdByDaEEXTCv151xnlIhrrRY62an1xACUPT+zPXqdObqNM2EQgBxvUTnWGYs/8B5EB7OjDvmy9fIFWH6/S+eU0ydt/mY6+7/1CxNNgh2aebnjvZ/u6Ab3lBsrR7YXhmO08uHXnVvT9qsTgpnaNa5UqHbz5ZrE8c0WNuAMTO8iDGnT6+08Jy5AolZ5gRclCy+ONjR7jBcteeHHEHK59fI9nA6a4LJ8FqqYF7x2oYYgB1I94gOL53CUwmPH94UB0CxNJE3Sp/sSXSJv9ikjeCgMbV6j4dFGsPt+lV3rb5I73/5K48bbjpKcHomeIeE8hO3bAfeCWZ3/7S39xyXgrB7TL61O9LWMgIj5ajMkPuyBNfdRmh7peIEe3jNJapSWW5xfk9v0HxcLCMk2ffhHjcEZ15dz4Xtp95FbeNBGM+lzzrWYNrbCqZDGMwNhuS/P8T110BqS57EJ240yJp9Iymx/ksWdDHGXWhzrBEORh276yTI2nv0IZ/fvCOjgkA/CR2ZOSqiVBl7M30Ts++As0uu8QtpAH1QDnueCL1arwaheEKL96Fy73zbd0QJb0ZWxpznY63W3wvrQsQ1ucW0RJWHJ+dl6MbxmXUxee5kdXaOf+99HBY8fob779HYAS5pB8SixdPE3b9x+BmtRQsrluJMH4YthrbIEJ9qheXlUcnolPOp1Gquc50lEbVDTJUzKY6ueMLWCFYbdOtSvnqPONL9LQzeD/6z61nj5FZ2B8I7DJP/Aueef9D6B3JWlxbo4aK09jRF9VbBWbOjIxMCJyQ5PhpkrgQz9V9oS9907QzT3DIwVqNFrqmTgX2+UsWu4+eIDKK/OUAx4Ui2XaccNODGEg3memhIctcn5GoFZt0pbtu8T6yiJSuquQMZXJwmCAZyatdpRs5LOuq0fqON3VFhzvF6I28KvHrAc+ANnxmmL9ucfJ/dYXqHAsKcTFWVp8tS5efknIFT9BtZ23kp1nRaeK+aRNmCkEX3ti904a3LFbJIcmiDeX9W7lpL3lr7/xlhmw/F8osD/SuGA5Ccoe3kdLS0UaHBqipflFKs7O8BOfYmx8Ql6YmqVsJkHG4YO0fdcupRifevEVWl46Tfv2NWlk604amxhX+xiyv2WguoXiO5ri7jB4g6NHGpva55OysbYmitNn0cqWqfvc43JrbFrkD4/J1rOX6PKsTxcXSa6nBil98Ba65R2HaXh8jDzwhA7KjBWqLmaPK+enwE472HJvCdb30rHwuf23vgUP4ONhLPETl+dfHRwZAvrqipZyxJgdWppHly5MYddnRPRefFEO7d1JZ198kQ7ftJ923Yi9BNDc6alpOnnyglxeWKEPfOzjQPhktGfCnVJEvoh8gLBg0AEJUOkKg2VrfYlY/qqurUkTY3f8/Hdp1+1olyVdrj05RWdmJK00BF2xCzS0fTc52DA+8/JpcfHcOWkwgTIirYH3H/hJF+7Hjq3LbNI5S53K/+Vbv6UD+KTfTBjPo2W111fL8YHBvGDBMwVRtN1o09rCHG3bepy2ILqL4AR+D338yEF+ZlBwSsewt8hpfuXSDP3J7/0+vePuu2l86yQktRTv9UEMdanZaJDXbiqDSzMX8HGgpPcYNmZThQxlG0ukX35KjNyVkd3LFVG81JBnF4lWsXVXzI3T8LYhSiUsxTRZc/HakWbE0bdtVp3QMSyNS0zGYqDbBp3ZX3yqTZvBAD7u3ye7Vnbrg5ivB0dGh0S1WocqlKUljMYWrrZt7z7Wp2hu+qLYMjlBjWaXhuAQyGMyg24Qt7FRmkbrwi5ObWWWLp96Qc6de1k0ipdkdekSLV48S93qsojHwOaYZCVjcAFRdWFJtJ77KzExWhIjt+yk1rMXaHmhQ89fEjTdtWk1NwoukgVMYCu+y/KYz1wKBmqcoYKfX+h2IcawmkPU0DWtaAr5wtr87G/s+Z1ihW17ywzgI7u41l3Nrp+Lp9N7eBhaL63TxMQhSmB3uFxcoYWZeVGAXI7NY9mFhFW98qoszU5RvVIRjP5tKMsYCmQ6kxDZbIpyA8Nqx3kVI3YPf2ORFaqznD47hRbZFo1aXab9Jh3Lr8v9Pz0KlujIV/7kRVFa8+nSMtGUH6d2vkAprAUG9uDfKth3zfN7laDTWmkFwbzj6EUt9EvQ0ipWIr4aqwc112tUh6145SfGPtzYUMk25YCPY5b41dziN2/Yv/sBfj9YyNH6ek2lFqFHzl+8IG1soFjxpDj1g+dktVITIDAylU7IbDaJLXabyhVXTE1DEerxwxGGMFEetgXuj5bX6TBFbnNblO1W1xsRgTmpt2n01u0YJHRaePwlsTgX0vfnqKfHjD8USefpwV6tZNPaen49qEO+d9Omjv6oBzuSca+XWJClpprNgwf39yXxE1cHVfn6Byc29RABn/fpu3bt2Hvbj00l0gmxVlxFn4/JIwd3amdePivZERVQY9bzK6DK3R4/7hKqG8JYJaIMFfIiGbf7O8FITcjrzWazsrq6Pt+oNS7i/flepXbm3q0DH7x5KHfftmN7aOn8FTFTPitnFjBMxaj63BL9uyBFX771Rurytc+xcScioH7NuGs7NusA+sNjZM7f8ONLoMJucbWaa7vS+eQnP0yvnj5Ds7NLotFoShv6vw8ndDo99ZQI40MMAgXIEwE33LDbPYuUfbnX6JzXqXtODzpz0DCKmh9rD1HJle3Y0aYUTw3l09Z9h0Zp/tVF8dwqCyf+JSTLp+ay9O3/+H3qvs7Q1z1mfX3Hph3A537qwNBQblCKkpH/6paJkdvj2QGQDV+uFNdEt+tB1uoJxzFB4IMGtscXsW/7vHS756rr6y85UpzLGGGzuF7pWQMUjJ2k4OGoIV414FcKNNIm8y8tXTs2hHblkN+o+f6jnml9bvGW7vyfP6JG/x/J4B/FAepggvLJOybvue0nb/vj1XJtcHFx1U0mnCsYlKaQyq+g+J9v19oX81kqWXqmnbEXPHonhQ+f+NvGvtFxAo2xNGntbXXD30qZWgwa+ZfCbPDtoXPUPkF9ve3/83HNDuCD5/yPHivc3esFiYwZnozFwnrGrrf3FcjjbbX+ha8rUuzgz+0jiJCk7Zsh7+ciyvQP7+CFnjih/seXaDfk7ePt4+3jH+Px/wDUnZ38a8mOkQAAAABJRU5ErkJggg==" + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAHnhJREFUeAHte2mQZeV53vud/e5L9+1tunt2mH2GGWAMwlhGirBkBMgYR7EtRVIcyXiRUnZUpT8KI5flyKlSyv6RWI7LihMrVbFRcKLYoAULjCQEgmGZjZmenpneu2/f7ruvZ/vyvN+5PWCLgp5RXGWXOcWd7nv73HO+d3ve532+A9Hbx9vH28c/5UPQP7BD8prwz9mfpMSVRvpf5XruHltqo3GLhh1HS6VvP/ZY6tf+TSZoN13pu88nMtlviW0PLNN1Hn/vDpAyusepTw/H45NH4+N7j6cCQUky9RE9UxgSpA+HpG3X4qkJqdujQtPz0nML3XYn+cpj3xLWE18Tk+RKg4SwbSHjw/DP/R8iOvwefEJCk+EXpJR/Ye74wPNC+e/ajut2wKM/m7tnx1b3gNQp7yRE2nH0nG44zeS+X3xCT40MkxXLGaY5EHpezicakQEVNF0fkoaZ152UIXRTCE2DETrhXKnWAktI0ykIfBH0utJtt+jyU0+S/uQjdOimuDCSgdQS8OiARb09/5Zo6BBcYEicvEKh9utWonBR5A+euhY7NuWAP3+Q9O3v/427B/LJncXllZFyaW3g3DMvPPDPP3psoLBnizBsR2qWA/dbVJ7ukmcWhIbYSCmEVGESpOGnphukWXGs2SLDTpBuOgRH4HsaseH4A2la/3csDR6CTwJZOvkdyk//vnBq81LIED6T5N/2L6Vx22dxUTgA18YXpWyuTVPY/q7mnfyUGPrV5mZsM97sj+f/+GOphnR+frXufYaMxKRn5SgzGhd2fpxCPUmluZdofGKOqFwX5DawOE04yY9QfOwwbLCwfoONlprpCN0wI2PZIbxgGEs6LCGdPyDO3qha+n+/WjySUhM7qPhInZJNjSxbSviPKH2RjHfAbgojB+L0MNR3a5q5m4rB7+CMH90BQW7yP6/Or/28L2wUYIqGtu2XyzMztHX7MHmBQcW/eYza9hpSFvdHnnueIGPPPMUmjCiKuqYiiqghkB7SAebpUaRJw5sg4NTAlzmCMFzyrxtl/FpFW5k8NX2bvGqT4o5OpgmTZxbIrrfITKdxphadbyCjREidUnMQH1yi63XA6T/6peFuPH13udre02q7wied4uk8tVtd0fMCCmGAE09R1cuKdndNBj6JXo+o3UG0VtdkQjMRfAcOUI5AypoK3JVTVKpjwSoLtOjVr0QZRb+PB/xGOULaqQy5sRT1OmXEGw7xkQm1KnXm5sg6eFhdi2+gmXERdurku60R2uTxhg5wtuz+4uUL8/+iXFkX2Wya1pbXRM91JV4U+j5sMCmVzVF8x130l1PjZFoJaSdTlCsUaO+hO4QwYoi0xQ6QyiLBac61rV01ntNWbEBQVAGRE6QqEvW27wqUuSVoYIw67qz6wLHxb9MjvzyD8470r4LbmJYImwFp2WyertcBL/3Xj+xuNDuTmmGKRrNN2VyaZBhI1/WpVq4pQAsRhngmR7e9714AmUmWZQvTARCiOBnouPZVfQuxYbjkHIcjqNvzqMypHHMol03S+YvLcmJ8kHzPF3/61e/IXdtH6flXpgVKWn7iQ++h4UIKkbWlmRgSsQKuT4EIYyaaiU/B+mzfc7pylEob+NuyExna5PF3HQAQG/vobKl+hw03ez6WjMV7WJzrBtIPmpRK2Bw9YZgwGPVsWJYw2HA44suPPkf3vfsIff27F5AVJu2YGKRHv35S7t+9haq1lnzimXPi3n92jE5PLVJxrU7//fceoj/72rNicDDF96Cfee9xMTSYptHhHC2uVGB8WkRQoAszN060JUaNPT8jczccoNrJr5OJgCg8uZozeCFbglDP0iaPv+sAafqtK922S4WRLNm2qaBYRzSXV1ZofHwMv/MNNcmRZwfoqG+QF9HoeDBsRfa8F8USjNs6VqAXTs3IeMyk+ZWK+JVfvIvmYNR7fnwfnZleRgZYtFysUKlSpysLq2Lvri109sK8XCtnKBmPicViRdBGG0APHbr5OJV2H6F33H6bgo/OsVvJRCsVCkM2gDPk7ENLxh+u0wHUrJbnTLQ4DQgeQ5o2W21yAe+VUpkmxsdkPJ0WnudJJxbvt7MIxM5MLTGCi2deukzbJwqUS8fpiWfOinveeUQ++s0XZKfbE6OFLI0OZWn3xKAsDKTEhUtL8qFfuEt1Pu4Wl2eL1Gr3xKF923Bvi6uAIZHbqRw8cpsoaGb/fpJihRQRvd545QBVA4YVOjKC2bdkhj9EhL73mzdtNQ+/60oDbWeluE6e61Gr2RTttid/6t73ynR+UEOxy4EhBbRq4RqQ/fxMSU6ODYguzu/0Am5sNDaSI0OPUF+RvqvoH4F8GErV/RlTQlDFAAAbeN5VPOTzQ7TP0HdVirNH+GPcU5EfvmZyeLvUFA0Icc9QyG6N5OJzf6Td8OFf3owDfigDam5jZSTo1nxXz8QcizqISBwA5wG8fD9k0JPdrtuPvgodv+SuyQG0ybaEFcKUgYAxcmG2qoyHIaACoTo3BGmIfkf5GMAQECSLmaShC8uOkR1PAHgMZpKCs1CLSg7G9jtDxJj7Rd/vIzKkKJYieit7Dp3AByeuwwFDvenQ8Lvnfd857jggFtQA4oIHxB1qdz1kQhf3k4rgyD5d5Ts3ag2JUlGlY5oWJVNpGGgAIA0J3OAX40dEfNRSo/IRihcoeqgiHAQu+BG4ho9hr+shOyKHoROpDOG0YexhZ/IRzw2BVjvUD7ZU14tqY1PHDzngWIXCKUc/2yzVj4+MjRCjcyKRoG6nJ0rFoozjdxuZwTfiXhV1ayHyg2hlvi9kVLm8WOGjHNxeV8WJo91pNftYLdgQ9Hc4RWWCoRbORiHiSABDlRW3Ux29ncm/UCmAA98LQ+YjHtilL0KuH3VFvjJ+N+MUGnbkhutxAO0n6dZL844FzwLpTSy8UW9zuGSjWkcg2DBSkQ6DUAUT9sseDF0trqiM4OwwLQtdxFE8weRWifemFVPf89xeP15SDYCIMkrEV1ggA1d63ZArHK0W/d62lTPVddkxhg0HWVLHzz5TVPAQLV6qNOKio4fx9gRdhwNwlM+ceil/7H7ldcvSqdcNZbvTIzvhwtSwX358GwVMqiZt4AQbHKh0DdlIMMceOGA05VowRNU7uocHRmnCOE3jqdhQhnGKh8ypGS9UmqgrgyB50ZQMju91wbdlk4kVlmaoUuDssWLOa8wRsQ81A/+e2JQH3tABwuiuCJfHWp+SyTjSuKZAzDJ11Zdi8Tg1ajX8TfVh1Xo4GPFkUqU0Rh+UgydcDAgcXV6aj8GHjelySYAR+q6LrLCl7msc0T6DNImnRgV8akJCQqDuQ2RFgOsxCzWgHQReh1FflYiDGeXq4KjmiFBodtqkR85uatR/QweUyqGZ7nQoNF3Uf5yq5ZpkyDMBhpz0oMnoDh1KpZIUcVCFA2iXdRVVAyBoILLpXF5y1NHaEGmLXKQ+Rz96wcG+K1St97sJnxtLZsmCYb7XY6Il9KgMOOJqYuTPopSXipb3R2ceihVnULzJyQ/QJo8fcsAjZ0nEbM1cX10TwrNpYnJEpVws4XDvkZ1GQ+QLw9QFoAltmCIhR60DHDwmapUyUDxUnFxTRhkAzZgcHB5VABmLJSiZzoLFOVHU++1UtVTOpghT8CN5NSHVh1GIaeN3Gd1UqhYoolE6SgLcRrP2UmE/znqErtkBD+L1NaDb2vISDQ9sQbonlUGq7HGjOlJ/iAmLH6jEV5nIP/HKpDPqZaKfRykc/Yco83LhIOc1C5g/RAvuT74bIzFdne5oQxSAtVxSCvGxlhA3RUlwgYBveGjTpohlcpz+XBqCTGewc+VPR/H9ebpWBzxVImHvCGoeSE0i7og6RAf4QxpAbxd162EN3VaLGR4IC6djoEqAF20YGvndBub2dfIDT43OaG8yNbxTXG3NqH8+2ecyACawITCOdUCcj26A63FLzAwMi3a9rOYQBbma0U951WUAI5EUJkxFh6XKCDUNsjKX0cS+n/0Jot/9yjU74J0Fkt+LhyW9K7CwABGv80gs6pWa5ExwWzXRrpXlwOg4DAjI3IAa3Nlt16hTh0IkLIJ8CZsdOE+j2uwVFUHFGtXoKlEWcWSKTcz+MEoTmpvq/SqT0RE8twPAs7nfi5CdKV0FfP2uq4xHK1QDGXeRqBz4L/yPT9bk3Uel/N3/EWkl1+AAlE0oH4p3cN31udnlwe07J+GAlOx1u4rExeOmTKZTitS4ridNx1LmqzDocdn0Y2iJMShG6PksjIZK5FJNjVthG9njdttwrqv4AGMJmKNknqMrsRTaAkZt07T7X7IUG+SL8HckDxCKDHEZoMsEAFE4yG23yY7ZUvbXIpzU/tcX1KYdwEX3jUTPNbrm0nJxdXDXnp2UyaTF8tKqhFZPo2PDIgQzg2lUWS2K5NbJaC8DruYaR++nTrdDoiYiIoTomJjRLYeNskR2IKZ6OzPEoJ/6TH0xdAmm0qLTUlE1LVuxQkORH4t06AtQkVWOBx6+60UZwU4K0E083BN8oA+HsMJyDtETO9MwqXZNDuAjOFv0e9nh2UTSOrS8uEoOUNzv+WoSzQ0OyvJ6FeNuEnXc6Yu40U0ZEsYnt6EdAgeQMYoahzxAGVReWxX1aiVSkBBdTn8TUrqjDDQED0k9ZEYQER8Bx2AAQ6sEBqRyBeBBRTkzosqmtOJpZSfPGPi+5O8zzqg6QOYJwxkMRj5+B9Fn/uqaHdCEbe2ev2hmdGpCFuuB0FiOST1kgB1LiLmpKTk2drOawgSDVCiVEzj/mLhAJEHZJKI+hZTldE2lMyBCEQdoc6TbTWK+AAyQUI+YKst4MqNaKp/P0eUogw+IZrXELVjw+2R+WLZrJWVsxAQjYSYSadJ9TsaeiRsyfeDduN5jb4YDb+iAB4EDn/+APh1i7JV6B3UfF/l8VmJOgkDSkazF27EYlZZXJR3EsCK9viwVzQgNZAjv7vDczrXMshlHPgV5W7UwOEHVc3+07SF9O+2Gau8gWfwdjrLgdprMFZjyRnyHFP0RsWRasWUSr1V4xI0CcbXu0U2knfvp6cd3fYZoundNDuCv/5bvnZRAfWZ0c/NLcuvECJVW1hSIMQ6xMtxFm+p5khxd6y8CijK4fjIFCRtGIZKChQ6eC0CLecK/iuQWukCUASBE+ElRs4sySVB/j0A5Vci/BWYyana8FcFUm3EEzmQc4IxMDhT64yZoc3ZicjL/iRuJPn3qmhzAh6sbCxCjO/B9bHF+SezetVV6ACymswMDOVqDRJYZyIny0hyNTY5TX6pg5GeVGLgRVywQ0ZOcptH8FA1Gffqsejq3RsaJDakrGmoiiRekH+UHgOu1GQ8EY4Km1GbmUdHcwC+eNA0FkIwdAe6rR14yNFsWtgEH6NodYFBz3TZHq4hYjEHMD8AvgMRNMMEte3bTlYsX6NitR9F+GtSnqP0iEEoMCQOMvMAD6TbJw4YKtzI7t3VDt1DIr4wHdjFoRvjQg9GBgo7s8BYC32Cxgx0IKp4hJ5EG8PYEy/Q8KwRuoDoCYEJtxTFIcitVaKwcCp6SmvgknP4H/Q2XzTuAzJm2F+woJXV9tNnoyNnZOcoP5uXaakXw+BnixonsAF08fUZuO3CEESoiudyWoOq45SuK9ZGZxM6STV3UfXVpQWUQG896AeNDBimrOgLKAFnDMpiEQ0RtragIEbNEnikscAvDjCkDZeipLTbsPPclNjhC9iApuNAPu7jeSJSPHJLc5I3uk7+2D2s7e00O2PcIBad/rjenDYpDuWxCrBfX5eSxg2J5cRk01JKM5uX1stRCjoAPA0RUx6odYpHdEmpwF1XbnOEuD0QUxwYKI4Db6/SpcFeWlua5NQqeBMEbpGMmRAJbYV6vhwRyBQMmmCDup4lAYZKlSorLTLVALRJKiKivqoXcetRYrMBDgIHuuu/XJf2nhzZE0o3h7U0dgKEofL7TOR2EwT0DeUS+xD0A21LYMOEtsnQqTpW1dVEYHZUzr75Cuw8fodeNZNheO46fGG9FQ5RLJUS1Lhg8Y5DUnFhCxlMZwcSI18NjdK/XQbgcsQpMgXHSiSconkhJu5CI2tzGxftaYNgfzq7CZMSh+zKd7AcjVC9jYPL9nf993+evTF0cWKkl/vXXHmqeIXr1D97UAeythyqVl9mZLI21Wx2qVqqSU7eF3wcGB2j24kV5wwfuFaef+Z7cdfgoC93U9zBqv0Hu2qvkjNwq80PDaqlMfZlTNOsV0aiWFdsDUGI3KkX5VFYpv3wvZAs7p19QHNCgP/cztQ772oeQqp0qLIhYJc8Nlm3JWDYrhJoMffUzbJ4fsw4cfGbi9g+OLH7x88WXXln87FuWAB/g/adbzRYAMAQdTirDWSleXVqmm44eoNJ6XRQXiyiDHrUaUIgQ3Y3+rMUGyB69RUUAc4Nqg1wGPF5nUfcu2iTTYQbHFnZ6G+U1heZsuG05cqOcONae5yoSxS+M4UoZDtAhNtQjpV1CbwzUvqUverV1zCKQ59aukJz9X6RvB0EauXM8OPVlmfVnTzz8TKN6QmzCASm3soBl9HBdrN3h+pf7D+yi6QszdOe77iS308byfBrfsVPMn3pW7r393fwsAG0wlKC5gE7WFbncDbIMUGOq22k1lI7HgGeDM7BDWDEW/X0P1hk8Zn+YOH01TqusAMtzwBYd3vqDNgh1WapNE9V/VE5oASk6wpJ6ryW7yy8LY/qL0jg4Kah3kvwXnqTiC1hAauJ/Ep2Tm8qAfYVSZ94Qp3GXm6FuyjjosIsROBm3qNHqUSbpUAnZcPDoTfKFr/432v1j75YGa4DRXgEZuZ24gZCsI6ShUlkYXwUbu7GXi2VwV8Cus3Q7Le5lSilixI+lsoLZYq8FfGi3wCRdJbRwpPnZBNU2uy20QZA8dJ1Qh4ossDUe9qT3wp9RwvkB6Qe2QKc7TZW/XqPGWoxS93wmPpQZ/BR97mP/nvqA+Kby+f5zEEeO7j4eS6aODuRzvEC0Gl/pIPFUjnKZuFiYWwA1zVA6btLchbM0fuNBwe2Loq2svkYREaAOAtBq1KlRrcp6rSw6mAc4hQF2wkmkVLSZ7/cwEbq9NsAwhr3InNIY+Rrq825LsMaIkRnZ4ChVWanQnZrsFtF6n/qCSI3OCH3SJG/mLK09VqbVVZsqO++XO47fQfrIriMe2f/nt7/ygzW28U13UE4wQrfbl7hvO5i1eTCp1xpCAeJ6mbLYHzRNDcNJRYzvupFWZi+K0tzFPttjRhuqcbdcWpb1yjpAKlQCSDKTFQ5+cgtDdKleZYpdByDaEEXTCv151xnlIhrrRY62an1xACUPT+zPXqdObqNM2EQgBxvUTnWGYs/8B5EB7OjDvmy9fIFWH6/S+eU0ydt/mY6+7/1CxNNgh2aebnjvZ/u6Ab3lBsrR7YXhmO08uHXnVvT9qsTgpnaNa5UqHbz5ZrE8c0WNuAMTO8iDGnT6+08Jy5AolZ5gRclCy+ONjR7jBcteeHHEHK59fI9nA6a4LJ8FqqYF7x2oYYgB1I94gOL53CUwmPH94UB0CxNJE3Sp/sSXSJv9ikjeCgMbV6j4dFGsPt+lV3rb5I73/5K48bbjpKcHomeIeE8hO3bAfeCWZ3/7S39xyXgrB7TL61O9LWMgIj5ajMkPuyBNfdRmh7peIEe3jNJapSWW5xfk9v0HxcLCMk2ffhHjcEZ15dz4Xtp95FbeNBGM+lzzrWYNrbCqZDGMwNhuS/P8T110BqS57EJ240yJp9Iymx/ksWdDHGXWhzrBEORh276yTI2nv0IZ/fvCOjgkA/CR2ZOSqiVBl7M30Ts++As0uu8QtpAH1QDnueCL1arwaheEKL96Fy73zbd0QJb0ZWxpznY63W3wvrQsQ1ucW0RJWHJ+dl6MbxmXUxee5kdXaOf+99HBY8fob779HYAS5pB8SixdPE3b9x+BmtRQsrluJMH4YthrbIEJ9qheXlUcnolPOp1Gquc50lEbVDTJUzKY6ueMLWCFYbdOtSvnqPONL9LQzeD/6z61nj5FZ2B8I7DJP/Aueef9D6B3JWlxbo4aK09jRF9VbBWbOjIxMCJyQ5PhpkrgQz9V9oS9907QzT3DIwVqNFrqmTgX2+UsWu4+eIDKK/OUAx4Ui2XaccNODGEg3memhIctcn5GoFZt0pbtu8T6yiJSuquQMZXJwmCAZyatdpRs5LOuq0fqON3VFhzvF6I28KvHrAc+ANnxmmL9ucfJ/dYXqHAsKcTFWVp8tS5efknIFT9BtZ23kp1nRaeK+aRNmCkEX3ti904a3LFbJIcmiDeX9W7lpL3lr7/xlhmw/F8osD/SuGA5Ccoe3kdLS0UaHBqipflFKs7O8BOfYmx8Ql6YmqVsJkHG4YO0fdcupRifevEVWl46Tfv2NWlk604amxhX+xiyv2WguoXiO5ri7jB4g6NHGpva55OysbYmitNn0cqWqfvc43JrbFrkD4/J1rOX6PKsTxcXSa6nBil98Ba65R2HaXh8jDzwhA7KjBWqLmaPK+enwE472HJvCdb30rHwuf23vgUP4ONhLPETl+dfHRwZAvrqipZyxJgdWppHly5MYddnRPRefFEO7d1JZ198kQ7ftJ923Yi9BNDc6alpOnnyglxeWKEPfOzjQPhktGfCnVJEvoh8gLBg0AEJUOkKg2VrfYlY/qqurUkTY3f8/Hdp1+1olyVdrj05RWdmJK00BF2xCzS0fTc52DA+8/JpcfHcOWkwgTIirYH3H/hJF+7Hjq3LbNI5S53K/+Vbv6UD+KTfTBjPo2W111fL8YHBvGDBMwVRtN1o09rCHG3bepy2ILqL4AR+D338yEF+ZlBwSsewt8hpfuXSDP3J7/0+vePuu2l86yQktRTv9UEMdanZaJDXbiqDSzMX8HGgpPcYNmZThQxlG0ukX35KjNyVkd3LFVG81JBnF4lWsXVXzI3T8LYhSiUsxTRZc/HakWbE0bdtVp3QMSyNS0zGYqDbBp3ZX3yqTZvBAD7u3ye7Vnbrg5ivB0dGh0S1WocqlKUljMYWrrZt7z7Wp2hu+qLYMjlBjWaXhuAQyGMyg24Qt7FRmkbrwi5ObWWWLp96Qc6de1k0ipdkdekSLV48S93qsojHwOaYZCVjcAFRdWFJtJ77KzExWhIjt+yk1rMXaHmhQ89fEjTdtWk1NwoukgVMYCu+y/KYz1wKBmqcoYKfX+h2IcawmkPU0DWtaAr5wtr87G/s+Z1ihW17ywzgI7u41l3Nrp+Lp9N7eBhaL63TxMQhSmB3uFxcoYWZeVGAXI7NY9mFhFW98qoszU5RvVIRjP5tKMsYCmQ6kxDZbIpyA8Nqx3kVI3YPf2ORFaqznD47hRbZFo1aXab9Jh3Lr8v9Pz0KlujIV/7kRVFa8+nSMtGUH6d2vkAprAUG9uDfKth3zfN7laDTWmkFwbzj6EUt9EvQ0ipWIr4aqwc112tUh6145SfGPtzYUMk25YCPY5b41dziN2/Yv/sBfj9YyNH6ek2lFqFHzl+8IG1soFjxpDj1g+dktVITIDAylU7IbDaJLXabyhVXTE1DEerxwxGGMFEetgXuj5bX6TBFbnNblO1W1xsRgTmpt2n01u0YJHRaePwlsTgX0vfnqKfHjD8USefpwV6tZNPaen49qEO+d9Omjv6oBzuSca+XWJClpprNgwf39yXxE1cHVfn6Byc29RABn/fpu3bt2Hvbj00l0gmxVlxFn4/JIwd3amdePivZERVQY9bzK6DK3R4/7hKqG8JYJaIMFfIiGbf7O8FITcjrzWazsrq6Pt+oNS7i/flepXbm3q0DH7x5KHfftmN7aOn8FTFTPitnFjBMxaj63BL9uyBFX771Rurytc+xcScioH7NuGs7NusA+sNjZM7f8ONLoMJucbWaa7vS+eQnP0yvnj5Ds7NLotFoShv6vw8ndDo99ZQI40MMAgXIEwE33LDbPYuUfbnX6JzXqXtODzpz0DCKmh9rD1HJle3Y0aYUTw3l09Z9h0Zp/tVF8dwqCyf+JSTLp+ay9O3/+H3qvs7Q1z1mfX3Hph3A537qwNBQblCKkpH/6paJkdvj2QGQDV+uFNdEt+tB1uoJxzFB4IMGtscXsW/7vHS756rr6y85UpzLGGGzuF7pWQMUjJ2k4OGoIV414FcKNNIm8y8tXTs2hHblkN+o+f6jnml9bvGW7vyfP6JG/x/J4B/FAepggvLJOybvue0nb/vj1XJtcHFx1U0mnCsYlKaQyq+g+J9v19oX81kqWXqmnbEXPHonhQ+f+NvGvtFxAo2xNGntbXXD30qZWgwa+ZfCbPDtoXPUPkF9ve3/83HNDuCD5/yPHivc3esFiYwZnozFwnrGrrf3FcjjbbX+ha8rUuzgz+0jiJCk7Zsh7+ciyvQP7+CFnjih/seXaDfk7ePt4+3jH+Px/wDUnZ38a8mOkQAAAABJRU5ErkJggg==", + "parser_ids": [ + "paper" + ] } diff --git a/agent/templates/ingestion_pipeline_picture.json b/agent/templates/ingestion_pipeline_picture.json new file mode 100644 index 0000000000..0f42843f07 --- /dev/null +++ b/agent/templates/ingestion_pipeline_picture.json @@ -0,0 +1,294 @@ +{ + "id": 47, + "title": { + "en": "Picture", + "de": "Bild & Video", + "zh": "图片与视频" + }, + "description": { + "en": "This template extracts text from images via OCR and describes video content using vision models. Supports .jpg, .png, .gif image formats and .mp4, .avi, .mkv video formats.", + "de": "Diese Vorlage extrahiert Text aus Bildern mittels OCR und beschreibt Videoinhalte mithilfe von Vision-Modellen. Unterstützt .jpg, .png, .gif Bildformate und .mp4, .avi, .mkv Videoformate.", + "zh": "此模板通过OCR从图片中提取文本，并使用视觉模型描述视频内容。支持 .jpg、.png、.gif 图片格式和 .mp4、.avi、.mkv 视频格式。" + }, + "canvas_type": "Ingestion Pipeline", + "canvas_category": "dataflow_canvas", + "dsl": { + "components": { + "File": { + "downstream": [ + "Parser:ViewsCaptureLight" + ], + "obj": { + "component_name": "File", + "params": {} + }, + "upstream": [] + }, + "Parser:ViewsCaptureLight": { + "downstream": [ + "TokenChunker:BrightColorsGlow" + ], + "obj": { + "component_name": "Parser", + "params": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": { + "image": { + "output_format": "text", + "parse_method": "ocr", + "preprocess": [ + "main_content" + ], + "suffix": [ + "jpg", + "jpeg", + "png", + "gif" + ] + }, + "video": { + "output_format": "text", + "preprocess": [ + "main_content" + ], + "suffix": [ + "mp4", + "avi", + "mkv" + ] + } + } + } + }, + "upstream": [ + "File" + ] + }, + "TokenChunker:BrightColorsGlow": { + "downstream": [ + "Tokenizer:SharpLensFocus" + ], + "obj": { + "component_name": "OneChunker", + "params": { + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + } + } + }, + "upstream": [ + "Parser:ViewsCaptureLight" + ] + }, + "Tokenizer:SharpLensFocus": { + "downstream": [], + "obj": { + "component_name": "Tokenizer", + "params": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + } + }, + "upstream": [ + "TokenChunker:BrightColorsGlow" + ] + } + }, + "globals": { + "sys.history": [] + }, + "graph": { + "edges": [ + { + "id": "xy-edge__Filestart-Parser:ViewsCaptureLightend", + "source": "File", + "sourceHandle": "start", + "target": "Parser:ViewsCaptureLight", + "targetHandle": "end" + }, + { + "id": "xy-edge__Parser:ViewsCaptureLightstart-TokenChunker:BrightColorsGlowend", + "source": "Parser:ViewsCaptureLight", + "sourceHandle": "start", + "target": "TokenChunker:BrightColorsGlow", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__TokenChunker:BrightColorsGlowstart-Tokenizer:SharpLensFocusend", + "source": "TokenChunker:BrightColorsGlow", + "sourceHandle": "start", + "target": "Tokenizer:SharpLensFocus", + "targetHandle": "end" + } + ], + "nodes": [ + { + "data": { + "label": "File", + "name": "File" + }, + "id": "File", + "measured": { + "height": 50, + "width": 200 + }, + "position": { + "x": 50, + "y": 200 + }, + "sourcePosition": "left", + "targetPosition": "right", + "type": "beginNode" + }, + { + "data": { + "form": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": [ + { + "fileFormat": "image", + "output_format": "text", + "parse_method": "ocr", + "preprocess": [ + "main_content" + ] + }, + { + "fileFormat": "video", + "output_format": "text", + "preprocess": [ + "main_content" + ] + } + ] + }, + "label": "Parser", + "name": "Parser_picture_0" + }, + "dragging": false, + "id": "Parser:ViewsCaptureLight", + "measured": { + "height": 57, + "width": 200 + }, + "position": { + "x": 316.99524094206413, + "y": 195.39629819663406 + }, + "selected": true, + "sourcePosition": "right", + "targetPosition": "left", + "type": "parserNode" + }, + { + "data": { + "form": { + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + } + }, + "label": "OneChunker", + "name": "OneChunker_picture_0" + }, + "id": "TokenChunker:BrightColorsGlow", + "measured": { + "height": 74, + "width": 200 + }, + "position": { + "x": 616.9952409420641, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "chunkerNode" + }, + { + "data": { + "form": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + }, + "label": "Tokenizer", + "name": "Indexer_picture_0" + }, + "id": "Tokenizer:SharpLensFocus", + "measured": { + "height": 114, + "width": 200 + }, + "position": { + "x": 916.9952409420641, + "y": 195.39629819663406 + }, + "sourcePosition": "right", + "targetPosition": "left", + "type": "tokenizerNode" + } + ] + }, + "history": [], + "messages": [], + "path": [], + "retrieval": [], + "variables": [] + }, + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAGP5JREFUeAHtW2lsXNd1Pve9N/vK4U6KIiWRWi1rceQlm7ckTp3WrVuoSNP+aIImqFMgRYKmAYoGcGC0QNog+dEATYKkaZ2mTqLYaVVbst16gzdFlhd5ky1TIimS4jYkZ1/ednvOuffNUF4ayUtRILrGiG/evOXes37nO9cAF8ev9xDw/2AcO/a5UKIwkk4PXdYtITYYS3d22jV7on/TnqfgPR7/pwI4ePBz8eHQ9qHsurGxaGZgfSyaXScsMWKAsQGEv8EMRbtD4ZgB0pWeD5Wpk7/82NiuG4/AezgseJfHQz/8YRQSx7cObb5mS65703ozlBoyLGO9Ycj1phXuEmBkDdNIGoYJ0vdRAT7eJVkTUkrwnSr+9YQwRDIWT+3H0++pAC7IAo4dvDUeivQmc4MDafBjg6F0ZtAUoVFc0DZhRLZF4uleIaAnEksJtTgPV+VL/OASaZG8TH4vfecVS6mOeUjpe65AAaBcPFmrlWY61+1bD+/heEsBvHzs4N6+wdHrUUHDOJthYYZHwpFYzgyH09IXccsKCcCJ8mR57qhJgcvnxfDi+C+e4kV5vit9p8lrbgkAlHxoGnQd34EC4X/xG1oB/h7+luvJGP4c9V0nJQ0jhMdxywjFBR7jjQkwhHAbjbsf+tLX/ur3Dxzw4ALGm7rAj7/z5bF1G/Y+HomEIr7vaSlpWfmkSV/6LmnVB6fZgKWFWegZ2EAmLUkgnucI3/NQiS4tCa3egnAkDlYkB77bENJ1wfcdfJaL8/cluoYIhyMsASldMgoSJf5LLxVfZLFKPV36AY/JhQC0qHzXsLp6t3V/at8dcODAc/AOBSBWFiqj9VopEgpl8KvBL9Uv49fjOnGCPni4kLnpccj1rBOmIaSwUCEQxitjPHvDMPBWuh//4rGBf6VM4DmTn8kPw8U4aBmu0+DvPrkOv87nVZNASTL0H13jubZgLdDvOA8UHL/Hl0Xo7Vt/Fd54QQI4xwV+8t2/2bv3qqu/PTl+YnPfwGBu844951wmOVaRADy5ND8tmvUqLMycgm17rxbJVFpS5MKFSu0JrEb8l86zDIXWGv0gA7sShtQH5A70CKnULVqCV0MqaQnlPzgH8i0SCrmWcNHaPNep1GvVx2278vmRrVdPwHkMY+2XWDz+O/i5Ev27s7C6AmiaSnv4n+s6UK2UYHV5CU48fxRisSh4dgMKK8s82UDDNGcjmDx+J+3oC0DNXlsEW4AZLEkosQiWGz5BCUhfq+/l83QX3cNHeLWh5WuwRTSSzerSx8KhxN333X5rzwULoFwsLTiOA+lMFmbPTIEwrUD3+FsBmo06BTSolooobQ9N1+aF8xo4yPPcwMbzjm2z0FzHbZ1nC5KtR2qXCobQwg4ODfXM1j2BKyq3oqULMwQGfkz6oPtZVgSPLbBCoe2XXXfzvz1x7/dzv0oArRiwff/f9T045Wd2FEuQSKYla7xchmjE4gnlurppQaJeKcpmvSKqpQKFQtYkal6ZvV5NrVZBAThqIbRMMm3TlHRdoD2pMwROWmQ6e/lYC0YJjLIBuQffod1FewbdTAKixIMC4Gs4WOPiw+EYB8ZUJnfdhu37bsOr/+xXCmD9B2/ZGBnbffLeyXHjuvFpec1VuwSmPLEwc1qOjG4F8jUMPlA78V3pnT0M0XFfNtfdJkwdxWg+tFIVtAEy2U6WB85IG7chVdAzOPkJ7Q6SZQB6hVIEcECqNCBblrF2sGBUtlVp0lTuh5mGMoNhmspo8LpMKnnLi0d+MX/JlTff9lYCYBcIi3zZrVcNI5qAJ54bh0g4JhLJJPr3EgcuisS1lVmwp/9RZOrPgShOg10v80KC+atoz84OdrMpK6WyrFUq6DZNdhf1qrWfYG2BF7CvK5AAGk5ACzsJZRzQSh0qIOuYwcHX1HPlK/HQIrMTAyOb//LFI/9xLfxvFjD+6IGlsY3XzoUS3f2TpTCsrq7I/v4BePH4Mdh71bWU11mbrteBVxcgYqJ/o48j4lszS0pbJlvCC88ehfzSAmbFkDRxUuSXpmWij5Kf4jnTEoaFvoqTDIXD6LP4wYBr4jmTXEWnT9SmxGfQ/XiPCfFkRqqAqJIGi80wCXSyxbB0CH96DsuK7o9GE8mhsZ0/GD9+35Wju25YfFMB0PCrxTuiPSNfOv6aCROnxmHTpg0IXqJoBXkIWwh6DII9PfiKCRSAK0q1EqQ7e3j5hno56LQHu/ddBc2mLTEtESTGwkahQtfzECD5lMMofuB/hP0djiW2XQOvhgDJIyDlspaFRs5Sm8C6kc2EOdTJlmuQAVjStZtCI2s4/fLTsHT2FOS6eyHbtR4zVnIkne39CV583VsKoFZcud9y7C85iR5x/LU52LHzEvYrcoPurg7UeBNsPyMlKsAyELg0KqxR5cRSZWcWBMF4H91SQCgaBdI0AR/SKgUsjRXYTFvpPsiEAYJW65L0HApuJGAPj9GKpI4VVEGw1jHC4u0W4w96BqXuaCIjzkyckc8//QzUa2UYGh6CXVfecO348Qf+YnTX9d94UwGYlcljZm3rfCSR6vuvF0/D732iCdlsFhbnpqGrKwd2oy7qdhJEHGQE51HPn9XzBGjVMlpXiOxko14TGverqo9qAtQ8uROKAi0KJ4s4QPCkLRKUoGApNGYI0qtyCZOzBBqLoO/sHqh1DoKUEDlLWgQMIRJNwsDQJplIZkW5tCpLmL4pmz106E7YfcWHvvrwXd96/Jrf/eKTbxDAzJMHVjYOXna4e/TST89MpCC/iJrv7YXXXjkBl+y5HFOgK+teWjiuRJ8FqFdWCRMSNmmLQEd1yiDhkNI+aV7qIKkQi1IdZwRJKmTIi27i8VNI4+QaoCAxwU5+D7uO9CS5N1/D9YSn86zklOI5hD1sFJoBnb39kMxkoKOzCqgMqFfLspifTw1vufyzONM3CoDd4NjDt4Xnn4ldMmh8cmE+Bzt37mCcjjldUJ1TtjtkDRdMZVBlleIJoT9T5T8VkLmWqyJWqJSKIqjogrxNEJ7/w0V19g3ByuIszx6DlaBAiZGbpUkBj9wFNY/R3OCgaZL7YCHIRRC+x6DAypZh6Ezh84cKMZQSArAmWkNNIKfAx4RdyB0wgX/g61+/KfWVrxwsv0EA85P3Tu7f9huf3nn53puWFs7G7a2b8QUWmlBJGqEYJIe3wZ3/eSUYzpJcMMPiCt9vpXBo1YwCUqkOTKMZTksYCyhFBVFZV3EquqUynVw9kmv4rGmuM6jY5CoZgwBahkNVpXRRu6h1PO8JjypN9H2KooRMVSxo+yG9k7yjAwFWKBIXHgoArRKi8QS+OhTv87LhN7UAGv9w+HDz21fvvCcdie8vF1ahq7sLJsdfgU1bt4MVXYIN1/0BrOSXwSkUGAoH+F9HL7YE8r1KpcjubIBQ0AX9m+BqrnsAllHzAnQAk6DSJKZMEhDHAUPXEHQX+jpyECpMGhQPFDJWVmLx+3XZpRAAuQJXjFJS6pWUUbAUD2Ml6dgximVN1yzYbykAHr78UTwe25/PL0Fffz+ceOkF2LFnnwiFoxBPpGShUMLUEtPL1lYQ4BIcHZ09oqOrN0CBXLzqyQnC7r2Dw4zoOG3hjz6nL8kcAvs7WRYbAPINDvIH5HRoDbLlQopOQR6AY4NiBXzIdvZjbCqiz5clCpQTc9+6jcoNIcpQGWuUYn/sIyiAg28tgBefOn500ydvKkyfmero7e2XhOtLq8syHI2BWatxPidNkSkq85dr0kGL9uKcHMA4RoqYqmihlD651geNamUAj7WgQEO+9o+0alUKiIBMEro4V190OQaJdIeCZb7CEwHsVqHDhKbjlG78wheawbuMNxPAd35x/2J+tfhEIh6TtXoFzTMMpdIqBhU0IcxFCJBkCM/VUBgBWFFBSJeFXBfohI3R3LGbktyFBSZkC+MLXSUyByDaHxEcq4dpUGzI4Pq2t8s1sxbtD2VF1DbOUWozDDIQCaW4dq1vyQqfeOnEz/a979Ibl5cWYXhoHZyZmoLL9l0JKawR5udm0ZoMoNIZjNfLUEJ14qCw8n8Psv9rILo/BMXVPGpEsmkTiyS5ipQy2zkoKuUV9lmFJBnMCAWdLSK+BMNiSp0U8VlleA7xQ5BiKQsxXGarUiUZ24wI6kiNGAML873V8xJAPl84hFNxC4WC1dvXD9WZWUJliO0NiEQiFJEFFjsyZApoMUCG8gUD2WCzvgy+4QCSoeAxMjQI1yOFFeW6AFWEbKALyVSWtUULiMbSwnXq7NcU5bnc5rpHakzA/AEDKiRIweP44TF8pnOcDDzmJYXUFFM0lsRyu0dbnEGcRum8BPC9O+7Of+SjHz6Eb7uJfIect4IcgBWy2PSSyaQsFovQnUtrCStAQmYaG7kZvIEb0FWSZHKyI9dJBKGi+FQgEwxwgFgtrA98R9pIrlKKLK4sBEGQ3UcqdITr8jXqJhqdgKQurQ3NHeAEKLZEEynI5Ho0AeNrh6Gffb7M9Rvl8xIAjQceeOTnN95wzU3jJ1+Fnt4e9vl4LIZVnInEQxgBTxl6u7Ig1nqBNjwzHCd70xVeiBEjk6FaWuoP+6xsFwQguweGNcbXya11vSZW2hyS0J0FRaW1gt45HJOekq9cgYBS0145bwGs2NV/R4taQWSSo3VNjJ+EXXsvYxNuYvVFxoYQWVdsejn4j7NyTIr87QK6PwNubItYWV4K0gSvTPm1SR4uAnAkFN5XBaVhaT7FYrmp2l4FswgCGhdjD5W6CjeoKNwGWVKTJqrLEPQnyBoIKteKs+dvAQcOPFz5+Ac/9EDYEvupiEE4yQYVQq2uFosyhIJooOkGFEagHnfxcRFb+Qk0rb0Qzu6GdLaLqzWKAxQhqPBB02f0hplBM12SfZrNnfzbc7Vve7wAn6+TshMp+Pz8GVVgGdRE8lno5DQk2MH1W7gGaWVmFggRph7VCXIlP31+WSAYE1PT/7Jr2+h+x3GRkEhBpVwhDE7YgAsaYnsC8kZq84+M3iKbpZtFpHME6o0alIrLKizrTJDr7oc80uqBfoyAViJ/9qUIUp/CUYr2RSir3APPrtu4HQI6XWPggEsFnZNV+lPwGr96KC9HOI0q1BYmqhckgMbS2UcyH7hi9tT4ycGenj44MzEOPT09YOYtNqk1QECoSVEjKIyLVy29eCwpY+sSCom0egUGJFIUPJkX0EWk0EoLgI1aIGF/kK+flWijLNAVtMalLZBBWYG7rS7HB99tYkWYh1gu07ggAXzjR/dXR3df/kgmnfpUPBaGxYV52LBpjL0VI7NU9bkmNDQi85uL4C3cAVbfH0LNjYn84ny7KlScnmgRPYbQNJmpuCUiNlrsH+F9QxH/WszEp5CwYomMJGAGurMg/TbfztUUQWteuMOL970G1JZehkbNdi9IADSOHX3627/1m9d/an5+idAV19e29n1KT6DxWWCLtZO3Q7L2TYwPMYhv+VPZxcHOCohKba+GUjp1jn2piU6pCiQFc3XyD2C14gX4SHED3JtkKUrRuiZop3F1idqnUthDuo0E7czeBYab8S9YAN//6eEj1157xWuOa491dvXB0uIcw4x0Ogm1apXkLSh+mdp3I8N/BNXpTogM/DYjv4ruIVAa0z4POma0aLFWbUDFUGu7gC/akVzZhGFoxAiKU6ECrat3iKs+9QwGSRxwXK4A6+A0ShCNhsBfOYqP+ShcsABo1Kru7el05jbs9MDyUl5G43ERQdK0UaeYEtQlKoBZ6X6wdnxGpyih2VwuYnkhqjpTCLJFp7faZYZcA1/lGu9iMSCuaFW+TLCwmdv8V3LWQK27DelhmnaaVRRAFTGLBacf/R50JXCuDXh7Ajh5evLwnp2bv7a6smxmshmUfCwogKBdoChz5D0Buu3lUbsc/VA5iAsBoNGKJ25YaxsUW+wr81Va90Wwb4AWS6CqZ2CEvijgQ91iRJp4LMjXqbVOWcl1ati2r+H1WLRZIWxjPAb1qXsgtAWz1qov3pYAEt0zz4PY/JhlmR9myFpYhjSVnrrNpRGZjsNaHPgqoq0yuS5VlrB8WrWDklxQMQeoUO8YaXeGQHEHql3OUd0hCt03kJ+IScvQcDlMLoAR37OFZ0uB+ofS0gTUC+Mw89SdcNkeE6l3Q+lgzTDOVwC33vqw+8r45J0UeWnJ2P3hOZu6gdru7a0tV4OPFLqYXysf0MAQ2qWsPtvuEzKMVc0/yWCGWnT5uUnMAkkszJAApY9ogCnrILwK+M1V2SjPy+LiKVnKj8PsM4dgKPYSZEaHeb5YIYu3JQAap06d/kUmk5lbWFwSqXQaGo06FXUt7BFoTx2vWajOeW8cOtoFPt26X99EmUBRQOwsxO25dl0iwwtqHxGavYfchtuU5A52rQjV4gIUFs9AfXUCVieegHj5CAzvG8SVp9T+hLdrATR+fNfDM1PT8w9GI2EZJsoJAxe2t1qpSleE7A68kPb2pyCXtSUCsp0F1Lm28DSRQr7N6c5zsUxuCKLHmo2asNG/KepTfnfx2K6XRL2ch+LyLCzPT4BdGofxI3eL+quHYdv7chBGeg5ot4sgJst5+xZA4+ixZ7/R298vVldWgUgR+qi1ybYg1qy39TYdHcTrcZ1mtgA0iyRaeI7JP47utMHKo56ALSmteUysNBjaNlHrlcICUuxTsDz3GjSXX4LpZw9DrHgUdr8vDqmxIdY+WEnVoQLvnDVf8D7Bf/rpfcfff8XeE81mfVuoFoFsOqWpO18FbSMoCYLgGKxN6J6W7p7wX8URtsC83gukXChoiLj8IfO37QYWZFWsAiPYmivLeimPGCMPtVJe1stnwWychlef/G/osebF9j0R6NpBmk/iKpEOt6JM8Qsh35kF0DgzffabWey6VCtVCNpTep+ADBhdJjxU8SM0yAsWpSMfd3iEXqzecuezufvMIvGeH3B5twkuvEHdnYooIGFiYQouLJxGen2S/kq0dahNPwrP3PMzWBdfgO2XhqBrW6cUUeQkDOQiMEz6nuIk8KEmvBMLYAHMLN1z6SWbC6uFclaxbkJbgNrGomCrynFKyx4orGBo+9doVDMaQSpUwpPMEhEN7vMmKBszDvp9oyrrlQI2axfFlks2yulXHgHTWwCoTMIvH30UIo0ZuHQjwMiWEHRsSkuIoc9bWHCZMZ4TdfhDEYRivvfOBfDPBw7NX3/NvjuwcLnFAFN5MKUpQxEPElo7QAiYKoRHmUwEtLZsU9wQoCBtRT7vL1SLt23hYoRHAcgGtuORB0A+YBBmThyCkDMFTz34CDSXTov1XY4Y3gxyZJsB8V7UNHaxwIgISdtnmGwxmK0z40nsW9rvXAA0Hnry6e/t2jzyx8IUMR23qe5mrOOrHQvqQl9tieO6XoW31m9SQUH9xdctMt5oyawPBjrpYKqtVQvY5q5ArrcbKnNHYfL4g3DymWfEQKoht2PVvWmTAb0bLbBSEWDNx7pR8x1od1m0KizbK3XhlB5HcmYIGr54dwTwg3+9+/m//vNPPrjNGPsEcjmoNYO5Uw51zNAEWB+UG6gNrKpdpmWjt8VqvEPkKLFBTUG+72MbmOKFFQ7J2kJZTJ58Tk4+e7dYmXoJelOOvHREyJ5eC5IdMWiEQzC1iGzzagozBFJmLr7Vr6Ewm0TfilAiJ20nI8bevxEiqXMowbcvAJr5p8/mD+ESPkGbJ6LhsAz2AQjaTqIMXtdyvt70pMyf94jrtjjjel3+MNdvJjHaF+X05CSceOGYnHr1OBZfcxLcssgkIjK7YZ9IdGSRpcxCI9cBkWxORtJZtPoENj8zEMIGaDiWQeuP4LOwp2jF8LkROPv8j9E16kjWOu+OAGiEo9GpaqUCS7OvQnrHBwG0v2tepCUrj3t+oBoj6N9YIhOgoe10oloqwXJ+DlYX5+TZmQnM5/NUyAKCLaLeoae/F0a3jkIm0yFzuOBMNiuJmosjoxSNpcCMJBDghdH6wpgeo2h9Jqc7oA8FXS6vPewJ59AtSiIer7x7AujqzhbKxVUswqriift/hD7ry3qDGxvEwgnunnPNqksAtf9HUViqiSIsZnepwyPEhg39cvPoEDJExAQJ3gtgqNY6979L5bIolYr8fKr8hCZE6LlK+Lx/GJftqu9ADRkfSVxXjoxhPICt2Bucc941AVTLzVpPTxfW3Q0sTAQ0fU2Q8mYI7oWC0gLwZmpJJi4j7AZBHIRgryBe7Hi0kQoX5rgt4Kg32kq9sUIauiGq4otKoYpfI7aBg5BiGn0K/igNtLYGot+Xnl+B3NTjjxVWxJF3TQBLM5XT1ZpdqFRrHdTiDodMlHZMtvkK9T9JcItLUTiq/tesn0KGunfZ4hA0ZlI1MrSSidoGo0Cj5t/U5lR9Sug2pWz1Q1u/k6UkkpEmAre7Pv4nf3tOFBTwDseXP79/d39P92fxQVEpiJjQqxbMTikmmDZv+gaf4C2ubJ4IynzCaLz7X+0ZI/QYUOLBpkn6rvt+ajOlghyW3o5DzzGDa00WIvUrWfZInxE13Kg37NlyYeXBr37z5y+fwzNcHBfHxXFxXBwXx8Xxaz3+Bwejx8HM3R61AAAAAElFTkSuQmCC", + "parser_ids": [ + "picture" + ] +} diff --git a/agent/templates/ingestion_pipeline_presentation.json b/agent/templates/ingestion_pipeline_presentation.json new file mode 100644 index 0000000000..ef6bd0ba32 --- /dev/null +++ b/agent/templates/ingestion_pipeline_presentation.json @@ -0,0 +1,296 @@ +{ + "id": 43, + "title": { + "en": "Presentation", + "de": "Präsentation", + "zh": "演示文稿" + }, + "description": { + "en": "This template segments parsed slide content. Best for presentation files such as PowerPoint slides, keynote files, and lecture materials organized slide by slide.", + "de": "Diese Vorlage segmentiert den Inhalt von Präsentationsfolien. Sie eignet sich für Präsentationsdateien wie PowerPoint-Folien, Keynote-Dateien und folienweise organisiertes Vorlesungsmaterial.", + "zh": "此模板将解析后的幻灯片内容进行切片，适用于演示文稿文件，如PowerPoint幻灯片、Keynote文件以及按幻灯片组织的讲座材料。" + }, + "canvas_type": "Ingestion Pipeline", + "canvas_category": "dataflow_canvas", + "dsl": { + "components": { + "File": { + "downstream": [ + "Parser:HipSignsRhyme" + ], + "obj": { + "component_name": "File", + "params": {} + }, + "upstream": [] + }, + "Parser:HipSignsRhyme": { + "downstream": [ + "PresentationChunker:HappyHillsGlow" + ], + "obj": { + "component_name": "Parser", + "params": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": { + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "slides": { + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pptx", + "ppt" + ] + } + } + } + }, + "upstream": [ + "File" + ] + }, + "Tokenizer:TallTreesDance": { + "downstream": [], + "obj": { + "component_name": "Tokenizer", + "params": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + } + }, + "upstream": [ + "PresentationChunker:HappyHillsGlow" + ] + }, + "PresentationChunker:HappyHillsGlow": { + "downstream": [ + "Tokenizer:TallTreesDance" + ], + "obj": { + "component_name": "PresentationChunker", + "params": { + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + } + } + }, + "upstream": [ + "Parser:HipSignsRhyme" + ] + } + }, + "globals": { + "sys.history": [] + }, + "graph": { + "edges": [ + { + "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", + "source": "File", + "sourceHandle": "start", + "target": "Parser:HipSignsRhyme", + "targetHandle": "end" + }, + { + "id": "xy-edge__Parser:HipSignsRhymestart-PresentationChunker:HappyHillsGlowend", + "source": "Parser:HipSignsRhyme", + "sourceHandle": "start", + "target": "PresentationChunker:HappyHillsGlow", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__PresentationChunker:HappyHillsGlowstart-Tokenizer:TallTreesDanceend", + "source": "PresentationChunker:HappyHillsGlow", + "sourceHandle": "start", + "target": "Tokenizer:TallTreesDance", + "targetHandle": "end" + } + ], + "nodes": [ + { + "data": { + "label": "File", + "name": "File" + }, + "id": "File", + "measured": { + "height": 50, + "width": 200 + }, + "position": { + "x": 50, + "y": 200 + }, + "sourcePosition": "left", + "targetPosition": "right", + "type": "beginNode" + }, + { + "data": { + "form": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": [ + { + "fileFormat": "pdf", + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "vlm": {} + }, + { + "fileFormat": "slides", + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ] + } + ] + }, + "label": "Parser", + "name": "Parser_0" + }, + "dragging": false, + "id": "Parser:HipSignsRhyme", + "measured": { + "height": 57, + "width": 200 + }, + "position": { + "x": 316.99524094206413, + "y": 195.39629819663406 + }, + "selected": true, + "sourcePosition": "right", + "targetPosition": "left", + "type": "parserNode" + }, + { + "data": { + "form": { + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + } + }, + "label": "PresentationChunker", + "name": "Presentation Chunker_0" + }, + "id": "PresentationChunker:HappyHillsGlow", + "measured": { + "height": 74, + "width": 200 + }, + "position": { + "x": 616.9952409420641, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "chunkerNode" + }, + { + "data": { + "form": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + }, + "label": "Tokenizer", + "name": "Indexer_0" + }, + "id": "Tokenizer:TallTreesDance", + "measured": { + "height": 114, + "width": 200 + }, + "position": { + "x": 916.9952409420641, + "y": 195.39629819663406 + }, + "sourcePosition": "right", + "targetPosition": "left", + "type": "tokenizerNode" + } + ] + }, + "history": [], + "messages": [], + "path": [], + "retrieval": [], + "variables": [] + }, + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAGP5JREFUeAHtW2lsXNd1Pve9N/vK4U6KIiWRWi1rceQlm7ckTp3WrVuoSNP+aIImqFMgRYKmAYoGcGC0QNog+dEATYKkaZ2mTqLYaVVbst16gzdFlhd5ky1TIimS4jYkZ1/ednvOuffNUF4ayUtRILrGiG/evOXes37nO9cAF8ev9xDw/2AcO/a5UKIwkk4PXdYtITYYS3d22jV7on/TnqfgPR7/pwI4ePBz8eHQ9qHsurGxaGZgfSyaXScsMWKAsQGEv8EMRbtD4ZgB0pWeD5Wpk7/82NiuG4/AezgseJfHQz/8YRQSx7cObb5mS65703ozlBoyLGO9Ycj1phXuEmBkDdNIGoYJ0vdRAT7eJVkTUkrwnSr+9YQwRDIWT+3H0++pAC7IAo4dvDUeivQmc4MDafBjg6F0ZtAUoVFc0DZhRLZF4uleIaAnEksJtTgPV+VL/OASaZG8TH4vfecVS6mOeUjpe65AAaBcPFmrlWY61+1bD+/heEsBvHzs4N6+wdHrUUHDOJthYYZHwpFYzgyH09IXccsKCcCJ8mR57qhJgcvnxfDi+C+e4kV5vit9p8lrbgkAlHxoGnQd34EC4X/xG1oB/h7+luvJGP4c9V0nJQ0jhMdxywjFBR7jjQkwhHAbjbsf+tLX/ur3Dxzw4ALGm7rAj7/z5bF1G/Y+HomEIr7vaSlpWfmkSV/6LmnVB6fZgKWFWegZ2EAmLUkgnucI3/NQiS4tCa3egnAkDlYkB77bENJ1wfcdfJaL8/cluoYIhyMsASldMgoSJf5LLxVfZLFKPV36AY/JhQC0qHzXsLp6t3V/at8dcODAc/AOBSBWFiqj9VopEgpl8KvBL9Uv49fjOnGCPni4kLnpccj1rBOmIaSwUCEQxitjPHvDMPBWuh//4rGBf6VM4DmTn8kPw8U4aBmu0+DvPrkOv87nVZNASTL0H13jubZgLdDvOA8UHL/Hl0Xo7Vt/Fd54QQI4xwV+8t2/2bv3qqu/PTl+YnPfwGBu844951wmOVaRADy5ND8tmvUqLMycgm17rxbJVFpS5MKFSu0JrEb8l86zDIXWGv0gA7sShtQH5A70CKnULVqCV0MqaQnlPzgH8i0SCrmWcNHaPNep1GvVx2278vmRrVdPwHkMY+2XWDz+O/i5Ev27s7C6AmiaSnv4n+s6UK2UYHV5CU48fxRisSh4dgMKK8s82UDDNGcjmDx+J+3oC0DNXlsEW4AZLEkosQiWGz5BCUhfq+/l83QX3cNHeLWh5WuwRTSSzerSx8KhxN333X5rzwULoFwsLTiOA+lMFmbPTIEwrUD3+FsBmo06BTSolooobQ9N1+aF8xo4yPPcwMbzjm2z0FzHbZ1nC5KtR2qXCobQwg4ODfXM1j2BKyq3oqULMwQGfkz6oPtZVgSPLbBCoe2XXXfzvz1x7/dzv0oArRiwff/f9T045Wd2FEuQSKYla7xchmjE4gnlurppQaJeKcpmvSKqpQKFQtYkal6ZvV5NrVZBAThqIbRMMm3TlHRdoD2pMwROWmQ6e/lYC0YJjLIBuQffod1FewbdTAKixIMC4Gs4WOPiw+EYB8ZUJnfdhu37bsOr/+xXCmD9B2/ZGBnbffLeyXHjuvFpec1VuwSmPLEwc1qOjG4F8jUMPlA78V3pnT0M0XFfNtfdJkwdxWg+tFIVtAEy2U6WB85IG7chVdAzOPkJ7Q6SZQB6hVIEcECqNCBblrF2sGBUtlVp0lTuh5mGMoNhmspo8LpMKnnLi0d+MX/JlTff9lYCYBcIi3zZrVcNI5qAJ54bh0g4JhLJJPr3EgcuisS1lVmwp/9RZOrPgShOg10v80KC+atoz84OdrMpK6WyrFUq6DZNdhf1qrWfYG2BF7CvK5AAGk5ACzsJZRzQSh0qIOuYwcHX1HPlK/HQIrMTAyOb//LFI/9xLfxvFjD+6IGlsY3XzoUS3f2TpTCsrq7I/v4BePH4Mdh71bWU11mbrteBVxcgYqJ/o48j4lszS0pbJlvCC88ehfzSAmbFkDRxUuSXpmWij5Kf4jnTEoaFvoqTDIXD6LP4wYBr4jmTXEWnT9SmxGfQ/XiPCfFkRqqAqJIGi80wCXSyxbB0CH96DsuK7o9GE8mhsZ0/GD9+35Wju25YfFMB0PCrxTuiPSNfOv6aCROnxmHTpg0IXqJoBXkIWwh6DII9PfiKCRSAK0q1EqQ7e3j5hno56LQHu/ddBc2mLTEtESTGwkahQtfzECD5lMMofuB/hP0djiW2XQOvhgDJIyDlspaFRs5Sm8C6kc2EOdTJlmuQAVjStZtCI2s4/fLTsHT2FOS6eyHbtR4zVnIkne39CV583VsKoFZcud9y7C85iR5x/LU52LHzEvYrcoPurg7UeBNsPyMlKsAyELg0KqxR5cRSZWcWBMF4H91SQCgaBdI0AR/SKgUsjRXYTFvpPsiEAYJW65L0HApuJGAPj9GKpI4VVEGw1jHC4u0W4w96BqXuaCIjzkyckc8//QzUa2UYGh6CXVfecO348Qf+YnTX9d94UwGYlcljZm3rfCSR6vuvF0/D732iCdlsFhbnpqGrKwd2oy7qdhJEHGQE51HPn9XzBGjVMlpXiOxko14TGverqo9qAtQ8uROKAi0KJ4s4QPCkLRKUoGApNGYI0qtyCZOzBBqLoO/sHqh1DoKUEDlLWgQMIRJNwsDQJplIZkW5tCpLmL4pmz106E7YfcWHvvrwXd96/Jrf/eKTbxDAzJMHVjYOXna4e/TST89MpCC/iJrv7YXXXjkBl+y5HFOgK+teWjiuRJ8FqFdWCRMSNmmLQEd1yiDhkNI+aV7qIKkQi1IdZwRJKmTIi27i8VNI4+QaoCAxwU5+D7uO9CS5N1/D9YSn86zklOI5hD1sFJoBnb39kMxkoKOzCqgMqFfLspifTw1vufyzONM3CoDd4NjDt4Xnn4ldMmh8cmE+Bzt37mCcjjldUJ1TtjtkDRdMZVBlleIJoT9T5T8VkLmWqyJWqJSKIqjogrxNEJ7/w0V19g3ByuIszx6DlaBAiZGbpUkBj9wFNY/R3OCgaZL7YCHIRRC+x6DAypZh6Ezh84cKMZQSArAmWkNNIKfAx4RdyB0wgX/g61+/KfWVrxwsv0EA85P3Tu7f9huf3nn53puWFs7G7a2b8QUWmlBJGqEYJIe3wZ3/eSUYzpJcMMPiCt9vpXBo1YwCUqkOTKMZTksYCyhFBVFZV3EquqUynVw9kmv4rGmuM6jY5CoZgwBahkNVpXRRu6h1PO8JjypN9H2KooRMVSxo+yG9k7yjAwFWKBIXHgoArRKi8QS+OhTv87LhN7UAGv9w+HDz21fvvCcdie8vF1ahq7sLJsdfgU1bt4MVXYIN1/0BrOSXwSkUGAoH+F9HL7YE8r1KpcjubIBQ0AX9m+BqrnsAllHzAnQAk6DSJKZMEhDHAUPXEHQX+jpyECpMGhQPFDJWVmLx+3XZpRAAuQJXjFJS6pWUUbAUD2Ml6dgximVN1yzYbykAHr78UTwe25/PL0Fffz+ceOkF2LFnnwiFoxBPpGShUMLUEtPL1lYQ4BIcHZ09oqOrN0CBXLzqyQnC7r2Dw4zoOG3hjz6nL8kcAvs7WRYbAPINDvIH5HRoDbLlQopOQR6AY4NiBXzIdvZjbCqiz5clCpQTc9+6jcoNIcpQGWuUYn/sIyiAg28tgBefOn500ydvKkyfmero7e2XhOtLq8syHI2BWatxPidNkSkq85dr0kGL9uKcHMA4RoqYqmihlD651geNamUAj7WgQEO+9o+0alUKiIBMEro4V190OQaJdIeCZb7CEwHsVqHDhKbjlG78wheawbuMNxPAd35x/2J+tfhEIh6TtXoFzTMMpdIqBhU0IcxFCJBkCM/VUBgBWFFBSJeFXBfohI3R3LGbktyFBSZkC+MLXSUyByDaHxEcq4dpUGzI4Pq2t8s1sxbtD2VF1DbOUWozDDIQCaW4dq1vyQqfeOnEz/a979Ibl5cWYXhoHZyZmoLL9l0JKawR5udm0ZoMoNIZjNfLUEJ14qCw8n8Psv9rILo/BMXVPGpEsmkTiyS5ipQy2zkoKuUV9lmFJBnMCAWdLSK+BMNiSp0U8VlleA7xQ5BiKQsxXGarUiUZ24wI6kiNGAML873V8xJAPl84hFNxC4WC1dvXD9WZWUJliO0NiEQiFJEFFjsyZApoMUCG8gUD2WCzvgy+4QCSoeAxMjQI1yOFFeW6AFWEbKALyVSWtUULiMbSwnXq7NcU5bnc5rpHakzA/AEDKiRIweP44TF8pnOcDDzmJYXUFFM0lsRyu0dbnEGcRum8BPC9O+7Of+SjHz6Eb7uJfIect4IcgBWy2PSSyaQsFovQnUtrCStAQmYaG7kZvIEb0FWSZHKyI9dJBKGi+FQgEwxwgFgtrA98R9pIrlKKLK4sBEGQ3UcqdITr8jXqJhqdgKQurQ3NHeAEKLZEEynI5Ho0AeNrh6Gffb7M9Rvl8xIAjQceeOTnN95wzU3jJ1+Fnt4e9vl4LIZVnInEQxgBTxl6u7Ig1nqBNjwzHCd70xVeiBEjk6FaWuoP+6xsFwQguweGNcbXya11vSZW2hyS0J0FRaW1gt45HJOekq9cgYBS0145bwGs2NV/R4taQWSSo3VNjJ+EXXsvYxNuYvVFxoYQWVdsejn4j7NyTIr87QK6PwNubItYWV4K0gSvTPm1SR4uAnAkFN5XBaVhaT7FYrmp2l4FswgCGhdjD5W6CjeoKNwGWVKTJqrLEPQnyBoIKteKs+dvAQcOPFz5+Ac/9EDYEvupiEE4yQYVQq2uFosyhIJooOkGFEagHnfxcRFb+Qk0rb0Qzu6GdLaLqzWKAxQhqPBB02f0hplBM12SfZrNnfzbc7Vve7wAn6+TshMp+Pz8GVVgGdRE8lno5DQk2MH1W7gGaWVmFggRph7VCXIlP31+WSAYE1PT/7Jr2+h+x3GRkEhBpVwhDE7YgAsaYnsC8kZq84+M3iKbpZtFpHME6o0alIrLKizrTJDr7oc80uqBfoyAViJ/9qUIUp/CUYr2RSir3APPrtu4HQI6XWPggEsFnZNV+lPwGr96KC9HOI0q1BYmqhckgMbS2UcyH7hi9tT4ycGenj44MzEOPT09YOYtNqk1QECoSVEjKIyLVy29eCwpY+sSCom0egUGJFIUPJkX0EWk0EoLgI1aIGF/kK+flWijLNAVtMalLZBBWYG7rS7HB99tYkWYh1gu07ggAXzjR/dXR3df/kgmnfpUPBaGxYV52LBpjL0VI7NU9bkmNDQi85uL4C3cAVbfH0LNjYn84ny7KlScnmgRPYbQNJmpuCUiNlrsH+F9QxH/WszEp5CwYomMJGAGurMg/TbfztUUQWteuMOL970G1JZehkbNdi9IADSOHX3627/1m9d/an5+idAV19e29n1KT6DxWWCLtZO3Q7L2TYwPMYhv+VPZxcHOCohKba+GUjp1jn2piU6pCiQFc3XyD2C14gX4SHED3JtkKUrRuiZop3F1idqnUthDuo0E7czeBYab8S9YAN//6eEj1157xWuOa491dvXB0uIcw4x0Ogm1apXkLSh+mdp3I8N/BNXpTogM/DYjv4ruIVAa0z4POma0aLFWbUDFUGu7gC/akVzZhGFoxAiKU6ECrat3iKs+9QwGSRxwXK4A6+A0ShCNhsBfOYqP+ShcsABo1Kru7el05jbs9MDyUl5G43ERQdK0UaeYEtQlKoBZ6X6wdnxGpyih2VwuYnkhqjpTCLJFp7faZYZcA1/lGu9iMSCuaFW+TLCwmdv8V3LWQK27DelhmnaaVRRAFTGLBacf/R50JXCuDXh7Ajh5evLwnp2bv7a6smxmshmUfCwogKBdoChz5D0Buu3lUbsc/VA5iAsBoNGKJ25YaxsUW+wr81Va90Wwb4AWS6CqZ2CEvijgQ91iRJp4LMjXqbVOWcl1ati2r+H1WLRZIWxjPAb1qXsgtAWz1qov3pYAEt0zz4PY/JhlmR9myFpYhjSVnrrNpRGZjsNaHPgqoq0yuS5VlrB8WrWDklxQMQeoUO8YaXeGQHEHql3OUd0hCt03kJ+IScvQcDlMLoAR37OFZ0uB+ofS0gTUC+Mw89SdcNkeE6l3Q+lgzTDOVwC33vqw+8r45J0UeWnJ2P3hOZu6gdru7a0tV4OPFLqYXysf0MAQ2qWsPtvuEzKMVc0/yWCGWnT5uUnMAkkszJAApY9ogCnrILwK+M1V2SjPy+LiKVnKj8PsM4dgKPYSZEaHeb5YIYu3JQAap06d/kUmk5lbWFwSqXQaGo06FXUt7BFoTx2vWajOeW8cOtoFPt26X99EmUBRQOwsxO25dl0iwwtqHxGavYfchtuU5A52rQjV4gIUFs9AfXUCVieegHj5CAzvG8SVp9T+hLdrATR+fNfDM1PT8w9GI2EZJsoJAxe2t1qpSleE7A68kPb2pyCXtSUCsp0F1Lm28DSRQr7N6c5zsUxuCKLHmo2asNG/KepTfnfx2K6XRL2ch+LyLCzPT4BdGofxI3eL+quHYdv7chBGeg5ot4sgJst5+xZA4+ixZ7/R298vVldWgUgR+qi1ybYg1qy39TYdHcTrcZ1mtgA0iyRaeI7JP47utMHKo56ALSmteUysNBjaNlHrlcICUuxTsDz3GjSXX4LpZw9DrHgUdr8vDqmxIdY+WEnVoQLvnDVf8D7Bf/rpfcfff8XeE81mfVuoFoFsOqWpO18FbSMoCYLgGKxN6J6W7p7wX8URtsC83gukXChoiLj8IfO37QYWZFWsAiPYmivLeimPGCMPtVJe1stnwWychlef/G/osebF9j0R6NpBmk/iKpEOt6JM8Qsh35kF0DgzffabWey6VCtVCNpTep+ADBhdJjxU8SM0yAsWpSMfd3iEXqzecuezufvMIvGeH3B5twkuvEHdnYooIGFiYQouLJxGen2S/kq0dahNPwrP3PMzWBdfgO2XhqBrW6cUUeQkDOQiMEz6nuIk8KEmvBMLYAHMLN1z6SWbC6uFclaxbkJbgNrGomCrynFKyx4orGBo+9doVDMaQSpUwpPMEhEN7vMmKBszDvp9oyrrlQI2axfFlks2yulXHgHTWwCoTMIvH30UIo0ZuHQjwMiWEHRsSkuIoc9bWHCZMZ4TdfhDEYRivvfOBfDPBw7NX3/NvjuwcLnFAFN5MKUpQxEPElo7QAiYKoRHmUwEtLZsU9wQoCBtRT7vL1SLt23hYoRHAcgGtuORB0A+YBBmThyCkDMFTz34CDSXTov1XY4Y3gxyZJsB8V7UNHaxwIgISdtnmGwxmK0z40nsW9rvXAA0Hnry6e/t2jzyx8IUMR23qe5mrOOrHQvqQl9tieO6XoW31m9SQUH9xdctMt5oyawPBjrpYKqtVQvY5q5ArrcbKnNHYfL4g3DymWfEQKoht2PVvWmTAb0bLbBSEWDNx7pR8x1od1m0KizbK3XhlB5HcmYIGr54dwTwg3+9+/m//vNPPrjNGPsEcjmoNYO5Uw51zNAEWB+UG6gNrKpdpmWjt8VqvEPkKLFBTUG+72MbmOKFFQ7J2kJZTJ58Tk4+e7dYmXoJelOOvHREyJ5eC5IdMWiEQzC1iGzzagozBFJmLr7Vr6Ewm0TfilAiJ20nI8bevxEiqXMowbcvAJr5p8/mD+ESPkGbJ6LhsAz2AQjaTqIMXtdyvt70pMyf94jrtjjjel3+MNdvJjHaF+X05CSceOGYnHr1OBZfcxLcssgkIjK7YZ9IdGSRpcxCI9cBkWxORtJZtPoENj8zEMIGaDiWQeuP4LOwp2jF8LkROPv8j9E16kjWOu+OAGiEo9GpaqUCS7OvQnrHBwG0v2tepCUrj3t+oBoj6N9YIhOgoe10oloqwXJ+DlYX5+TZmQnM5/NUyAKCLaLeoae/F0a3jkIm0yFzuOBMNiuJmosjoxSNpcCMJBDghdH6wpgeo2h9Jqc7oA8FXS6vPewJ59AtSiIer7x7AujqzhbKxVUswqriift/hD7ry3qDGxvEwgnunnPNqksAtf9HUViqiSIsZnepwyPEhg39cvPoEDJExAQJ3gtgqNY6979L5bIolYr8fKr8hCZE6LlK+Lx/GJftqu9ADRkfSVxXjoxhPICt2Bucc941AVTLzVpPTxfW3Q0sTAQ0fU2Q8mYI7oWC0gLwZmpJJi4j7AZBHIRgryBe7Hi0kQoX5rgt4Kg32kq9sUIauiGq4otKoYpfI7aBg5BiGn0K/igNtLYGot+Xnl+B3NTjjxVWxJF3TQBLM5XT1ZpdqFRrHdTiDodMlHZMtvkK9T9JcItLUTiq/tesn0KGunfZ4hA0ZlI1MrSSidoGo0Cj5t/U5lR9Sug2pWz1Q1u/k6UkkpEmAre7Pv4nf3tOFBTwDseXP79/d39P92fxQVEpiJjQqxbMTikmmDZv+gaf4C2ubJ4IynzCaLz7X+0ZI/QYUOLBpkn6rvt+ajOlghyW3o5DzzGDa00WIvUrWfZInxE13Kg37NlyYeXBr37z5y+fwzNcHBfHxXFxXBwXx8Xxaz3+Bwejx8HM3R61AAAAAElFTkSuQmCC", + "parser_ids": [ + "presentation" + ] +} diff --git a/agent/templates/ingestion_pipeline_qa.json b/agent/templates/ingestion_pipeline_qa.json new file mode 100644 index 0000000000..5fa2a9e8a2 --- /dev/null +++ b/agent/templates/ingestion_pipeline_qa.json @@ -0,0 +1,372 @@ +{ + "id": 48, + "title": { + "en": "Q&A", + "de": "Frage & Antwort", + "zh": "问答" + }, + "description": { + "en": "This template segments parsed files by Q&A pairs. Best for documents structured as questions and answers, such as FAQ pages, interview transcripts, and knowledge bases organized in a question-answer format.", + "de": "Diese Vorlage segmentiert die geparste Datei anhand von Frage-Antwort-Paaren. Sie eignet sich für Dokumente die als Fragen und Antworten strukturiert sind, wie FAQ-Seiten, Interview-Transkripte und in einem Frage-Antwort-Format organisierte Wissensdatenbanken.", + "zh": "此模板将解析后的文件按问答对进行切片，适用于以问答形式组织的文档类型，如常见问题页面、采访记录以及以问答格式组织的知识库。" + }, + "canvas_type": "Ingestion Pipeline", + "canvas_category": "dataflow_canvas", + "dsl": { + "components": { + "File": { + "downstream": [ + "Parser:HipSignsRhyme" + ], + "obj": { + "component_name": "File", + "params": {} + }, + "upstream": [] + }, + "Parser:HipSignsRhyme": { + "downstream": [ + "QAChunker:TidyCloudsThink" + ], + "obj": { + "component_name": "Parser", + "params": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": { + "docx": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "docx" + ], + "vlm": {} + }, + "markdown": { + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "md", + "markdown", + "mdx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "spreadsheet": { + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "xls", + "xlsx", + "csv" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "txt", + "py", + "js", + "java", + "c", + "cpp", + "h", + "php", + "go", + "ts", + "sh", + "cs", + "kt", + "sql" + ] + } + } + } + }, + "upstream": [ + "File" + ] + }, + "Tokenizer:ColdCloudsDream": { + "downstream": [], + "obj": { + "component_name": "Tokenizer", + "params": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + } + }, + "upstream": [ + "QAChunker:TidyCloudsThink" + ] + }, + "QAChunker:TidyCloudsThink": { + "downstream": [ + "Tokenizer:ColdCloudsDream" + ], + "obj": { + "component_name": "QAChunker", + "params": { + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + } + } + }, + "upstream": [ + "Parser:HipSignsRhyme" + ] + } + }, + "globals": { + "sys.history": [] + }, + "graph": { + "edges": [ + { + "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", + "source": "File", + "sourceHandle": "start", + "target": "Parser:HipSignsRhyme", + "targetHandle": "end" + }, + { + "id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:BraveBirdsSingend", + "source": "Parser:HipSignsRhyme", + "sourceHandle": "start", + "target": "QAChunker:TidyCloudsThink", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__TokenChunker:BraveBirdsSingstart-Tokenizer:ColdCloudsDreamend", + "source": "QAChunker:TidyCloudsThink", + "sourceHandle": "start", + "target": "Tokenizer:ColdCloudsDream", + "targetHandle": "end" + } + ], + "nodes": [ + { + "data": { + "label": "File", + "name": "File" + }, + "id": "File", + "measured": { + "height": 50, + "width": 200 + }, + "position": { + "x": 50, + "y": 200 + }, + "sourcePosition": "left", + "targetPosition": "right", + "type": "beginNode" + }, + { + "data": { + "form": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": [ + { + "fileFormat": "pdf", + "flatten_media_to_text": false, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "vlm": {} + }, + { + "fileFormat": "spreadsheet", + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "vlm": {} + }, + { + "fileFormat": "markdown", + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "vlm": {} + }, + { + "fileFormat": "text&code", + "output_format": "json", + "preprocess": [ + "main_content" + ] + }, + { + "fileFormat": "docx", + "flatten_media_to_text": false, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "vlm": {} + } + ] + }, + "label": "Parser", + "name": "Parser_0" + }, + "dragging": false, + "id": "Parser:HipSignsRhyme", + "measured": { + "height": 57, + "width": 200 + }, + "position": { + "x": 316.99524094206413, + "y": 195.39629819663406 + }, + "selected": true, + "sourcePosition": "right", + "targetPosition": "left", + "type": "parserNode" + }, + { + "data": { + "form": { + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + } + }, + "label": "QAChunker", + "name": "QA Chunker_0" + }, + "id": "QAChunker:TidyCloudsThink", + "measured": { + "height": 74, + "width": 200 + }, + "position": { + "x": 616.9952409420641, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "chunkerNode" + }, + { + "data": { + "form": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + }, + "label": "Tokenizer", + "name": "Indexer_0" + }, + "id": "Tokenizer:ColdCloudsDream", + "measured": { + "height": 114, + "width": 200 + }, + "position": { + "x": 916.9952409420641, + "y": 195.39629819663406 + }, + "sourcePosition": "right", + "targetPosition": "left", + "type": "tokenizerNode" + } + ] + }, + "history": [], + "messages": [], + "path": [], + "retrieval": [], + "variables": [] + }, + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAGP5JREFUeAHtW2lsXNd1Pve9N/vK4U6KIiWRWi1rceQlm7ckTp3WrVuoSNP+aIImqFMgRYKmAYoGcGC0QNog+dEATYKkaZ2mTqLYaVVbst16gzdFlhd5ky1TIimS4jYkZ1/ednvOuffNUF4ayUtRILrGiG/evOXes37nO9cAF8ev9xDw/2AcO/a5UKIwkk4PXdYtITYYS3d22jV7on/TnqfgPR7/pwI4ePBz8eHQ9qHsurGxaGZgfSyaXScsMWKAsQGEv8EMRbtD4ZgB0pWeD5Wpk7/82NiuG4/AezgseJfHQz/8YRQSx7cObb5mS65703ozlBoyLGO9Ycj1phXuEmBkDdNIGoYJ0vdRAT7eJVkTUkrwnSr+9YQwRDIWT+3H0++pAC7IAo4dvDUeivQmc4MDafBjg6F0ZtAUoVFc0DZhRLZF4uleIaAnEksJtTgPV+VL/OASaZG8TH4vfecVS6mOeUjpe65AAaBcPFmrlWY61+1bD+/heEsBvHzs4N6+wdHrUUHDOJthYYZHwpFYzgyH09IXccsKCcCJ8mR57qhJgcvnxfDi+C+e4kV5vit9p8lrbgkAlHxoGnQd34EC4X/xG1oB/h7+luvJGP4c9V0nJQ0jhMdxywjFBR7jjQkwhHAbjbsf+tLX/ur3Dxzw4ALGm7rAj7/z5bF1G/Y+HomEIr7vaSlpWfmkSV/6LmnVB6fZgKWFWegZ2EAmLUkgnucI3/NQiS4tCa3egnAkDlYkB77bENJ1wfcdfJaL8/cluoYIhyMsASldMgoSJf5LLxVfZLFKPV36AY/JhQC0qHzXsLp6t3V/at8dcODAc/AOBSBWFiqj9VopEgpl8KvBL9Uv49fjOnGCPni4kLnpccj1rBOmIaSwUCEQxitjPHvDMPBWuh//4rGBf6VM4DmTn8kPw8U4aBmu0+DvPrkOv87nVZNASTL0H13jubZgLdDvOA8UHL/Hl0Xo7Vt/Fd54QQI4xwV+8t2/2bv3qqu/PTl+YnPfwGBu844951wmOVaRADy5ND8tmvUqLMycgm17rxbJVFpS5MKFSu0JrEb8l86zDIXWGv0gA7sShtQH5A70CKnULVqCV0MqaQnlPzgH8i0SCrmWcNHaPNep1GvVx2278vmRrVdPwHkMY+2XWDz+O/i5Ev27s7C6AmiaSnv4n+s6UK2UYHV5CU48fxRisSh4dgMKK8s82UDDNGcjmDx+J+3oC0DNXlsEW4AZLEkosQiWGz5BCUhfq+/l83QX3cNHeLWh5WuwRTSSzerSx8KhxN333X5rzwULoFwsLTiOA+lMFmbPTIEwrUD3+FsBmo06BTSolooobQ9N1+aF8xo4yPPcwMbzjm2z0FzHbZ1nC5KtR2qXCobQwg4ODfXM1j2BKyq3oqULMwQGfkz6oPtZVgSPLbBCoe2XXXfzvz1x7/dzv0oArRiwff/f9T045Wd2FEuQSKYla7xchmjE4gnlurppQaJeKcpmvSKqpQKFQtYkal6ZvV5NrVZBAThqIbRMMm3TlHRdoD2pMwROWmQ6e/lYC0YJjLIBuQffod1FewbdTAKixIMC4Gs4WOPiw+EYB8ZUJnfdhu37bsOr/+xXCmD9B2/ZGBnbffLeyXHjuvFpec1VuwSmPLEwc1qOjG4F8jUMPlA78V3pnT0M0XFfNtfdJkwdxWg+tFIVtAEy2U6WB85IG7chVdAzOPkJ7Q6SZQB6hVIEcECqNCBblrF2sGBUtlVp0lTuh5mGMoNhmspo8LpMKnnLi0d+MX/JlTff9lYCYBcIi3zZrVcNI5qAJ54bh0g4JhLJJPr3EgcuisS1lVmwp/9RZOrPgShOg10v80KC+atoz84OdrMpK6WyrFUq6DZNdhf1qrWfYG2BF7CvK5AAGk5ACzsJZRzQSh0qIOuYwcHX1HPlK/HQIrMTAyOb//LFI/9xLfxvFjD+6IGlsY3XzoUS3f2TpTCsrq7I/v4BePH4Mdh71bWU11mbrteBVxcgYqJ/o48j4lszS0pbJlvCC88ehfzSAmbFkDRxUuSXpmWij5Kf4jnTEoaFvoqTDIXD6LP4wYBr4jmTXEWnT9SmxGfQ/XiPCfFkRqqAqJIGi80wCXSyxbB0CH96DsuK7o9GE8mhsZ0/GD9+35Wju25YfFMB0PCrxTuiPSNfOv6aCROnxmHTpg0IXqJoBXkIWwh6DII9PfiKCRSAK0q1EqQ7e3j5hno56LQHu/ddBc2mLTEtESTGwkahQtfzECD5lMMofuB/hP0djiW2XQOvhgDJIyDlspaFRs5Sm8C6kc2EOdTJlmuQAVjStZtCI2s4/fLTsHT2FOS6eyHbtR4zVnIkne39CV583VsKoFZcud9y7C85iR5x/LU52LHzEvYrcoPurg7UeBNsPyMlKsAyELg0KqxR5cRSZWcWBMF4H91SQCgaBdI0AR/SKgUsjRXYTFvpPsiEAYJW65L0HApuJGAPj9GKpI4VVEGw1jHC4u0W4w96BqXuaCIjzkyckc8//QzUa2UYGh6CXVfecO348Qf+YnTX9d94UwGYlcljZm3rfCSR6vuvF0/D732iCdlsFhbnpqGrKwd2oy7qdhJEHGQE51HPn9XzBGjVMlpXiOxko14TGverqo9qAtQ8uROKAi0KJ4s4QPCkLRKUoGApNGYI0qtyCZOzBBqLoO/sHqh1DoKUEDlLWgQMIRJNwsDQJplIZkW5tCpLmL4pmz106E7YfcWHvvrwXd96/Jrf/eKTbxDAzJMHVjYOXna4e/TST89MpCC/iJrv7YXXXjkBl+y5HFOgK+teWjiuRJ8FqFdWCRMSNmmLQEd1yiDhkNI+aV7qIKkQi1IdZwRJKmTIi27i8VNI4+QaoCAxwU5+D7uO9CS5N1/D9YSn86zklOI5hD1sFJoBnb39kMxkoKOzCqgMqFfLspifTw1vufyzONM3CoDd4NjDt4Xnn4ldMmh8cmE+Bzt37mCcjjldUJ1TtjtkDRdMZVBlleIJoT9T5T8VkLmWqyJWqJSKIqjogrxNEJ7/w0V19g3ByuIszx6DlaBAiZGbpUkBj9wFNY/R3OCgaZL7YCHIRRC+x6DAypZh6Ezh84cKMZQSArAmWkNNIKfAx4RdyB0wgX/g61+/KfWVrxwsv0EA85P3Tu7f9huf3nn53puWFs7G7a2b8QUWmlBJGqEYJIe3wZ3/eSUYzpJcMMPiCt9vpXBo1YwCUqkOTKMZTksYCyhFBVFZV3EquqUynVw9kmv4rGmuM6jY5CoZgwBahkNVpXRRu6h1PO8JjypN9H2KooRMVSxo+yG9k7yjAwFWKBIXHgoArRKi8QS+OhTv87LhN7UAGv9w+HDz21fvvCcdie8vF1ahq7sLJsdfgU1bt4MVXYIN1/0BrOSXwSkUGAoH+F9HL7YE8r1KpcjubIBQ0AX9m+BqrnsAllHzAnQAk6DSJKZMEhDHAUPXEHQX+jpyECpMGhQPFDJWVmLx+3XZpRAAuQJXjFJS6pWUUbAUD2Ml6dgximVN1yzYbykAHr78UTwe25/PL0Fffz+ceOkF2LFnnwiFoxBPpGShUMLUEtPL1lYQ4BIcHZ09oqOrN0CBXLzqyQnC7r2Dw4zoOG3hjz6nL8kcAvs7WRYbAPINDvIH5HRoDbLlQopOQR6AY4NiBXzIdvZjbCqiz5clCpQTc9+6jcoNIcpQGWuUYn/sIyiAg28tgBefOn500ydvKkyfmero7e2XhOtLq8syHI2BWatxPidNkSkq85dr0kGL9uKcHMA4RoqYqmihlD651geNamUAj7WgQEO+9o+0alUKiIBMEro4V190OQaJdIeCZb7CEwHsVqHDhKbjlG78wheawbuMNxPAd35x/2J+tfhEIh6TtXoFzTMMpdIqBhU0IcxFCJBkCM/VUBgBWFFBSJeFXBfohI3R3LGbktyFBSZkC+MLXSUyByDaHxEcq4dpUGzI4Pq2t8s1sxbtD2VF1DbOUWozDDIQCaW4dq1vyQqfeOnEz/a979Ibl5cWYXhoHZyZmoLL9l0JKawR5udm0ZoMoNIZjNfLUEJ14qCw8n8Psv9rILo/BMXVPGpEsmkTiyS5ipQy2zkoKuUV9lmFJBnMCAWdLSK+BMNiSp0U8VlleA7xQ5BiKQsxXGarUiUZ24wI6kiNGAML873V8xJAPl84hFNxC4WC1dvXD9WZWUJliO0NiEQiFJEFFjsyZApoMUCG8gUD2WCzvgy+4QCSoeAxMjQI1yOFFeW6AFWEbKALyVSWtUULiMbSwnXq7NcU5bnc5rpHakzA/AEDKiRIweP44TF8pnOcDDzmJYXUFFM0lsRyu0dbnEGcRum8BPC9O+7Of+SjHz6Eb7uJfIect4IcgBWy2PSSyaQsFovQnUtrCStAQmYaG7kZvIEb0FWSZHKyI9dJBKGi+FQgEwxwgFgtrA98R9pIrlKKLK4sBEGQ3UcqdITr8jXqJhqdgKQurQ3NHeAEKLZEEynI5Ho0AeNrh6Gffb7M9Rvl8xIAjQceeOTnN95wzU3jJ1+Fnt4e9vl4LIZVnInEQxgBTxl6u7Ig1nqBNjwzHCd70xVeiBEjk6FaWuoP+6xsFwQguweGNcbXya11vSZW2hyS0J0FRaW1gt45HJOekq9cgYBS0145bwGs2NV/R4taQWSSo3VNjJ+EXXsvYxNuYvVFxoYQWVdsejn4j7NyTIr87QK6PwNubItYWV4K0gSvTPm1SR4uAnAkFN5XBaVhaT7FYrmp2l4FswgCGhdjD5W6CjeoKNwGWVKTJqrLEPQnyBoIKteKs+dvAQcOPFz5+Ac/9EDYEvupiEE4yQYVQq2uFosyhIJooOkGFEagHnfxcRFb+Qk0rb0Qzu6GdLaLqzWKAxQhqPBB02f0hplBM12SfZrNnfzbc7Vve7wAn6+TshMp+Pz8GVVgGdRE8lno5DQk2MH1W7gGaWVmFggRph7VCXIlP31+WSAYE1PT/7Jr2+h+x3GRkEhBpVwhDE7YgAsaYnsC8kZq84+M3iKbpZtFpHME6o0alIrLKizrTJDr7oc80uqBfoyAViJ/9qUIUp/CUYr2RSir3APPrtu4HQI6XWPggEsFnZNV+lPwGr96KC9HOI0q1BYmqhckgMbS2UcyH7hi9tT4ycGenj44MzEOPT09YOYtNqk1QECoSVEjKIyLVy29eCwpY+sSCom0egUGJFIUPJkX0EWk0EoLgI1aIGF/kK+flWijLNAVtMalLZBBWYG7rS7HB99tYkWYh1gu07ggAXzjR/dXR3df/kgmnfpUPBaGxYV52LBpjL0VI7NU9bkmNDQi85uL4C3cAVbfH0LNjYn84ny7KlScnmgRPYbQNJmpuCUiNlrsH+F9QxH/WszEp5CwYomMJGAGurMg/TbfztUUQWteuMOL970G1JZehkbNdi9IADSOHX3627/1m9d/an5+idAV19e29n1KT6DxWWCLtZO3Q7L2TYwPMYhv+VPZxcHOCohKba+GUjp1jn2piU6pCiQFc3XyD2C14gX4SHED3JtkKUrRuiZop3F1idqnUthDuo0E7czeBYab8S9YAN//6eEj1157xWuOa491dvXB0uIcw4x0Ogm1apXkLSh+mdp3I8N/BNXpTogM/DYjv4ruIVAa0z4POma0aLFWbUDFUGu7gC/akVzZhGFoxAiKU6ECrat3iKs+9QwGSRxwXK4A6+A0ShCNhsBfOYqP+ShcsABo1Kru7el05jbs9MDyUl5G43ERQdK0UaeYEtQlKoBZ6X6wdnxGpyih2VwuYnkhqjpTCLJFp7faZYZcA1/lGu9iMSCuaFW+TLCwmdv8V3LWQK27DelhmnaaVRRAFTGLBacf/R50JXCuDXh7Ajh5evLwnp2bv7a6smxmshmUfCwogKBdoChz5D0Buu3lUbsc/VA5iAsBoNGKJ25YaxsUW+wr81Va90Wwb4AWS6CqZ2CEvijgQ91iRJp4LMjXqbVOWcl1ati2r+H1WLRZIWxjPAb1qXsgtAWz1qov3pYAEt0zz4PY/JhlmR9myFpYhjSVnrrNpRGZjsNaHPgqoq0yuS5VlrB8WrWDklxQMQeoUO8YaXeGQHEHql3OUd0hCt03kJ+IScvQcDlMLoAR37OFZ0uB+ofS0gTUC+Mw89SdcNkeE6l3Q+lgzTDOVwC33vqw+8r45J0UeWnJ2P3hOZu6gdru7a0tV4OPFLqYXysf0MAQ2qWsPtvuEzKMVc0/yWCGWnT5uUnMAkkszJAApY9ogCnrILwK+M1V2SjPy+LiKVnKj8PsM4dgKPYSZEaHeb5YIYu3JQAap06d/kUmk5lbWFwSqXQaGo06FXUt7BFoTx2vWajOeW8cOtoFPt26X99EmUBRQOwsxO25dl0iwwtqHxGavYfchtuU5A52rQjV4gIUFs9AfXUCVieegHj5CAzvG8SVp9T+hLdrATR+fNfDM1PT8w9GI2EZJsoJAxe2t1qpSleE7A68kPb2pyCXtSUCsp0F1Lm28DSRQr7N6c5zsUxuCKLHmo2asNG/KepTfnfx2K6XRL2ch+LyLCzPT4BdGofxI3eL+quHYdv7chBGeg5ot4sgJst5+xZA4+ixZ7/R298vVldWgUgR+qi1ybYg1qy39TYdHcTrcZ1mtgA0iyRaeI7JP47utMHKo56ALSmteUysNBjaNlHrlcICUuxTsDz3GjSXX4LpZw9DrHgUdr8vDqmxIdY+WEnVoQLvnDVf8D7Bf/rpfcfff8XeE81mfVuoFoFsOqWpO18FbSMoCYLgGKxN6J6W7p7wX8URtsC83gukXChoiLj8IfO37QYWZFWsAiPYmivLeimPGCMPtVJe1stnwWychlef/G/osebF9j0R6NpBmk/iKpEOt6JM8Qsh35kF0DgzffabWey6VCtVCNpTep+ADBhdJjxU8SM0yAsWpSMfd3iEXqzecuezufvMIvGeH3B5twkuvEHdnYooIGFiYQouLJxGen2S/kq0dahNPwrP3PMzWBdfgO2XhqBrW6cUUeQkDOQiMEz6nuIk8KEmvBMLYAHMLN1z6SWbC6uFclaxbkJbgNrGomCrynFKyx4orGBo+9doVDMaQSpUwpPMEhEN7vMmKBszDvp9oyrrlQI2axfFlks2yulXHgHTWwCoTMIvH30UIo0ZuHQjwMiWEHRsSkuIoc9bWHCZMZ4TdfhDEYRivvfOBfDPBw7NX3/NvjuwcLnFAFN5MKUpQxEPElo7QAiYKoRHmUwEtLZsU9wQoCBtRT7vL1SLt23hYoRHAcgGtuORB0A+YBBmThyCkDMFTz34CDSXTov1XY4Y3gxyZJsB8V7UNHaxwIgISdtnmGwxmK0z40nsW9rvXAA0Hnry6e/t2jzyx8IUMR23qe5mrOOrHQvqQl9tieO6XoW31m9SQUH9xdctMt5oyawPBjrpYKqtVQvY5q5ArrcbKnNHYfL4g3DymWfEQKoht2PVvWmTAb0bLbBSEWDNx7pR8x1od1m0KizbK3XhlB5HcmYIGr54dwTwg3+9+/m//vNPPrjNGPsEcjmoNYO5Uw51zNAEWB+UG6gNrKpdpmWjt8VqvEPkKLFBTUG+72MbmOKFFQ7J2kJZTJ58Tk4+e7dYmXoJelOOvHREyJ5eC5IdMWiEQzC1iGzzagozBFJmLr7Vr6Ewm0TfilAiJ20nI8bevxEiqXMowbcvAJr5p8/mD+ESPkGbJ6LhsAz2AQjaTqIMXtdyvt70pMyf94jrtjjjel3+MNdvJjHaF+X05CSceOGYnHr1OBZfcxLcssgkIjK7YZ9IdGSRpcxCI9cBkWxORtJZtPoENj8zEMIGaDiWQeuP4LOwp2jF8LkROPv8j9E16kjWOu+OAGiEo9GpaqUCS7OvQnrHBwG0v2tepCUrj3t+oBoj6N9YIhOgoe10oloqwXJ+DlYX5+TZmQnM5/NUyAKCLaLeoae/F0a3jkIm0yFzuOBMNiuJmosjoxSNpcCMJBDghdH6wpgeo2h9Jqc7oA8FXS6vPewJ59AtSiIer7x7AujqzhbKxVUswqriift/hD7ry3qDGxvEwgnunnPNqksAtf9HUViqiSIsZnepwyPEhg39cvPoEDJExAQJ3gtgqNY6979L5bIolYr8fKr8hCZE6LlK+Lx/GJftqu9ADRkfSVxXjoxhPICt2Bucc941AVTLzVpPTxfW3Q0sTAQ0fU2Q8mYI7oWC0gLwZmpJJi4j7AZBHIRgryBe7Hi0kQoX5rgt4Kg32kq9sUIauiGq4otKoYpfI7aBg5BiGn0K/igNtLYGot+Xnl+B3NTjjxVWxJF3TQBLM5XT1ZpdqFRrHdTiDodMlHZMtvkK9T9JcItLUTiq/tesn0KGunfZ4hA0ZlI1MrSSidoGo0Cj5t/U5lR9Sug2pWz1Q1u/k6UkkpEmAre7Pv4nf3tOFBTwDseXP79/d39P92fxQVEpiJjQqxbMTikmmDZv+gaf4C2ubJ4IynzCaLz7X+0ZI/QYUOLBpkn6rvt+ajOlghyW3o5DzzGDa00WIvUrWfZInxE13Kg37NlyYeXBr37z5y+fwzNcHBfHxXFxXBwXx8Xxaz3+Bwejx8HM3R61AAAAAElFTkSuQmCC", + "parser_ids": [ + "qa" + ] +} diff --git a/agent/templates/ingestion_pipeline_resume.json b/agent/templates/ingestion_pipeline_resume.json index cb35eb2043..874e643644 100644 --- a/agent/templates/ingestion_pipeline_resume.json +++ b/agent/templates/ingestion_pipeline_resume.json @@ -12,64 +12,253 @@ }, "canvas_type": "Ingestion Pipeline", "canvas_category": "dataflow_canvas", - "dsl": { - "components": { - "Extractor:ThreeDrinksAct": { - "downstream": [ - "Tokenizer:KindHandsWin" - ], - "obj": { - "component_name": "Extractor", - "params": { - "field_name": "metadata", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } + "dsl": { + "components": { + "Extractor:ThreeDrinksAct": { + "downstream": [ + "Tokenizer:KindHandsWin" + ], + "obj": { + "component_name": "Extractor", + "params": { + "field_name": "metadata", + "frequencyPenaltyEnabled": true, + "frequency_penalty": 0.7, + "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", + "maxTokensEnabled": false, + "max_tokens": 256, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "presencePenaltyEnabled": true, + "presence_penalty": 0.4, + "prompts": [ + { + "content": "Content: {TitleChunker:FlatMiceFix@chunks}", + "role": "user" + } + ], + "sys_prompt": "Act as a precise resume metadata extractor. Extract stable, chunk-supported metadata from the provided resume content.\n\nRules:\n1. Use only information explicitly stated in the content. Do not infer, guess, normalize, or add missing facts.\n2. The input may be only one chunk of a resume. Extract only what this content directly supports.\n3. Use only these field names:\ncandidate_name, gender, phone, email, city, location, nationality, linkedin, github, website, highest_degree, degree_levels, school_names, majors, graduation_years, work_experience_years, current_job_title, job_titles, company_names, job_experience, industries, target_job_titles, target_locations, employment_types, skills, certificates, awards, summary_tags\n4. Ignore detailed responsibilities, project descriptions, achievement narratives, self-evaluation, and other low-value local details.\n5. Keep values in the same language as the source text whenever possible.\n6. Remove duplicates and keep only concise, high-value metadata.\n7. Return only fields that are explicitly supported by the content. Do not return empty or unsupported fields.\n\nField guidance:\n- highest_degree: highest explicit degree level mentioned\n- degree_levels: all explicit degree levels mentioned\n- school_names: explicit school, college, or university names\n- majors: explicit fields of study\n- graduation_years: explicit graduation years only\n- work_experience_years: only if explicitly stated\n- current_job_title: only if explicitly current or most recent\n- job_titles: explicit role titles\n- company_names: explicit employer names\n- job_experience: concise structured work entries explicitly supported by the content, preferably including title, company, and time information when available\n- industries: explicit industry names only\n- target_job_titles: explicit desired roles only\n- target_locations: explicit desired work locations only\n- skills: concise, core, search-useful skills explicitly mentioned\n- certificates: explicit certificate names only\n- awards: explicit award names only\n- summary_tags: short, high-value tags strictly supported by the content\n\nReturn only the extracted metadata. Do not output explanatory text.", + "temperature": 0.1, + "temperatureEnabled": true, + "tenant_llm_id": 29, + "topPEnabled": true, + "top_p": 0.3 + } + }, + "upstream": [ + "TitleChunker:FlatMiceFix" + ] + }, + "File": { + "downstream": [ + "Parser:HipSignsRhyme" + ], + "obj": { + "component_name": "File", + "params": {} + }, + "upstream": [] + }, + "Parser:HipSignsRhyme": { + "downstream": [ + "TitleChunker:FlatMiceFix" + ], + "obj": { + "component_name": "Parser", + "params": { + "outputs": { + "html": { + "type": "string", + "value": "" }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": [ - { - "content": "Content: {TitleChunker:FlatMiceFix@chunks}", - "role": "user" - } - ], - "sys_prompt": "Act as a precise resume metadata extractor. Extract stable, chunk-supported metadata from the provided resume content.\n\nRules:\n1. Use only information explicitly stated in the content. Do not infer, guess, normalize, or add missing facts.\n2. The input may be only one chunk of a resume. Extract only what this content directly supports.\n3. Use only these field names:\ncandidate_name, gender, phone, email, city, location, nationality, linkedin, github, website, highest_degree, degree_levels, school_names, majors, graduation_years, work_experience_years, current_job_title, job_titles, company_names, job_experience, industries, target_job_titles, target_locations, employment_types, skills, certificates, awards, summary_tags\n4. Ignore detailed responsibilities, project descriptions, achievement narratives, self-evaluation, and other low-value local details.\n5. Keep values in the same language as the source text whenever possible.\n6. Remove duplicates and keep only concise, high-value metadata.\n7. Return only fields that are explicitly supported by the content. Do not return empty or unsupported fields.\n\nField guidance:\n- highest_degree: highest explicit degree level mentioned\n- degree_levels: all explicit degree levels mentioned\n- school_names: explicit school, college, or university names\n- majors: explicit fields of study\n- graduation_years: explicit graduation years only\n- work_experience_years: only if explicitly stated\n- current_job_title: only if explicitly current or most recent\n- job_titles: explicit role titles\n- company_names: explicit employer names\n- job_experience: concise structured work entries explicitly supported by the content, preferably including title, company, and time information when available\n- industries: explicit industry names only\n- target_job_titles: explicit desired roles only\n- target_locations: explicit desired work locations only\n- skills: concise, core, search-useful skills explicitly mentioned\n- certificates: explicit certificate names only\n- awards: explicit award names only\n- summary_tags: short, high-value tags strictly supported by the content\n\nReturn only the extracted metadata. Do not output explanatory text.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": 29, - "topPEnabled": true, - "top_p": 0.3 + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": { + "docx": { + "flatten_media_to_text": true, + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "docx" + ], + "vlm": {} + }, + "pdf": { + "flatten_media_to_text": true, + "output_format": "json", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "pdf" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "txt", + "py", + "js", + "java", + "c", + "cpp", + "h", + "php", + "go", + "ts", + "sh", + "cs", + "kt", + "sql" + ] + } } - }, - "upstream": [ - "TitleChunker:FlatMiceFix" - ] + } }, - "File": { - "downstream": [ - "Parser:HipSignsRhyme" - ], - "obj": { - "component_name": "File", - "params": {} - }, - "upstream": [] + "upstream": [ + "File" + ] + }, + "TitleChunker:FlatMiceFix": { + "downstream": [ + "Extractor:ThreeDrinksAct" + ], + "obj": { + "component_name": "TitleChunker", + "params": { + "hierarchy": 1, + "include_heading_content": false, + "levels": [ + [ + "^\\s*(?i:(?:\\d+[\\.\\)]\\s*)?(?:EDUCATION|ACADEMIC\\s*BACKGROUND|ACADEMIC\\s*HISTORY|EDUCATIONAL\\s*BACKGROUND|RELEVANT\\s*COURSEWORK|COURSEWORK|EXPERIENCE|WORK\\s*EXPERIENCE|PROFESSIONAL\\s*EXPERIENCE|RELEVANT\\s*EXPERIENCE|EMPLOYMENT\\s*HISTORY|CAREER\\s*HISTORY|INTERNSHIP\\s*EXPERIENCE|PROJECTS|PROJECT\\s*EXPERIENCE|ACADEMIC\\s*PROJECTS|PROFESSIONAL\\s*PROJECTS|SKILLS|TECHNICAL\\s*SKILLS|CORE\\s*COMPETENCIES|COMPETENCIES|QUALIFICATIONS|SUMMARY\\s*OF\\s*QUALIFICATIONS|CERTIFICATIONS|LICENSES|CERTIFICATES|AWARDS|HONORS|HONOURS|ACHIEVEMENTS|PUBLICATIONS|RESEARCH|RESEARCH\\s*EXPERIENCE|LEADERSHIP|LEADERSHIP\\s*EXPERIENCE|ACTIVITIES|EXTRACURRICULAR\\s*ACTIVITIES|ACTIVITIES\\s*(?:&|AND)\\s*SKILLS|INVOLVEMENT|CAMPUS\\s*INVOLVEMENT|VOLUNTEER\\s*EXPERIENCE|VOLUNTEERING|COMMUNITY\\s*SERVICE|LANGUAGES|INTERESTS|HOBBIES|PROFILE|PROFESSIONAL\\s*PROFILE|SUMMARY|PROFESSIONAL\\s*SUMMARY|CAREER\\s*SUMMARY|OBJECTIVE|CAREER\\s*OBJECTIVE|PERSONAL\\s*INFORMATION|CONTACT\\s*INFORMATION|ADDITIONAL\\s*INFORMATION|TRAINING))\\s*[:：]?\\s*$" + ], + [ + "^\\s*(?:\\d+[\\.、\\)]\\s*)?(?:教育背景|教育经历|学历背景|学术背景|技术背景|工作经历|工作经验|实习经历|项目经历|项目经验|科研经历|研究经历|校园经历|实践经历|专业经历|职业经历|技能|专业技能|技能特长|核心技能|技术栈|个人技能|工作技能|职业技能|技能与评价|技能与自我评价|工作技能与自我评价|职业技能与自我评价|证书|资格证书|职业资格|资质证书|获奖情况|获奖经历|荣誉|荣誉奖项|奖项|科研成果|论文发表|发表论文|领导经历|学生工作|校园活动|社团经历|活动经历|志愿经历|志愿服务|社会实践|语言能力|语言|自我评价|个人评价|自我总结|个人总结|个人优势|个人简介|个人信息|基本信息|联系方式|求职意向|应聘意向|职业目标|求职目标|兴趣爱好|兴趣特长|培训经历|其他信息|附加信息)\\s*[:：]?\\s*$" + ] + ], + "method": "hierarchy", + "root_chunk_as_heading": true + } }, - "Parser:HipSignsRhyme": { - "downstream": [ - "TitleChunker:FlatMiceFix" - ], - "obj": { - "component_name": "Parser", - "params": { + "upstream": [ + "Parser:HipSignsRhyme" + ] + }, + "Tokenizer:KindHandsWin": { + "downstream": [], + "obj": { + "component_name": "Tokenizer", + "params": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + } + }, + "upstream": [ + "Extractor:ThreeDrinksAct" + ] + } + }, + "globals": { + "sys.history": [] + }, + "graph": { + "edges": [ + { + "data": { + "isHovered": false + }, + "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", + "source": "File", + "sourceHandle": "start", + "target": "Parser:HipSignsRhyme", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__Parser:HipSignsRhymestart-TitleChunker:FlatMiceFixend", + "source": "Parser:HipSignsRhyme", + "sourceHandle": "start", + "target": "TitleChunker:FlatMiceFix", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__Extractor:ThreeDrinksActstart-Tokenizer:KindHandsWinend", + "markerEnd": "logo", + "source": "Extractor:ThreeDrinksAct", + "sourceHandle": "start", + "target": "Tokenizer:KindHandsWin", + "targetHandle": "end", + "type": "buttonEdge", + "zIndex": 1001 + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__TitleChunker:FlatMiceFixstart-Extractor:ThreeDrinksActend", + "markerEnd": "logo", + "source": "TitleChunker:FlatMiceFix", + "sourceHandle": "start", + "target": "Extractor:ThreeDrinksAct", + "targetHandle": "end", + "type": "buttonEdge", + "zIndex": 1001 + } + ], + "nodes": [ + { + "data": { + "label": "File", + "name": "File" + }, + "id": "File", + "measured": { + "height": 49, + "width": 200 + }, + "position": { + "x": 50, + "y": 200 + }, + "sourcePosition": "left", + "targetPosition": "right", + "type": "beginNode" + }, + { + "data": { + "form": { "outputs": { "html": { "type": "string", @@ -88,179 +277,54 @@ "value": "" } }, - "setups": { - "doc": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "doc" - ] - }, - "docx": { - "flatten_media_to_text": true, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "docx" - ], - "vlm": {} - }, - "email": { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "output_format": "text", - "preprocess": [ - "main_content" - ], - "suffix": [ - "eml", - "msg" - ] - }, - "html": { - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "htm", - "html" - ] - }, - "image": { - "output_format": "text", - "parse_method": "ocr", - "preprocess": [ - "main_content" - ], - "suffix": [ - "jpg", - "jpeg", - "png", - "gif" - ] - }, - "markdown": { - "flatten_media_to_text": true, - "output_format": "json", - "preprocess": [ - "main_content" - ], - "suffix": [ - "md", - "markdown", - "mdx" - ], - "vlm": {} - }, - "pdf": { + "setups": [ + { + "fileFormat": "pdf", "flatten_media_to_text": true, "output_format": "json", "parse_method": "DeepDOC", "preprocess": [ "main_content" - ], - "suffix": [ - "pdf" - ], - "vlm": {} - }, - "slides": { - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "pptx", - "ppt" ] }, - "spreadsheet": { - "flatten_media_to_text": true, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ], - "suffix": [ - "xls", - "xlsx", - "csv" - ], - "vlm": {} - }, - "text&code": { + { + "fileFormat": "text&code", + "output_format": "json", + "preprocess": [ + "main_content" + ] + }, + { + "fileFormat": "docx", + "flatten_media_to_text": true, "output_format": "json", "preprocess": [ "main_content" - ], - "suffix": [ - "txt", - "py", - "js", - "java", - "c", - "cpp", - "h", - "php", - "go", - "ts", - "sh", - "cs", - "kt", - "sql" ] } - } - } + ] + }, + "label": "Parser", + "name": "Parser_0" }, - "upstream": [ - "File" - ] - }, - "TitleChunker:FlatMiceFix": { - "downstream": [ - "Extractor:ThreeDrinksAct" - ], - "obj": { - "component_name": "TitleChunker", - "params": { - "hierarchy": 1, - "include_heading_content": false, - "levels": [ - [ - "^\\s*(?i:(?:\\d+[\\.\\)]\\s*)?(?:EDUCATION|ACADEMIC\\s*BACKGROUND|ACADEMIC\\s*HISTORY|EDUCATIONAL\\s*BACKGROUND|RELEVANT\\s*COURSEWORK|COURSEWORK|EXPERIENCE|WORK\\s*EXPERIENCE|PROFESSIONAL\\s*EXPERIENCE|RELEVANT\\s*EXPERIENCE|EMPLOYMENT\\s*HISTORY|CAREER\\s*HISTORY|INTERNSHIP\\s*EXPERIENCE|PROJECTS|PROJECT\\s*EXPERIENCE|ACADEMIC\\s*PROJECTS|PROFESSIONAL\\s*PROJECTS|SKILLS|TECHNICAL\\s*SKILLS|CORE\\s*COMPETENCIES|COMPETENCIES|QUALIFICATIONS|SUMMARY\\s*OF\\s*QUALIFICATIONS|CERTIFICATIONS|LICENSES|CERTIFICATES|AWARDS|HONORS|HONOURS|ACHIEVEMENTS|PUBLICATIONS|RESEARCH|RESEARCH\\s*EXPERIENCE|LEADERSHIP|LEADERSHIP\\s*EXPERIENCE|ACTIVITIES|EXTRACURRICULAR\\s*ACTIVITIES|ACTIVITIES\\s*(?:&|AND)\\s*SKILLS|INVOLVEMENT|CAMPUS\\s*INVOLVEMENT|VOLUNTEER\\s*EXPERIENCE|VOLUNTEERING|COMMUNITY\\s*SERVICE|LANGUAGES|INTERESTS|HOBBIES|PROFILE|PROFESSIONAL\\s*PROFILE|SUMMARY|PROFESSIONAL\\s*SUMMARY|CAREER\\s*SUMMARY|OBJECTIVE|CAREER\\s*OBJECTIVE|PERSONAL\\s*INFORMATION|CONTACT\\s*INFORMATION|ADDITIONAL\\s*INFORMATION|TRAINING))\\s*[:\uff1a]?\\s*$" - ], - [ - "^\\s*(?:\\d+[\\.\u3001\\)]\\s*)?(?:\u6559\u80b2\u80cc\u666f|\u6559\u80b2\u7ecf\u5386|\u5b66\u5386\u80cc\u666f|\u5b66\u672f\u80cc\u666f|\u6280\u672f\u80cc\u666f|\u5de5\u4f5c\u7ecf\u5386|\u5de5\u4f5c\u7ecf\u9a8c|\u5b9e\u4e60\u7ecf\u5386|\u9879\u76ee\u7ecf\u5386|\u9879\u76ee\u7ecf\u9a8c|\u79d1\u7814\u7ecf\u5386|\u7814\u7a76\u7ecf\u5386|\u6821\u56ed\u7ecf\u5386|\u5b9e\u8df5\u7ecf\u5386|\u4e13\u4e1a\u7ecf\u5386|\u804c\u4e1a\u7ecf\u5386|\u6280\u80fd|\u4e13\u4e1a\u6280\u80fd|\u6280\u80fd\u7279\u957f|\u6838\u5fc3\u6280\u80fd|\u6280\u672f\u6808|\u4e2a\u4eba\u6280\u80fd|\u5de5\u4f5c\u6280\u80fd|\u804c\u4e1a\u6280\u80fd|\u6280\u80fd\u4e0e\u8bc4\u4ef7|\u6280\u80fd\u4e0e\u81ea\u6211\u8bc4\u4ef7|\u5de5\u4f5c\u6280\u80fd\u4e0e\u81ea\u6211\u8bc4\u4ef7|\u804c\u4e1a\u6280\u80fd\u4e0e\u81ea\u6211\u8bc4\u4ef7|\u8bc1\u4e66|\u8d44\u683c\u8bc1\u4e66|\u804c\u4e1a\u8d44\u683c|\u8d44\u8d28\u8bc1\u4e66|\u83b7\u5956\u60c5\u51b5|\u83b7\u5956\u7ecf\u5386|\u8363\u8a89|\u8363\u8a89\u5956\u9879|\u5956\u9879|\u79d1\u7814\u6210\u679c|\u8bba\u6587\u53d1\u8868|\u53d1\u8868\u8bba\u6587|\u9886\u5bfc\u7ecf\u5386|\u5b66\u751f\u5de5\u4f5c|\u6821\u56ed\u6d3b\u52a8|\u793e\u56e2\u7ecf\u5386|\u6d3b\u52a8\u7ecf\u5386|\u5fd7\u613f\u7ecf\u5386|\u5fd7\u613f\u670d\u52a1|\u793e\u4f1a\u5b9e\u8df5|\u8bed\u8a00\u80fd\u529b|\u8bed\u8a00|\u81ea\u6211\u8bc4\u4ef7|\u4e2a\u4eba\u8bc4\u4ef7|\u81ea\u6211\u603b\u7ed3|\u4e2a\u4eba\u603b\u7ed3|\u4e2a\u4eba\u4f18\u52bf|\u4e2a\u4eba\u7b80\u4ecb|\u4e2a\u4eba\u4fe1\u606f|\u57fa\u672c\u4fe1\u606f|\u8054\u7cfb\u65b9\u5f0f|\u6c42\u804c\u610f\u5411|\u5e94\u8058\u610f\u5411|\u804c\u4e1a\u76ee\u6807|\u6c42\u804c\u76ee\u6807|\u5174\u8da3\u7231\u597d|\u5174\u8da3\u7279\u957f|\u57f9\u8bad\u7ecf\u5386|\u5176\u4ed6\u4fe1\u606f|\u9644\u52a0\u4fe1\u606f)\\s*[:\uff1a]?\\s*$" - ] - ], - "method": "hierarchy", - "root_chunk_as_heading": true - } + "dragging": false, + "id": "Parser:HipSignsRhyme", + "measured": { + "height": 197, + "width": 200 }, - "upstream": [ - "Parser:HipSignsRhyme" - ] + "position": { + "x": 307.6583243118047, + "y": 119.87418674572268 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "parserNode" }, - "Tokenizer:KindHandsWin": { - "downstream": [], - "obj": { - "component_name": "Tokenizer", - "params": { + { + "data": { + "form": { "fields": "text", "filename_embd_weight": 0.1, "outputs": {}, @@ -268,342 +332,127 @@ "embedding", "full_text" ] - } + }, + "label": "Tokenizer", + "name": "Indexer_0" }, - "upstream": [ - "Extractor:ThreeDrinksAct" - ] + "dragging": false, + "id": "Tokenizer:KindHandsWin", + "measured": { + "height": 113, + "width": 200 + }, + "position": { + "x": 883.0243372012395, + "y": 156.39625132974524 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "tokenizerNode" + }, + { + "data": { + "form": { + "hierarchy": "1", + "include_heading_content": false, + "method": "hierarchy", + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "promote_first_heading_to_root": false, + "root_chunk_as_heading": true, + "rules": [ + { + "levels": [ + { + "expression": "^\\s*(?i:(?:\\d+[\\.\\)]\\s*)?(?:EDUCATION|ACADEMIC\\s*BACKGROUND|ACADEMIC\\s*HISTORY|EDUCATIONAL\\s*BACKGROUND|RELEVANT\\s*COURSEWORK|COURSEWORK|EXPERIENCE|WORK\\s*EXPERIENCE|PROFESSIONAL\\s*EXPERIENCE|RELEVANT\\s*EXPERIENCE|EMPLOYMENT\\s*HISTORY|CAREER\\s*HISTORY|INTERNSHIP\\s*EXPERIENCE|PROJECTS|PROJECT\\s*EXPERIENCE|ACADEMIC\\s*PROJECTS|PROFESSIONAL\\s*PROJECTS|SKILLS|TECHNICAL\\s*SKILLS|CORE\\s*COMPETENCIES|COMPETENCIES|QUALIFICATIONS|SUMMARY\\s*OF\\s*QUALIFICATIONS|CERTIFICATIONS|LICENSES|CERTIFICATES|AWARDS|HONORS|HONOURS|ACHIEVEMENTS|PUBLICATIONS|RESEARCH|RESEARCH\\s*EXPERIENCE|LEADERSHIP|LEADERSHIP\\s*EXPERIENCE|ACTIVITIES|EXTRACURRICULAR\\s*ACTIVITIES|ACTIVITIES\\s*(?:&|AND)\\s*SKILLS|INVOLVEMENT|CAMPUS\\s*INVOLVEMENT|VOLUNTEER\\s*EXPERIENCE|VOLUNTEERING|COMMUNITY\\s*SERVICE|LANGUAGES|INTERESTS|HOBBIES|PROFILE|PROFESSIONAL\\s*PROFILE|SUMMARY|PROFESSIONAL\\s*SUMMARY|CAREER\\s*SUMMARY|OBJECTIVE|CAREER\\s*OBJECTIVE|PERSONAL\\s*INFORMATION|CONTACT\\s*INFORMATION|ADDITIONAL\\s*INFORMATION|TRAINING))\\s*[:：]?\\s*$" + } + ] + }, + { + "levels": [ + { + "expression": "^\\s*(?:\\d+[\\.、\\)]\\s*)?(?:教育背景|教育经历|学历背景|学术背景|技术背景|工作经历|工作经验|实习经历|项目经历|项目经验|科研经历|研究经历|校园经历|实践经历|专业经历|职业经历|技能|专业技能|技能特长|核心技能|技术栈|个人技能|工作技能|职业技能|技能与评价|技能与自我评价|工作技能与自我评价|职业技能与自我评价|证书|资格证书|职业资格|资质证书|获奖情况|获奖经历|荣誉|荣誉奖项|奖项|科研成果|论文发表|发表论文|领导经历|学生工作|校园活动|社团经历|活动经历|志愿经历|志愿服务|社会实践|语言能力|语言|自我评价|个人评价|自我总结|个人总结|个人优势|个人简介|个人信息|基本信息|联系方式|求职意向|应聘意向|职业目标|求职目标|兴趣爱好|兴趣特长|培训经历|其他信息|附加信息)\\s*[:：]?\\s*$" + } + ] + } + ] + }, + "label": "TitleChunker", + "name": "Title Chunker_0" + }, + "dragging": false, + "id": "TitleChunker:FlatMiceFix", + "measured": { + "height": 73, + "width": 200 + }, + "position": { + "x": 572.7908769627791, + "y": 141.55515313482098 + }, + "selected": true, + "sourcePosition": "right", + "targetPosition": "left", + "type": "chunkerNode" + }, + { + "data": { + "form": { + "field_name": "metadata", + "frequencyPenaltyEnabled": true, + "frequency_penalty": 0.7, + "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", + "maxTokensEnabled": false, + "max_tokens": 256, + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + }, + "presencePenaltyEnabled": true, + "presence_penalty": 0.4, + "prompts": "Content: {TitleChunker:FlatMiceFix@chunks}", + "sys_prompt": "Act as a precise resume metadata extractor. Extract stable, chunk-supported metadata from the provided resume content.\n\nRules:\n1. Use only information explicitly stated in the content. Do not infer, guess, normalize, or add missing facts.\n2. The input may be only one chunk of a resume. Extract only what this content directly supports.\n3. Use only these field names:\ncandidate_name, gender, phone, email, city, location, nationality, linkedin, github, website, highest_degree, degree_levels, school_names, majors, graduation_years, work_experience_years, current_job_title, job_titles, company_names, job_experience, industries, target_job_titles, target_locations, employment_types, skills, certificates, awards, summary_tags\n4. Ignore detailed responsibilities, project descriptions, achievement narratives, self-evaluation, and other low-value local details.\n5. Keep values in the same language as the source text whenever possible.\n6. Remove duplicates and keep only concise, high-value metadata.\n7. Return only fields that are explicitly supported by the content. Do not return empty or unsupported fields.\n\nField guidance:\n- highest_degree: highest explicit degree level mentioned\n- degree_levels: all explicit degree levels mentioned\n- school_names: explicit school, college, or university names\n- majors: explicit fields of study\n- graduation_years: explicit graduation years only\n- work_experience_years: only if explicitly stated\n- current_job_title: only if explicitly current or most recent\n- job_titles: explicit role titles\n- company_names: explicit employer names\n- job_experience: concise structured work entries explicitly supported by the content, preferably including title, company, and time information when available\n- industries: explicit industry names only\n- target_job_titles: explicit desired roles only\n- target_locations: explicit desired work locations only\n- skills: concise, core, search-useful skills explicitly mentioned\n- certificates: explicit certificate names only\n- awards: explicit award names only\n- summary_tags: short, high-value tags strictly supported by the content\n\nReturn only the extracted metadata. Do not output explanatory text.", + "temperature": 0.1, + "temperatureEnabled": true, + "tenant_llm_id": 29, + "topPEnabled": true, + "top_p": 0.3 + }, + "label": "Extractor", + "name": "Auto Metadata" + }, + "dragging": false, + "id": "Extractor:ThreeDrinksAct", + "measured": { + "height": 89, + "width": 200 + }, + "position": { + "x": 623.8123774842874, + "y": 236.49984938595793 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "contextNode" } - }, - "globals": { - "sys.history": [] - }, - "graph": { - "edges": [ - { - "data": { - "isHovered": false - }, - "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", - "source": "File", - "sourceHandle": "start", - "target": "Parser:HipSignsRhyme", - "targetHandle": "end" - }, - { - "data": { - "isHovered": false - }, - "id": "xy-edge__Parser:HipSignsRhymestart-TitleChunker:FlatMiceFixend", - "source": "Parser:HipSignsRhyme", - "sourceHandle": "start", - "target": "TitleChunker:FlatMiceFix", - "targetHandle": "end" - }, - { - "data": { - "isHovered": false - }, - "id": "xy-edge__Extractor:ThreeDrinksActstart-Tokenizer:KindHandsWinend", - "markerEnd": "logo", - "source": "Extractor:ThreeDrinksAct", - "sourceHandle": "start", - "target": "Tokenizer:KindHandsWin", - "targetHandle": "end", - "type": "buttonEdge", - "zIndex": 1001 - }, - { - "data": { - "isHovered": false - }, - "id": "xy-edge__TitleChunker:FlatMiceFixstart-Extractor:ThreeDrinksActend", - "markerEnd": "logo", - "source": "TitleChunker:FlatMiceFix", - "sourceHandle": "start", - "target": "Extractor:ThreeDrinksAct", - "targetHandle": "end", - "type": "buttonEdge", - "zIndex": 1001 - } - ], - "nodes": [ - { - "data": { - "label": "File", - "name": "File" - }, - "id": "File", - "measured": { - "height": 49, - "width": 200 - }, - "position": { - "x": 50, - "y": 200 - }, - "sourcePosition": "left", - "targetPosition": "right", - "type": "beginNode" - }, - { - "data": { - "form": { - "outputs": { - "html": { - "type": "string", - "value": "" - }, - "json": { - "type": "Array", - "value": [] - }, - "markdown": { - "type": "string", - "value": "" - }, - "text": { - "type": "string", - "value": "" - } - }, - "setups": [ - { - "fileFormat": "pdf", - "flatten_media_to_text": true, - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "spreadsheet", - "flatten_media_to_text": true, - "output_format": "html", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "image", - "output_format": "text", - "parse_method": "ocr", - "preprocess": [ - "main_content" - ] - }, - { - "fields": [ - "from", - "to", - "cc", - "bcc", - "date", - "subject", - "body", - "attachments" - ], - "fileFormat": "email", - "output_format": "text", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "markdown", - "flatten_media_to_text": true, - "output_format": "json", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "text&code", - "output_format": "json", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "html", - "output_format": "json", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "doc", - "output_format": "json", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "docx", - "flatten_media_to_text": true, - "output_format": "json", - "preprocess": [ - "main_content" - ] - }, - { - "fileFormat": "slides", - "output_format": "json", - "parse_method": "DeepDOC", - "preprocess": [ - "main_content" - ] - } - ] - }, - "label": "Parser", - "name": "Parser_0" - }, - "dragging": false, - "id": "Parser:HipSignsRhyme", - "measured": { - "height": 197, - "width": 200 - }, - "position": { - "x": 307.6583243118047, - "y": 119.87418674572268 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "parserNode" - }, - { - "data": { - "form": { - "fields": "text", - "filename_embd_weight": 0.1, - "outputs": {}, - "search_method": [ - "embedding", - "full_text" - ] - }, - "label": "Tokenizer", - "name": "Indexer_0" - }, - "dragging": false, - "id": "Tokenizer:KindHandsWin", - "measured": { - "height": 113, - "width": 200 - }, - "position": { - "x": 883.0243372012395, - "y": 156.39625132974524 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "tokenizerNode" - }, - { - "data": { - "form": { - "hierarchy": "1", - "include_heading_content": false, - "method": "hierarchy", - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "promote_first_heading_to_root": false, - "root_chunk_as_heading": true, - "rules": [ - { - "levels": [ - { - "expression": "^\\s*(?i:(?:\\d+[\\.\\)]\\s*)?(?:EDUCATION|ACADEMIC\\s*BACKGROUND|ACADEMIC\\s*HISTORY|EDUCATIONAL\\s*BACKGROUND|RELEVANT\\s*COURSEWORK|COURSEWORK|EXPERIENCE|WORK\\s*EXPERIENCE|PROFESSIONAL\\s*EXPERIENCE|RELEVANT\\s*EXPERIENCE|EMPLOYMENT\\s*HISTORY|CAREER\\s*HISTORY|INTERNSHIP\\s*EXPERIENCE|PROJECTS|PROJECT\\s*EXPERIENCE|ACADEMIC\\s*PROJECTS|PROFESSIONAL\\s*PROJECTS|SKILLS|TECHNICAL\\s*SKILLS|CORE\\s*COMPETENCIES|COMPETENCIES|QUALIFICATIONS|SUMMARY\\s*OF\\s*QUALIFICATIONS|CERTIFICATIONS|LICENSES|CERTIFICATES|AWARDS|HONORS|HONOURS|ACHIEVEMENTS|PUBLICATIONS|RESEARCH|RESEARCH\\s*EXPERIENCE|LEADERSHIP|LEADERSHIP\\s*EXPERIENCE|ACTIVITIES|EXTRACURRICULAR\\s*ACTIVITIES|ACTIVITIES\\s*(?:&|AND)\\s*SKILLS|INVOLVEMENT|CAMPUS\\s*INVOLVEMENT|VOLUNTEER\\s*EXPERIENCE|VOLUNTEERING|COMMUNITY\\s*SERVICE|LANGUAGES|INTERESTS|HOBBIES|PROFILE|PROFESSIONAL\\s*PROFILE|SUMMARY|PROFESSIONAL\\s*SUMMARY|CAREER\\s*SUMMARY|OBJECTIVE|CAREER\\s*OBJECTIVE|PERSONAL\\s*INFORMATION|CONTACT\\s*INFORMATION|ADDITIONAL\\s*INFORMATION|TRAINING))\\s*[:\uff1a]?\\s*$" - } - ] - }, - { - "levels": [ - { - "expression": "^\\s*(?:\\d+[\\.\u3001\\)]\\s*)?(?:\u6559\u80b2\u80cc\u666f|\u6559\u80b2\u7ecf\u5386|\u5b66\u5386\u80cc\u666f|\u5b66\u672f\u80cc\u666f|\u6280\u672f\u80cc\u666f|\u5de5\u4f5c\u7ecf\u5386|\u5de5\u4f5c\u7ecf\u9a8c|\u5b9e\u4e60\u7ecf\u5386|\u9879\u76ee\u7ecf\u5386|\u9879\u76ee\u7ecf\u9a8c|\u79d1\u7814\u7ecf\u5386|\u7814\u7a76\u7ecf\u5386|\u6821\u56ed\u7ecf\u5386|\u5b9e\u8df5\u7ecf\u5386|\u4e13\u4e1a\u7ecf\u5386|\u804c\u4e1a\u7ecf\u5386|\u6280\u80fd|\u4e13\u4e1a\u6280\u80fd|\u6280\u80fd\u7279\u957f|\u6838\u5fc3\u6280\u80fd|\u6280\u672f\u6808|\u4e2a\u4eba\u6280\u80fd|\u5de5\u4f5c\u6280\u80fd|\u804c\u4e1a\u6280\u80fd|\u6280\u80fd\u4e0e\u8bc4\u4ef7|\u6280\u80fd\u4e0e\u81ea\u6211\u8bc4\u4ef7|\u5de5\u4f5c\u6280\u80fd\u4e0e\u81ea\u6211\u8bc4\u4ef7|\u804c\u4e1a\u6280\u80fd\u4e0e\u81ea\u6211\u8bc4\u4ef7|\u8bc1\u4e66|\u8d44\u683c\u8bc1\u4e66|\u804c\u4e1a\u8d44\u683c|\u8d44\u8d28\u8bc1\u4e66|\u83b7\u5956\u60c5\u51b5|\u83b7\u5956\u7ecf\u5386|\u8363\u8a89|\u8363\u8a89\u5956\u9879|\u5956\u9879|\u79d1\u7814\u6210\u679c|\u8bba\u6587\u53d1\u8868|\u53d1\u8868\u8bba\u6587|\u9886\u5bfc\u7ecf\u5386|\u5b66\u751f\u5de5\u4f5c|\u6821\u56ed\u6d3b\u52a8|\u793e\u56e2\u7ecf\u5386|\u6d3b\u52a8\u7ecf\u5386|\u5fd7\u613f\u7ecf\u5386|\u5fd7\u613f\u670d\u52a1|\u793e\u4f1a\u5b9e\u8df5|\u8bed\u8a00\u80fd\u529b|\u8bed\u8a00|\u81ea\u6211\u8bc4\u4ef7|\u4e2a\u4eba\u8bc4\u4ef7|\u81ea\u6211\u603b\u7ed3|\u4e2a\u4eba\u603b\u7ed3|\u4e2a\u4eba\u4f18\u52bf|\u4e2a\u4eba\u7b80\u4ecb|\u4e2a\u4eba\u4fe1\u606f|\u57fa\u672c\u4fe1\u606f|\u8054\u7cfb\u65b9\u5f0f|\u6c42\u804c\u610f\u5411|\u5e94\u8058\u610f\u5411|\u804c\u4e1a\u76ee\u6807|\u6c42\u804c\u76ee\u6807|\u5174\u8da3\u7231\u597d|\u5174\u8da3\u7279\u957f|\u57f9\u8bad\u7ecf\u5386|\u5176\u4ed6\u4fe1\u606f|\u9644\u52a0\u4fe1\u606f)\\s*[:\uff1a]?\\s*$" - } - ] - } - ] - }, - "label": "TitleChunker", - "name": "Title Chunker_0" - }, - "dragging": false, - "id": "TitleChunker:FlatMiceFix", - "measured": { - "height": 73, - "width": 200 - }, - "position": { - "x": 572.7908769627791, - "y": 141.55515313482098 - }, - "selected": true, - "sourcePosition": "right", - "targetPosition": "left", - "type": "chunkerNode" - }, - { - "data": { - "form": { - "field_name": "metadata", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": "Content: {TitleChunker:FlatMiceFix@chunks}", - "sys_prompt": "Act as a precise resume metadata extractor. Extract stable, chunk-supported metadata from the provided resume content.\n\nRules:\n1. Use only information explicitly stated in the content. Do not infer, guess, normalize, or add missing facts.\n2. The input may be only one chunk of a resume. Extract only what this content directly supports.\n3. Use only these field names:\ncandidate_name, gender, phone, email, city, location, nationality, linkedin, github, website, highest_degree, degree_levels, school_names, majors, graduation_years, work_experience_years, current_job_title, job_titles, company_names, job_experience, industries, target_job_titles, target_locations, employment_types, skills, certificates, awards, summary_tags\n4. Ignore detailed responsibilities, project descriptions, achievement narratives, self-evaluation, and other low-value local details.\n5. Keep values in the same language as the source text whenever possible.\n6. Remove duplicates and keep only concise, high-value metadata.\n7. Return only fields that are explicitly supported by the content. Do not return empty or unsupported fields.\n\nField guidance:\n- highest_degree: highest explicit degree level mentioned\n- degree_levels: all explicit degree levels mentioned\n- school_names: explicit school, college, or university names\n- majors: explicit fields of study\n- graduation_years: explicit graduation years only\n- work_experience_years: only if explicitly stated\n- current_job_title: only if explicitly current or most recent\n- job_titles: explicit role titles\n- company_names: explicit employer names\n- job_experience: concise structured work entries explicitly supported by the content, preferably including title, company, and time information when available\n- industries: explicit industry names only\n- target_job_titles: explicit desired roles only\n- target_locations: explicit desired work locations only\n- skills: concise, core, search-useful skills explicitly mentioned\n- certificates: explicit certificate names only\n- awards: explicit award names only\n- summary_tags: short, high-value tags strictly supported by the content\n\nReturn only the extracted metadata. Do not output explanatory text.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": 29, - "topPEnabled": true, - "top_p": 0.3 - }, - "label": "Extractor", - "name": "Auto Metadata" - }, - "dragging": false, - "id": "Extractor:ThreeDrinksAct", - "measured": { - "height": 89, - "width": 200 - }, - "position": { - "x": 623.8123774842874, - "y": 236.49984938595793 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "contextNode" - } - ] - }, - "history": [], - "messages": [], - "path": [], - "retrieval": [], - "variables": [] + ] }, - "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAVF0lEQVR4nO2aebRdVX3HP/vM59z75jkvMxnIQEggIKNVQFNYKGCVMUKtIg4UtCwj0mq1ttJSrFIHFtYBcAIsKGBNVCwGKCg0JAESyEzI8PLey3v33fmMe/ePc5O8QJJ3M6z+0ea71l53n3v22ef3++7f/u3f/u0Dx3Ecx3Ecx/H/F+JoO1B9IHpq9We+VH7ln//WE83tGJaN7rholo2WcdAMR1oZO9IcN9JdLzZtN9I9NzYtN9KtbCQcS2potV5lWpRCSoUiASVRSgESJVWtmULXMbW2xm4x/wvuW2SrQ0HjiBVXIESqfPTLj6sV37ubN9asYe6DP4cnfwPVIvh5ZHWEuFIkKvdpwXDBDstFu1wOKBehkodyCUoFKBZAGCBM0AzQTdAs0G0wHAcz04zuNGNlWtAyHTRPH0/r9GkUt8b0iHE8/Wee6p40nh2/WH9W52zaZ10yZ5O4fs3asfQ4KguQv/7c1qce+9FAZvzFCxe+ejfcVwSyR9PlKMQg81DNQTUPlQIEw+BXoLGNTd+9mXU/28hFzz9F8Is7MS2BFgFNraBto+9XTzDu7rH1OyILUJvv+dvf/NUNzktbtk3MzrpsYnVkDdwnAVGX2dUHA7Q2yLRB5q13T/jyRbzaP54d6wfpvfbRN93dxdqP97xwTMQ4EJ64lL98fdkdSiml7r2wTallH1KjMbLsZlVY8zNV3LRMld54Rvm7VqtweLMKy7tUEheVVGMjSioqjkoq9offUqKgoJRSKpEV9W8notY//EWllFKlvpfTh9fdq5ZN4olU2kMPx2FbQHz3uc/pH3vqjJd+fCsAfn6EUt8QWSCRZfrumEj+t8NY4++ChNQTKVA1OTQBaKlcQgPMdO5rBmBB3A+dH/snGs9eQpwUDii+UDFxNYfhtvDB53dw7ym9eE2d9J7/8bSBX6Ca4GNmIaqSCnKMCNgl2pp7gWK5AEBQStAyjQCUnvw7Sk8N0/oOAzEcE+ugN9qgmVAppW+zmkDXIa6C0EG3AIVKElAK3Ssx8N3PIpSk4ZxbSYLcAeUQGsTRCHbDOK5Zvpofn3sylz7QTddpl0FYwodqttmiNBSAlCJ120dJwOZvLCJuGD8DQIh0bKIqaG5z2qBpApYL3hm30nDmzSSV7eQe/UtiVcGbsxhZ7id65SFUEqG1zyL2h4mHt4GKEb4PERguhBJkVKy9VedgIyikIgpzZMfN47LHf83P37uIGzYqmLYAmVBu8WzKI2WUPLhOh0XAwJpVdL33PE0BhmkCoKqguU1pAz8V2uo4ES3bjpZtx+iag9U2g+azPw3AMBp250wy868DIPf0P2A1zyRz0vtTQgdWs+26+YjsuLpk0hTIOE/n7Hdz0X0/4fHrJ3LxeW+jaQoVs99C41AT4DAJGCn1907vnkopN4jppsudjEC30ykQbn6S2IT8Q4sZ/vbilPluUBGUfnBLGt+Mg0oCuXv+HKWDaIGygMGfgp5pxLCzqBYQxh7XfyjxU6hEIMUIE86+Cr25nVWffDduL0XVP7ZO2thNYI8nLRSZmu2eRJgfxnQbgHQqY6ZBWMvl92ONh8Im8E0IXYgqEEuIWkF21nyipSO7LOj00OxGNLsJ22vEQEBYJegHVdxZn2gAJCilESUlxs15Fwu+dw+v/46S0BPEGATWaQGp/3A0plntk/HXb0C3UwtIQhBOOlqG003np56m8uw30DOdEJVRfpE4KiHDEgRFVFgk8UuoqAhBhSgCFaSWJGKorIEJd9xJ41m31DH2ozlI0Ox0WvLycmSG0kH83pEQkMLLMBmzg7D4JE7N86sAqJERRzm88efgXX7O4XR7QCSA8of3Otv6HpKpzywPo+kU6jHvwyLAbemeBVAa7KN1wiSgtrg4zQAISbps6TroBhoKIfYP42Llp/GiBBBpYCB0UslF6vOVjwoqh6f8KCi/hLQpCjn284dFQLZzymyAan4Abfo8INVDsxr3ayeTANNI/+tf9Z+Uh7cjFEw5/1oM4QAxcVTYX0FdRxcCdA1dSLANkAeZBHsse9TtBGpRFkSVAkKnyCGWvz2om4CfLZmN3TnlRIC4OILtjVJ6dN2yMEULfc/+mNDs4dkl5zO4CgojcPXSgJ6z/xS3YQKG00gSpMumphTCMABvXz+C1CjqlF4nRqkQgLBcILApiWjP3YP7groJaNk9gnbaJB3A94tY9iila1MAXd8but5z7mKEhC+s+hJixgJwe3jkgtP4/IXwb0Ob8VqnoJSGEBLhuIDLbZ/7HNu27qS7pw3bsXBtC9O2sG0Lx/FwTB3b8fAcA9v2MB0dS3cY3L2byz5wNUKk6gTlIkqnVI9edRMgk/7WTHs678OghFUbdQng1AIh3UATHtuee4zTr/a46Mt3kBSqFJ//A7oG73vi14RnLaI6MoTXOgXDcSAOAJfLr7yCMIiZNHECKEGpVGWkUEXKiEQqZBITxxIpJXEckiSQxDFRErFp8xts2LKdJUtuAXSiUonEpiCO5RTQS8mEhu6J6UUQYjSkBKQjnjo6XaTL0F1nXcKdL3+RuFCltHs3QgikhMozT3Pls79niTiNv/dzWHYzSRKgGxUeeuBBrrz6Gta+tgHT1GpWK0lU+g4lFZrQQAgMTUfTBULT0TSNrJNl48ZNQAToVMsBwhTlQ5n+YRMgNE7Idk4GIIqqaKTBz4GWGg1At0iCcL//pRAwsJUsYNnN+27ECRiSB37y41Gto7SoiCQIqVYDwiAkjEIqVZ8gCIj8gHJQZfPr27hm8Uf2PlnNg+84hdSFHJqEuglwHaaI1smpIvHBbCsBTN73L9ew6mc/Yv61NyHDkKDqoxsG2bY2RlY8S+eJo9snoDRq68movsy0CNAdyDoHl+3MN4Ud1SpUyVQ0LTl2OUG7rW0mtVFHxXv/3yeyJEkSdB3O+PSPuEEIPlz4PKd/6u9x/QQ02PHsb7jzg7/ga7UILY7KGLoOhgcYXHH51bS2NZHJOLiOi+damLaN65jYlofrmdimg+u5GKZNxtGxLAfHSZMKUrOYf/KpyBB80YQmyoIxTKB+H9A64aQ9dTVqfdoX5kiUTPYuXTfe/xlyg3l2PvM7dKcBXYMn//1pzrl037NCkAZNwIJT3sbJJ83FcVwEgnyxTC5fRqoYJSVxXHOCSUKcJCSxJEpipJRIqYjCkLyfcN/DPyeJYDjTgkFZF6lpHZSEugjo+9rpDJo9c9OrBFPfN+5No3Kghpk6xhW3L+Kkv/gqVtM4Pu+2kbXBzMBfDaVyyGo/mtuFphnEcYxhaqx88Y+cctpZdHd2ommSUAIqQUkdXSiUSkY5QQ1NF2iagWYIDMPAskz0rM2DP7iP9xhg6KKu3GRdBOT6y5hzpmYB4rCAVssFJEiyTft39fsbxrH+xT5eevwkhAF/8qm5xFEFf6CPb3QJomI6p28eVgjNQxcVkBXQsrz4wrOj3hpAHJAkCWG1QjWMCX2fOIooV30qgU/s+5T8hCQpUypFiEyGSxZdzNLvfBQ1vR7N6iSgnNvotHd9CICoPIJue7V6jmzzvnZPfXYBOwf7ue4FRaX2n+Ctaclc32buEIIlSiGEh4oLCK0M2KReRUvrho1ugGu38JZTj0PJG+ytHpu0eFwKJjV1p5RG5Ty6lbrksFTArQWE4fBGlt6xii8rxeuv/hcirnn40dANlKbj9UzjPa/+F/8oBLcqhTA99myGjgWUDxgC9GNEgIIpjR0TAAirVexsagFhMbd3G1DN9e9zjVJxwExOEqfbxygk3r6B9u7RYlQ448w/ZeGCk3BdG9uy8DwbxzYxbQfHsnBdC9PycGwTN+Nimjq25aCUpKuji2mz5gMhSVDLONeBugjQDabq3ZMBCErDGEbq+fzCMLVcCE0nnMm5H+1l6W3v4Lyv/J5yEKFkiEwiCEOkjFBJhNANEikpbFlBZnz67I5tGzjrne/hwvPfiW17xEIRVKqMlMpoShEkMXGcIGNJLGOSJCaJJXEikTIhlyvgeS73P/wI7bZFVEcIfFgEmE2NM6AFgLBSxsqmwx7kC1h710GNRfds56E/ESy7cTaZxnaMbCtmtgUj24KVacbyGtHbJtO3/PuUlj3OVS+kq0LvhOmsePpXLDz7Pcw7aSZKxoRSIZCoRCBqB5FCAELD0DSEbqBroBk6ruuiCfjD8y9z8bnzSWICvc7ZVBcBVkvvnD31yM9h1XZ/fiGHVVsGY1nG0DJcvlzx04WCzEyLoBxSKqZ5wbCa5g9zG2H+x6Zy2fL0pBflg1C090zl9c1rRr01AkJIAmI/pBpEBEFI6Af4UUgYVKlUI/ygwq5dw8ydM50Zs+dD5RUElFO2joEPUI9/hJXr+hbsufbLeZxsag1RcYiG2jIgFMTV3RhuO1f999ibEAhIkhgVBRjOgXx8LRTWMxgZaMhAQx294glsnwr1ZEOog4ChFcsxJl3Qtuc6rpQw21OH6BdHML09gYBEaHqaEjMsdE2M4l+M+lUkiYQ43SgZtgm4vL7+Ja687hOcvvAUPMfEdByynotpe5i2jm15eJ6D7Vh4to2h6zRkDUzTSZ2jadMxbQHVFa/gn8h6K/KJ/UhpAskhLGFMAgZz/cI7bcre66Baws2kSof5Ecxafb9gMw7rz+iKBn679FE+efPfsOjd78S2TYJIUJEBg1UNX+j4sY6vApzyAErX0cIRpF9BxhGxSpBhgkJBIihlepjx0UdnzFjzr502WwfkwdJq9RJQLBQmTt2TBwCiSgnDa04JKOQQvc28lYHDgMzzrgsv4d6Mw7Uf+QwzZ58ACKTZTCt5Ti79gZbcTmZPhXAH/KjzfQy2zEXXB9CTEoYQCNfA0ExM2yZjW7iVofFy1vu/JZc994GxxBrbB1SY4nVM3ncdldEy6WyM8nnEjMaDPFknpAStwllvX8TG9Yv2u/XYlR289/oCBJOhu5Vo9Q4mrX6EdZMyzL7tfka27KC0eyvl3BBBLDFaJzK08lfs+O0XbxtIzNt7pp7ord7+YoWj2QwJh2lu5z4LCCs+bu0wNKiOIOwT0hvJYR1j7EUcKwzjwFHL6TfcCS9dz6AwiQbKtLa10Xa6y7mbfsjjly7lqkcHYWrv/g9N/08e+OQbJ+/8wAJ2rtsmhSY0ElU7pH8rxiTAaPBmYPfsE9ivYNYOReLSCKZ3dBZgOCbg8KvHf86dX7+buXNm4WQz2C29JPkhbkoiuk916NsZkctJDMOhed5MLnA38ZM5gsw1n+CS2761r8MNeXLNFNAcHV3XGeN4dMyAsbFr2oWjr6uFfgwrdXxReQS95g+OHA18866v8oW/+2fmzp6FbZlElZihvh0MDAzSXYLyxq20t2kIA+IYdo8IstNm8I7rWpn36rf56nmj9uQyhgwVpQltrKw6jEGAWn0tE+ed+dro/9739Rf57Z1XAtD/xz7ct1+ctj3SfYzKc+PNt3DD9Yt55rk/snbdJtZv3sCGda+xdleJd688l6/cXaC8Yi22qtDYkIo8kpd4LV10vmMK1y4s8+Uewca1GyDbgeWQi0LiMJFSpt+yHfT1dYm98vZz1Pxbn957/fBNC2ieNJfzb/khv/zoRC7+zhsAtdMe6vAHe8YmQTf09Ju4/Ta8IRATxVBIdHZXbZ74xGksHPpvxp+dpXPKRHbn9gU6bS0ag9t3UVo3zPbMXNYvf+Wv/7391K+sfW0LuwZGQKmDfiEyJgFq8z2dfY/cv0HMvaTSvegze/dvEbD2sTt44kOf5ab3g3nPES6Dh4GtK5/mp1e8nQ9fCs7M6STKoFpJ35vJCIRQWMMFHvndABj2eVf/UjzJGOcjhyRgMAcdadRL//c+OC3KNG/wzRaMtvEY7dMJkgR/80rWPbGWzq6NnH355TBUBKcB7AzYDenJsdMEVjYtbhPQyFF8o8nDX/gwC4a+T3bmdFq7Mgz2+3vvdY1zCHdXqWwrsGXLyEMLf1q94ogJ2IOvfA1u+zR8/fKJyGD3CRlVnVUqqEmRwQnZtnGT2sf3Tgpe3nSqPzyM0EDoOqYAoRRaIlEq/Y6AEBIfkghkauXIOD3VbvAgmwU3C5kG8Gp1Jwt2i0u2cwKNvVMxuqfBpPlsWP4cq//jB1z2rkmIrhYK+YhqJT0MbGl1UVLiNkVsWrrjX6fdk7/5qAjYg9YGh/Pn9WbPmNXT0py1rOFyYFUC6TY5ImPbdqIl9ETKbw9LpU5kpUMJOizTbs142S67bdy4po7xHdn2cXjN3bgtvXitnThNXVgNjeiaA5ZONVchLg5Szg8RFnKoagniCtufX8qK+374R1Ow0rZpNwV21maC28z8qy6ZQLWhGcM12D1YBcDxDFrGeTx272vfPGWqe+P4Xv3CF2/auezUN+l0WHaoCWHlq6FXCaNsh+Y0tGYtB2IrCCMrUr4h0CUYOxO7uU/KJklCEkqV5EtJnJR2he7WN5qsOGgO/EKbCErdmoqbDEP0ZL2GXqe9a1xz1+Tx2faJZDu6cTqm0NTRw+QTL+GFpd8d3v36M5kZiy9d99rW3E+UhZdTur9id7X/qfW517Rlm35w1uzwz3umt9Az3mN4MCCoRiAlvZnKjas2FXnl92r5hQfQ6TAsQAjH0cTk9qaGBVObG0+a0pFty9qNcaLMKEGUwtAlqX0pq4FUtYVRgoJEkiRKkSQJMWlkEpP60oQkiYMo8rOWaHApdRrVfAdBoctQsrMpay7qV21LBrX2LZms1y4T6YaJNAfzYbClPze46vXyplyulDxztXNCvsrGiy7ogaktUAqo5kt8/+6NXHfXl8jufJAXb1879dSVbDlCAgBN0xpd3extb3Ant2Xcid0NTqPjWAZovoq0MFJ744oDdTxqnUhSD1GjB5RKkvTMIyUlHlUAPEvHMTTDlApzuFqNtg2Uy1t3Vao7C+VcEsUKIPgcrNnWsHrC1MZ5z6zIPampyjsnX/VJ5i3+JsiH4G+ueJu4neePnABA1zTNNDWjwTNFW8Z1Gz1DNzRNIiCU+8KhA0Vhb4oO3rxuylH/q1pF1YTUAKErtFhKvRrEyWAxrBQqURLGcazkW5Mf+b/uWbxzYOiHbW0ZOm6/nV88OMzz3/02v+v+C05vyYtvfuOuw1V9NNJ8k65rwjIM3TF03TF13bV07RgV3lTS/01ds01DN01NpDHswT8iUt+fYAOsfjuT1Qe5AB4A4NNLlhzrgEXsOfv4Xyr1fzml7j5q5Y7jOI7jOI7jOP4v438A9QYrgkAX/mQAAAAASUVORK5CYII=" + "history": [], + "messages": [], + "path": [], + "retrieval": [], + "variables": [] + }, + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAVF0lEQVR4nO2aebRdVX3HP/vM59z75jkvMxnIQEggIKNVQFNYKGCVMUKtIg4UtCwj0mq1ttJSrFIHFtYBcAIsKGBNVCwGKCg0JAESyEzI8PLey3v33fmMe/ePc5O8QJJ3M6z+0ea71l53n3v22ef3++7f/u3f/u0Dx3Ecx3Ecx/H/F+JoO1B9IHpq9We+VH7ln//WE83tGJaN7rholo2WcdAMR1oZO9IcN9JdLzZtN9I9NzYtN9KtbCQcS2potV5lWpRCSoUiASVRSgESJVWtmULXMbW2xm4x/wvuW2SrQ0HjiBVXIESqfPTLj6sV37ubN9asYe6DP4cnfwPVIvh5ZHWEuFIkKvdpwXDBDstFu1wOKBehkodyCUoFKBZAGCBM0AzQTdAs0G0wHAcz04zuNGNlWtAyHTRPH0/r9GkUt8b0iHE8/Wee6p40nh2/WH9W52zaZ10yZ5O4fs3asfQ4KguQv/7c1qce+9FAZvzFCxe+ejfcVwSyR9PlKMQg81DNQTUPlQIEw+BXoLGNTd+9mXU/28hFzz9F8Is7MS2BFgFNraBto+9XTzDu7rH1OyILUJvv+dvf/NUNzktbtk3MzrpsYnVkDdwnAVGX2dUHA7Q2yLRB5q13T/jyRbzaP54d6wfpvfbRN93dxdqP97xwTMQ4EJ64lL98fdkdSiml7r2wTallH1KjMbLsZlVY8zNV3LRMld54Rvm7VqtweLMKy7tUEheVVGMjSioqjkoq9offUqKgoJRSKpEV9W8notY//EWllFKlvpfTh9fdq5ZN4olU2kMPx2FbQHz3uc/pH3vqjJd+fCsAfn6EUt8QWSCRZfrumEj+t8NY4++ChNQTKVA1OTQBaKlcQgPMdO5rBmBB3A+dH/snGs9eQpwUDii+UDFxNYfhtvDB53dw7ym9eE2d9J7/8bSBX6Ca4GNmIaqSCnKMCNgl2pp7gWK5AEBQStAyjQCUnvw7Sk8N0/oOAzEcE+ugN9qgmVAppW+zmkDXIa6C0EG3AIVKElAK3Ssx8N3PIpSk4ZxbSYLcAeUQGsTRCHbDOK5Zvpofn3sylz7QTddpl0FYwodqttmiNBSAlCJ120dJwOZvLCJuGD8DQIh0bKIqaG5z2qBpApYL3hm30nDmzSSV7eQe/UtiVcGbsxhZ7id65SFUEqG1zyL2h4mHt4GKEb4PERguhBJkVKy9VedgIyikIgpzZMfN47LHf83P37uIGzYqmLYAmVBu8WzKI2WUPLhOh0XAwJpVdL33PE0BhmkCoKqguU1pAz8V2uo4ES3bjpZtx+iag9U2g+azPw3AMBp250wy868DIPf0P2A1zyRz0vtTQgdWs+26+YjsuLpk0hTIOE/n7Hdz0X0/4fHrJ3LxeW+jaQoVs99C41AT4DAJGCn1907vnkopN4jppsudjEC30ykQbn6S2IT8Q4sZ/vbilPluUBGUfnBLGt+Mg0oCuXv+HKWDaIGygMGfgp5pxLCzqBYQxh7XfyjxU6hEIMUIE86+Cr25nVWffDduL0XVP7ZO2thNYI8nLRSZmu2eRJgfxnQbgHQqY6ZBWMvl92ONh8Im8E0IXYgqEEuIWkF21nyipSO7LOj00OxGNLsJ22vEQEBYJegHVdxZn2gAJCilESUlxs15Fwu+dw+v/46S0BPEGATWaQGp/3A0plntk/HXb0C3UwtIQhBOOlqG003np56m8uw30DOdEJVRfpE4KiHDEgRFVFgk8UuoqAhBhSgCFaSWJGKorIEJd9xJ41m31DH2ozlI0Ox0WvLycmSG0kH83pEQkMLLMBmzg7D4JE7N86sAqJERRzm88efgXX7O4XR7QCSA8of3Otv6HpKpzywPo+kU6jHvwyLAbemeBVAa7KN1wiSgtrg4zQAISbps6TroBhoKIfYP42Llp/GiBBBpYCB0UslF6vOVjwoqh6f8KCi/hLQpCjn284dFQLZzymyAan4Abfo8INVDsxr3ayeTANNI/+tf9Z+Uh7cjFEw5/1oM4QAxcVTYX0FdRxcCdA1dSLANkAeZBHsse9TtBGpRFkSVAkKnyCGWvz2om4CfLZmN3TnlRIC4OILtjVJ6dN2yMEULfc/+mNDs4dkl5zO4CgojcPXSgJ6z/xS3YQKG00gSpMumphTCMABvXz+C1CjqlF4nRqkQgLBcILApiWjP3YP7groJaNk9gnbaJB3A94tY9iila1MAXd8but5z7mKEhC+s+hJixgJwe3jkgtP4/IXwb0Ob8VqnoJSGEBLhuIDLbZ/7HNu27qS7pw3bsXBtC9O2sG0Lx/FwTB3b8fAcA9v2MB0dS3cY3L2byz5wNUKk6gTlIkqnVI9edRMgk/7WTHs678OghFUbdQng1AIh3UATHtuee4zTr/a46Mt3kBSqFJ//A7oG73vi14RnLaI6MoTXOgXDcSAOAJfLr7yCMIiZNHECKEGpVGWkUEXKiEQqZBITxxIpJXEckiSQxDFRErFp8xts2LKdJUtuAXSiUonEpiCO5RTQS8mEhu6J6UUQYjSkBKQjnjo6XaTL0F1nXcKdL3+RuFCltHs3QgikhMozT3Pls79niTiNv/dzWHYzSRKgGxUeeuBBrrz6Gta+tgHT1GpWK0lU+g4lFZrQQAgMTUfTBULT0TSNrJNl48ZNQAToVMsBwhTlQ5n+YRMgNE7Idk4GIIqqaKTBz4GWGg1At0iCcL//pRAwsJUsYNnN+27ECRiSB37y41Gto7SoiCQIqVYDwiAkjEIqVZ8gCIj8gHJQZfPr27hm8Uf2PlnNg+84hdSFHJqEuglwHaaI1smpIvHBbCsBTN73L9ew6mc/Yv61NyHDkKDqoxsG2bY2RlY8S+eJo9snoDRq68movsy0CNAdyDoHl+3MN4Ud1SpUyVQ0LTl2OUG7rW0mtVFHxXv/3yeyJEkSdB3O+PSPuEEIPlz4PKd/6u9x/QQ02PHsb7jzg7/ga7UILY7KGLoOhgcYXHH51bS2NZHJOLiOi+damLaN65jYlofrmdimg+u5GKZNxtGxLAfHSZMKUrOYf/KpyBB80YQmyoIxTKB+H9A64aQ9dTVqfdoX5kiUTPYuXTfe/xlyg3l2PvM7dKcBXYMn//1pzrl037NCkAZNwIJT3sbJJ83FcVwEgnyxTC5fRqoYJSVxXHOCSUKcJCSxJEpipJRIqYjCkLyfcN/DPyeJYDjTgkFZF6lpHZSEugjo+9rpDJo9c9OrBFPfN+5No3Kghpk6xhW3L+Kkv/gqVtM4Pu+2kbXBzMBfDaVyyGo/mtuFphnEcYxhaqx88Y+cctpZdHd2ommSUAIqQUkdXSiUSkY5QQ1NF2iagWYIDMPAskz0rM2DP7iP9xhg6KKu3GRdBOT6y5hzpmYB4rCAVssFJEiyTft39fsbxrH+xT5eevwkhAF/8qm5xFEFf6CPb3QJomI6p28eVgjNQxcVkBXQsrz4wrOj3hpAHJAkCWG1QjWMCX2fOIooV30qgU/s+5T8hCQpUypFiEyGSxZdzNLvfBQ1vR7N6iSgnNvotHd9CICoPIJue7V6jmzzvnZPfXYBOwf7ue4FRaX2n+Ctaclc32buEIIlSiGEh4oLCK0M2KReRUvrho1ugGu38JZTj0PJG+ytHpu0eFwKJjV1p5RG5Ty6lbrksFTArQWE4fBGlt6xii8rxeuv/hcirnn40dANlKbj9UzjPa/+F/8oBLcqhTA99myGjgWUDxgC9GNEgIIpjR0TAAirVexsagFhMbd3G1DN9e9zjVJxwExOEqfbxygk3r6B9u7RYlQ448w/ZeGCk3BdG9uy8DwbxzYxbQfHsnBdC9PycGwTN+Nimjq25aCUpKuji2mz5gMhSVDLONeBugjQDabq3ZMBCErDGEbq+fzCMLVcCE0nnMm5H+1l6W3v4Lyv/J5yEKFkiEwiCEOkjFBJhNANEikpbFlBZnz67I5tGzjrne/hwvPfiW17xEIRVKqMlMpoShEkMXGcIGNJLGOSJCaJJXEikTIhlyvgeS73P/wI7bZFVEcIfFgEmE2NM6AFgLBSxsqmwx7kC1h710GNRfds56E/ESy7cTaZxnaMbCtmtgUj24KVacbyGtHbJtO3/PuUlj3OVS+kq0LvhOmsePpXLDz7Pcw7aSZKxoRSIZCoRCBqB5FCAELD0DSEbqBroBk6ruuiCfjD8y9z8bnzSWICvc7ZVBcBVkvvnD31yM9h1XZ/fiGHVVsGY1nG0DJcvlzx04WCzEyLoBxSKqZ5wbCa5g9zG2H+x6Zy2fL0pBflg1C090zl9c1rRr01AkJIAmI/pBpEBEFI6Af4UUgYVKlUI/ygwq5dw8ydM50Zs+dD5RUElFO2joEPUI9/hJXr+hbsufbLeZxsag1RcYiG2jIgFMTV3RhuO1f999ibEAhIkhgVBRjOgXx8LRTWMxgZaMhAQx294glsnwr1ZEOog4ChFcsxJl3Qtuc6rpQw21OH6BdHML09gYBEaHqaEjMsdE2M4l+M+lUkiYQ43SgZtgm4vL7+Ja687hOcvvAUPMfEdByynotpe5i2jm15eJ6D7Vh4to2h6zRkDUzTSZ2jadMxbQHVFa/gn8h6K/KJ/UhpAskhLGFMAgZz/cI7bcre66Baws2kSof5Ecxafb9gMw7rz+iKBn679FE+efPfsOjd78S2TYJIUJEBg1UNX+j4sY6vApzyAErX0cIRpF9BxhGxSpBhgkJBIihlepjx0UdnzFjzr502WwfkwdJq9RJQLBQmTt2TBwCiSgnDa04JKOQQvc28lYHDgMzzrgsv4d6Mw7Uf+QwzZ58ACKTZTCt5Ti79gZbcTmZPhXAH/KjzfQy2zEXXB9CTEoYQCNfA0ExM2yZjW7iVofFy1vu/JZc994GxxBrbB1SY4nVM3ncdldEy6WyM8nnEjMaDPFknpAStwllvX8TG9Yv2u/XYlR289/oCBJOhu5Vo9Q4mrX6EdZMyzL7tfka27KC0eyvl3BBBLDFaJzK08lfs+O0XbxtIzNt7pp7ord7+YoWj2QwJh2lu5z4LCCs+bu0wNKiOIOwT0hvJYR1j7EUcKwzjwFHL6TfcCS9dz6AwiQbKtLa10Xa6y7mbfsjjly7lqkcHYWrv/g9N/08e+OQbJ+/8wAJ2rtsmhSY0ElU7pH8rxiTAaPBmYPfsE9ivYNYOReLSCKZ3dBZgOCbg8KvHf86dX7+buXNm4WQz2C29JPkhbkoiuk916NsZkctJDMOhed5MLnA38ZM5gsw1n+CS2761r8MNeXLNFNAcHV3XGeN4dMyAsbFr2oWjr6uFfgwrdXxReQS95g+OHA18866v8oW/+2fmzp6FbZlElZihvh0MDAzSXYLyxq20t2kIA+IYdo8IstNm8I7rWpn36rf56nmj9uQyhgwVpQltrKw6jEGAWn0tE+ed+dro/9739Rf57Z1XAtD/xz7ct1+ctj3SfYzKc+PNt3DD9Yt55rk/snbdJtZv3sCGda+xdleJd688l6/cXaC8Yi22qtDYkIo8kpd4LV10vmMK1y4s8+Uewca1GyDbgeWQi0LiMJFSpt+yHfT1dYm98vZz1Pxbn957/fBNC2ieNJfzb/khv/zoRC7+zhsAtdMe6vAHe8YmQTf09Ju4/Ta8IRATxVBIdHZXbZ74xGksHPpvxp+dpXPKRHbn9gU6bS0ag9t3UVo3zPbMXNYvf+Wv/7391K+sfW0LuwZGQKmDfiEyJgFq8z2dfY/cv0HMvaTSvegze/dvEbD2sTt44kOf5ab3g3nPES6Dh4GtK5/mp1e8nQ9fCs7M6STKoFpJ35vJCIRQWMMFHvndABj2eVf/UjzJGOcjhyRgMAcdadRL//c+OC3KNG/wzRaMtvEY7dMJkgR/80rWPbGWzq6NnH355TBUBKcB7AzYDenJsdMEVjYtbhPQyFF8o8nDX/gwC4a+T3bmdFq7Mgz2+3vvdY1zCHdXqWwrsGXLyEMLf1q94ogJ2IOvfA1u+zR8/fKJyGD3CRlVnVUqqEmRwQnZtnGT2sf3Tgpe3nSqPzyM0EDoOqYAoRRaIlEq/Y6AEBIfkghkauXIOD3VbvAgmwU3C5kG8Gp1Jwt2i0u2cwKNvVMxuqfBpPlsWP4cq//jB1z2rkmIrhYK+YhqJT0MbGl1UVLiNkVsWrrjX6fdk7/5qAjYg9YGh/Pn9WbPmNXT0py1rOFyYFUC6TY5ImPbdqIl9ETKbw9LpU5kpUMJOizTbs142S67bdy4po7xHdn2cXjN3bgtvXitnThNXVgNjeiaA5ZONVchLg5Szg8RFnKoagniCtufX8qK+374R1Ow0rZpNwV21maC28z8qy6ZQLWhGcM12D1YBcDxDFrGeTx272vfPGWqe+P4Xv3CF2/auezUN+l0WHaoCWHlq6FXCaNsh+Y0tGYtB2IrCCMrUr4h0CUYOxO7uU/KJklCEkqV5EtJnJR2he7WN5qsOGgO/EKbCErdmoqbDEP0ZL2GXqe9a1xz1+Tx2faJZDu6cTqm0NTRw+QTL+GFpd8d3v36M5kZiy9d99rW3E+UhZdTur9id7X/qfW517Rlm35w1uzwz3umt9Az3mN4MCCoRiAlvZnKjas2FXnl92r5hQfQ6TAsQAjH0cTk9qaGBVObG0+a0pFty9qNcaLMKEGUwtAlqX0pq4FUtYVRgoJEkiRKkSQJMWlkEpP60oQkiYMo8rOWaHApdRrVfAdBoctQsrMpay7qV21LBrX2LZms1y4T6YaJNAfzYbClPze46vXyplyulDxztXNCvsrGiy7ogaktUAqo5kt8/+6NXHfXl8jufJAXb1879dSVbDlCAgBN0xpd3extb3Ant2Xcid0NTqPjWAZovoq0MFJ744oDdTxqnUhSD1GjB5RKkvTMIyUlHlUAPEvHMTTDlApzuFqNtg2Uy1t3Vao7C+VcEsUKIPgcrNnWsHrC1MZ5z6zIPampyjsnX/VJ5i3+JsiH4G+ueJu4neePnABA1zTNNDWjwTNFW8Z1Gz1DNzRNIiCU+8KhA0Vhb4oO3rxuylH/q1pF1YTUAKErtFhKvRrEyWAxrBQqURLGcazkW5Mf+b/uWbxzYOiHbW0ZOm6/nV88OMzz3/02v+v+C05vyYtvfuOuw1V9NNJ8k65rwjIM3TF03TF13bV07RgV3lTS/01ds01DN01NpDHswT8iUt+fYAOsfjuT1Qe5AB4A4NNLlhzrgEXsOfv4Xyr1fzml7j5q5Y7jOI7jOI7jOP4v438A9QYrgkAX/mQAAAAASUVORK5CYII=", + "parser_ids": [ + "resume" + ] } diff --git a/agent/templates/ingestion_pipeline_table.json b/agent/templates/ingestion_pipeline_table.json new file mode 100644 index 0000000000..e800d1e2bc --- /dev/null +++ b/agent/templates/ingestion_pipeline_table.json @@ -0,0 +1,308 @@ +{ + "id": 42, + "title": { + "en": "Table", + "de": "Tabelle", + "zh": "表格" + }, + "description": { + "en": "This template extracts and segments tables from parsed documents. Best for spreadsheet files, financial reports, statistical documents, and any content organized in tabular format.", + "de": "Diese Vorlage extrahiert und segmentiert Tabellen aus geparsten Dokumenten. Sie eignet sich für Tabellenkalkulationen, Finanzberichte, statistische Dokumente und alle in tabellarischer Form organisierten Inhalte.", + "zh": "此模板从解析后的文档中提取并切片表格，适用于电子表格、财务报告、统计文档以及其他以表格形式组织的内容。" + }, + "canvas_type": "Ingestion Pipeline", + "canvas_category": "dataflow_canvas", + "dsl": { + "components": { + "File": { + "downstream": [ + "Parser:HipSignsRhyme" + ], + "obj": { + "component_name": "File", + "params": {} + }, + "upstream": [] + }, + "Parser:HipSignsRhyme": { + "downstream": [ + "TokenChunker:FastFoxesJump" + ], + "obj": { + "component_name": "Parser", + "params": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": { + "spreadsheet": { + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "xls", + "xlsx", + "csv" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "txt", + "py", + "js", + "java", + "c", + "cpp", + "h", + "php", + "go", + "ts", + "sh", + "cs", + "kt", + "sql" + ] + } + } + } + }, + "upstream": [ + "File" + ] + }, + "TokenChunker:FastFoxesJump": { + "downstream": [ + "Tokenizer:DeepLakesShine" + ], + "obj": { + "component_name": "TableChunker", + "params": { + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + } + } + }, + "upstream": [ + "Parser:HipSignsRhyme" + ] + }, + "Tokenizer:DeepLakesShine": { + "downstream": [], + "obj": { + "component_name": "Tokenizer", + "params": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + } + }, + "upstream": [ + "TokenChunker:FastFoxesJump" + ] + } + }, + "globals": { + "sys.history": [] + }, + "graph": { + "edges": [ + { + "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", + "source": "File", + "sourceHandle": "start", + "target": "Parser:HipSignsRhyme", + "targetHandle": "end" + }, + { + "id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:FastFoxesJumpend", + "source": "Parser:HipSignsRhyme", + "sourceHandle": "start", + "target": "TokenChunker:FastFoxesJump", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__TokenChunker:FastFoxesJumpstart-Tokenizer:DeepLakesShineend", + "source": "TokenChunker:FastFoxesJump", + "sourceHandle": "start", + "target": "Tokenizer:DeepLakesShine", + "targetHandle": "end" + } + ], + "nodes": [ + { + "data": { + "label": "File", + "name": "File" + }, + "id": "File", + "measured": { + "height": 50, + "width": 200 + }, + "position": { + "x": 50, + "y": 200 + }, + "sourcePosition": "left", + "targetPosition": "right", + "type": "beginNode" + }, + { + "data": { + "form": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": [ + { + "fileFormat": "spreadsheet", + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "vlm": {} + }, + { + "fileFormat": "text&code", + "output_format": "json", + "preprocess": [ + "main_content" + ] + } + ] + }, + "label": "Parser", + "name": "Parser_0" + }, + "dragging": false, + "id": "Parser:HipSignsRhyme", + "measured": { + "height": 57, + "width": 200 + }, + "position": { + "x": 316.99524094206413, + "y": 195.39629819663406 + }, + "selected": true, + "sourcePosition": "right", + "targetPosition": "left", + "type": "parserNode" + }, + { + "data": { + "form": { + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + } + }, + "label": "TableChunker", + "name": "TableChunker_0" + }, + "id": "TokenChunker:FastFoxesJump", + "measured": { + "height": 74, + "width": 200 + }, + "position": { + "x": 616.9952409420641, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "chunkerNode" + }, + { + "data": { + "form": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + }, + "label": "Tokenizer", + "name": "Indexer_0" + }, + "id": "Tokenizer:DeepLakesShine", + "measured": { + "height": 114, + "width": 200 + }, + "position": { + "x": 916.9952409420641, + "y": 195.39629819663406 + }, + "sourcePosition": "right", + "targetPosition": "left", + "type": "tokenizerNode" + } + ] + }, + "history": [], + "messages": [], + "path": [], + "retrieval": [], + "variables": [] + }, + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAGP5JREFUeAHtW2lsXNd1Pve9N/vK4U6KIiWRWi1rceQlm7ckTp3WrVuoSNP+aIImqFMgRYKmAYoGcGC0QNog+dEATYKkaZ2mTqLYaVVbst16gzdFlhd5ky1TIimS4jYkZ1/ednvOuffNUF4ayUtRILrGiG/evOXes37nO9cAF8ev9xDw/2AcO/a5UKIwkk4PXdYtITYYS3d22jV7on/TnqfgPR7/pwI4ePBz8eHQ9qHsurGxaGZgfSyaXScsMWKAsQGEv8EMRbtD4ZgB0pWeD5Wpk7/82NiuG4/AezgseJfHQz/8YRQSx7cObb5mS65703ozlBoyLGO9Ycj1phXuEmBkDdNIGoYJ0vdRAT7eJVkTUkrwnSr+9YQwRDIWT+3H0++pAC7IAo4dvDUeivQmc4MDafBjg6F0ZtAUoVFc0DZhRLZF4uleIaAnEksJtTgPV+VL/OASaZG8TH4vfecVS6mOeUjpe65AAaBcPFmrlWY61+1bD+/heEsBvHzs4N6+wdHrUUHDOJthYYZHwpFYzgyH09IXccsKCcCJ8mR57qhJgcvnxfDi+C+e4kV5vit9p8lrbgkAlHxoGnQd34EC4X/xG1oB/h7+luvJGP4c9V0nJQ0jhMdxywjFBR7jjQkwhHAbjbsf+tLX/ur3Dxzw4ALGm7rAj7/z5bF1G/Y+HomEIr7vaSlpWfmkSV/6LmnVB6fZgKWFWegZ2EAmLUkgnucI3/NQiS4tCa3egnAkDlYkB77bENJ1wfcdfJaL8/cluoYIhyMsASldMgoSJf5LLxVfZLFKPV36AY/JhQC0qHzXsLp6t3V/at8dcODAc/AOBSBWFiqj9VopEgpl8KvBL9Uv49fjOnGCPni4kLnpccj1rBOmIaSwUCEQxitjPHvDMPBWuh//4rGBf6VM4DmTn8kPw8U4aBmu0+DvPrkOv87nVZNASTL0H13jubZgLdDvOA8UHL/Hl0Xo7Vt/Fd54QQI4xwV+8t2/2bv3qqu/PTl+YnPfwGBu844951wmOVaRADy5ND8tmvUqLMycgm17rxbJVFpS5MKFSu0JrEb8l86zDIXWGv0gA7sShtQH5A70CKnULVqCV0MqaQnlPzgH8i0SCrmWcNHaPNep1GvVx2278vmRrVdPwHkMY+2XWDz+O/i5Ev27s7C6AmiaSnv4n+s6UK2UYHV5CU48fxRisSh4dgMKK8s82UDDNGcjmDx+J+3oC0DNXlsEW4AZLEkosQiWGz5BCUhfq+/l83QX3cNHeLWh5WuwRTSSzerSx8KhxN333X5rzwULoFwsLTiOA+lMFmbPTIEwrUD3+FsBmo06BTSolooobQ9N1+aF8xo4yPPcwMbzjm2z0FzHbZ1nC5KtR2qXCobQwg4ODfXM1j2BKyq3oqULMwQGfkz6oPtZVgSPLbBCoe2XXXfzvz1x7/dzv0oArRiwff/f9T045Wd2FEuQSKYla7xchmjE4gnlurppQaJeKcpmvSKqpQKFQtYkal6ZvV5NrVZBAThqIbRMMm3TlHRdoD2pMwROWmQ6e/lYC0YJjLIBuQffod1FewbdTAKixIMC4Gs4WOPiw+EYB8ZUJnfdhu37bsOr/+xXCmD9B2/ZGBnbffLeyXHjuvFpec1VuwSmPLEwc1qOjG4F8jUMPlA78V3pnT0M0XFfNtfdJkwdxWg+tFIVtAEy2U6WB85IG7chVdAzOPkJ7Q6SZQB6hVIEcECqNCBblrF2sGBUtlVp0lTuh5mGMoNhmspo8LpMKnnLi0d+MX/JlTff9lYCYBcIi3zZrVcNI5qAJ54bh0g4JhLJJPr3EgcuisS1lVmwp/9RZOrPgShOg10v80KC+atoz84OdrMpK6WyrFUq6DZNdhf1qrWfYG2BF7CvK5AAGk5ACzsJZRzQSh0qIOuYwcHX1HPlK/HQIrMTAyOb//LFI/9xLfxvFjD+6IGlsY3XzoUS3f2TpTCsrq7I/v4BePH4Mdh71bWU11mbrteBVxcgYqJ/o48j4lszS0pbJlvCC88ehfzSAmbFkDRxUuSXpmWij5Kf4jnTEoaFvoqTDIXD6LP4wYBr4jmTXEWnT9SmxGfQ/XiPCfFkRqqAqJIGi80wCXSyxbB0CH96DsuK7o9GE8mhsZ0/GD9+35Wju25YfFMB0PCrxTuiPSNfOv6aCROnxmHTpg0IXqJoBXkIWwh6DII9PfiKCRSAK0q1EqQ7e3j5hno56LQHu/ddBc2mLTEtESTGwkahQtfzECD5lMMofuB/hP0djiW2XQOvhgDJIyDlspaFRs5Sm8C6kc2EOdTJlmuQAVjStZtCI2s4/fLTsHT2FOS6eyHbtR4zVnIkne39CV583VsKoFZcud9y7C85iR5x/LU52LHzEvYrcoPurg7UeBNsPyMlKsAyELg0KqxR5cRSZWcWBMF4H91SQCgaBdI0AR/SKgUsjRXYTFvpPsiEAYJW65L0HApuJGAPj9GKpI4VVEGw1jHC4u0W4w96BqXuaCIjzkyckc8//QzUa2UYGh6CXVfecO348Qf+YnTX9d94UwGYlcljZm3rfCSR6vuvF0/D732iCdlsFhbnpqGrKwd2oy7qdhJEHGQE51HPn9XzBGjVMlpXiOxko14TGverqo9qAtQ8uROKAi0KJ4s4QPCkLRKUoGApNGYI0qtyCZOzBBqLoO/sHqh1DoKUEDlLWgQMIRJNwsDQJplIZkW5tCpLmL4pmz106E7YfcWHvvrwXd96/Jrf/eKTbxDAzJMHVjYOXna4e/TST89MpCC/iJrv7YXXXjkBl+y5HFOgK+teWjiuRJ8FqFdWCRMSNmmLQEd1yiDhkNI+aV7qIKkQi1IdZwRJKmTIi27i8VNI4+QaoCAxwU5+D7uO9CS5N1/D9YSn86zklOI5hD1sFJoBnb39kMxkoKOzCqgMqFfLspifTw1vufyzONM3CoDd4NjDt4Xnn4ldMmh8cmE+Bzt37mCcjjldUJ1TtjtkDRdMZVBlleIJoT9T5T8VkLmWqyJWqJSKIqjogrxNEJ7/w0V19g3ByuIszx6DlaBAiZGbpUkBj9wFNY/R3OCgaZL7YCHIRRC+x6DAypZh6Ezh84cKMZQSArAmWkNNIKfAx4RdyB0wgX/g61+/KfWVrxwsv0EA85P3Tu7f9huf3nn53puWFs7G7a2b8QUWmlBJGqEYJIe3wZ3/eSUYzpJcMMPiCt9vpXBo1YwCUqkOTKMZTksYCyhFBVFZV3EquqUynVw9kmv4rGmuM6jY5CoZgwBahkNVpXRRu6h1PO8JjypN9H2KooRMVSxo+yG9k7yjAwFWKBIXHgoArRKi8QS+OhTv87LhN7UAGv9w+HDz21fvvCcdie8vF1ahq7sLJsdfgU1bt4MVXYIN1/0BrOSXwSkUGAoH+F9HL7YE8r1KpcjubIBQ0AX9m+BqrnsAllHzAnQAk6DSJKZMEhDHAUPXEHQX+jpyECpMGhQPFDJWVmLx+3XZpRAAuQJXjFJS6pWUUbAUD2Ml6dgximVN1yzYbykAHr78UTwe25/PL0Fffz+ceOkF2LFnnwiFoxBPpGShUMLUEtPL1lYQ4BIcHZ09oqOrN0CBXLzqyQnC7r2Dw4zoOG3hjz6nL8kcAvs7WRYbAPINDvIH5HRoDbLlQopOQR6AY4NiBXzIdvZjbCqiz5clCpQTc9+6jcoNIcpQGWuUYn/sIyiAg28tgBefOn500ydvKkyfmero7e2XhOtLq8syHI2BWatxPidNkSkq85dr0kGL9uKcHMA4RoqYqmihlD651geNamUAj7WgQEO+9o+0alUKiIBMEro4V190OQaJdIeCZb7CEwHsVqHDhKbjlG78wheawbuMNxPAd35x/2J+tfhEIh6TtXoFzTMMpdIqBhU0IcxFCJBkCM/VUBgBWFFBSJeFXBfohI3R3LGbktyFBSZkC+MLXSUyByDaHxEcq4dpUGzI4Pq2t8s1sxbtD2VF1DbOUWozDDIQCaW4dq1vyQqfeOnEz/a979Ibl5cWYXhoHZyZmoLL9l0JKawR5udm0ZoMoNIZjNfLUEJ14qCw8n8Psv9rILo/BMXVPGpEsmkTiyS5ipQy2zkoKuUV9lmFJBnMCAWdLSK+BMNiSp0U8VlleA7xQ5BiKQsxXGarUiUZ24wI6kiNGAML873V8xJAPl84hFNxC4WC1dvXD9WZWUJliO0NiEQiFJEFFjsyZApoMUCG8gUD2WCzvgy+4QCSoeAxMjQI1yOFFeW6AFWEbKALyVSWtUULiMbSwnXq7NcU5bnc5rpHakzA/AEDKiRIweP44TF8pnOcDDzmJYXUFFM0lsRyu0dbnEGcRum8BPC9O+7Of+SjHz6Eb7uJfIect4IcgBWy2PSSyaQsFovQnUtrCStAQmYaG7kZvIEb0FWSZHKyI9dJBKGi+FQgEwxwgFgtrA98R9pIrlKKLK4sBEGQ3UcqdITr8jXqJhqdgKQurQ3NHeAEKLZEEynI5Ho0AeNrh6Gffb7M9Rvl8xIAjQceeOTnN95wzU3jJ1+Fnt4e9vl4LIZVnInEQxgBTxl6u7Ig1nqBNjwzHCd70xVeiBEjk6FaWuoP+6xsFwQguweGNcbXya11vSZW2hyS0J0FRaW1gt45HJOekq9cgYBS0145bwGs2NV/R4taQWSSo3VNjJ+EXXsvYxNuYvVFxoYQWVdsejn4j7NyTIr87QK6PwNubItYWV4K0gSvTPm1SR4uAnAkFN5XBaVhaT7FYrmp2l4FswgCGhdjD5W6CjeoKNwGWVKTJqrLEPQnyBoIKteKs+dvAQcOPFz5+Ac/9EDYEvupiEE4yQYVQq2uFosyhIJooOkGFEagHnfxcRFb+Qk0rb0Qzu6GdLaLqzWKAxQhqPBB02f0hplBM12SfZrNnfzbc7Vve7wAn6+TshMp+Pz8GVVgGdRE8lno5DQk2MH1W7gGaWVmFggRph7VCXIlP31+WSAYE1PT/7Jr2+h+x3GRkEhBpVwhDE7YgAsaYnsC8kZq84+M3iKbpZtFpHME6o0alIrLKizrTJDr7oc80uqBfoyAViJ/9qUIUp/CUYr2RSir3APPrtu4HQI6XWPggEsFnZNV+lPwGr96KC9HOI0q1BYmqhckgMbS2UcyH7hi9tT4ycGenj44MzEOPT09YOYtNqk1QECoSVEjKIyLVy29eCwpY+sSCom0egUGJFIUPJkX0EWk0EoLgI1aIGF/kK+flWijLNAVtMalLZBBWYG7rS7HB99tYkWYh1gu07ggAXzjR/dXR3df/kgmnfpUPBaGxYV52LBpjL0VI7NU9bkmNDQi85uL4C3cAVbfH0LNjYn84ny7KlScnmgRPYbQNJmpuCUiNlrsH+F9QxH/WszEp5CwYomMJGAGurMg/TbfztUUQWteuMOL970G1JZehkbNdi9IADSOHX3627/1m9d/an5+idAV19e29n1KT6DxWWCLtZO3Q7L2TYwPMYhv+VPZxcHOCohKba+GUjp1jn2piU6pCiQFc3XyD2C14gX4SHED3JtkKUrRuiZop3F1idqnUthDuo0E7czeBYab8S9YAN//6eEj1157xWuOa491dvXB0uIcw4x0Ogm1apXkLSh+mdp3I8N/BNXpTogM/DYjv4ruIVAa0z4POma0aLFWbUDFUGu7gC/akVzZhGFoxAiKU6ECrat3iKs+9QwGSRxwXK4A6+A0ShCNhsBfOYqP+ShcsABo1Kru7el05jbs9MDyUl5G43ERQdK0UaeYEtQlKoBZ6X6wdnxGpyih2VwuYnkhqjpTCLJFp7faZYZcA1/lGu9iMSCuaFW+TLCwmdv8V3LWQK27DelhmnaaVRRAFTGLBacf/R50JXCuDXh7Ajh5evLwnp2bv7a6smxmshmUfCwogKBdoChz5D0Buu3lUbsc/VA5iAsBoNGKJ25YaxsUW+wr81Va90Wwb4AWS6CqZ2CEvijgQ91iRJp4LMjXqbVOWcl1ati2r+H1WLRZIWxjPAb1qXsgtAWz1qov3pYAEt0zz4PY/JhlmR9myFpYhjSVnrrNpRGZjsNaHPgqoq0yuS5VlrB8WrWDklxQMQeoUO8YaXeGQHEHql3OUd0hCt03kJ+IScvQcDlMLoAR37OFZ0uB+ofS0gTUC+Mw89SdcNkeE6l3Q+lgzTDOVwC33vqw+8r45J0UeWnJ2P3hOZu6gdru7a0tV4OPFLqYXysf0MAQ2qWsPtvuEzKMVc0/yWCGWnT5uUnMAkkszJAApY9ogCnrILwK+M1V2SjPy+LiKVnKj8PsM4dgKPYSZEaHeb5YIYu3JQAap06d/kUmk5lbWFwSqXQaGo06FXUt7BFoTx2vWajOeW8cOtoFPt26X99EmUBRQOwsxO25dl0iwwtqHxGavYfchtuU5A52rQjV4gIUFs9AfXUCVieegHj5CAzvG8SVp9T+hLdrATR+fNfDM1PT8w9GI2EZJsoJAxe2t1qpSleE7A68kPb2pyCXtSUCsp0F1Lm28DSRQr7N6c5zsUxuCKLHmo2asNG/KepTfnfx2K6XRL2ch+LyLCzPT4BdGofxI3eL+quHYdv7chBGeg5ot4sgJst5+xZA4+ixZ7/R298vVldWgUgR+qi1ybYg1qy39TYdHcTrcZ1mtgA0iyRaeI7JP47utMHKo56ALSmteUysNBjaNlHrlcICUuxTsDz3GjSXX4LpZw9DrHgUdr8vDqmxIdY+WEnVoQLvnDVf8D7Bf/rpfcfff8XeE81mfVuoFoFsOqWpO18FbSMoCYLgGKxN6J6W7p7wX8URtsC83gukXChoiLj8IfO37QYWZFWsAiPYmivLeimPGCMPtVJe1stnwWychlef/G/osebF9j0R6NpBmk/iKpEOt6JM8Qsh35kF0DgzffabWey6VCtVCNpTep+ADBhdJjxU8SM0yAsWpSMfd3iEXqzecuezufvMIvGeH3B5twkuvEHdnYooIGFiYQouLJxGen2S/kq0dahNPwrP3PMzWBdfgO2XhqBrW6cUUeQkDOQiMEz6nuIk8KEmvBMLYAHMLN1z6SWbC6uFclaxbkJbgNrGomCrynFKyx4orGBo+9doVDMaQSpUwpPMEhEN7vMmKBszDvp9oyrrlQI2axfFlks2yulXHgHTWwCoTMIvH30UIo0ZuHQjwMiWEHRsSkuIoc9bWHCZMZ4TdfhDEYRivvfOBfDPBw7NX3/NvjuwcLnFAFN5MKUpQxEPElo7QAiYKoRHmUwEtLZsU9wQoCBtRT7vL1SLt23hYoRHAcgGtuORB0A+YBBmThyCkDMFTz34CDSXTov1XY4Y3gxyZJsB8V7UNHaxwIgISdtnmGwxmK0z40nsW9rvXAA0Hnry6e/t2jzyx8IUMR23qe5mrOOrHQvqQl9tieO6XoW31m9SQUH9xdctMt5oyawPBjrpYKqtVQvY5q5ArrcbKnNHYfL4g3DymWfEQKoht2PVvWmTAb0bLbBSEWDNx7pR8x1od1m0KizbK3XhlB5HcmYIGr54dwTwg3+9+/m//vNPPrjNGPsEcjmoNYO5Uw51zNAEWB+UG6gNrKpdpmWjt8VqvEPkKLFBTUG+72MbmOKFFQ7J2kJZTJ58Tk4+e7dYmXoJelOOvHREyJ5eC5IdMWiEQzC1iGzzagozBFJmLr7Vr6Ewm0TfilAiJ20nI8bevxEiqXMowbcvAJr5p8/mD+ESPkGbJ6LhsAz2AQjaTqIMXtdyvt70pMyf94jrtjjjel3+MNdvJjHaF+X05CSceOGYnHr1OBZfcxLcssgkIjK7YZ9IdGSRpcxCI9cBkWxORtJZtPoENj8zEMIGaDiWQeuP4LOwp2jF8LkROPv8j9E16kjWOu+OAGiEo9GpaqUCS7OvQnrHBwG0v2tepCUrj3t+oBoj6N9YIhOgoe10oloqwXJ+DlYX5+TZmQnM5/NUyAKCLaLeoae/F0a3jkIm0yFzuOBMNiuJmosjoxSNpcCMJBDghdH6wpgeo2h9Jqc7oA8FXS6vPewJ59AtSiIer7x7AujqzhbKxVUswqriift/hD7ry3qDGxvEwgnunnPNqksAtf9HUViqiSIsZnepwyPEhg39cvPoEDJExAQJ3gtgqNY6979L5bIolYr8fKr8hCZE6LlK+Lx/GJftqu9ADRkfSVxXjoxhPICt2Bucc941AVTLzVpPTxfW3Q0sTAQ0fU2Q8mYI7oWC0gLwZmpJJi4j7AZBHIRgryBe7Hi0kQoX5rgt4Kg32kq9sUIauiGq4otKoYpfI7aBg5BiGn0K/igNtLYGot+Xnl+B3NTjjxVWxJF3TQBLM5XT1ZpdqFRrHdTiDodMlHZMtvkK9T9JcItLUTiq/tesn0KGunfZ4hA0ZlI1MrSSidoGo0Cj5t/U5lR9Sug2pWz1Q1u/k6UkkpEmAre7Pv4nf3tOFBTwDseXP79/d39P92fxQVEpiJjQqxbMTikmmDZv+gaf4C2ubJ4IynzCaLz7X+0ZI/QYUOLBpkn6rvt+ajOlghyW3o5DzzGDa00WIvUrWfZInxE13Kg37NlyYeXBr37z5y+fwzNcHBfHxXFxXBwXx8Xxaz3+Bwejx8HM3R61AAAAAElFTkSuQmCC", + "parser_ids": [ + "table" + ] +} diff --git a/agent/templates/ingestion_pipeline_tag.json b/agent/templates/ingestion_pipeline_tag.json new file mode 100644 index 0000000000..9397841577 --- /dev/null +++ b/agent/templates/ingestion_pipeline_tag.json @@ -0,0 +1,308 @@ +{ + "id": 46, + "title": { + "en": "Tag", + "de": "Tag", + "zh": "标签" + }, + "description": { + "en": "This template segments parsed files by tag-based structures. Best for knowledge bases organized with tagging and categorization systems, enabling retrieval by label categories and tag hierarchies.", + "de": "Diese Vorlage segmentiert die geparste Datei anhand von Tag-basierten Strukturen. Sie eignet sich für Wissensdatenbanken die mit Tagging- und Kategorisierungssystemen organisiert sind und ermöglicht den Abruf nach Label-Kategorien und Tag-Hierarchien.", + "zh": "此模板将解析后的文件按标签结构进行切片，适用于以标签和分类系统组织的知识库，支持按标签类别和标签层级进行检索。" + }, + "canvas_type": "Ingestion Pipeline", + "canvas_category": "dataflow_canvas", + "dsl": { + "components": { + "File": { + "downstream": [ + "Parser:HipSignsRhyme" + ], + "obj": { + "component_name": "File", + "params": {} + }, + "upstream": [] + }, + "Parser:HipSignsRhyme": { + "downstream": [ + "TagChunker:NewNoonsGlow" + ], + "obj": { + "component_name": "Parser", + "params": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": { + "spreadsheet": { + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "suffix": [ + "xls", + "xlsx", + "csv" + ], + "vlm": {} + }, + "text&code": { + "output_format": "json", + "preprocess": [ + "main_content" + ], + "suffix": [ + "txt", + "py", + "js", + "java", + "c", + "cpp", + "h", + "php", + "go", + "ts", + "sh", + "cs", + "kt", + "sql" + ] + } + } + } + }, + "upstream": [ + "File" + ] + }, + "TagChunker:NewNoonsGlow": { + "downstream": [ + "Tokenizer:OldOwlsWatch" + ], + "obj": { + "component_name": "TagChunker", + "params": { + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + } + } + }, + "upstream": [ + "Parser:HipSignsRhyme" + ] + }, + "Tokenizer:OldOwlsWatch": { + "downstream": [], + "obj": { + "component_name": "Tokenizer", + "params": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + } + }, + "upstream": [ + "TagChunker:NewNoonsGlow" + ] + } + }, + "globals": { + "sys.history": [] + }, + "graph": { + "edges": [ + { + "id": "xy-edge__Filestart-Parser:HipSignsRhymeend", + "source": "File", + "sourceHandle": "start", + "target": "Parser:HipSignsRhyme", + "targetHandle": "end" + }, + { + "id": "xy-edge__Parser:HipSignsRhymestart-TagChunker:NewNoonsGlowend", + "source": "Parser:HipSignsRhyme", + "sourceHandle": "start", + "target": "TagChunker:NewNoonsGlow", + "targetHandle": "end" + }, + { + "data": { + "isHovered": false + }, + "id": "xy-edge__TagChunker:NewNoonsGlowstart-Tokenizer:OldOwlsWatchend", + "source": "TagChunker:NewNoonsGlow", + "sourceHandle": "start", + "target": "Tokenizer:OldOwlsWatch", + "targetHandle": "end" + } + ], + "nodes": [ + { + "data": { + "label": "File", + "name": "File" + }, + "id": "File", + "measured": { + "height": 50, + "width": 200 + }, + "position": { + "x": 50, + "y": 200 + }, + "sourcePosition": "left", + "targetPosition": "right", + "type": "beginNode" + }, + { + "data": { + "form": { + "outputs": { + "html": { + "type": "string", + "value": "" + }, + "json": { + "type": "Array", + "value": [] + }, + "markdown": { + "type": "string", + "value": "" + }, + "text": { + "type": "string", + "value": "" + } + }, + "setups": [ + { + "fileFormat": "spreadsheet", + "flatten_media_to_text": false, + "output_format": "html", + "parse_method": "DeepDOC", + "preprocess": [ + "main_content" + ], + "vlm": {} + }, + { + "fileFormat": "text&code", + "output_format": "json", + "preprocess": [ + "main_content" + ] + } + ] + }, + "label": "Parser", + "name": "Parser_0" + }, + "dragging": false, + "id": "Parser:HipSignsRhyme", + "measured": { + "height": 57, + "width": 200 + }, + "position": { + "x": 316.99524094206413, + "y": 195.39629819663406 + }, + "selected": true, + "sourcePosition": "right", + "targetPosition": "left", + "type": "parserNode" + }, + { + "data": { + "form": { + "outputs": { + "chunks": { + "type": "Array", + "value": [] + } + } + }, + "label": "TagChunker", + "name": "Tag Chunker_0" + }, + "id": "TagChunker:NewNoonsGlow", + "measured": { + "height": 74, + "width": 200 + }, + "position": { + "x": 616.9952409420641, + "y": 195.39629819663406 + }, + "selected": false, + "sourcePosition": "right", + "targetPosition": "left", + "type": "chunkerNode" + }, + { + "data": { + "form": { + "fields": "text", + "filename_embd_weight": 0.1, + "outputs": {}, + "search_method": [ + "embedding", + "full_text" + ] + }, + "label": "Tokenizer", + "name": "Indexer_0" + }, + "id": "Tokenizer:OldOwlsWatch", + "measured": { + "height": 114, + "width": 200 + }, + "position": { + "x": 916.9952409420641, + "y": 195.39629819663406 + }, + "sourcePosition": "right", + "targetPosition": "left", + "type": "tokenizerNode" + } + ] + }, + "history": [], + "messages": [], + "path": [], + "retrieval": [], + "variables": [] + }, + "avatar": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAGP5JREFUeAHtW2lsXNd1Pve9N/vK4U6KIiWRWi1rceQlm7ckTp3WrVuoSNP+aIImqFMgRYKmAYoGcGC0QNog+dEATYKkaZ2mTqLYaVVbst16gzdFlhd5ky1TIimS4jYkZ1/ednvOuffNUF4ayUtRILrGiG/evOXes37nO9cAF8ev9xDw/2AcO/a5UKIwkk4PXdYtITYYS3d22jV7on/TnqfgPR7/pwI4ePBz8eHQ9qHsurGxaGZgfSyaXScsMWKAsQGEv8EMRbtD4ZgB0pWeD5Wpk7/82NiuG4/AezgseJfHQz/8YRQSx7cObb5mS65703ozlBoyLGO9Ycj1phXuEmBkDdNIGoYJ0vdRAT7eJVkTUkrwnSr+9YQwRDIWT+3H0++pAC7IAo4dvDUeivQmc4MDafBjg6F0ZtAUoVFc0DZhRLZF4uleIaAnEksJtTgPV+VL/OASaZG8TH4vfecVS6mOeUjpe65AAaBcPFmrlWY61+1bD+/heEsBvHzs4N6+wdHrUUHDOJthYYZHwpFYzgyH09IXccsKCcCJ8mR57qhJgcvnxfDi+C+e4kV5vit9p8lrbgkAlHxoGnQd34EC4X/xG1oB/h7+luvJGP4c9V0nJQ0jhMdxywjFBR7jjQkwhHAbjbsf+tLX/ur3Dxzw4ALGm7rAj7/z5bF1G/Y+HomEIr7vaSlpWfmkSV/6LmnVB6fZgKWFWegZ2EAmLUkgnucI3/NQiS4tCa3egnAkDlYkB77bENJ1wfcdfJaL8/cluoYIhyMsASldMgoSJf5LLxVfZLFKPV36AY/JhQC0qHzXsLp6t3V/at8dcODAc/AOBSBWFiqj9VopEgpl8KvBL9Uv49fjOnGCPni4kLnpccj1rBOmIaSwUCEQxitjPHvDMPBWuh//4rGBf6VM4DmTn8kPw8U4aBmu0+DvPrkOv87nVZNASTL0H13jubZgLdDvOA8UHL/Hl0Xo7Vt/Fd54QQI4xwV+8t2/2bv3qqu/PTl+YnPfwGBu844951wmOVaRADy5ND8tmvUqLMycgm17rxbJVFpS5MKFSu0JrEb8l86zDIXWGv0gA7sShtQH5A70CKnULVqCV0MqaQnlPzgH8i0SCrmWcNHaPNep1GvVx2278vmRrVdPwHkMY+2XWDz+O/i5Ev27s7C6AmiaSnv4n+s6UK2UYHV5CU48fxRisSh4dgMKK8s82UDDNGcjmDx+J+3oC0DNXlsEW4AZLEkosQiWGz5BCUhfq+/l83QX3cNHeLWh5WuwRTSSzerSx8KhxN333X5rzwULoFwsLTiOA+lMFmbPTIEwrUD3+FsBmo06BTSolooobQ9N1+aF8xo4yPPcwMbzjm2z0FzHbZ1nC5KtR2qXCobQwg4ODfXM1j2BKyq3oqULMwQGfkz6oPtZVgSPLbBCoe2XXXfzvz1x7/dzv0oArRiwff/f9T045Wd2FEuQSKYla7xchmjE4gnlurppQaJeKcpmvSKqpQKFQtYkal6ZvV5NrVZBAThqIbRMMm3TlHRdoD2pMwROWmQ6e/lYC0YJjLIBuQffod1FewbdTAKixIMC4Gs4WOPiw+EYB8ZUJnfdhu37bsOr/+xXCmD9B2/ZGBnbffLeyXHjuvFpec1VuwSmPLEwc1qOjG4F8jUMPlA78V3pnT0M0XFfNtfdJkwdxWg+tFIVtAEy2U6WB85IG7chVdAzOPkJ7Q6SZQB6hVIEcECqNCBblrF2sGBUtlVp0lTuh5mGMoNhmspo8LpMKnnLi0d+MX/JlTff9lYCYBcIi3zZrVcNI5qAJ54bh0g4JhLJJPr3EgcuisS1lVmwp/9RZOrPgShOg10v80KC+atoz84OdrMpK6WyrFUq6DZNdhf1qrWfYG2BF7CvK5AAGk5ACzsJZRzQSh0qIOuYwcHX1HPlK/HQIrMTAyOb//LFI/9xLfxvFjD+6IGlsY3XzoUS3f2TpTCsrq7I/v4BePH4Mdh71bWU11mbrteBVxcgYqJ/o48j4lszS0pbJlvCC88ehfzSAmbFkDRxUuSXpmWij5Kf4jnTEoaFvoqTDIXD6LP4wYBr4jmTXEWnT9SmxGfQ/XiPCfFkRqqAqJIGi80wCXSyxbB0CH96DsuK7o9GE8mhsZ0/GD9+35Wju25YfFMB0PCrxTuiPSNfOv6aCROnxmHTpg0IXqJoBXkIWwh6DII9PfiKCRSAK0q1EqQ7e3j5hno56LQHu/ddBc2mLTEtESTGwkahQtfzECD5lMMofuB/hP0djiW2XQOvhgDJIyDlspaFRs5Sm8C6kc2EOdTJlmuQAVjStZtCI2s4/fLTsHT2FOS6eyHbtR4zVnIkne39CV583VsKoFZcud9y7C85iR5x/LU52LHzEvYrcoPurg7UeBNsPyMlKsAyELg0KqxR5cRSZWcWBMF4H91SQCgaBdI0AR/SKgUsjRXYTFvpPsiEAYJW65L0HApuJGAPj9GKpI4VVEGw1jHC4u0W4w96BqXuaCIjzkyckc8//QzUa2UYGh6CXVfecO348Qf+YnTX9d94UwGYlcljZm3rfCSR6vuvF0/D732iCdlsFhbnpqGrKwd2oy7qdhJEHGQE51HPn9XzBGjVMlpXiOxko14TGverqo9qAtQ8uROKAi0KJ4s4QPCkLRKUoGApNGYI0qtyCZOzBBqLoO/sHqh1DoKUEDlLWgQMIRJNwsDQJplIZkW5tCpLmL4pmz106E7YfcWHvvrwXd96/Jrf/eKTbxDAzJMHVjYOXna4e/TST89MpCC/iJrv7YXXXjkBl+y5HFOgK+teWjiuRJ8FqFdWCRMSNmmLQEd1yiDhkNI+aV7qIKkQi1IdZwRJKmTIi27i8VNI4+QaoCAxwU5+D7uO9CS5N1/D9YSn86zklOI5hD1sFJoBnb39kMxkoKOzCqgMqFfLspifTw1vufyzONM3CoDd4NjDt4Xnn4ldMmh8cmE+Bzt37mCcjjldUJ1TtjtkDRdMZVBlleIJoT9T5T8VkLmWqyJWqJSKIqjogrxNEJ7/w0V19g3ByuIszx6DlaBAiZGbpUkBj9wFNY/R3OCgaZL7YCHIRRC+x6DAypZh6Ezh84cKMZQSArAmWkNNIKfAx4RdyB0wgX/g61+/KfWVrxwsv0EA85P3Tu7f9huf3nn53puWFs7G7a2b8QUWmlBJGqEYJIe3wZ3/eSUYzpJcMMPiCt9vpXBo1YwCUqkOTKMZTksYCyhFBVFZV3EquqUynVw9kmv4rGmuM6jY5CoZgwBahkNVpXRRu6h1PO8JjypN9H2KooRMVSxo+yG9k7yjAwFWKBIXHgoArRKi8QS+OhTv87LhN7UAGv9w+HDz21fvvCcdie8vF1ahq7sLJsdfgU1bt4MVXYIN1/0BrOSXwSkUGAoH+F9HL7YE8r1KpcjubIBQ0AX9m+BqrnsAllHzAnQAk6DSJKZMEhDHAUPXEHQX+jpyECpMGhQPFDJWVmLx+3XZpRAAuQJXjFJS6pWUUbAUD2Ml6dgximVN1yzYbykAHr78UTwe25/PL0Fffz+ceOkF2LFnnwiFoxBPpGShUMLUEtPL1lYQ4BIcHZ09oqOrN0CBXLzqyQnC7r2Dw4zoOG3hjz6nL8kcAvs7WRYbAPINDvIH5HRoDbLlQopOQR6AY4NiBXzIdvZjbCqiz5clCpQTc9+6jcoNIcpQGWuUYn/sIyiAg28tgBefOn500ydvKkyfmero7e2XhOtLq8syHI2BWatxPidNkSkq85dr0kGL9uKcHMA4RoqYqmihlD651geNamUAj7WgQEO+9o+0alUKiIBMEro4V190OQaJdIeCZb7CEwHsVqHDhKbjlG78wheawbuMNxPAd35x/2J+tfhEIh6TtXoFzTMMpdIqBhU0IcxFCJBkCM/VUBgBWFFBSJeFXBfohI3R3LGbktyFBSZkC+MLXSUyByDaHxEcq4dpUGzI4Pq2t8s1sxbtD2VF1DbOUWozDDIQCaW4dq1vyQqfeOnEz/a979Ibl5cWYXhoHZyZmoLL9l0JKawR5udm0ZoMoNIZjNfLUEJ14qCw8n8Psv9rILo/BMXVPGpEsmkTiyS5ipQy2zkoKuUV9lmFJBnMCAWdLSK+BMNiSp0U8VlleA7xQ5BiKQsxXGarUiUZ24wI6kiNGAML873V8xJAPl84hFNxC4WC1dvXD9WZWUJliO0NiEQiFJEFFjsyZApoMUCG8gUD2WCzvgy+4QCSoeAxMjQI1yOFFeW6AFWEbKALyVSWtUULiMbSwnXq7NcU5bnc5rpHakzA/AEDKiRIweP44TF8pnOcDDzmJYXUFFM0lsRyu0dbnEGcRum8BPC9O+7Of+SjHz6Eb7uJfIect4IcgBWy2PSSyaQsFovQnUtrCStAQmYaG7kZvIEb0FWSZHKyI9dJBKGi+FQgEwxwgFgtrA98R9pIrlKKLK4sBEGQ3UcqdITr8jXqJhqdgKQurQ3NHeAEKLZEEynI5Ho0AeNrh6Gffb7M9Rvl8xIAjQceeOTnN95wzU3jJ1+Fnt4e9vl4LIZVnInEQxgBTxl6u7Ig1nqBNjwzHCd70xVeiBEjk6FaWuoP+6xsFwQguweGNcbXya11vSZW2hyS0J0FRaW1gt45HJOekq9cgYBS0145bwGs2NV/R4taQWSSo3VNjJ+EXXsvYxNuYvVFxoYQWVdsejn4j7NyTIr87QK6PwNubItYWV4K0gSvTPm1SR4uAnAkFN5XBaVhaT7FYrmp2l4FswgCGhdjD5W6CjeoKNwGWVKTJqrLEPQnyBoIKteKs+dvAQcOPFz5+Ac/9EDYEvupiEE4yQYVQq2uFosyhIJooOkGFEagHnfxcRFb+Qk0rb0Qzu6GdLaLqzWKAxQhqPBB02f0hplBM12SfZrNnfzbc7Vve7wAn6+TshMp+Pz8GVVgGdRE8lno5DQk2MH1W7gGaWVmFggRph7VCXIlP31+WSAYE1PT/7Jr2+h+x3GRkEhBpVwhDE7YgAsaYnsC8kZq84+M3iKbpZtFpHME6o0alIrLKizrTJDr7oc80uqBfoyAViJ/9qUIUp/CUYr2RSir3APPrtu4HQI6XWPggEsFnZNV+lPwGr96KC9HOI0q1BYmqhckgMbS2UcyH7hi9tT4ycGenj44MzEOPT09YOYtNqk1QECoSVEjKIyLVy29eCwpY+sSCom0egUGJFIUPJkX0EWk0EoLgI1aIGF/kK+flWijLNAVtMalLZBBWYG7rS7HB99tYkWYh1gu07ggAXzjR/dXR3df/kgmnfpUPBaGxYV52LBpjL0VI7NU9bkmNDQi85uL4C3cAVbfH0LNjYn84ny7KlScnmgRPYbQNJmpuCUiNlrsH+F9QxH/WszEp5CwYomMJGAGurMg/TbfztUUQWteuMOL970G1JZehkbNdi9IADSOHX3627/1m9d/an5+idAV19e29n1KT6DxWWCLtZO3Q7L2TYwPMYhv+VPZxcHOCohKba+GUjp1jn2piU6pCiQFc3XyD2C14gX4SHED3JtkKUrRuiZop3F1idqnUthDuo0E7czeBYab8S9YAN//6eEj1157xWuOa491dvXB0uIcw4x0Ogm1apXkLSh+mdp3I8N/BNXpTogM/DYjv4ruIVAa0z4POma0aLFWbUDFUGu7gC/akVzZhGFoxAiKU6ECrat3iKs+9QwGSRxwXK4A6+A0ShCNhsBfOYqP+ShcsABo1Kru7el05jbs9MDyUl5G43ERQdK0UaeYEtQlKoBZ6X6wdnxGpyih2VwuYnkhqjpTCLJFp7faZYZcA1/lGu9iMSCuaFW+TLCwmdv8V3LWQK27DelhmnaaVRRAFTGLBacf/R50JXCuDXh7Ajh5evLwnp2bv7a6smxmshmUfCwogKBdoChz5D0Buu3lUbsc/VA5iAsBoNGKJ25YaxsUW+wr81Va90Wwb4AWS6CqZ2CEvijgQ91iRJp4LMjXqbVOWcl1ati2r+H1WLRZIWxjPAb1qXsgtAWz1qov3pYAEt0zz4PY/JhlmR9myFpYhjSVnrrNpRGZjsNaHPgqoq0yuS5VlrB8WrWDklxQMQeoUO8YaXeGQHEHql3OUd0hCt03kJ+IScvQcDlMLoAR37OFZ0uB+ofS0gTUC+Mw89SdcNkeE6l3Q+lgzTDOVwC33vqw+8r45J0UeWnJ2P3hOZu6gdru7a0tV4OPFLqYXysf0MAQ2qWsPtvuEzKMVc0/yWCGWnT5uUnMAkkszJAApY9ogCnrILwK+M1V2SjPy+LiKVnKj8PsM4dgKPYSZEaHeb5YIYu3JQAap06d/kUmk5lbWFwSqXQaGo06FXUt7BFoTx2vWajOeW8cOtoFPt26X99EmUBRQOwsxO25dl0iwwtqHxGavYfchtuU5A52rQjV4gIUFs9AfXUCVieegHj5CAzvG8SVp9T+hLdrATR+fNfDM1PT8w9GI2EZJsoJAxe2t1qpSleE7A68kPb2pyCXtSUCsp0F1Lm28DSRQr7N6c5zsUxuCKLHmo2asNG/KepTfnfx2K6XRL2ch+LyLCzPT4BdGofxI3eL+quHYdv7chBGeg5ot4sgJst5+xZA4+ixZ7/R298vVldWgUgR+qi1ybYg1qy39TYdHcTrcZ1mtgA0iyRaeI7JP47utMHKo56ALSmteUysNBjaNlHrlcICUuxTsDz3GjSXX4LpZw9DrHgUdr8vDqmxIdY+WEnVoQLvnDVf8D7Bf/rpfcfff8XeE81mfVuoFoFsOqWpO18FbSMoCYLgGKxN6J6W7p7wX8URtsC83gukXChoiLj8IfO37QYWZFWsAiPYmivLeimPGCMPtVJe1stnwWychlef/G/osebF9j0R6NpBmk/iKpEOt6JM8Qsh35kF0DgzffabWey6VCtVCNpTep+ADBhdJjxU8SM0yAsWpSMfd3iEXqzecuezufvMIvGeH3B5twkuvEHdnYooIGFiYQouLJxGen2S/kq0dahNPwrP3PMzWBdfgO2XhqBrW6cUUeQkDOQiMEz6nuIk8KEmvBMLYAHMLN1z6SWbC6uFclaxbkJbgNrGomCrynFKyx4orGBo+9doVDMaQSpUwpPMEhEN7vMmKBszDvp9oyrrlQI2axfFlks2yulXHgHTWwCoTMIvH30UIo0ZuHQjwMiWEHRsSkuIoc9bWHCZMZ4TdfhDEYRivvfOBfDPBw7NX3/NvjuwcLnFAFN5MKUpQxEPElo7QAiYKoRHmUwEtLZsU9wQoCBtRT7vL1SLt23hYoRHAcgGtuORB0A+YBBmThyCkDMFTz34CDSXTov1XY4Y3gxyZJsB8V7UNHaxwIgISdtnmGwxmK0z40nsW9rvXAA0Hnry6e/t2jzyx8IUMR23qe5mrOOrHQvqQl9tieO6XoW31m9SQUH9xdctMt5oyawPBjrpYKqtVQvY5q5ArrcbKnNHYfL4g3DymWfEQKoht2PVvWmTAb0bLbBSEWDNx7pR8x1od1m0KizbK3XhlB5HcmYIGr54dwTwg3+9+/m//vNPPrjNGPsEcjmoNYO5Uw51zNAEWB+UG6gNrKpdpmWjt8VqvEPkKLFBTUG+72MbmOKFFQ7J2kJZTJ58Tk4+e7dYmXoJelOOvHREyJ5eC5IdMWiEQzC1iGzzagozBFJmLr7Vr6Ewm0TfilAiJ20nI8bevxEiqXMowbcvAJr5p8/mD+ESPkGbJ6LhsAz2AQjaTqIMXtdyvt70pMyf94jrtjjjel3+MNdvJjHaF+X05CSceOGYnHr1OBZfcxLcssgkIjK7YZ9IdGSRpcxCI9cBkWxORtJZtPoENj8zEMIGaDiWQeuP4LOwp2jF8LkROPv8j9E16kjWOu+OAGiEo9GpaqUCS7OvQnrHBwG0v2tepCUrj3t+oBoj6N9YIhOgoe10oloqwXJ+DlYX5+TZmQnM5/NUyAKCLaLeoae/F0a3jkIm0yFzuOBMNiuJmosjoxSNpcCMJBDghdH6wpgeo2h9Jqc7oA8FXS6vPewJ59AtSiIer7x7AujqzhbKxVUswqriift/hD7ry3qDGxvEwgnunnPNqksAtf9HUViqiSIsZnepwyPEhg39cvPoEDJExAQJ3gtgqNY6979L5bIolYr8fKr8hCZE6LlK+Lx/GJftqu9ADRkfSVxXjoxhPICt2Bucc941AVTLzVpPTxfW3Q0sTAQ0fU2Q8mYI7oWC0gLwZmpJJi4j7AZBHIRgryBe7Hi0kQoX5rgt4Kg32kq9sUIauiGq4otKoYpfI7aBg5BiGn0K/igNtLYGot+Xnl+B3NTjjxVWxJF3TQBLM5XT1ZpdqFRrHdTiDodMlHZMtvkK9T9JcItLUTiq/tesn0KGunfZ4hA0ZlI1MrSSidoGo0Cj5t/U5lR9Sug2pWz1Q1u/k6UkkpEmAre7Pv4nf3tOFBTwDseXP79/d39P92fxQVEpiJjQqxbMTikmmDZv+gaf4C2ubJ4IynzCaLz7X+0ZI/QYUOLBpkn6rvt+ajOlghyW3o5DzzGDa00WIvUrWfZInxE13Kg37NlyYeXBr37z5y+fwzNcHBfHxXFxXBwXx8Xxaz3+Bwejx8HM3R61AAAAAElFTkSuQmCC", + "parser_ids": [ + "tag" + ] +} diff --git a/go.mod b/go.mod index 20b551fd9a..da9706e6d4 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/iromli/go-itsdangerous v0.0.0-20220223194502-9c8bef8dac6a github.com/json-iterator/go v1.1.12 github.com/kaptinlin/jsonrepair v0.4.8 + github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 github.com/lib/pq v1.10.9 github.com/minio/minio-go/v7 v7.0.99 github.com/nats-io/nats-server/v2 v2.14.3 diff --git a/go.sum b/go.sum index fdc2f34f06..82ef6f96fc 100644 --- a/go.sum +++ b/go.sum @@ -336,6 +336,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ExRDqGQltzXqN/xypdKP86niVn8= +github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= diff --git a/internal/agent/dsl/normalize.go b/internal/agent/dsl/normalize.go index 77773e28fa..f6187bcdce 100644 --- a/internal/agent/dsl/normalize.go +++ b/internal/agent/dsl/normalize.go @@ -588,36 +588,40 @@ func stringsToAny(s []string) []any { // type is still rendered (just possibly in a generic shape). The user // can re-pick a type from the operator palette to refine. var componentNameToNodeTypeMap = map[string]string{ - "Begin": "beginNode", - "Retrieval": "ragNode", - "Categorize": "categorizeNode", - "Message": "messageNode", - "Answer": "messageNode", - "RewriteQuestion": "rewriteNode", - "ExeSQL": "toolNode", - "Switch": "switchNode", - "Agent": "agentNode", - "Tool": "toolNode", - "File": "fileNode", - "Parser": "parserNode", - "Tokenizer": "tokenizerNode", - "TokenChunker": "chunkerNode", - "TitleChunker": "chunkerNode", - "Extractor": "contextNode", - "Loop": "loopNode", - "LoopStart": "loopStartNode", - "ExitLoop": "exitLoopNode", - "Iteration": "iterationNode", - "IterationStart": "iterationStartNode", - "Parallel": "parallelNode", - "DataOperations": "dataOperationsNode", - "ListOperations": "listOperationsNode", - "VariableAssigner": "variableAssignerNode", - "VariableAggregator": "variableAggregatorNode", - "Keyword": "keywordNode", - "Note": "noteNode", - "Placeholder": "placeholderNode", - "Code": "toolNode", + "Begin": "beginNode", + "Retrieval": "ragNode", + "Categorize": "categorizeNode", + "Message": "messageNode", + "Answer": "messageNode", + "RewriteQuestion": "rewriteNode", + "ExeSQL": "toolNode", + "Switch": "switchNode", + "Agent": "agentNode", + "Tool": "toolNode", + "File": "fileNode", + "Parser": "parserNode", + "Tokenizer": "tokenizerNode", + "TokenChunker": "chunkerNode", + "TitleChunker": "chunkerNode", + "OneChunker": "chunkerNode", + "TagChunker": "chunkerNode", + "TableChunker": "chunkerNode", + "PresentationChunker": "chunkerNode", + "Extractor": "contextNode", + "Loop": "loopNode", + "LoopStart": "loopStartNode", + "ExitLoop": "exitLoopNode", + "Iteration": "iterationNode", + "IterationStart": "iterationStartNode", + "Parallel": "parallelNode", + "DataOperations": "dataOperationsNode", + "ListOperations": "listOperationsNode", + "VariableAssigner": "variableAssignerNode", + "VariableAggregator": "variableAggregatorNode", + "Keyword": "keywordNode", + "Note": "noteNode", + "Placeholder": "placeholderNode", + "Code": "toolNode", } func componentNameToNodeType(name string) string { diff --git a/internal/dao/canvas_template_seed.go b/internal/dao/canvas_template_seed.go index 7b01f7d1ad..3042272698 100644 --- a/internal/dao/canvas_template_seed.go +++ b/internal/dao/canvas_template_seed.go @@ -42,6 +42,15 @@ func SeedCanvasTemplates() error { return nil } + // The canvas_template table may have been created by the Python backend, + // which has no parser_ids column, and the Go server is often started with + // migration disabled so GORM AutoMigrate never adds it. Ensure the column + // exists before inserting, otherwise every INSERT below fails with + // "Unknown column 'parser_ids'" and the catalogue ends up empty. + if err := addColumnIfNotExists(DB, "canvas_template", "parser_ids", "LONGTEXT NULL"); err != nil { + return fmt.Errorf("failed to ensure canvas_template.parser_ids column: %w", err) + } + entries, err := os.ReadDir(dir) if err != nil { return fmt.Errorf("failed to read agent templates directory %s: %w", dir, err) diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go index 0bfe46428a..3eb490618c 100644 --- a/internal/engine/elasticsearch/chunk.go +++ b/internal/engine/elasticsearch/chunk.go @@ -1378,6 +1378,7 @@ func searchAfterPaginate( } chunk["_score"] = hit.Score chunk["_id"] = hit.ID + chunk["id"] = hit.ID // shim id to _id, matching Python es_conn.py behavior chunk["_index"] = hit.Index collected = append(collected, chunk) collectedTake++ @@ -2836,6 +2837,7 @@ func convertESResponse(esResp *SearchResponse, vectorFieldName string) []map[str } chunks[i]["_score"] = hit.Score chunks[i]["_id"] = hit.ID + chunks[i]["id"] = hit.ID // shim id to _id, matching Python es_conn.py behavior chunks[i]["_index"] = hit.Index if len(hit.Highlight) > 0 { chunks[i]["highlight"] = hit.Highlight diff --git a/internal/engine/elasticsearch/chunk_test.go b/internal/engine/elasticsearch/chunk_test.go index 9bd6f06c58..e8a6c140c7 100644 --- a/internal/engine/elasticsearch/chunk_test.go +++ b/internal/engine/elasticsearch/chunk_test.go @@ -18,8 +18,6 @@ package elasticsearch import ( "context" - "reflect" - "sort" "testing" "ragflow/internal/common" @@ -217,11 +215,13 @@ func TestSearchAfterPaginateSkipsDeepOffset(t *testing.T) { } // The first returned hit must be h-10500, not h-0 — that is the // entire point of the search_after path. - if id, _ := got[0]["id"].(int); id != offset { - t.Errorf("first hit id = %d, want %d (search_after must skip the deep offset)", id, offset) + wantFirstID := "h-" + itoa(offset) + if id, _ := got[0]["id"].(string); id != wantFirstID { + t.Errorf("first hit id = %s, want %s (search_after must skip the deep offset)", id, wantFirstID) } - if id, _ := got[limit-1]["id"].(int); id != offset+limit-1 { - t.Errorf("last hit id = %d, want %d", id, offset+limit-1) + wantLastID := "h-" + itoa(offset+limit-1) + if id, _ := got[limit-1]["id"].(string); id != wantLastID { + t.Errorf("last hit id = %s, want %s", id, wantLastID) } // 12 fetches: 10 full skip + 1 partial-skip (trims scripted 510 // down to 500) + 1 partial-take (the 10 leftover hits the take @@ -348,23 +348,13 @@ func TestSearchAfterPaginateLimitLargerThanBatchSize(t *testing.T) { t.Errorf("expected 3 take fetches, got %d", m.idx) } // Hits should be in order h-0..h-2499. - wantIDs := make([]int, limit) - for i := range wantIDs { - wantIDs[i] = i - } - gotIDs := make([]int, len(got)) for i, h := range got { - gotIDs[i] = h["id"].(int) + wantID := "h-" + itoa(i) + if id, _ := h["id"].(string); id != wantID { + t.Errorf("hit[%d].id = %s, want %s", i, id, wantID) + break + } } - if !reflect.DeepEqual(gotIDs, wantIDs) { - t.Errorf("hit order wrong: got %v, want %v", sortedCopy(gotIDs), wantIDs) - } -} - -func sortedCopy(in []int) []int { - out := append([]int(nil), in...) - sort.Ints(out) - return out } // TestBuildBoolQueryFromConditionIDFilter is the regression for the diff --git a/internal/entity/kb.go b/internal/entity/kb.go index 7a0fe65009..d1fea1c3ec 100644 --- a/internal/entity/kb.go +++ b/internal/entity/kb.go @@ -58,7 +58,6 @@ const ( ParserTypeOne ParserType = "one" ParserTypeAudio ParserType = "audio" ParserTypeEmail ParserType = "email" - ParserTypeKG ParserType = "knowledge_graph" ParserTypeTag ParserType = "tag" ) diff --git a/internal/entity/models/types.go b/internal/entity/models/types.go index 9e04679e06..0926075c47 100644 --- a/internal/entity/models/types.go +++ b/internal/entity/models/types.go @@ -202,6 +202,7 @@ type OCRConfig struct { } type ParseFileConfig struct { + ParseMethod string `json:"parse_method"` } // EmbeddingModel wraps a ModelDriver with embedding-specific configuration diff --git a/internal/handler/components_testpkg/components_handler_test.go b/internal/handler/components_testpkg/components_handler_test.go index d2a15bcd4e..59c9f2df47 100644 --- a/internal/handler/components_testpkg/components_handler_test.go +++ b/internal/handler/components_testpkg/components_handler_test.go @@ -133,8 +133,8 @@ func TestComponentsHandler_NoFilter(t *testing.T) { } // TestComponentsHandler_FilterIngestion verifies the -// ?category=ingestion filter returns the 8 ingestion components -// (Extractor, File, Parser, Tokenizer + 4 chunker variants). Names +// ?category=ingestion filter returns the ingestion components +// (Extractor, File, Parser, Tokenizer + 9 chunker variants). Names // must be sorted ascending (plan §4 task 1 stable output). func TestComponentsHandler_FilterIngestion(t *testing.T) { eng := newComponentsTestRig(t) @@ -147,7 +147,8 @@ func TestComponentsHandler_FilterIngestion(t *testing.T) { wantNames := []string{ "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", - "parser", "titlechunker", "tokenchunker", "tokenizer", + "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", + "titlechunker", "tokenchunker", "tokenizer", } assertNameSet(t, "ingestion", data, wantNames) @@ -172,7 +173,8 @@ func TestComponentsHandler_FilterMultiple(t *testing.T) { wantNames := []string{ "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", - "parser", "titlechunker", "tokenchunker", "tokenizer", + "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", + "titlechunker", "tokenchunker", "tokenizer", } assertNameSet(t, "ingestion,shared", data, wantNames) } @@ -272,7 +274,8 @@ func TestComponentsHandler_CaseInsensitive(t *testing.T) { _, _, data := decodeEnvelope(t, w.Body.Bytes()) wantNames := []string{ "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", - "parser", "titlechunker", "tokenchunker", "tokenizer", + "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", + "titlechunker", "tokenchunker", "tokenizer", } assertNameSet(t, "INGESTION (case-folded)", data, wantNames) } diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go index 4dc0b472c2..e12a6a93f2 100644 --- a/internal/ingestion/component/chunker/common.go +++ b/internal/ingestion/component/chunker/common.go @@ -43,6 +43,16 @@ func newChunkerByName(name string, params map[string]any) (runtime.Component, er return NewGroupTitleChunker(params) case ComponentNameHierarchyTitleChunker: return NewHierarchyTitleChunker(params) + case ComponentNameQAChunker: + return NewQAChunker(params) + case ComponentNameOneChunker: + return NewOneChunker(params) + case ComponentNameTagChunker: + return NewTagChunker(params) + case ComponentNameTableChunker: + return NewTableChunker(params) + case ComponentNamePresentationChunker: + return NewPresentationChunker(params) default: return nil, fmt.Errorf("chunker: unknown component %q", name) } diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.go index f4e0d7db71..e3a652de31 100644 --- a/internal/ingestion/component/chunker/group.go +++ b/internal/ingestion/component/chunker/group.go @@ -20,9 +20,8 @@ // text records into chunks that span multiple body records while // staying inside one heading section. // -// - PARALLELISM: Parallelism() advertises a fan-out hint to outer -// executors. Heading detection stays sequential; grouping work is -// local to one invocation. +// Heading detection stays sequential; grouping work is local to +// one invocation. // // - MIRRORS python `_build_section_ids` + `GroupTitleChunker.build_chunks`: // consecutive records with the same (target_level-derived) sec_id @@ -35,6 +34,7 @@ package chunker import ( "context" + "encoding/json" "fmt" "strings" @@ -107,24 +107,20 @@ func invokeGroup(_ context.Context, inputs map[string]any, p *titleChunkerParam) if len(records) == 0 { return emptyOutputs(), nil } - lines := make([]string, len(records)) - for i, r := range records { - lines[i] = r.text - } - ctx := newLevelContext(lines, p) + ctx := newLevelContext(records, p) levels := ctx.Levels() - targetLevel := resolveTargetLevel(levels, hierarchyOr(p, ctx.mostLevel)) + // Mirror python group_chunker._resolve_group_target_level: when + // `hierarchy` is unset the target level is `most_level` directly + // (NOT resolve_target_level — that would re-rank the distinct + // heading levels and pick the wrong depth when the heading levels + // are not contiguous from 1). When `hierarchy` is set, it defers to + // resolve_target_level. + targetLevel := resolveGroupTargetLevel(levels, p, ctx.mostLevel) secIDs := buildSectionIDs(levels, targetLevel) groups := groupRecords(records, secIDs, p) - if p.RootChunkAsHeading && len(groups) > 1 { - groups = applyRootAsHeading(groups) - } - chunks := make([]map[string]any, 0, len(groups)) - for _, g := range groups { - chunks = append(chunks, map[string]any{"text": joinGroupText(g)}) - } + chunks := buildChunksFromRecordGroups(groups, p, isPlainTextFormat(inputs)) if len(chunks) == 0 { return emptyOutputs(), nil } @@ -185,20 +181,9 @@ func groupRecords(records []lineRecord, secIDs []int, p *titleChunkerParam) [][] return recordGroups } -// applyRootAsHeading mirrors the `root_chunk_as_heading` branch in -// common.py:build_chunks_from_record_groups — prepending the root -// text to every following chunk and dropping the root chunk itself. -func applyRootAsHeading(groups [][]lineRecord) [][]lineRecord { - if len(groups) < 2 { - return groups - } - rootText := joinGroupText(groups[0]) - for i := 1; i < len(groups); i++ { - groups[i] = prependJoin(groups[i], rootText) - } - return groups[1:] -} - +// joinGroupText mirrors python's `"".join(record["text"] + "\n" for +// record in records)` — every record's text followed by a newline +// (including the last), matching the python text join exactly. func joinGroupText(g []lineRecord) string { var sb strings.Builder for _, r := range g { @@ -208,18 +193,52 @@ func joinGroupText(g []lineRecord) string { return sb.String() } -func prependJoin(g []lineRecord, prefix string) []lineRecord { - if prefix == "" { - return g +// isPlainTextFormat mirrors the `output_format in ["markdown", "text", +// "html"]` branch of common.py:build_chunks_from_record_groups. Plain +// payloads emit only "text"; structured payloads (chunks/json) also +// carry doc_type_kwd and img_id. +func isPlainTextFormat(inputs map[string]any) bool { + if f, ok := inputs["output_format"].(string); ok { + return f == "markdown" || f == "text" || f == "html" } - extra := lineRecord{text: prefix, docType: "text"} - if len(g) == 0 { - return []lineRecord{extra} + return false +} + +// buildChunksFromRecordGroups mirrors common.py:build_chunks_from_record_groups +// (minus the deepdoc-only remove_tag / merge_pdf_positions steps): +// - plain payloads: {"text": joined} +// - structured payloads: text plus doc_type_kwd / img_id from the +// group's leading record. +// +// root_chunk_as_heading is applied here, exactly as python does (post +// materialisation): the root chunk's text is prepended to every +// following chunk and the root chunk is dropped. +func buildChunksFromRecordGroups(groups [][]lineRecord, p *titleChunkerParam, plain bool) []map[string]any { + chunks := make([]map[string]any, 0, len(groups)) + for _, g := range groups { + if len(g) == 0 { + continue + } + chunk := map[string]any{"text": joinGroupText(g)} + if !plain { + first := g[0] + if first.docType != "" { + chunk["doc_type_kwd"] = first.docType + } + if first.imgID != nil { + chunk["img_id"] = *first.imgID + } + } + chunks = append(chunks, chunk) } - out := make([]lineRecord, 0, len(g)+1) - out = append(out, extra) - out = append(out, g...) - return out + if p.RootChunkAsHeading && len(chunks) > 1 { + rootText := toString(chunks[0]["text"]) + for i := 1; i < len(chunks); i++ { + chunks[i]["text"] = rootText + "\n" + toString(chunks[i]["text"]) + } + chunks = chunks[1:] + } + return chunks } // extractLineRecords reads the chunker inputs in the same order the @@ -257,21 +276,42 @@ func recordsFromStructured(items []schema.ChunkDoc) []lineRecord { img := it.ImgID imgID = &img } + meta := make(map[string]any) + if it.ContentLtks != "" { + meta["content_ltks"] = it.ContentLtks + } + if it.ContentSmLtks != "" { + meta["content_sm_ltks"] = it.ContentSmLtks + } + if it.ContentWithWeight != "" { + meta["content_with_weight"] = it.ContentWithWeight + } + if it.TitleTks != "" { + meta["title_tks"] = it.TitleTks + } + if it.TitleSmTks != "" { + meta["title_sm_tks"] = it.TitleSmTks + } + for k, v := range it.Extra { + meta[k] = json.RawMessage(v) + } out = append(out, lineRecord{ - text: text, - docType: dt, - imgID: imgID, - layout: it.Layout, + text: text, + docType: dt, + imgID: imgID, + layout: it.Layout, + parentMeta: meta, }) } return out } -// hierarchyOr returns the param's hierarchy value (if set), falling -// back to the `mostLevel` computed from the level-frequency pass. -func hierarchyOr(p *titleChunkerParam, mostLevel int) int { +// resolveGroupTargetLevel mirrors group_chunker._resolve_group_target_level: +// when `hierarchy` is set (>0) the target depth is resolve_target_level, +// otherwise it is the frequency-derived `most_level` directly. +func resolveGroupTargetLevel(levels []int, p *titleChunkerParam, mostLevel int) int { if p.Hierarchy != nil && *p.Hierarchy > 0 { - return *p.Hierarchy + return resolveTargetLevel(levels, *p.Hierarchy) } return mostLevel } @@ -303,7 +343,6 @@ func NewGroupTitleChunker(params map[string]any) (runtime.Component, error) { }, nil } -func (c *GroupTitleChunkerComponent) Parallelism() int { return 2 } func (c *GroupTitleChunkerComponent) Inputs() map[string]string { return ChunkerInputs } @@ -312,23 +351,21 @@ func (c *GroupTitleChunkerComponent) Outputs() map[string]string { } func (c *GroupTitleChunkerComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { - return runtime.TrackElapsed(ComponentNameGroupTitleChunker, func() (map[string]any, error) { - if inputs == nil { - inputs = map[string]any{} - } - // `name` is read from the workflow-wide Globals bag (seeded at - // pipeline start, published by the File component), not from the - // upstream output map. - name := globals.GlobalOrInput(ctx, inputs, "name", "") - if name == "" { - return map[string]any{ - "output_format": "chunks", - "chunks": []map[string]any{}, - "_ERROR": "GroupTitleChunker: missing required upstream field \"name\"", - }, nil - } - return invokeGroup(ctx, withName(inputs, name), &c.param) - }) + if inputs == nil { + inputs = map[string]any{} + } + // `name` is read from the workflow-wide Globals bag (seeded at + // pipeline start, published by the File component), not from the + // upstream output map. + name := globals.GlobalOrInput(ctx, inputs, "name", "") + if name == "" { + return map[string]any{ + "output_format": "chunks", + "chunks": []map[string]any{}, + "_ERROR": "GroupTitleChunker: missing required upstream field \"name\"", + }, nil + } + return invokeGroup(ctx, withName(inputs, name), &c.param) } // init registers GroupTitleChunker under CategoryIngestion. diff --git a/internal/ingestion/component/chunker/group_test.go b/internal/ingestion/component/chunker/group_test.go index f191bff9f6..b1206e71d8 100644 --- a/internal/ingestion/component/chunker/group_test.go +++ b/internal/ingestion/component/chunker/group_test.go @@ -21,6 +21,7 @@ import ( "testing" "ragflow/internal/agent/runtime" + "ragflow/internal/ingestion/component/schema" ) func TestGroupTitleChunker_Registered(t *testing.T) { @@ -42,22 +43,6 @@ func TestGroupTitleChunker_Registered(t *testing.T) { } } -func TestGroupTitleChunker_Parallelism(t *testing.T) { - c, err := NewGroupTitleChunker(map[string]any{ - "levels": [][]string{{`^# `}}, - }) - if err != nil { - t.Fatalf("NewGroupTitleChunker: %v", err) - } - gc, ok := c.(*GroupTitleChunkerComponent) - if !ok { - t.Fatalf("NewGroupTitleChunker returned %T", c) - } - if got := gc.Parallelism(); got != 2 { - t.Errorf("Parallelism() = %d, want 2", got) - } -} - func TestGroupTitleChunker_InputsOutputs_NonEmpty(t *testing.T) { _, _, meta, ok := runtime.DefaultRegistry.Lookup("GroupTitleChunker") if !ok { @@ -146,6 +131,63 @@ func TestGroupTitleChunker_RootChunkAsHeading_StillSingleGroup(t *testing.T) { } } +// TestResolveGroupTargetLevel_UsesMostLevelDirectly pins Gap F: when +// `hierarchy` is unset the group target level is `most_level` DIRECTLY, +// not resolve_target_level (which would re-rank the distinct heading +// levels and pick the wrong depth when levels are not contiguous from +// 1). With levels {2,2,3} most_level is 2, but resolve_target_level +// would return 3. +func TestResolveGroupTargetLevel_UsesMostLevelDirectly(t *testing.T) { + levels := []int{2, 2, 3, bodyLevel, bodyLevel} + pUnset := &titleChunkerParam{TitleChunkerParam: schema.TitleChunkerParam{Method: "group"}} + if got := resolveGroupTargetLevel(levels, pUnset, 2); got != 2 { + t.Errorf("unset hierarchy: got %d, want 2 (most_level directly)", got) + } + h := 2 + pSet := &titleChunkerParam{TitleChunkerParam: schema.TitleChunkerParam{Method: "group", Hierarchy: &h}} + if got := resolveGroupTargetLevel(levels, pSet, 2); got != 3 { + t.Errorf("hierarchy=2: got %d, want 3 (resolve_target_level)", got) + } +} + +// TestGroupChunker_StructuredMetadata pins Gap E: for a structured +// (output_format=chunks) payload, non-text records keep their +// doc_type_kwd and img_id on the emitted chunk. +func TestGroupChunker_StructuredMetadata(t *testing.T) { + c, err := NewGroupTitleChunker(map[string]any{ + "levels": [][]string{{`^# `}}, + }) + if err != nil { + t.Fatalf("NewGroupTitleChunker: %v", err) + } + items := []map[string]any{ + {"text": "# Heading", "doc_type_kwd": "text"}, + {"text": "body line", "doc_type_kwd": "text"}, + {"text": "an image caption", "doc_type_kwd": "image", "img_id": "img-9"}, + } + out, err := c.Invoke(context.Background(), map[string]any{ + "name": "doc", + "output_format": "chunks", + "chunks": items, + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + found := false + for _, ck := range chunks { + if dt, _ := ck["doc_type_kwd"].(string); dt == "image" { + found = true + if ck["img_id"] != "img-9" { + t.Errorf("image chunk img_id = %v, want img-9", ck["img_id"]) + } + } + } + if !found { + t.Fatal("no image chunk emitted") + } +} + func TestGroupTitleChunker_InvokeDeterministic(t *testing.T) { c, err := NewGroupTitleChunker(map[string]any{ "levels": [][]string{ diff --git a/internal/ingestion/component/chunker/hierarchy.go b/internal/ingestion/component/chunker/hierarchy.go index a135416731..6f761cbc86 100644 --- a/internal/ingestion/component/chunker/hierarchy.go +++ b/internal/ingestion/component/chunker/hierarchy.go @@ -14,14 +14,13 @@ // limitations under the License. // -// SCOPE (honest) for hierarchy.go (Phase 2.3d): +// SCOPE (honest) for hierarchy.go: // // - Implements the HierarchyTitleChunker variant: builds a heading // tree from per-line levels, then walks the tree in DFS to emit // chunks that respect hierarchical scope (a sub-tree's body // chunks include the heading texts of all ancestor nodes). // -// - PARALLELISM: 2 goroutines (plan §4 Phase 2 row 2.3d). The // tree-build pass is sequential; the subtree-to-chunk conversion // runs across 2 goroutines, then results are merged in DFS // traversal order (deterministic, plan §8 R8). @@ -42,7 +41,6 @@ package chunker import ( "context" "fmt" - "strings" "ragflow/internal/agent/runtime" "ragflow/internal/ingestion/component/globals" @@ -142,151 +140,84 @@ func invokeHierarchy(_ context.Context, inputs map[string]any, p *titleChunkerPa if len(records) == 0 { return emptyOutputs(), nil } - lines := make([]string, len(records)) - textRecords := make([]int, 0, len(records)) - textLines := make([]string, 0, len(records)) - for i, r := range records { - lines[i] = r.text - if r.isText() { - textLines = append(textLines, r.text) - textRecords = append(textRecords, i) - } - } - ctx := newLevelContext(textLines, p) + ctx := newLevelContext(records, p) levels := ctx.Levels() - // Re-map per-text-record levels onto the global record index so - // non-text records carry the sentinel bodyLevel (matching python - // `flush_text_records` entry pre-condition). - recordLevels := make([]int, len(records)) - for i, lvl := range levels { - recordLevels[textRecords[i]] = lvl - } - for i := range recordLevels { - if records[i].isText() && recordLevels[i] == 0 { - recordLevels[i] = bodyLevel + + // Mirror python HierarchyTitleChunker.build_chunks: accumulate + // contiguous text records into a run; on each non-text record flush + // the run (build the heading tree + emit its paths) and emit the + // non-text record as its own single-record group. A final flush + // handles the trailing run. + // + // The target level is resolved per run via + // resolve_target_level(text_levels, hierarchy) — the exact call + // python makes inside flush_text_records (Gap H: the hierarchy + // pointer is dereferenced defensively so a nil never panics). + var recordGroups [][]lineRecord + var textRun []lineRecord + var textLevels []int + + flush := func() { + if len(textRun) == 0 { + return } - } - - // Same loop as python `flush_text_records` — but here we - // sequence-fan-out across 2 workers to honour Plan §4 - // Phase 2 row 2.3d. In practice the work is dominated by the - // single tree build + DFS walk, so we keep the code simple and - // parallel by chunk, not by record. - - // Split text records into contiguous runs (so flush_text_records - // semantics survive). - var runs [][]int - var curRun []int - for i := range recordLevels { - if !records[i].isText() { - if len(curRun) > 0 { - runs = append(runs, curRun) - curRun = nil - } - curRun = nil - continue + h := 1 + if p.Hierarchy != nil { + h = *p.Hierarchy } - curRun = append(curRun, i) - } - if len(curRun) > 0 { - runs = append(runs, curRun) - } - - // Parallelism fan-out happens implicitly via goroutines; see comment above. - - // For each run: build tree, get paths, expand to record-index lists. - runRecords := make([][][]int, len(runs)) - for i, run := range runs { - indexed := make([]indexedLine, 0, len(run)) - for j, recIdx := range run { - indexed = append(indexed, indexedLine{level: recordLevels[recIdx], index: j}) - } - targetLevel := resolveTargetLevel(recordLevels, *p.Hierarchy) + targetLevel := resolveTargetLevel(textLevels, h) if targetLevel == 0 { - // Fall back to flat behaviour. - var all []int - all = append(all, run...) - runRecords[i] = [][]int{all} - continue - } - root := buildTree(indexed, targetLevel) - var pathIndexes [][]int - root.getPaths(&pathIndexes, nil, targetLevel, p.IncludeHeadingContent) - // Map the per-run text-record indexes back to global record indexes. - expanded := make([][]int, 0, len(pathIndexes)) - for _, path := range pathIndexes { - out := make([]int, 0, len(path)) - for _, idx := range path { - if idx >= 0 && idx < len(run) { - out = append(out, run[idx]) + // resolve_target_level returned None (no heading levels in + // the run): emit the whole run as a single group, exactly + // like python's `record_groups.append(text_records.copy())`. + runCopy := make([]lineRecord, len(textRun)) + copy(runCopy, textRun) + recordGroups = append(recordGroups, runCopy) + } else { + indexed := make([]indexedLine, len(textRun)) + for j := range textRun { + indexed[j] = indexedLine{level: textLevels[j], index: j} + } + root := buildTree(indexed, targetLevel) + var pathIndexes [][]int + root.getPaths(&pathIndexes, nil, targetLevel, p.IncludeHeadingContent) + for _, path := range pathIndexes { + if len(path) == 0 { + continue } - } - if len(out) > 0 { - expanded = append(expanded, out) + grp := make([]lineRecord, len(path)) + for k, idx := range path { + grp[k] = textRun[idx] + } + recordGroups = append(recordGroups, grp) } } - runRecords[i] = expanded + textRun = textRun[:0] + textLevels = textLevels[:0] } - // Interleave: text runs are batched; non-text records flow inline. - var combined [][]int - runIdx := 0 - for i, lvl := range recordLevels { - if i > 0 && !records[i].isText() { - _ = lvl - } - if records[i].isText() { + for i, rec := range records { + if rec.isText() { + textRun = append(textRun, rec) + textLevels = append(textLevels, levels[i]) continue } - // Flush current run's records. - if runIdx < len(runRecords) { - combined = append(combined, runRecords[runIdx]...) - runIdx++ - } - combined = append(combined, []int{i}) - } - if runIdx < len(runRecords) { - for i := runIdx; i < len(runRecords); i++ { - combined = append(combined, runRecords[i]...) - } + flush() + recordGroups = append(recordGroups, []lineRecord{rec}) } + flush() - out2 := make([]map[string]any, 0, len(combined)) - for _, path := range combined { - var sb strings.Builder - for _, idx := range path { - sb.WriteString(records[idx].text) - sb.WriteString("\n") - } - out2 = append(out2, map[string]any{"text": sb.String()}) - } - if p.RootChunkAsHeading && len(out2) > 1 { - out2 = applyRootAsHeadingMaps(out2) - } - if len(out2) == 0 { + chunks := buildChunksFromRecordGroups(recordGroups, p, isPlainTextFormat(inputs)) + if len(chunks) == 0 { return emptyOutputs(), nil } out := map[string]any{ "output_format": "chunks", - "chunks": out2, + "chunks": chunks, } return out, nil } -// applyRootAsHeadingMaps mirrors the root_chunk_as_heading branch -// for output []map[string]any. We prepend the root text to every -// following chunk and drop the root chunk. -func applyRootAsHeadingMaps(chunks []map[string]any) []map[string]any { - if len(chunks) < 2 { - return chunks - } - rootText := toString(chunks[0]["text"]) - for i := 1; i < len(chunks); i++ { - chunks[i]["text"] = rootText + "\n" + toString(chunks[i]["text"]) - } - return chunks[1:] -} - // HierarchyTitleChunkerComponent is the standalone variant entry point. type HierarchyTitleChunkerComponent struct { name string @@ -311,7 +242,6 @@ func NewHierarchyTitleChunker(params map[string]any) (runtime.Component, error) }, nil } -func (c *HierarchyTitleChunkerComponent) Parallelism() int { return 2 } func (c *HierarchyTitleChunkerComponent) Inputs() map[string]string { return ChunkerInputs } @@ -320,23 +250,21 @@ func (c *HierarchyTitleChunkerComponent) Outputs() map[string]string { } func (c *HierarchyTitleChunkerComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { - return runtime.TrackElapsed(ComponentNameHierarchyTitleChunker, func() (map[string]any, error) { - if inputs == nil { - inputs = map[string]any{} - } - // `name` is read from the workflow-wide Globals bag (seeded at - // pipeline start, published by the File component), not from the - // upstream output map. - name := globals.GlobalOrInput(ctx, inputs, "name", "") - if name == "" { - return map[string]any{ - "output_format": "chunks", - "chunks": []map[string]any{}, - "_ERROR": "HierarchyTitleChunker: missing required upstream field \"name\"", - }, nil - } - return invokeHierarchy(ctx, withName(inputs, name), &c.param) - }) + if inputs == nil { + inputs = map[string]any{} + } + // `name` is read from the workflow-wide Globals bag (seeded at + // pipeline start, published by the File component), not from the + // upstream output map. + name := globals.GlobalOrInput(ctx, inputs, "name", "") + if name == "" { + return map[string]any{ + "output_format": "chunks", + "chunks": []map[string]any{}, + "_ERROR": "HierarchyTitleChunker: missing required upstream field \"name\"", + }, nil + } + return invokeHierarchy(ctx, withName(inputs, name), &c.param) } // init registers HierarchyTitleChunker under CategoryIngestion. diff --git a/internal/ingestion/component/chunker/hierarchy_test.go b/internal/ingestion/component/chunker/hierarchy_test.go index 9d6479f176..6428721f9c 100644 --- a/internal/ingestion/component/chunker/hierarchy_test.go +++ b/internal/ingestion/component/chunker/hierarchy_test.go @@ -18,6 +18,7 @@ package chunker import ( "context" + "strings" "testing" "ragflow/internal/agent/runtime" @@ -42,23 +43,6 @@ func TestHierarchyTitleChunker_Registered(t *testing.T) { } } -func TestHierarchyTitleChunker_Parallelism(t *testing.T) { - c, err := NewHierarchyTitleChunker(map[string]any{ - "hierarchy": 1, - "levels": [][]string{{`^# `}, {`^## `}}, - }) - if err != nil { - t.Fatalf("NewHierarchyTitleChunker: %v", err) - } - hc, ok := c.(*HierarchyTitleChunkerComponent) - if !ok { - t.Fatalf("NewHierarchyTitleChunker returned %T", c) - } - if got := hc.Parallelism(); got != 2 { - t.Errorf("Parallelism() = %d, want 2", got) - } -} - func TestHierarchyTitleChunker_InputsOutputs_NonEmpty(t *testing.T) { _, _, meta, ok := runtime.DefaultRegistry.Lookup("HierarchyTitleChunker") if !ok { @@ -158,6 +142,45 @@ func TestHierarchyTitleChunker_LeafOnlyDefault(t *testing.T) { } } +// TestHierarchyChunker_StructuredMetadata pins Gap E for the hierarchy +// strategy: a structured (output_format=chunks) image record keeps its +// doc_type_kwd and img_id on the emitted chunk. +func TestHierarchyChunker_StructuredMetadata(t *testing.T) { + c, err := NewHierarchyTitleChunker(map[string]any{ + "hierarchy": 1, + "levels": [][]string{{`^# `}}, + }) + if err != nil { + t.Fatalf("NewHierarchyTitleChunker: %v", err) + } + items := []map[string]any{ + {"text": "# H1", "doc_type_kwd": "text"}, + {"text": "body one", "doc_type_kwd": "text"}, + {"text": "imgA caption", "doc_type_kwd": "image", "img_id": "a"}, + } + out, err := c.Invoke(context.Background(), map[string]any{ + "name": "doc", + "output_format": "chunks", + "chunks": items, + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + found := false + for _, ck := range chunks { + if dt, _ := ck["doc_type_kwd"].(string); dt == "image" { + found = true + if ck["img_id"] != "a" { + t.Errorf("image chunk img_id = %v, want a", ck["img_id"]) + } + } + } + if !found { + t.Fatal("no image chunk emitted") + } +} + func TestHierarchyTitleChunker_InvokeDeterministic(t *testing.T) { c, err := NewHierarchyTitleChunker(map[string]any{ "hierarchy": 2, @@ -197,3 +220,52 @@ func TestHierarchyTitleChunker_InvokeDeterministic(t *testing.T) { } } } + +// TestHierarchyTitleChunker_ConsecutiveNonTextOrder pins Gap G: two +// consecutive non-text records must stay in input order and ahead of +// the following text run. Python's flush_text_records flushes the +// preceding text run on the first non-text and is a no-op for the next +// non-text, yielding [..., N1, N2, run, ...]; the old loop flushed the +// trailing run early, yielding [..., N1, run, N2]. +func TestHierarchyTitleChunker_ConsecutiveNonTextOrder(t *testing.T) { + c, err := NewHierarchyTitleChunker(map[string]any{ + "hierarchy": 1, + "levels": [][]string{{`^# `}}, + }) + if err != nil { + t.Fatalf("NewHierarchyTitleChunker: %v", err) + } + items := []map[string]any{ + {"text": "# H1", "doc_type_kwd": "text"}, + {"text": "body one", "doc_type_kwd": "text"}, + {"text": "imgA caption", "doc_type_kwd": "image", "img_id": "a"}, + {"text": "imgB caption", "doc_type_kwd": "image", "img_id": "b"}, + {"text": "# H2", "doc_type_kwd": "text"}, + {"text": "body two", "doc_type_kwd": "text"}, + } + out, err := c.Invoke(context.Background(), map[string]any{ + "name": "doc", + "chunks": items, + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + idx := func(sub string) int { + for i, ck := range chunks { + if text, _ := ck["text"].(string); strings.Contains(text, sub) { + return i + } + } + return -1 + } + iA := idx("imgA caption") + iB := idx("imgB caption") + iBody := idx("body two") + if iA < 0 || iB < 0 || iBody < 0 { + t.Fatalf("missing expected chunks: imgA=%d imgB=%d body=%d (chunks=%d)", iA, iB, iBody, len(chunks)) + } + if !(iA < iB && iB < iBody) { + t.Errorf("non-text order wrong: imgA=%d imgB=%d body=%d, want imgA 0 && len(img2) > 0 && &img1[0] == &img2[0] { + return img1 + } + // Nil / empty guard (Python: img1 and not img2 → img1, etc.). + if len(img1) == 0 && len(img2) == 0 { + return nil + } + if len(img1) == 0 { + return img2 + } + if len(img2) == 0 { + return img1 + } + + // Decode both images. + dec1, _, err1 := image.Decode(bytes.NewReader(img1)) + dec2, _, err2 := image.Decode(bytes.NewReader(img2)) + if err1 != nil { + if err2 != nil { + return nil + } + return img2 + } + if err2 != nil { + return img1 + } + + // Pixel-data equality guard (Python: img1.tobytes() == img2.tobytes()). + if img1data, img2data := imgBytes(dec1), imgBytes(dec2); img1data != nil && img2data != nil && bytes.Equal(img1data, img2data) { + return img1 + } + + // Compute dimensions. + bounds1 := dec1.Bounds() + bounds2 := dec2.Bounds() + w1, h1 := bounds1.Dx(), bounds1.Dy() + w2, h2 := bounds2.Dx(), bounds2.Dy() + + newW := w1 + if w2 > newW { + newW = w2 + } + newH := h1 + h2 + + // Create new RGBA image and paste img1 at top, img2 below. + dst := image.NewRGBA(image.Rect(0, 0, newW, newH)) + draw.Draw(dst, dst.Bounds(), image.White, image.Point{}, draw.Src) + draw.Draw(dst, image.Rect(0, 0, w1, h1), dec1, bounds1.Min, draw.Over) + draw.Draw(dst, image.Rect(0, h1, w2, h1+h2), dec2, bounds2.Min, draw.Over) + + var buf bytes.Buffer + if err := png.Encode(&buf, dst); err != nil { + return nil + } + return buf.Bytes() +} + +// imgBytes returns the raw RGBA pixel data of img as a []byte. Returns nil +// when img is not convertible (should not happen for formats decoded by +// the Go image library, which always produce RGBA or NRGBA). +func imgBytes(img image.Image) []byte { + b := img.Bounds() + rgba := image.NewRGBA(b) + draw.Draw(rgba, b, img, b.Min, draw.Src) + return rgba.Pix +} diff --git a/internal/ingestion/component/chunker/one.go b/internal/ingestion/component/chunker/one.go new file mode 100644 index 0000000000..8abc084f04 --- /dev/null +++ b/internal/ingestion/component/chunker/one.go @@ -0,0 +1,173 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// OneChunker emits a single chunk per upstream document. It is the +// faithful Go port of the Python `one` chunk method +// (rag/app/one.py) and also covers the `picture` / `audio` methods, +// whose Python chunk() functions return exactly one chunk per file +// (rag/app/picture.py, rag/app/audio.py) — the latter additionally +// carrying the raw image / media context, which this chunker preserves. +// +// Unlike TokenChunker in "one" mode (which drops per-item media +// context), OneChunker carries the image attachment and doc_type of a +// single upstream item through unchanged, so picture/audio pipelines +// keep their media payload. +package chunker + +import ( + "context" + "fmt" + "strings" + + "ragflow/internal/agent/runtime" + "ragflow/internal/ingestion/component/schema" +) + +const ComponentNameOneChunker = "OneChunker" + +type oneChunkerParam struct{} + +func (p *oneChunkerParam) Update(conf map[string]any) {} + +func (oneChunkerParam) Defaults() oneChunkerParam { return oneChunkerParam{} } + +func (oneChunkerParam) Validate() error { return nil } + +type OneChunkerComponent struct { + name string + param oneChunkerParam +} + +func NewOneChunker(params map[string]any) (runtime.Component, error) { + p := oneChunkerParam{}.Defaults() + (&p).Update(params) + if err := p.Validate(); err != nil { + return nil, err + } + return &OneChunkerComponent{ + name: ComponentNameOneChunker, + param: p, + }, nil +} +func (c *OneChunkerComponent) Inputs() map[string]string { return ChunkerInputs } + +func (c *OneChunkerComponent) Outputs() map[string]string { return ChunkerOutputs } + +func (c *OneChunkerComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { + return c.invoke(ctx, inputs) +} + +func (c *OneChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) { + if inputs == nil { + return emptyOutputs(), nil + } + upstream, err := decodeChunkerFromUpstream(inputs) + if err != nil { + return map[string]any{ + "output_format": "chunks", + "chunks": []map[string]any{}, + "_ERROR": fmt.Sprintf("Input error: %v", err), + }, nil + } + + switch upstream.OutputFormat { + case schema.PayloadFormatMarkdown: + if upstream.MarkdownResult == nil { + return emptyOutputs(), nil + } + return emitOne(*upstream.MarkdownResult, "text"), nil + case schema.PayloadFormatText: + if upstream.TextResult == nil { + return emptyOutputs(), nil + } + return emitOne(*upstream.TextResult, "text"), nil + case schema.PayloadFormatHTML: + if upstream.HTMLResult == nil { + return emptyOutputs(), nil + } + return emitOne(*upstream.HTMLResult, "text"), nil + default: + return emitOneFromItems(upstream.JSONResult, upstream.Chunks), nil + } +} + +// emitOne wraps a single text payload as one chunk. +func emitOne(text, docType string) map[string]any { + if strings.TrimSpace(text) == "" { + return emptyOutputs() + } + return chunkOutputs([]schema.ChunkDoc{{ + Text: text, + DocType: docType, + CKType: docType, + }}) +} + +// emitOneFromItems collapses a structured upstream payload into a single +// chunk. When the payload is a single item, its media context (image) +// and doc_type are preserved. When it is many items, their text is +// concatenated and the first available image attachment is carried over — +// mirroring the Python "one chunk per file" behavior for picture/audio, +// where each upstream item is one page/slide/transcript segment of the +// same source file. +func emitOneFromItems(items, chunks []schema.ChunkDoc) map[string]any { + src := items + if len(src) == 0 { + src = chunks + } + if len(src) == 0 { + return emptyOutputs() + } + if len(src) == 1 { + it := src[0] + docType := itemDocType(it) + text := itemTextOrFallback(it) + if strings.TrimSpace(text) == "" && it.Image == "" { + return emptyOutputs() + } + out := schema.ChunkDoc{ + Text: text, + DocType: docType, + CKType: docType, + Image: it.Image, + } + return chunkOutputs([]schema.ChunkDoc{out}) + } + + var parts []string + var img string + for _, it := range src { + if t := itemTextOrFallback(it); t != "" { + parts = append(parts, t) + } + if img == "" && it.Image != "" { + img = it.Image + } + } + merged := strings.Join(parts, "\n") + if strings.TrimSpace(merged) == "" && img == "" { + return emptyOutputs() + } + out := schema.ChunkDoc{Text: merged, DocType: "text", CKType: "text"} + if img != "" { + out.Image = img + } + return chunkOutputs([]schema.ChunkDoc{out}) +} + +func init() { + MustRegisterChunker(ComponentNameOneChunker) +} diff --git a/internal/ingestion/component/chunker/one_test.go b/internal/ingestion/component/chunker/one_test.go new file mode 100644 index 0000000000..0cd371da8c --- /dev/null +++ b/internal/ingestion/component/chunker/one_test.go @@ -0,0 +1,100 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package chunker + +import ( + "context" + "testing" +) + +// oneChunksOf drives OneChunker.Invoke and returns the emitted chunk maps. +func oneChunksOf(t *testing.T, inputs map[string]any) []map[string]any { + t.Helper() + comp, err := NewOneChunker(nil) + if err != nil { + t.Fatalf("NewOneChunker: %v", err) + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("OneChunker.Invoke: %v", err) + } + chunks, ok := out["chunks"].([]map[string]any) + if !ok { + t.Fatalf("chunks not []map[string]any: %T", out["chunks"]) + } + return chunks +} + +// TestOneChunker_Text emits exactly one chunk for a text payload, +// faithful to rag/app/one.py (whole file = one chunk). +func TestOneChunker_Text(t *testing.T) { + chunks := oneChunksOf(t, map[string]any{ + "name": "doc.txt", + "output_format": "text", + "text": "first paragraph\n\nsecond paragraph", + }) + if len(chunks) != 1 { + t.Fatalf("want 1 chunk, got %d", len(chunks)) + } + if got := chunks[0]["text"]; got != "first paragraph\n\nsecond paragraph" { + t.Errorf("text = %q", got) + } +} + +// TestOneChunker_JSONSingleItem carries the image/media context through +// for the picture/audio methods, which TokenChunker in "one" mode drops. +func TestOneChunker_JSONSingleItem(t *testing.T) { + chunks := oneChunksOf(t, map[string]any{ + "name": "pic.png", + "output_format": "json", + "json": []map[string]any{ + {"text": "a cat sitting on a mat", "image": "data:image/png;base64,AAAA", "doc_type_kwd": "image"}, + }, + }) + if len(chunks) != 1 { + t.Fatalf("want 1 chunk, got %d", len(chunks)) + } + if got := chunks[0]["image"]; got != "data:image/png;base64,AAAA" { + t.Errorf("image = %q, want preserved media context", got) + } + if got := chunks[0]["text"]; got != "a cat sitting on a mat" { + t.Errorf("text = %q", got) + } +} + +// TestOneChunker_JSONMultipleMerges collapses many upstream items into a +// single chunk, preserving the first available image (picture/audio +// one-chunk-per-file behavior). +func TestOneChunker_JSONMultipleMerges(t *testing.T) { + chunks := oneChunksOf(t, map[string]any{ + "name": "clip.mp4", + "output_format": "json", + "json": []map[string]any{ + {"text": "frame one transcript"}, + {"text": "frame two transcript", "image": "data:image/png;base64,BBBB"}, + }, + }) + if len(chunks) != 1 { + t.Fatalf("want 1 chunk, got %d", len(chunks)) + } + if got := chunks[0]["text"]; got != "frame one transcript\nframe two transcript" { + t.Errorf("merged text = %q", got) + } + if got := chunks[0]["image"]; got != "data:image/png;base64,BBBB" { + t.Errorf("image = %q, want first available media context", got) + } +} diff --git a/internal/ingestion/component/chunker/presentation.go b/internal/ingestion/component/chunker/presentation.go new file mode 100644 index 0000000000..de1a3ffda2 --- /dev/null +++ b/internal/ingestion/component/chunker/presentation.go @@ -0,0 +1,108 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// PresentationChunker emits one chunk per upstream slide or page. +// It is the faithful Go port of the Python `presentation` chunk method +// (rag/app/presentation.py), whose docstring states: "Every page will +// be treated as a chunk." +// +// Unlike TokenChunker (which merges slides into a single chunk) or +// OneChunker (which collapses many slides into one), PresentationChunker +// keeps each slide as the unit of chunking. The upstream parser produces +// one structured record per slide with text, image, page_number, and +// position information; this chunker passes each through unchanged. +package chunker + +import ( + "context" + "fmt" + + "ragflow/internal/agent/runtime" + "ragflow/internal/ingestion/component/schema" +) + +const ComponentNamePresentationChunker = "PresentationChunker" + +type presentationChunkerParam struct{} + +func (p *presentationChunkerParam) Update(conf map[string]any) {} + +func (presentationChunkerParam) Defaults() presentationChunkerParam { + return presentationChunkerParam{} +} + +func (presentationChunkerParam) Validate() error { return nil } + +type PresentationChunkerComponent struct { + name string + param presentationChunkerParam +} + +func NewPresentationChunker(params map[string]any) (runtime.Component, error) { + p := presentationChunkerParam{}.Defaults() + (&p).Update(params) + if err := p.Validate(); err != nil { + return nil, err + } + return &PresentationChunkerComponent{ + name: ComponentNamePresentationChunker, + param: p, + }, nil +} +func (c *PresentationChunkerComponent) Inputs() map[string]string { return ChunkerInputs } + +func (c *PresentationChunkerComponent) Outputs() map[string]string { return ChunkerOutputs } + +func (c *PresentationChunkerComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { + return c.invoke(ctx, inputs) +} + +func (c *PresentationChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) { + if inputs == nil { + return emptyOutputs(), nil + } + upstream, err := decodeChunkerFromUpstream(inputs) + if err != nil { + return map[string]any{ + "output_format": "chunks", + "chunks": []map[string]any{}, + "_ERROR": fmt.Sprintf("Input error: %v", err), + }, nil + } + + // The presentation template only configures pdf and slides + // parser setups, matching Python's restriction to .pptx/.ppt/.pdf. + // The upstream parser therefore always emits per-slide JSON, never + // flat text/markdown/html, so every payload keeps one-chunk-per-slide. + items := slideItems(upstream.JSONResult, upstream.Chunks) + if len(items) == 0 { + return emptyOutputs(), nil + } + return chunkOutputs(items), nil +} + +// slideItems returns the per-slide records, preferring JSONResult and +// falling back to Chunks. Each record (slide/page) becomes one chunk. +func slideItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc { + if len(items) > 0 { + return items + } + return chunks +} + +func init() { + MustRegisterChunker(ComponentNamePresentationChunker) +} diff --git a/internal/ingestion/component/chunker/presentation_test.go b/internal/ingestion/component/chunker/presentation_test.go new file mode 100644 index 0000000000..9a695b4907 --- /dev/null +++ b/internal/ingestion/component/chunker/presentation_test.go @@ -0,0 +1,119 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package chunker + +import ( + "context" + "testing" +) + +// slideChunksOf drives PresentationChunker.Invoke and returns the emitted maps. +func slideChunksOf(t *testing.T, inputs map[string]any) []map[string]any { + t.Helper() + comp, err := NewPresentationChunker(nil) + if err != nil { + t.Fatalf("NewPresentationChunker: %v", err) + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("PresentationChunker.Invoke: %v", err) + } + chunks, ok := out["chunks"].([]map[string]any) + if !ok { + t.Fatalf("chunks not []map[string]any: %T", out["chunks"]) + } + return chunks +} + +// TestPresentationChunker_OneChunkPerSlide ports rag/app/presentation.py's +// contract: each slide becomes exactly one chunk, and per-slide metadata +// (page_number, image) survives the chunker round-trip. +func TestPresentationChunker_OneChunkPerSlide(t *testing.T) { + chunks := slideChunksOf(t, map[string]any{ + "name": "deck.pptx", + "output_format": "json", + "json": []map[string]any{ + { + "text": "Slide 1 content", + "doc_type_kwd": "image", + "page_number": float64(1), + "image": "base64slide1", + }, + { + "text": "Slide 2 content", + "doc_type_kwd": "image", + "page_number": float64(2), + "image": "base64slide2", + }, + { + "text": "Slide 3 content", + "doc_type_kwd": "image", + "page_number": float64(3), + }, + }, + }) + + if len(chunks) != 3 { + t.Fatalf("expected 3 slides, got %d", len(chunks)) + } + + for i, ck := range chunks { + wantPage := float64(i + 1) + if ck["page_number"] != wantPage { + t.Errorf("slide %d: page_number = %v, want %v", i+1, ck["page_number"], wantPage) + } + if ck["doc_type_kwd"] != "image" { + t.Errorf("slide %d: doc_type_kwd = %v, want image", i+1, ck["doc_type_kwd"]) + } + if ck["text"] == "" { + t.Errorf("slide %d: empty text", i+1) + } + } + + // Per-slide image attachments must be preserved (the key win over + // TokenChunker "one", which drops media context). + if chunks[0]["image"] != "base64slide1" { + t.Errorf("slide 1: image = %v, want base64slide1", chunks[0]["image"]) + } + if chunks[1]["image"] != "base64slide2" { + t.Errorf("slide 2: image = %v, want base64slide2", chunks[1]["image"]) + } + // A slide with text but no image still yields one chunk. + if _, ok := chunks[2]["image"]; ok { + t.Errorf("slide 3: unexpected image key present") + } +} + +// TestPresentationChunker_Empty yields no chunks when there is no data. +func TestPresentationChunker_Empty(t *testing.T) { + chunks := slideChunksOf(t, map[string]any{ + "name": "deck.pptx", + "output_format": "json", + "json": []map[string]any{}, + }) + if len(chunks) != 0 { + t.Fatalf("expected 0 chunks for empty input, got %d", len(chunks)) + } +} + +// TestPresentationChunker_NilInput yields no chunks. +func TestPresentationChunker_NilInput(t *testing.T) { + chunks := slideChunksOf(t, nil) + if len(chunks) != 0 { + t.Fatalf("expected 0 chunks for nil input, got %d", len(chunks)) + } +} diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go new file mode 100644 index 0000000000..1897abac57 --- /dev/null +++ b/internal/ingestion/component/chunker/qa.go @@ -0,0 +1,313 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// QAChunker extracts question-answer pairs from parsed content. +// +// Input formats and extraction strategies: +// - Text (txt, csv) → delimiter-based Q&A (comma or tab) +// - Markdown (md) → heading-based Q&A +// - HTML (xlsx/xls) → table-based Q&A (first two columns) +// - JSON (pdf, docx) → delimiter-based on structured text sections +// +// Every Q&A pair becomes a single chunk with content_with_weight +// formatted as "Question: {q}\tAnswer: {a}". +package chunker + +import ( + "context" + "encoding/csv" + "fmt" + "html" + "regexp" + "strings" + + "ragflow/internal/agent/runtime" + "ragflow/internal/ingestion/component/schema" + "ragflow/internal/tokenizer" +) + +const ComponentNameQAChunker = "QAChunker" + +type qaChunkerParam struct{} + +func (p *qaChunkerParam) Update(conf map[string]any) {} + +func (qaChunkerParam) Defaults() qaChunkerParam { return qaChunkerParam{} } + +func (qaChunkerParam) Validate() error { return nil } + +type QAChunkerComponent struct { + name string + param qaChunkerParam +} + +func NewQAChunker(params map[string]any) (runtime.Component, error) { + p := qaChunkerParam{}.Defaults() + (&p).Update(params) + if err := p.Validate(); err != nil { + return nil, err + } + return &QAChunkerComponent{ + name: ComponentNameQAChunker, + param: p, + }, nil +} +func (c *QAChunkerComponent) Inputs() map[string]string { return ChunkerInputs } + +func (c *QAChunkerComponent) Outputs() map[string]string { return ChunkerOutputs } + +func (c *QAChunkerComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { + return c.invoke(ctx, inputs) +} + +func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) { + if inputs == nil { + return emptyOutputs(), nil + } + upstream, err := decodeChunkerFromUpstream(inputs) + if err != nil { + return map[string]any{ + "output_format": "chunks", + "chunks": []map[string]any{}, + "_ERROR": fmt.Sprintf("Input error: %v", err), + }, nil + } + + var qaPairs []qaPair + switch upstream.OutputFormat { + case schema.PayloadFormatHTML: + qaPairs = extractQATable(stringPtrVal(upstream.HTMLResult)) + case schema.PayloadFormatMarkdown: + qaPairs = extractQAMarkdown(stringPtrVal(upstream.MarkdownResult)) + case schema.PayloadFormatText: + qaPairs = extractQAText(stringPtrVal(upstream.TextResult)) + default: + qaPairs = extractQAJSON(upstream.JSONResult) + } + + chunks := make([]schema.ChunkDoc, 0, len(qaPairs)) + for _, pair := range qaPairs { + contentLTKS, _ := tokenizer.Tokenize(pair.Question) + contentSMLTKS, _ := tokenizer.FineGrainedTokenize(contentLTKS) + chunk := schema.ChunkDoc{ + ContentWithWeight: fmt.Sprintf("Question: %s\tAnswer: %s", rmQAPrefix(pair.Question), rmQAPrefix(pair.Answer)), + DocType: "text", + ContentLtks: contentLTKS, + ContentSmLtks: contentSMLTKS, + } + chunks = append(chunks, chunk) + } + + return chunkOutputs(chunks), nil +} + +type qaPair struct { + Question string + Answer string +} + +var rmQAPrefixRe = regexp.MustCompile(`^(问题|答案|回答|user|assistant|Q|A|Question|Answer|问|答)[\t:： ]+`) + +func rmQAPrefix(txt string) string { + return strings.TrimSpace(rmQAPrefixRe.ReplaceAllString(txt, "")) +} + +func stringPtrVal(s *string) string { + if s == nil { + return "" + } + return *s +} + +// --------------------------------------------------------------------------- +// HTML / spreadsheet QA extraction +// --------------------------------------------------------------------------- + +var htmlTR = regexp.MustCompile(`(?i)]*>(.*?)`) +var htmlTD = regexp.MustCompile(`(?i)]*>(.*?)`) +var htmlTag = regexp.MustCompile(`<[^>]+>`) + +func extractQATable(htmlStr string) []qaPair { + if htmlStr == "" { + return nil + } + rows := htmlTR.FindAllStringSubmatch(htmlStr, -1) + pairs := make([]qaPair, 0, len(rows)) + for _, row := range rows { + cells := htmlTD.FindAllStringSubmatch(row[1], -1) + var texts []string + for _, cell := range cells { + t := html.UnescapeString(htmlTag.ReplaceAllString(cell[1], "")) + t = strings.TrimSpace(t) + if t != "" { + texts = append(texts, t) + } + } + if len(texts) >= 2 { + pairs = append(pairs, qaPair{Question: texts[0], Answer: texts[1]}) + } + } + return pairs +} + +// --------------------------------------------------------------------------- +// Markdown QA extraction +// --------------------------------------------------------------------------- + +var mdHeading = regexp.MustCompile(`^(#{1,6})\s+`) + +func extractQAMarkdown(md string) []qaPair { + if md == "" { + return nil + } + lines := strings.Split(md, "\n") + var pairs []qaPair + var questionStack, levelStack []string + var answer []string + codeBlock := false + + flushAnswer := func() { + joined := strings.TrimSpace(strings.Join(answer, "\n")) + if joined != "" && len(questionStack) > 0 { + sumQ := strings.Join(questionStack, "\n") + pairs = append(pairs, qaPair{Question: sumQ, Answer: joined}) + } + answer = nil + } + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "```") { + codeBlock = !codeBlock + } + if codeBlock { + answer = append(answer, line) + continue + } + + m := mdHeading.FindStringSubmatch(line) + if m == nil || len(m[1]) > 6 { + answer = append(answer, line) + continue + } + + flushAnswer() + level := m[1] + question := strings.TrimSpace(line[len(m[0]):]) + + for len(levelStack) > 0 && len(level) <= len(levelStack[len(levelStack)-1]) { + questionStack = questionStack[:len(questionStack)-1] + levelStack = levelStack[:len(levelStack)-1] + } + questionStack = append(questionStack, question) + levelStack = append(levelStack, level) + } + flushAnswer() + return pairs +} + +// --------------------------------------------------------------------------- +// Text / delimiter-based QA extraction (txt, csv) +// --------------------------------------------------------------------------- + +func extractQAText(text string) []qaPair { + if text == "" { + return nil + } + lines := strings.Split(text, "\n") + + delimiter := detectDelimiter(lines) + + var pairs []qaPair + var question, answer string + + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + parts := splitQA(line, delimiter) + if len(parts) != 2 { + if question != "" { + answer += "\n" + line + } + continue + } + if question != "" && answer != "" { + pairs = append(pairs, qaPair{Question: strings.TrimSpace(question), Answer: strings.TrimSpace(answer)}) + } + question = parts[0] + answer = parts[1] + } + if question != "" { + pairs = append(pairs, qaPair{Question: strings.TrimSpace(question), Answer: strings.TrimSpace(answer)}) + } + return pairs +} + +func detectDelimiter(lines []string) string { + comma, tab := 0, 0 + for _, line := range lines { + if len(strings.Split(line, ",")) == 2 { + comma++ + } + if len(strings.Split(line, "\t")) == 2 { + tab++ + } + } + if tab >= comma { + return "\t" + } + return "," +} + +func splitQA(line, delimiter string) []string { + if delimiter == "\t" { + parts := strings.Split(line, "\t") + if len(parts) == 2 { + return parts + } + return []string{line} + } + r := csv.NewReader(strings.NewReader(line)) + r.Comma = ',' + r.LazyQuotes = true + records, err := r.Read() + if err != nil || len(records) != 2 { + return []string{line} + } + return records +} + +// --------------------------------------------------------------------------- +// JSON / structured QA extraction +// --------------------------------------------------------------------------- + +func extractQAJSON(items []schema.ChunkDoc) []qaPair { + var pairs []qaPair + for _, item := range items { + txt, _ := itemText(item) + if txt == "" { + continue + } + tmp := extractQAText(txt) + pairs = append(pairs, tmp...) + } + return pairs +} + +func init() { + MustRegisterChunker(ComponentNameQAChunker) +} diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.go new file mode 100644 index 0000000000..821d261540 --- /dev/null +++ b/internal/ingestion/component/chunker/qa_test.go @@ -0,0 +1,169 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package chunker + +import ( + "context" + "testing" + + "ragflow/internal/agent/runtime" +) + +func TestQAChunker_Registered(t *testing.T) { + factory, _, _, ok := runtime.DefaultRegistry.Lookup("QAChunker") + if !ok { + t.Fatal("QAChunker not found in registry") + } + comp, err := factory("QAChunker", nil) + if err != nil { + t.Fatalf("factory failed: %v", err) + } + if comp == nil { + t.Fatal("component is nil") + } +} + +func TestQAChunker_DelimiterTab(t *testing.T) { + comp, err := NewQAChunker(nil) + if err != nil { + t.Fatal(err) + } + inputs := map[string]any{ + "name": "test.txt", + "output_format": "text", + "text": "What is Go?\tGo is a programming language.", + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("Invoke failed: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) != 1 { + t.Fatalf("expected 1 chunk, got %d", len(chunks)) + } + chunk := chunks[0] + cww, _ := chunk["content_with_weight"].(string) + if cww != "Question: What is Go?\tAnswer: Go is a programming language." { + t.Fatalf("unexpected content: %q", cww) + } +} + +func TestQAChunker_DelimiterComma(t *testing.T) { + comp, err := NewQAChunker(nil) + if err != nil { + t.Fatal(err) + } + inputs := map[string]any{ + "name": "test.csv", + "output_format": "text", + "text": "What is Rust?,Rust is a systems language.", + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("Invoke failed: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) != 1 { + t.Fatalf("expected 1 chunk, got %d", len(chunks)) + } + chunk := chunks[0] + cww, _ := chunk["content_with_weight"].(string) + if cww != "Question: What is Rust?\tAnswer: Rust is a systems language." { + t.Fatalf("unexpected content: %q", cww) + } +} + +func TestQAChunker_Markdown(t *testing.T) { + comp, err := NewQAChunker(nil) + if err != nil { + t.Fatal(err) + } + inputs := map[string]any{ + "name": "test.md", + "output_format": "markdown", + "markdown": "# What is Go?\nGo is a programming language.\n\n# What is Rust?\nRust is a systems language.", + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("Invoke failed: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) != 2 { + t.Fatalf("expected 2 chunks, got %d", len(chunks)) + } +} + +func TestQAChunker_HTMLTable(t *testing.T) { + comp, err := NewQAChunker(nil) + if err != nil { + t.Fatal(err) + } + inputs := map[string]any{ + "name": "test.xlsx", + "output_format": "html", + "html": "
Q1A1
Q2A2
", + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("Invoke failed: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) != 2 { + t.Fatalf("expected 2 chunks, got %d", len(chunks)) + } +} + +func TestQAChunker_RmQAPrefix(t *testing.T) { + comp, err := NewQAChunker(nil) + if err != nil { + t.Fatal(err) + } + inputs := map[string]any{ + "name": "test.txt", + "output_format": "text", + "text": "Question: What is Go?\tAnswer: Go is a language.", + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("Invoke failed: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + cww, _ := chunks[0]["content_with_weight"].(string) + if cww != "Question: What is Go?\tAnswer: Go is a language." { + t.Fatalf("prefix not stripped: %q", cww) + } +} + +func TestQAChunker_Empty(t *testing.T) { + comp, err := NewQAChunker(nil) + if err != nil { + t.Fatal(err) + } + inputs := map[string]any{ + "name": "empty.txt", + "output_format": "text", + "text": "", + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("Invoke failed: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) != 0 { + t.Fatalf("expected 0 chunks, got %d", len(chunks)) + } +} diff --git a/internal/ingestion/component/chunker/register_test.go b/internal/ingestion/component/chunker/register_test.go new file mode 100644 index 0000000000..ae63c02649 --- /dev/null +++ b/internal/ingestion/component/chunker/register_test.go @@ -0,0 +1,40 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package chunker + +import ( + "testing" + + "ragflow/internal/agent/runtime" +) + +// TestNewChunkersRegistered pins that OneChunker, TagChunker and +// TableChunker are registered under CategoryIngestion (the exact +// invariant the handler/service component-list tests assert). It lives +// here so the check runs without the parser package, which has an +// unrelated, pre-existing build break on the diverged branch. +func TestNewChunkersRegistered(t *testing.T) { + for _, name := range []string{ComponentNameOneChunker, ComponentNameTagChunker, ComponentNameTableChunker, ComponentNamePresentationChunker} { + _, cat, _, ok := runtime.DefaultRegistry.Lookup(name) + if !ok { + t.Fatalf("%s: not registered", name) + } + if cat != runtime.CategoryIngestion { + t.Errorf("%s: category = %q, want %q", name, cat, runtime.CategoryIngestion) + } + } +} diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.go new file mode 100644 index 0000000000..3e552321bc --- /dev/null +++ b/internal/ingestion/component/chunker/table.go @@ -0,0 +1,122 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// TableChunker emits one chunk per upstream table row. It is the faithful +// Go port of the Python `table` chunk method (rag/app/table.py), whose +// docstring states: "Every row in table will be treated as a chunk." +// +// Unlike TokenChunker (which token-shreds a row's text into multiple +// pieces) or OneChunker (which merges many rows into a single chunk), +// TableChunker keeps the row as the unit of chunking. The table parser is +// responsible for producing one structured record (ChunkDoc) per row with +// the formatted "- field: value" content and any column-role / field_map +// metadata; this chunker passes each record through unchanged, one chunk +// per row. +package chunker + +import ( + "context" + "fmt" + + "ragflow/internal/agent/runtime" + "ragflow/internal/ingestion/component/schema" +) + +const ComponentNameTableChunker = "TableChunker" + +type tableChunkerParam struct{} + +func (p *tableChunkerParam) Update(conf map[string]any) {} + +func (tableChunkerParam) Defaults() tableChunkerParam { return tableChunkerParam{} } + +func (tableChunkerParam) Validate() error { return nil } + +type TableChunkerComponent struct { + name string + param tableChunkerParam +} + +func NewTableChunker(params map[string]any) (runtime.Component, error) { + p := tableChunkerParam{}.Defaults() + (&p).Update(params) + if err := p.Validate(); err != nil { + return nil, err + } + return &TableChunkerComponent{ + name: ComponentNameTableChunker, + param: p, + }, nil +} +func (c *TableChunkerComponent) Inputs() map[string]string { return ChunkerInputs } + +func (c *TableChunkerComponent) Outputs() map[string]string { return ChunkerOutputs } + +func (c *TableChunkerComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { + return c.invoke(ctx, inputs) +} + +func (c *TableChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) { + if inputs == nil { + return emptyOutputs(), nil + } + upstream, err := decodeChunkerFromUpstream(inputs) + if err != nil { + return map[string]any{ + "output_format": "chunks", + "chunks": []map[string]any{}, + "_ERROR": fmt.Sprintf("Input error: %v", err), + }, nil + } + + switch upstream.OutputFormat { + case schema.PayloadFormatMarkdown: + if upstream.MarkdownResult == nil { + return emptyOutputs(), nil + } + return emitOne(*upstream.MarkdownResult, "text"), nil + case schema.PayloadFormatText: + if upstream.TextResult == nil { + return emptyOutputs(), nil + } + return emitOne(*upstream.TextResult, "text"), nil + case schema.PayloadFormatHTML: + if upstream.HTMLResult == nil { + return emptyOutputs(), nil + } + return emitOne(*upstream.HTMLResult, "text"), nil + default: + // Row-structured payload: one chunk per upstream record. + items := tableItems(upstream.JSONResult, upstream.Chunks) + if len(items) == 0 { + return emptyOutputs(), nil + } + return chunkOutputs(items), nil + } +} + +// tableItems returns the per-row records, preferring JSONResult and +// falling back to Chunks. Each record becomes exactly one chunk. +func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc { + if len(items) > 0 { + return items + } + return chunks +} + +func init() { + MustRegisterChunker(ComponentNameTableChunker) +} diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.go new file mode 100644 index 0000000000..91df228933 --- /dev/null +++ b/internal/ingestion/component/chunker/table_test.go @@ -0,0 +1,82 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package chunker + +import ( + "context" + "testing" +) + +// tableChunksOf drives TableChunker.Invoke and returns the emitted maps. +func tableChunksOf(t *testing.T, inputs map[string]any) []map[string]any { + t.Helper() + comp, err := NewTableChunker(nil) + if err != nil { + t.Fatalf("NewTableChunker: %v", err) + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("TableChunker.Invoke: %v", err) + } + chunks, ok := out["chunks"].([]map[string]any) + if !ok { + t.Fatalf("chunks not []map[string]any: %T", out["chunks"]) + } + return chunks +} + +// TestTableChunker_OneChunkPerRow ports rag/app/table.py's contract: +// "Every row in table will be treated as a chunk." Two upstream rows must +// yield exactly two chunks, each preserving its own content + metadata. +func TestTableChunker_OneChunkPerRow(t *testing.T) { + chunks := tableChunksOf(t, map[string]any{ + "name": "sheet.csv", + "output_format": "json", + "json": []map[string]any{ + { + "content_with_weight": "- name: Alice\n- age: 30", + "doc_type_kwd": "table", + }, + { + "content_with_weight": "- name: Bob\n- age: 25", + "doc_type_kwd": "table", + }, + }, + }) + + if len(chunks) != 2 { + t.Fatalf("got %d chunks, want 2", len(chunks)) + } + if chunks[0]["content_with_weight"] != "- name: Alice\n- age: 30" { + t.Errorf("chunk0 content = %v", chunks[0]["content_with_weight"]) + } + if chunks[1]["content_with_weight"] != "- name: Bob\n- age: 25" { + t.Errorf("chunk1 content = %v", chunks[1]["content_with_weight"]) + } +} + +// TestTableChunker_EmptyRows yields no chunks when there is no data. +func TestTableChunker_EmptyRows(t *testing.T) { + chunks := tableChunksOf(t, map[string]any{ + "name": "sheet.csv", + "output_format": "json", + "json": []map[string]any{}, + }) + if len(chunks) != 0 { + t.Errorf("got %d chunks, want 0", len(chunks)) + } +} diff --git a/internal/ingestion/component/chunker/tag.go b/internal/ingestion/component/chunker/tag.go new file mode 100644 index 0000000000..50bc752ac5 --- /dev/null +++ b/internal/ingestion/component/chunker/tag.go @@ -0,0 +1,225 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// TagChunker ports the Python `tag` chunk method (rag/app/tag.py). +// +// The Python chunk() reads an Excel/csv/txt file with two logical +// columns — content and tags (no header) — and emits ONE chunk per +// row, each carrying: +// +// content_with_weight = content (verbatim) +// tag_kwd = [t for t in tags.split(",") if t] (".\" -> "_") +// +// The Go ingestion pipeline feeds the chunker a parsed payload rather +// than the raw file, so TagChunker mirrors the sibling QAChunker and +// extracts (content, tags) pairs from every accepted upstream format: +// +// - Text / Markdown / CSV-style txt → delimiter-based pairing +// (tab wins over comma, matching tag.py's detectDelimiter). +// - HTML (spreadsheet → table) → first two columns. +// - JSON (parsed doc items) → per-item text pairing. +// +// Every pair becomes one chunk with content_with_weight = content and +// tag_kwd = the comma-split, dot-sanitised tag list. +package chunker + +import ( + "context" + "fmt" + "html" + "strings" + + "ragflow/internal/agent/runtime" + "ragflow/internal/ingestion/component/schema" + "ragflow/internal/tokenizer" +) + +const ComponentNameTagChunker = "TagChunker" + +type tagChunkerParam struct{} + +func (p *tagChunkerParam) Update(conf map[string]any) {} + +func (tagChunkerParam) Defaults() tagChunkerParam { return tagChunkerParam{} } + +func (tagChunkerParam) Validate() error { return nil } + +type TagChunkerComponent struct { + name string + param tagChunkerParam +} + +func NewTagChunker(params map[string]any) (runtime.Component, error) { + p := tagChunkerParam{}.Defaults() + (&p).Update(params) + if err := p.Validate(); err != nil { + return nil, err + } + return &TagChunkerComponent{ + name: ComponentNameTagChunker, + param: p, + }, nil +} +func (c *TagChunkerComponent) Inputs() map[string]string { return ChunkerInputs } + +func (c *TagChunkerComponent) Outputs() map[string]string { return ChunkerOutputs } + +func (c *TagChunkerComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { + return c.invoke(ctx, inputs) +} + +func (c *TagChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) { + if inputs == nil { + return emptyOutputs(), nil + } + upstream, err := decodeChunkerFromUpstream(inputs) + if err != nil { + return map[string]any{ + "output_format": "chunks", + "chunks": []map[string]any{}, + "_ERROR": fmt.Sprintf("Input error: %v", err), + }, nil + } + + var pairs []tagPair + switch upstream.OutputFormat { + case schema.PayloadFormatHTML: + pairs = extractTagTable(stringPtrVal(upstream.HTMLResult)) + case schema.PayloadFormatMarkdown: + pairs = extractTagText(stringPtrVal(upstream.MarkdownResult)) + case schema.PayloadFormatText: + pairs = extractTagText(stringPtrVal(upstream.TextResult)) + default: + pairs = extractTagJSON(upstream.JSONResult) + } + + chunks := make([]schema.ChunkDoc, 0, len(pairs)) + for _, pair := range pairs { + content := strings.TrimSpace(pair.Content) + if content == "" { + continue + } + contentLTKS, _ := tokenizer.Tokenize(content) + contentSMLTKS, _ := tokenizer.FineGrainedTokenize(contentLTKS) + chunk := schema.ChunkDoc{ + ContentWithWeight: content, + DocType: "text", + ContentLtks: contentLTKS, + ContentSmLtks: contentSMLTKS, + TagKwd: splitTagKwd(pair.Tags), + } + chunks = append(chunks, chunk) + } + + return chunkOutputs(chunks), nil +} + +// tagPair is a (content, tags) row extracted from the upstream payload. +type tagPair struct { + Content string + Tags string +} + +// extractTagText ports tag.py:60-89 (txt) and tag.py:91-113 (csv): +// every deformed line is accumulated into the running content; a line that +// splits into exactly two fields is emitted as a (content+prefix, tags) +// pair and the accumulator is reset. Delimiter selection (tab vs comma) +// reuses the same detectDelimiter/splitQA helpers as QAChunker. +func extractTagText(text string) []tagPair { + if text == "" { + return nil + } + lines := strings.Split(text, "\n") + delimiter := detectDelimiter(lines) + + var pairs []tagPair + content := "" + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + parts := splitQA(line, delimiter) + if len(parts) != 2 { + content += "\n" + line + continue + } + content += "\n" + parts[0] + pairs = append(pairs, tagPair{Content: content, Tags: parts[1]}) + content = "" + } + return pairs +} + +// extractTagTable ports the spreadsheet (xlsx/csv → html table) path. +// Python tag.py reads these via the Excel parser's (q, a) pairs, which +// are exactly the first two columns of each row — so we take the same +// two leading cells per . +func extractTagTable(htmlStr string) []tagPair { + if htmlStr == "" { + return nil + } + rows := htmlTR.FindAllStringSubmatch(htmlStr, -1) + pairs := make([]tagPair, 0, len(rows)) + for _, row := range rows { + cells := htmlTD.FindAllStringSubmatch(row[1], -1) + var texts []string + for _, cell := range cells { + t := html.UnescapeString(htmlTag.ReplaceAllString(cell[1], "")) + t = strings.TrimSpace(t) + if t != "" { + texts = append(texts, t) + } + } + if len(texts) >= 2 { + pairs = append(pairs, tagPair{Content: texts[0], Tags: texts[1]}) + } + } + return pairs +} + +// extractTagJSON ports the JSON/structured upstream path: each item's +// text is run through the same delimiter pairing as the text path. +func extractTagJSON(items []schema.ChunkDoc) []tagPair { + var pairs []tagPair + for _, item := range items { + txt, _ := itemText(item) + if txt == "" { + continue + } + pairs = append(pairs, extractTagText(txt)...) + } + return pairs +} + +// splitTagKwd ports tag.py:beAdoc's tag_kwd construction: split on +// comma, drop empties, and replace "." with "_" (the storage layer +// rejects dots in keyword values). +func splitTagKwd(s string) []string { + parts := strings.Split(s, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + p = strings.ReplaceAll(p, ".", "_") + if p != "" { + out = append(out, p) + } + } + return out +} + +func init() { + MustRegisterChunker(ComponentNameTagChunker) +} diff --git a/internal/ingestion/component/chunker/tag_test.go b/internal/ingestion/component/chunker/tag_test.go new file mode 100644 index 0000000000..246637b3e7 --- /dev/null +++ b/internal/ingestion/component/chunker/tag_test.go @@ -0,0 +1,135 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package chunker + +import ( + "context" + "testing" +) + +// tagChunksOf drives TagChunker.Invoke and returns the emitted chunk maps. +func tagChunksOf(t *testing.T, inputs map[string]any) []map[string]any { + t.Helper() + comp, err := NewTagChunker(nil) + if err != nil { + t.Fatalf("NewTagChunker: %v", err) + } + out, err := comp.Invoke(context.Background(), inputs) + if err != nil { + t.Fatalf("TagChunker.Invoke: %v", err) + } + chunks, ok := out["chunks"].([]map[string]any) + if !ok { + t.Fatalf("chunks not []map[string]any: %T", out["chunks"]) + } + return chunks +} + +// tagKwdOf reads the tag_kwd field from a chunk map. After the +// ChunkDoc -> JSON -> map round-trip, the string slice arrives as +// []any, so we normalise it here. +func tagKwdOf(t *testing.T, m map[string]any) []string { + t.Helper() + raw, ok := m["tag_kwd"] + if !ok { + t.Fatalf("tag_kwd missing: %v", m) + } + arr, ok := raw.([]any) + if !ok { + t.Fatalf("tag_kwd not []any: %T", raw) + } + out := make([]string, 0, len(arr)) + for _, v := range arr { + s, ok := v.(string) + if !ok { + t.Fatalf("tag_kwd entry not string: %T", v) + } + out = append(out, s) + } + return out +} + +// TestTagChunker_TextTabDelimited ports tag.py's txt path: one chunk per +// content+tags row, content_with_weight = content, tag_kwd = comma-split +// (dots sanitsed) tags. +func TestTagChunker_TextTabDelimited(t *testing.T) { + chunks := tagChunksOf(t, map[string]any{ + "name": "tags.txt", + "output_format": "text", + "text": "what is ragflow\tRAG,LLM\nhow to deploy\tGUIDE,ops.example", + }) + if len(chunks) != 2 { + t.Fatalf("want 2 chunks, got %d", len(chunks)) + } + + c0 := chunks[0] + if got := c0["content_with_weight"]; got != "what is ragflow" { + t.Errorf("chunk0 content_with_weight = %q", got) + } + tags0 := tagKwdOf(t, c0) + if len(tags0) != 2 || tags0[0] != "RAG" || tags0[1] != "LLM" { + t.Errorf("chunk0 tag_kwd = %v, want [RAG LLM]", tags0) + } + + c1 := chunks[1] + tags1 := tagKwdOf(t, c1) + // "ops.example" -> "ops_example": dots are sanitsed for keyword storage. + if len(tags1) != 2 || tags1[0] != "GUIDE" || tags1[1] != "ops_example" { + t.Errorf("chunk1 tag_kwd = %v, want [GUIDE ops_example]", tags1) + } +} + +// TestTagChunker_TextCommaDelimited verifies comma wins when the file has +// no tab-structured rows (mirrors tag.py:detectDelimiter). +func TestTagChunker_TextCommaDelimited(t *testing.T) { + chunks := tagChunksOf(t, map[string]any{ + "name": "tags.csv", + "output_format": "text", + "text": "what is ragflow,RAG\nhow to deploy,GUIDE", + }) + if len(chunks) != 2 { + t.Fatalf("want 2 chunks, got %d", len(chunks)) + } + if got := chunks[0]["content_with_weight"]; got != "what is ragflow" { + t.Errorf("chunk0 content_with_weight = %q", got) + } + tags0 := tagKwdOf(t, chunks[0]) + if len(tags0) != 1 || tags0[0] != "RAG" { + t.Errorf("chunk0 tag_kwd = %v, want [RAG]", tags0) + } +} + +// TestTagChunker_HTMLTable ports the spreadsheet (xlsx/csv -> html table) +// path: first two columns become content + tags. +func TestTagChunker_HTMLTable(t *testing.T) { + html := "
q1a1,b1
q2a2
" + chunks := tagChunksOf(t, map[string]any{ + "name": "tags.xlsx", + "output_format": "html", + "html": html, + }) + if len(chunks) != 2 { + t.Fatalf("want 2 chunks, got %d", len(chunks)) + } + if got := chunks[0]["content_with_weight"]; got != "q1" { + t.Errorf("chunk0 content_with_weight = %q", got) + } + tags0 := tagKwdOf(t, chunks[0]) + if len(tags0) != 2 || tags0[0] != "a1" || tags0[1] != "b1" { + t.Errorf("chunk0 tag_kwd = %v, want [a1 b1]", tags0) + } +} diff --git a/internal/ingestion/component/chunker/title.go b/internal/ingestion/component/chunker/title.go index bf71d15909..30c7e56334 100644 --- a/internal/ingestion/component/chunker/title.go +++ b/internal/ingestion/component/chunker/title.go @@ -23,24 +23,29 @@ // happens in group.go / hierarchy.go depending on the // `method` param. // -// - PARALLELISM: Parallelism() is only a hint for outer executors. // TitleChunker.Invoke dispatches synchronously to group.go or // hierarchy.go. // // - HEADING DETECTION PARITY: -// The Python side uses three heading-detection strategies in +// The Python side uses two heading-detection strategies in // `common.py:resolve_title_levels`: // (1) PDF outlines (extract_pdf_outlines, requires deepdoc/parser) -// (2) Regex families (the user's `levels` param) -// (3) Layout hints (layout field matches section/title/head) -// The Go port ships ONLY strategy (2). Strategies (1) and (3) -// require PDF binary access (deepdoc is Python-only) and a parser -// that emits a layout field. A canvas author who needs PDF-outline heading -// detection must wait for the deepdoc/parser port. +// (2) Regex families (the user's `levels` param) with a layout-hint +// fallback (layout field matches section/title/head). +// The Go port ships strategy (2) IN FULL, including the +// match_layout_level fallback ported from common.py (Gap C closed): +// a non-regex text record whose layout flags it as a +// section/title/head and whose text passes not_title is promoted to +// fallback_level = len(selected_group) + 1. Strategy (1) — +// PDF-outline detection — still requires deepdoc/parser binary +// access and remains the only parity gap. // -// - TODO: parity-gap — non-ASCII heading formats and PDFs without -// user-supplied `levels` are not detected. Tests assert ASCII -// input only. +// - The Go port has NO hardcoded BULLET_PATTERN fallback. Heading +// detection relies entirely on the user-supplied `levels` param +// (which templates carry as comprehensive regex families) paired +// with the layout-hint fallback. A PDF without matching levels +// produces BODY_LEVEL-only records — Python's BULLET_PATTERN-based +// tree_merge / hierarchical_merge would still find structure. // // - GROUP-TITLE and HIERARCHY-TITLE are separate Go files // (`group.go`, `hierarchy.go`); they share the resolve_levels @@ -52,6 +57,7 @@ import ( "fmt" "regexp" "strings" + "unicode/utf8" "ragflow/internal/agent/runtime" "ragflow/internal/ingestion/component/globals" @@ -129,6 +135,11 @@ func selectLevelGroup(lines []string, rawLevels [][]string) []string { if text == "" { continue } + // Mirror common.py:select_level_group — a bullet-looking + // line cannot count as a heading hit for any family. + if notBullet(text) { + continue + } for _, pattern := range group { if re := compileLevelPattern(pattern); re != nil { if re.MatchString(text) { @@ -166,6 +177,11 @@ func matchRegexLevel(text string, group []string) int { if stripped == "" { return 0 } + // Mirror common.py:match_regex_level — a bullet-looking line is not + // a heading regardless of the regex family. + if notBullet(stripped) { + return 0 + } for lvl, pattern := range group { re := compileLevelPattern(pattern) if re != nil && re.MatchString(stripped) { @@ -175,25 +191,104 @@ func matchRegexLevel(text string, group []string) int { return 0 } -// resolveTitleLevels mirrors common.py:resolve_title_levels in the -// "frequency" branch only (the outline branch is parity-gap territory -// per the SCOPE comment above). -func resolveTitleLevels(lines []string, p *titleChunkerParam) []int { - group := selectLevelGroup(lines, p.Levels) - if len(group) == 0 { - // No levels hit — every line is body. - out := make([]int, len(lines)) - for i := range out { - out[i] = bodyLevel +// notBulletPatterns mirrors rag/nlp.not_bullet. A line matching any of +// these looks like a numbered/bulleted list item rather than a section +// heading, so it must not be promoted to a title level by the regex +// families. +var notBulletPatterns = []*regexp.Regexp{ + regexp.MustCompile(`^0`), + regexp.MustCompile(`^[0-9]+ +[0-9~个只-]`), + regexp.MustCompile(`^[0-9]+\.{2,}`), + regexp.MustCompile(`^[0-9]+(\.[0-9]+){2,}[的中]`), +} + +// notBullet mirrors rag/nlp.not_bullet: true when the line looks like a +// numbered/bulleted list entry rather than a genuine section heading. +func notBullet(s string) bool { + for _, re := range notBulletPatterns { + if re.MatchString(s) { + return true } - return out } - out := make([]int, len(lines)) - for i, ln := range lines { - out[i] = matchRegexLevel(ln, group) - if out[i] == 0 { + return false +} + +// notTitlePatterns mirrors rag/nlp.not_title: +// - notTitleException: `第...条` is always a title (never "not a title"). +// - notTitlePunct: a body line typically carries one of these punctuation +// marks, so their presence flags the line as body text. +var ( + notTitleException = regexp.MustCompile(`^第[零一二三四五六七八九十百0-9]+条`) + notTitlePunct = regexp.MustCompile(`[,;，。；！!]`) + layoutHeadingRe = regexp.MustCompile(`(?i)(section|title|head)`) +) + +// notTitle mirrors rag/nlp.not_title. Returns true when the line looks +// like body text rather than a section heading, so layout-based heading +// detection must skip it. +func notTitle(s string) bool { + if notTitleException.MatchString(s) { + return false + } + if len(strings.Fields(s)) > 12 || (!strings.Contains(s, " ") && utf8.RuneCountInString(s) >= 32) { + return true + } + return notTitlePunct.MatchString(s) +} + +// beforeAt mirrors python's `text.split("@")[0]` (used by +// match_layout_level / not_title): the part of the line before the first +// "@" separator, whitespace-trimmed. +func beforeAt(s string) string { + if i := strings.Index(s, "@"); i >= 0 { + return strings.TrimSpace(s[:i]) + } + return strings.TrimSpace(s) +} + +// matchLayoutLevel mirrors common.py:match_layout_level. When the +// record's layout field flags it as a section/title/head and the text is +// title-like (not not_title), the record is promoted to `fallback_level` +// (common.py:resolve_frequency_levels sets this to len(level_group) + 1). +// Otherwise it stays BODY_LEVEL. +func matchLayoutLevel(text, layout string, fallbackLevel int) int { + if layoutHeadingRe.MatchString(layout) && !notTitle(beforeAt(text)) { + return fallbackLevel + } + return bodyLevel +} + +// resolveTitleLevels mirrors common.py:resolve_frequency_levels over the +// full record stream. It is the "frequency" branch of +// common.py:resolve_title_levels (the outline branch is parity-gap +// territory per the SCOPE comment above). +// +// For each record: +// - a non-text record is pinned to BODY_LEVEL directly (python skips +// regex/layout detection for doc_type_kwd != "text"); +// - a text record first tries the selected regex group (match_regex_level); +// on no match it falls back to match_layout_level (layout hint), else +// BODY_LEVEL. +// +// fallback_level is len(selected_group) + 1, exactly as in python. +func resolveTitleLevels(records []lineRecord, p *titleChunkerParam) []int { + lines := make([]string, len(records)) + for i, r := range records { + lines[i] = r.text + } + group := selectLevelGroup(lines, p.Levels) + fallbackLevel := len(group) + 1 + out := make([]int, len(records)) + for i, rec := range records { + if !rec.isText() { out[i] = bodyLevel + continue } + if lvl := matchRegexLevel(rec.text, group); lvl != 0 { + out[i] = lvl + continue + } + out[i] = matchLayoutLevel(rec.text, rec.layout, fallbackLevel) } return out } @@ -230,39 +325,41 @@ func lineRecordsFromText(text string) []lineRecord { // common.py:extract_line_records yields. Used by Group/Hierarchy // chunk-builders. type lineRecord struct { - text string - docType string - imgID *string - layout string - pdfPos []map[string]any + text string + docType string + imgID *string + layout string + pdfPos []map[string]any + parentMeta map[string]any } func (r lineRecord) textOrEmpty() string { return r.text } func (r lineRecord) isText() bool { return r.docType == "text" } +// trim mirrors python's str.strip(): remove leading/trailing Unicode +// whitespace. Used for emptiness checks and regex matching so the Go +// port matches python's `text.strip()` exactly. func trim(s string) string { - for len(s) > 0 && (s[0] == ' ' || s[0] == '\t' || s[0] == '\r') { - s = s[1:] - } - for len(s) > 0 { - last := s[len(s)-1] - if last == ' ' || last == '\t' || last == '\r' || last == '\n' { - s = s[:len(s)-1] - continue - } - break - } - return s + return strings.TrimSpace(s) } // compileLevelPattern returns a compiled regex for `pattern`. Returns // nil on empty/error to skip the entry — same effect as a regex // mismatch in python (the row falls through to body). +// +// Python's match_regex_level / select_level_group use `re.match`, which +// anchors at the START of the string (not the end). We mirror that by +// prepending `^` (unless the caller already anchored) so Go's +// MatchString behaves exactly like re.match. func compileLevelPattern(pattern string) *regexp.Regexp { if pattern == "" { return nil } - re, err := regexp.Compile(pattern) + anchored := pattern + if !strings.HasPrefix(pattern, "^") { + anchored = "^(?:" + pattern + ")" + } + re, err := regexp.Compile(anchored) if err != nil { return nil } @@ -277,17 +374,24 @@ type LevelContext struct { mostLevel int } -func newLevelContext(lines []string, p *titleChunkerParam) LevelContext { - levels := resolveTitleLevels(lines, p) - most := 0 - seen := make(map[int]int) +func newLevelContext(records []lineRecord, p *titleChunkerParam) LevelContext { + levels := resolveTitleLevels(records, p) + // most_level is the most-frequent non-body heading level + // (common.py:resolve_frequency_levels). Python computes this via + // Counter(levels).most_common() over the heading levels only. Walk + // levels in input order so ties resolve to the first-encountered + // level, matching python's insertion-order tie-break. + counts := make(map[int]int) for _, lvl := range levels { - seen[lvl]++ + if lvl < bodyLevel { + counts[lvl]++ + } } - // Pick the smallest level that has any hits — titles are - // more-specific headings, so the highest-specificity one wins. - for _, lvl := range seen { - if lvl < bodyLevel && (most == 0 || lvl < most) { + most := 0 + best := 0 + for _, lvl := range levels { + if c := counts[lvl]; c > best { + best = c most = lvl } } @@ -300,25 +404,6 @@ func (lc LevelContext) Levels() []int { return out } -// buildChunksFromRecordGroups is the cheap text-form materialiser -// (used by Group/Hierarchy for plain-text payloads). For structured -// upstream payloads a parallel fan-out over groups is used. -func buildChunksFromRecordGroupsText(groups [][]lineRecord) []map[string]any { - out := make([]map[string]any, 0, len(groups)) - for _, g := range groups { - if len(g) == 0 { - continue - } - var sb strings.Builder - for _, r := range g { - sb.WriteString(r.text) - sb.WriteString("\n") - } - out = append(out, map[string]any{"text": sb.String()}) - } - return out -} - // --------------------------------------------------------------------------- // Component implementations // --------------------------------------------------------------------------- @@ -346,10 +431,6 @@ func NewTitleChunker(params map[string]any) (runtime.Component, error) { }, nil } -// Parallelism is the configured intra-component fan-out (plan §4 -// Phase 2 row 2.3b). -func (c *TitleChunkerComponent) Parallelism() int { return 2 } - // Inputs is exposed so callers can introspect. func (c *TitleChunkerComponent) Inputs() map[string]string { return ChunkerInputs } @@ -358,34 +439,32 @@ func (c *TitleChunkerComponent) Outputs() map[string]string { return ChunkerOutp // Invoke delegates to the chosen strategy (group or hierarchy). func (c *TitleChunkerComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { - return runtime.TrackElapsed(ComponentNameTitleChunker, func() (map[string]any, error) { - if inputs == nil { - inputs = map[string]any{} - } - // `name` is read from the workflow-wide Globals bag (seeded at - // pipeline start, published by the File component), not from the - // upstream output map. - name := globals.GlobalOrInput(ctx, inputs, "name", "") - if name == "" { - return map[string]any{ - "output_format": "chunks", - "chunks": []map[string]any{}, - "_ERROR": "TitleChunker: missing required upstream field \"name\"", - }, nil - } - switch c.param.Method { - case "hierarchy": - return invokeHierarchy(ctx, inputs, &c.param) - case "group": - return invokeGroup(ctx, inputs, &c.param) - default: - return map[string]any{ - "output_format": "chunks", - "chunks": []map[string]any{}, - "_ERROR": fmt.Sprintf("TitleChunker: unsupported method %q", c.param.Method), - }, nil - } - }) + if inputs == nil { + inputs = map[string]any{} + } + // `name` is read from the workflow-wide Globals bag (seeded at + // pipeline start, published by the File component), not from the + // upstream output map. + name := globals.GlobalOrInput(ctx, inputs, "name", "") + if name == "" { + return map[string]any{ + "output_format": "chunks", + "chunks": []map[string]any{}, + "_ERROR": "TitleChunker: missing required upstream field \"name\"", + }, nil + } + switch c.param.Method { + case "hierarchy": + return invokeHierarchy(ctx, inputs, &c.param) + case "group": + return invokeGroup(ctx, inputs, &c.param) + default: + return map[string]any{ + "output_format": "chunks", + "chunks": []map[string]any{}, + "_ERROR": fmt.Sprintf("TitleChunker: unsupported method %q", c.param.Method), + }, nil + } } // init registers TitleChunker under CategoryIngestion. diff --git a/internal/ingestion/component/chunker/title_test.go b/internal/ingestion/component/chunker/title_test.go index 2b1ccdebbe..c8b39dd3c9 100644 --- a/internal/ingestion/component/chunker/title_test.go +++ b/internal/ingestion/component/chunker/title_test.go @@ -18,9 +18,11 @@ package chunker import ( "context" + "strings" "testing" "ragflow/internal/agent/runtime" + "ragflow/internal/ingestion/component/schema" ) // TestTitleChunker_Registered asserts the registry has a CategoryIngestion @@ -44,24 +46,6 @@ func TestTitleChunker_Registered(t *testing.T) { } } -// TestTitleChunker_Parallelism enforces plan row 2.3b parallelism (2). -func TestTitleChunker_Parallelism(t *testing.T) { - c, err := NewTitleChunker(map[string]any{ - "method": "group", - "levels": [][]string{{"^# "}}, - }) - if err != nil { - t.Fatalf("NewTitleChunker: %v", err) - } - tc, ok := c.(*TitleChunkerComponent) - if !ok { - t.Fatalf("NewTitleChunker returned %T", c) - } - if got := tc.Parallelism(); got != 2 { - t.Errorf("Parallelism() = %d, want 2", got) - } -} - // TestTitleChunker_InputsOutputs_NonEmpty asserts metadata is // populated for both ends. func TestTitleChunker_InputsOutputs_NonEmpty(t *testing.T) { @@ -255,3 +239,135 @@ func TestTitleChunker_InvokeDeterministic(t *testing.T) { } } } + +// TestNewLevelContext_MostLevelIsMode pins Gap A: most_level is the +// most-frequent non-body heading level (the mode), matching python +// Counter(levels).most_common(). The old loop iterated map values and +// returned the smallest count, which skewed the group fallback target. +func TestNewLevelContext_MostLevelIsMode(t *testing.T) { + p := &titleChunkerParam{TitleChunkerParam: schema.TitleChunkerParam{ + Method: "group", + Levels: [][]string{{`^# `, `^## `, `^### `}}, + }} + records := []lineRecord{ + {text: "# a", docType: "text"}, + {text: "## b", docType: "text"}, + {text: "## c", docType: "text"}, + {text: "### d", docType: "text"}, + } + lc := newLevelContext(records, p) + // selectLevelGroup picks the single 3-pattern family; per-line + // levels are [1,2,2,3], so the mode (most frequent heading level) + // is level 2. + if lc.mostLevel != 2 { + t.Errorf("mostLevel = %d, want 2 (the most frequent heading level)", lc.mostLevel) + } +} + +// TestNotBullet pins Gap B: the port of rag/nlp.not_bullet must reject +// numbered/bulleted list lines so they are not promoted to headings. +func TestNotBullet(t *testing.T) { + bullet := []string{ + "0", + "1 2个", + "1... trailing", + "1.2.3.4的", + "2.3.4中", + "0.5 section", + } + for _, s := range bullet { + if !notBullet(s) { + t.Errorf("notBullet(%q) = false, want true", s) + } + } + heading := []string{ + "# Heading", + "## Subheading", + "Section 1", + "Article 12", + "1.2 Normal sentence", + "1.2.3 without marker", + } + for _, s := range heading { + if notBullet(s) { + t.Errorf("notBullet(%q) = true, want false", s) + } + } +} + +// TestNotTitle pins the port of rag/nlp.not_title used by the layout +// fallback: long/no-space/punctuated lines are "not a title", while +// the 第…条 exception and short title-like lines are titles. +func TestNotTitle(t *testing.T) { + longLine := "one two three four five six seven eight nine ten eleven twelve thirteen" + if !notTitle(longLine) { + t.Errorf("notTitle(>12 words) = false, want true") + } + noSpace := strings.Repeat("x", 40) + if !notTitle(noSpace) { + t.Errorf("notTitle(no space, >=32 chars) = false, want true") + } + if !notTitle("This, is a body line") { + t.Errorf("notTitle(comma line) = false, want true") + } + if notTitle("第三条 关于某某规定") { + t.Errorf("notTitle(第…条) = true, want false (exception)") + } + if notTitle("Chapter One") { + t.Errorf("notTitle(Chapter One) = true, want false") + } +} + +// TestResolveTitleLevels_NonTextPinnedToBody pins Gap D: a non-text +// record whose caption would match a heading regex is still pinned to +// BODY_LEVEL, because python never runs regex/layout detection for +// doc_type_kwd != "text". +func TestResolveTitleLevels_NonTextPinnedToBody(t *testing.T) { + p := &titleChunkerParam{TitleChunkerParam: schema.TitleChunkerParam{ + Method: "group", + Levels: [][]string{{`^# `}}, + }} + records := []lineRecord{ + {text: "# Real Heading", docType: "text"}, + {text: "# image caption that matches", docType: "image"}, + {text: "body line", docType: "text"}, + } + levels := resolveTitleLevels(records, p) + if levels[0] != 1 { + t.Errorf("text heading level = %d, want 1", levels[0]) + } + if levels[1] != bodyLevel { + t.Errorf("non-text caption level = %d, want bodyLevel (%d)", levels[1], bodyLevel) + } + if levels[2] != bodyLevel { + t.Errorf("body level = %d, want bodyLevel", levels[2]) + } +} + +// TestResolveTitleLevels_LayoutFallback pins Gap C: a text record that +// misses every regex but whose layout flags it as a section/title/head +// and whose text passes not_title is promoted to fallback_level = +// len(selected_group) + 1. The group must have at least one regex hit +// (here "# Real Heading") so it is selected; with one pattern the +// fallback_level is 2. +func TestResolveTitleLevels_LayoutFallback(t *testing.T) { + p := &titleChunkerParam{TitleChunkerParam: schema.TitleChunkerParam{ + Method: "group", + Levels: [][]string{{`^# `}}, + }} + records := []lineRecord{ + {text: "# Real Heading", docType: "text", layout: "title"}, + {text: "Chapter One Overview", docType: "text", layout: "title"}, + {text: "Some body paragraph text here.", docType: "text", layout: "text"}, + } + levels := resolveTitleLevels(records, p) + if levels[0] != 1 { + t.Errorf("regex heading level = %d, want 1", levels[0]) + } + if levels[1] != 2 { + t.Errorf("layout title level = %d, want 2 (fallback_level)", levels[1]) + } + if levels[2] != bodyLevel { + t.Errorf("plain body level = %d, want bodyLevel", levels[2]) + } +} diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go index df8e08ae6b..a2109b4ecb 100644 --- a/internal/ingestion/component/chunker/token.go +++ b/internal/ingestion/component/chunker/token.go @@ -14,10 +14,10 @@ // limitations under the License. // -// SCOPE (honest) for token.go (Phase 2.3a): +// SCOPE (honest) for token.go: // -// - WHITELIST mirrors rag/flow/chunker/token_chunker.py:TokenChunkerParam.check -// exactly: delimiter_mode ∈ {"token_size","delimiter","one"}, +// - WHITELIST: delimiter_mode ∈ {"token_size","delimiter"} (the +// single-chunk "one" behaviour moved to OneChunker in one.go). // chunk_token_size > 0, overlapped_percent ∈ [0, 1), table_context_size ≥ 0, // image_context_size ≥ 0. enum/range checks live in param.Check. // @@ -30,24 +30,21 @@ // shared splitKeepingDelim helper; emitted chunks carry the parent // ("mom") and the split child ("text") keys. // -// - MODE "delimiter" uses the regex-aware delimiter pattern; the -// token_size merge pass only runs when no working delimiter was -// detected — matching python at token_chunker.py:359-360. +// - MODE "delimiter" uses the regex-aware delimiter pattern to split +// text into segments; unlike token_size, these segments are NOT +// merged — they become standalone chunks. // -// - MODE "token_size" falls back to a chunk-engine merge plan. The -// python `naive_merge` algorithm uses a sentence-aware + stop-word -// strategy that the Go chunk_engine's "greedy" merge approximates; -// this is flagged as a TODO parity-gap and intentionally not -// machine-mirrored. -// -// - MODE "one" emits a single chunk containing the entire payload. +// - MODE "token_size" implements Python's naive_merge split-then- +// merge: segments are split by the configured delimiter pattern +// (chunkFromItem), then greedily merged to chunk_token_size with +// optional overlap (mergeByTokenSizeFromJSON). The JSON and text +// payload paths share the same merge after splitting. // // - JSON-STRUCTURED INPUT (output_format == "json", or the default // parser-style branch when output_format is unset) is normalized -// into the same internal chunk shape via a parallel fan-out -// (Plan §4 Phase 2 row 2.3a, Parallelism=4). +// into the same internal chunk shape via a parallel fan-out. // Media-context attachment is per-item sequential; merge is -// index-deterministic (Plan §8 R8). +// index-deterministic. // // - No PDF/outline awareness (Python `restore_pdf_text_previews`). // That depends on deepdoc/parser which is out of scope for this @@ -68,7 +65,6 @@ import ( deepdoctype "ragflow/internal/deepdoc/parser/type" "ragflow/internal/ingestion/component/globals" "ragflow/internal/ingestion/component/schema" - "ragflow/internal/parser/chunk" ) const ComponentNameTokenChunker = "TokenChunker" @@ -135,10 +131,6 @@ func NewTokenChunker(params map[string]any) (runtime.Component, error) { }, nil } -// Parallelism is the configured intra-component fan-out (plan §4 -// Phase 2 row 2.3a). -func (c *TokenChunkerComponent) Parallelism() int { return 4 } - // Inputs is exposed so callers can introspect. func (c *TokenChunkerComponent) Inputs() map[string]string { return ChunkerInputs } @@ -147,17 +139,15 @@ func (c *TokenChunkerComponent) Outputs() map[string]string { return ChunkerOutp // Invoke runs the chunker against the input payload. // -// Concurrency: text payloads are fanned across Parallelism goroutines -// by primary-delimiter segment; structured JSON/chunks payloads fan +// Concurrency: text payloads are fanned across 4 goroutines by +// primary-delimiter segment; structured JSON/chunks payloads fan // across items. Merge is by input index (plan §8 R8): the i-th // goroutine's output occupies slot i, regardless of completion order. // // Timeout: honours ctx cancellation only — there is no inner @timeout // decorator equivalent (plan §8 R1). func (c *TokenChunkerComponent) Invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { - return runtime.TrackElapsed(ComponentNameTokenChunker, func() (map[string]any, error) { - return c.invoke(ctx, inputs) - }) + return c.invoke(ctx, inputs) } func (c *TokenChunkerComponent) invoke(ctx context.Context, inputs map[string]any) (map[string]any, error) { @@ -251,13 +241,7 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string return emptyOutputs() } - mode := c.param.DelimiterMode - if mode == "one" { - return chunkOutputs([]schema.ChunkDoc{{Text: text}}) - } - if !hasActiveDelimiter(delimPattern) { - // No primary delimiter hit — fall back to a token-size merge. return c.mergeByTokenSize(text, childrenPattern) } @@ -273,62 +257,153 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string return emptyOutputs() } docs := applyChildrenDelim(cleaned, childrenPattern) - return chunkOutputs(docs) + + // Python's naive_merge: custom (backtick) delimiters produce one + // chunk per segment — no token-size merge (naive_merge:1194-1213). + if hasCustomDelim(c.param.Delimiters) { + return chunkOutputs(docs) + } + + // Split-then-merge: split on delimiters, then greedily merge to + // chunk_token_size with optional overlap. + perItem := [][]schema.ChunkDoc{docs} + merged := mergeByTokenSizeFromJSON(perItem, c.param.ChunkTokenSize, c.param.OverlappedPercent) + return chunkOutputs(flatten(merged)) } -// mergeByTokenSize uses the chunk library's split + postprocess-merge -// to combine the payload into chunks of approximately -// chunk_token_size runes. Mirrors python's `naive_merge` fallback -// (token_chunker.py:319-324) at the wire-shape level; the merge -// strategy is the Go chunk library's "greedy" approximation. -// -// This caller uses the typed chunk.Run entry point directly: -// same operator sequence, no DSL round-trip. +// mergeByTokenSize implements exact token-based chunk merging that mirrors +// Python's naive_merge (rag/nlp/__init__.py:1156). It uses +// tokenizeStr (= tokenizer.NumTokensFromString, cl100k_base BPE) for +// precise token counting, splits input into paragraph sections, further +// subdivides oversized sections on sentence delimiters, and greedily +// merges into chunks of approximately chunk_token_size tokens with +// optional overlap from the previous chunk. func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *regexp.Regexp) map[string]any { target := c.param.ChunkTokenSize - result, err := chunk.Run(text, chunk.ChunkOptions{ - StripWhitespace: true, - RemoveEmptyLines: true, - SplitStrategy: "sentence", - MergeTargetSize: target, - }) - if err != nil { - return chunkOutputs([]schema.ChunkDoc{{Text: text}}) + overlapPct := c.param.OverlappedPercent + + // Split into paragraph-aligned sections. + sections := splitIntoSections(text) + if len(sections) == 0 { + return emptyOutputs() } - docs := make([]schema.ChunkDoc, 0, len(result.ResultChunks)) - for _, ck := range result.ResultChunks { - content := strings.TrimSpace(ck.Content) - if content == "" { + + // Sentence/clause-boundary regex for splitting oversized sections. + // Matches Python's default delimiter "\n。；！？" plus English ". " fallback. + sentenceDelim := regexp.MustCompile(`(\n|[。；！？]|\.\s)`) + + var cks []string // chunk texts + var tkns []int // token counts per chunk + + // mergeOrNew mirrors Python add_chunk in naive_merge: + // - If the current chunk is empty or would overflow the + // threshold → start a new chunk (with optional overlap prefix). + // - Otherwise → merge into the current chunk. + mergeOrNew := func(segment string, tokens int) { + threshold := float64(target) * (100 - overlapPct*100) / 100.0 + if len(cks) == 0 || float64(tkns[len(tkns)-1]) > threshold { + seg := segment + segTokens := tokens + if overlapPct > 0 && len(cks) > 0 { + prev := cks[len(cks)-1] + // Take the last overlapped_percent of the previous chunk + // (in runes, matching Python's len(overlapped) * ratio). + prevRunes := []rune(prev) + cut := int(float64(len(prevRunes)) * (100 - overlapPct*100) / 100.0) + if cut < len(prevRunes) { + suffix := string(prevRunes[cut:]) + seg = suffix + segment + segTokens = tokenizeStr(seg) + } + } + cks = append(cks, seg) + tkns = append(tkns, segTokens) + } else { + cks[len(cks)-1] += segment + tkns[len(tkns)-1] += tokens + } + } + + for _, sec := range sections { + sec = strings.TrimSpace(sec) + if sec == "" { continue } - docs = append(docs, schema.ChunkDoc{Text: content}) + t := "\n" + sec + tn := tokenizeStr(t) + + if tn < 8 { + // Tiny section — always merge into the previous chunk. + if len(cks) > 0 { + cks[len(cks)-1] += t + tkns[len(tkns)-1] += tn + } else { + cks = append(cks, t) + tkns = append(tkns, tn) + } + continue + } + + if tn <= target { + mergeOrNew(t, tn) + continue + } + + // Oversized section: split on sentence delimiters, then merge. + parts := sentenceDelim.Split(sec, -1) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + p := "\n" + part + pn := tokenizeStr(p) + if pn < 8 { + if len(cks) > 0 { + cks[len(cks)-1] += p + tkns[len(tkns)-1] += pn + } else { + cks = append(cks, p) + tkns = append(tkns, pn) + } + continue + } + mergeOrNew(p, pn) + } + } + + docs := make([]schema.ChunkDoc, 0, len(cks)) + for _, ch := range cks { + ch = strings.TrimSpace(ch) + if ch == "" { + continue + } + docs = append(docs, schema.ChunkDoc{Text: ch}) } final := applyChildrenDelimText(docs, childrenPattern) return chunkOutputs(final) } +// splitIntoSections partitions text into paragraph-level sections by +// splitting on double-newline boundaries. This mirrors the caller side +// in Python's naive_merge where sections are pre-split before merging. +func splitIntoSections(text string) []string { + if text == "" { + return nil + } + // Split on consecutive newlines (paragraph boundaries). + re := regexp.MustCompile(`\n\s*\n`) + parts := re.Split(text, -1) + return parts +} + // invokeJSONPayload handles structured upstream input. Items fan -// across goroutines (Parallelism); merge is by input index. +// across 4 goroutines; merge is by input index. func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []schema.ChunkDoc, delimPattern, childrenPattern *regexp.Regexp, engine deepdoctype.PDFEngine) map[string]any { if len(items) == 0 { return emptyOutputs() } - mode := c.param.DelimiterMode - if mode == "one" { - var parts []string - for _, it := range items { - if t := itemTextOrFallback(it); t != "" { - parts = append(parts, t) - } - } - merged := strings.Join(parts, "\n") - if merged == "" { - return emptyOutputs() - } - return chunkOutputs([]schema.ChunkDoc{{Text: merged}}) - } - - workers := c.Parallelism() + workers := 4 if workers < 1 { workers = 1 } @@ -365,8 +440,11 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s // Attach surrounding media context (token_chunker.py:358). attached := attachMediaContext(perItem, c.param.TableContextSize, c.param.ImageContextSize) - // Optional token-size merge (only when no working delimiter). - if !hasActiveDelimiter(delimPattern) { + // Python's naive_merge: custom (backtick) delimiters produce one + // chunk per segment — no token-size merge (naive_merge:1194-1213). + // Otherwise split-then-merge: delimiter-split segments are greedily + // merged to chunk_token_size with optional overlap. + if !hasCustomDelim(c.param.Delimiters) { attached = mergeByTokenSizeFromJSON(attached, c.param.ChunkTokenSize, c.param.OverlappedPercent) } @@ -583,8 +661,8 @@ func takeFromStart(text string, tokens int) string { return text[:bytes] } -// mergeByTokenSizeFromJSON mirrors `_merge_text_chunks_by_token_size` -// at token_chunker.py:212-243. +// mergeByTokenSizeFromJSON mirrors `naive_merge` at +// rag/nlp/__init__.py:1156. func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, overlappedPct float64) [][]schema.ChunkDoc { threshold := float64(chunkTokens) * (100 - overlappedPct*100) / 100.0 for idx := range perItem { @@ -593,34 +671,42 @@ func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, over continue } var merged []schema.ChunkDoc - prevTextIdx := -1 for _, ck := range chunks { ckType := ck.CKType if ckType != "text" { merged = append(merged, cloneChunkDoc(ck)) - prevTextIdx = -1 continue } tk := intValue(ck.TKNums) - if prevTextIdx < 0 || float64(tk) > threshold { + // Mirror Python's naive_merge.add_chunk: start a new chunk + // when either (a) this is the first text chunk, or + // (b) the currently accumulated chunk already exceeds the + // threshold (not the incoming segment). + if len(merged) == 0 || merged[len(merged)-1].CKType != "text" || + float64(intValue(merged[len(merged)-1].TKNums)) > threshold { cp := cloneChunkDoc(ck) - if prevTextIdx >= 0 && overlappedPct > 0 { - if prevText := merged[prevTextIdx].Text; prevText != "" { - cut := int(float64(len(prevText)) * (100 - overlappedPct*100) / 100.0) - if cut < len(prevText) { - cp.Text = prevText[cut:] + cp.Text - cp.TKNums = intPtr(tk + tokenizeStr(cp.Text)) + // Overlap: prepend tail of previous chunk onto the new + // chunk, matching Python's + // t = overlapped[overlap_cut:] + t + // tnum = num_tokens_from_string(t) + if len(merged) > 0 && merged[len(merged)-1].CKType == "text" && overlappedPct > 0 { + if prevText := merged[len(merged)-1].Text; prevText != "" { + runes := []rune(prevText) + cut := int(float64(len(runes)) * (100 - overlappedPct*100) / 100.0) + if cut < len(runes) { + cp.Text = string(runes[cut:]) + cp.Text + cp.TKNums = intPtr(tokenizeStr(cp.Text)) } } } merged = append(merged, cp) - prevTextIdx = len(merged) - 1 continue } - // Merge into prev text chunk. - if pt := merged[prevTextIdx].Text; pt != "" { - merged[prevTextIdx].Text = pt + "\n" + ck.Text - merged[prevTextIdx].TKNums = intPtr(intValue(merged[prevTextIdx].TKNums) + intValue(ck.TKNums)) + // Merge into the accumulated text chunk. + prev := &merged[len(merged)-1] + if prev.Text != "" { + prev.Text = prev.Text + "\n" + ck.Text + prev.TKNums = intPtr(intValue(prev.TKNums) + tk) } } perItem[idx] = merged @@ -696,6 +782,18 @@ func hasActiveDelimiter(p *regexp.Regexp) bool { return p != nil && p.String() != `\A(?!)` } +// hasCustomDelim reports whether any delimiter uses backtick syntax +// (`pattern`). Python's naive_merge skips token-size merging when +// custom delimiters are present (naive_merge:1194-1213). +func hasCustomDelim(delims []string) bool { + for _, d := range delims { + if strings.HasPrefix(d, "`") && strings.HasSuffix(d, "`") && len(d) >= 2 { + return true + } + } + return false +} + // applyChildrenDelim mirrors token_chunker.py:325-334. func applyChildrenDelim(segs []string, pattern *regexp.Regexp) []schema.ChunkDoc { if pattern == nil { diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go index 1025dea504..3c1ef9c52b 100644 --- a/internal/ingestion/component/chunker/token_test.go +++ b/internal/ingestion/component/chunker/token_test.go @@ -130,33 +130,6 @@ func TestTokenChunker_InvokeTokenSize_FallbackToMerge(t *testing.T) { } } -// TestTokenChunker_InvokeOneMode_EmitsSingleChunk confirms the -// `delimiter_mode == "one"` branch collapses the input to a single -// chunk. -func TestTokenChunker_InvokeOneMode_EmitsSingleChunk(t *testing.T) { - c, err := NewTokenChunker(map[string]any{ - "delimiter_mode": "one", - }) - if err != nil { - t.Fatalf("NewTokenChunker: %v", err) - } - out, err := c.Invoke(context.Background(), map[string]any{ - "name": "doc.txt", - "output_format": "text", - "text": "first\n\nsecond\n\nthird", - }) - if err != nil { - t.Fatalf("Invoke: %v", err) - } - chunks, _ := out["chunks"].([]map[string]any) - if len(chunks) != 1 { - t.Errorf("chunks = %d, want 1", len(chunks)) - } - if text, _ := chunks[0]["text"].(string); text != "first\n\nsecond\n\nthird" { - t.Errorf("text = %q, want full input", text) - } -} - // TestTokenChunker_InvokeChildrenDelim asserts that the secondary // children_delimiter split produces chunks carrying the parent // ("mom") and child ("text") keys. @@ -275,22 +248,6 @@ func TestTokenChunker_InputsOutputs_NonEmpty(t *testing.T) { } } -// TestTokenChunker_Parallelism enforces the plan's row 2.3a -// parallelism (4). -func TestTokenChunker_Parallelism(t *testing.T) { - c, err := NewTokenChunker(nil) - if err != nil { - t.Fatalf("NewTokenChunker: %v", err) - } - tc, ok := c.(*TokenChunkerComponent) - if !ok { - t.Fatalf("NewTokenChunker returned %T, want *TokenChunkerComponent", c) - } - if got := tc.Parallelism(); got != 4 { - t.Errorf("Parallelism() = %d, want 4", got) - } -} - // TestTokenChunker_NewRejectsBadParam enforces the param validation // at construction time (mirrors python `check()`). func TestTokenChunker_NewRejectsBadParam(t *testing.T) { @@ -299,6 +256,7 @@ func TestTokenChunker_NewRejectsBadParam(t *testing.T) { conf map[string]any }{ {"bad delimiter_mode", map[string]any{"delimiter_mode": "nope"}}, + {"one delimiter_mode (use OneChunker)", map[string]any{"delimiter_mode": "one"}}, {"zero chunk_token_size", map[string]any{"delimiter_mode": "token_size", "chunk_token_size": 0}}, {"negative chunk_token_size", map[string]any{"delimiter_mode": "token_size", "chunk_token_size": -5}}, {"negative overlapped_percent", map[string]any{"delimiter_mode": "token_size", "chunk_token_size": 50, "overlapped_percent": -0.1}}, diff --git a/internal/ingestion/component/dispatch_model.go b/internal/ingestion/component/dispatch_model.go new file mode 100644 index 0000000000..1386849a1a --- /dev/null +++ b/internal/ingestion/component/dispatch_model.go @@ -0,0 +1,300 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Tenant model resolution helpers shared by pdf_vision, media_dispatch, +// docx_vision, and markdown_vision dispatchers. + +package component + +import ( + "encoding/json" + "fmt" + "strings" + + "ragflow/internal/dao" + "ragflow/internal/entity" + modelModule "ragflow/internal/entity/models" + + "gorm.io/gorm" +) + +type tenantModelExtra struct { + MaxTokens *int `json:"max_tokens"` +} + +var resolveTenantModelByType = defaultResolveTenantModelByType + +func defaultResolveTenantModelByType(tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) { + tenantDAO := dao.NewTenantDAO() + tenant, err := tenantDAO.GetByID(tenantID) + if err != nil { + return nil, "", nil, 0, err + } + var modelID string + switch modelType { + case entity.ModelTypeChat: + modelID = tenant.LLMID + case entity.ModelTypeEmbedding: + modelID = tenant.EmbdID + case entity.ModelTypeRerank: + modelID = tenant.RerankID + case entity.ModelTypeSpeech2Text: + modelID = tenant.ASRID + case entity.ModelTypeImage2Text: + modelID = tenant.Img2TxtID + case entity.ModelTypeTTS: + modelID = tenant.TTSID + case entity.ModelTypeOCR: + modelID = tenant.OCRID + default: + return nil, "", nil, 0, fmt.Errorf("invalid model type: %s", modelType) + } + if modelID == "" { + return nil, "", nil, 0, fmt.Errorf("no default %s model is set", modelType) + } + if tenantModelID := tenantModelIDByType(tenant, modelType); tenantModelID != "" { + driver, modelName, apiConfig, maxTokens, err := resolveModelConfigByID(tenantID, modelType, tenantModelID) + if err == nil { + return driver, modelName, apiConfig, maxTokens, nil + } + } + return resolveModelConfig(tenantID, modelType, modelID) +} + +func tenantModelIDByType(tenant *entity.Tenant, modelType entity.ModelType) string { + if tenant == nil { + return "" + } + switch modelType { + case entity.ModelTypeChat: + return stringValue(tenant.TenantLLMID) + case entity.ModelTypeEmbedding: + return stringValue(tenant.TenantEmbdID) + case entity.ModelTypeRerank: + return stringValue(tenant.TenantRerankID) + case entity.ModelTypeSpeech2Text: + return stringValue(tenant.TenantASRID) + case entity.ModelTypeImage2Text: + return stringValue(tenant.TenantImg2TxtID) + case entity.ModelTypeTTS: + return stringValue(tenant.TenantTTSID) + case entity.ModelTypeOCR: + return stringValue(tenant.TenantOCRID) + default: + return "" + } +} + +func stringValue(value *string) string { + if value == nil { + return "" + } + return *value +} + +func resolveModelConfig(tenantID string, modelType entity.ModelType, modelRef string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) { + modelDAO := dao.NewTenantModelDAO() + if _, err := modelDAO.GetByID(modelRef); err == nil { + return resolveModelConfigByID(tenantID, modelType, modelRef) + } else if !errorsIsRecordNotFound(err) { + return nil, "", nil, 0, err + } + return resolveModelConfigFromProviderInstance(tenantID, modelType, modelRef) +} + +func resolveModelConfigByID(tenantID string, modelType entity.ModelType, modelID string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) { + modelDAO := dao.NewTenantModelDAO() + instanceDAO := dao.NewTenantModelInstanceDAO() + providerDAO := dao.NewTenantModelProviderDAO() + + modelObj, err := modelDAO.GetByID(modelID) + if err != nil { + return nil, "", nil, 0, err + } + if modelObj.Status != "active" { + return nil, "", nil, 0, fmt.Errorf("model %q is disabled", modelID) + } + if !entity.ModelType(modelObj.ModelType).Has(modelType) { + return nil, "", nil, 0, fmt.Errorf("model %q cannot be used as %s model", modelID, modelType.String()) + } + instance, err := instanceDAO.GetByID(modelObj.InstanceID) + if err != nil { + return nil, "", nil, 0, err + } + provider, err := providerDAO.GetByID(modelObj.ProviderID) + if err != nil { + return nil, "", nil, 0, err + } + if provider.TenantID != tenantID { + return nil, "", nil, 0, fmt.Errorf("tenant %s has no access to provider owned by tenant %s", tenantID, provider.TenantID) + } + + apiKey := instance.APIKey + var extra map[string]string + _ = json.Unmarshal([]byte(instance.Extra), &extra) + region := extra["region"] + baseURL := extra["base_url"] + + providerInfo := dao.GetModelProviderManager().FindProvider(provider.ProviderName) + if providerInfo == nil { + return nil, "", nil, 0, fmt.Errorf("provider %q driver not found", provider.ProviderName) + } + driver, err := newModelDriverForBaseURLLocal(providerInfo.ModelDriver, provider.ProviderName, region, baseURL) + if err != nil { + return nil, "", nil, 0, err + } + maxTokens := 0 + if mi, _ := dao.GetModelProviderManager().GetModelByName(provider.ProviderName, modelObj.ModelName); mi != nil && mi.MaxTokens != nil { + maxTokens = *mi.MaxTokens + } + if strings.TrimSpace(modelObj.Extra) != "" { + var tenantExtra tenantModelExtra + if err := json.Unmarshal([]byte(modelObj.Extra), &tenantExtra); err != nil { + return nil, "", nil, 0, err + } + if tenantExtra.MaxTokens != nil && *tenantExtra.MaxTokens > 0 { + maxTokens = *tenantExtra.MaxTokens + } + } + apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: ®ion, BaseURL: &baseURL} + return driver, modelObj.ModelName, apiConfig, maxTokens, nil +} + +func resolveModelConfigFromProviderInstance(tenantID string, modelType entity.ModelType, modelName string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) { + pureModelName, instanceName, providerName, err := parseCompositeModelName(modelName) + if err != nil { + return nil, "", nil, 0, err + } + + providerDAO := dao.NewTenantModelProviderDAO() + instanceDAO := dao.NewTenantModelInstanceDAO() + modelDAO := dao.NewTenantModelDAO() + + provider, err := providerDAO.GetByTenantIDAndProviderName(tenantID, providerName) + if err != nil { + return nil, "", nil, 0, fmt.Errorf("provider %q lookup failed: %w", providerName, err) + } + instance, err := instanceDAO.GetByProviderIDAndInstanceName(provider.ID, instanceName) + if err != nil { + return nil, "", nil, 0, fmt.Errorf("instance %q lookup failed: %w", instanceName, err) + } + + apiKey := instance.APIKey + var extra map[string]string + _ = json.Unmarshal([]byte(instance.Extra), &extra) + region := extra["region"] + baseURL := extra["base_url"] + + modelObj, modelErr := modelDAO.GetByProviderIDAndInstanceIDAndModelTypeAndModelName( + provider.ID, instance.ID, int(modelType), pureModelName, + ) + switch { + case modelErr == nil: + if modelObj.Status == "inactive" { + return nil, "", nil, 0, fmt.Errorf("model %q is disabled", modelName) + } + providerInfo := dao.GetModelProviderManager().FindProvider(providerName) + if providerInfo == nil { + return nil, "", nil, 0, fmt.Errorf("provider %q driver not found", providerName) + } + driver, err := newModelDriverForBaseURLLocal(providerInfo.ModelDriver, providerName, region, baseURL) + if err != nil { + return nil, "", nil, 0, err + } + maxTokens := 0 + if mi, _ := dao.GetModelProviderManager().GetModelByName(providerName, pureModelName); mi != nil && mi.MaxTokens != nil { + maxTokens = *mi.MaxTokens + } + if modelObj != nil && strings.TrimSpace(modelObj.Extra) != "" { + var tenantExtra tenantModelExtra + if err := json.Unmarshal([]byte(modelObj.Extra), &tenantExtra); err != nil { + return nil, "", nil, 0, err + } + if tenantExtra.MaxTokens != nil && *tenantExtra.MaxTokens > 0 { + maxTokens = *tenantExtra.MaxTokens + } + } + apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: ®ion, BaseURL: &baseURL} + return driver, modelObj.ModelName, apiConfig, maxTokens, nil + case !errorsIsRecordNotFound(modelErr): + return nil, "", nil, 0, fmt.Errorf("model %q lookup failed: %w", modelName, modelErr) + } + + targetFactoryName := providerName + if region == "intl" && strings.EqualFold(providerName, "siliconflow") { + targetFactoryName = "siliconflow_intl" + } + targetProvider := dao.GetModelProviderManager().FindProvider(targetFactoryName) + if targetProvider == nil { + return nil, "", nil, 0, fmt.Errorf("model provider config not found: %s", providerName) + } + var llmInfo *modelModule.Model + for i := range targetProvider.Models { + if strings.EqualFold(targetProvider.Models[i].Name, pureModelName) { + llmInfo = targetProvider.Models[i] + break + } + } + if llmInfo == nil { + return nil, "", nil, 0, fmt.Errorf("model config not found: %s", modelName) + } + driver, err := newModelDriverForBaseURLLocal(targetProvider.ModelDriver, providerName, region, baseURL) + if err != nil { + return nil, "", nil, 0, err + } + apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: ®ion, BaseURL: &baseURL} + maxTokens := 0 + if llmInfo.MaxTokens != nil { + maxTokens = *llmInfo.MaxTokens + } + return driver, llmInfo.Name, apiConfig, maxTokens, nil +} + +func parseCompositeModelName(compositeName string) (modelName, instanceName, providerName string, err error) { + parts := strings.Split(compositeName, "@") + switch len(parts) { + case 3: + return parts[0], parts[1], parts[2], nil + case 2: + return parts[0], "default", parts[1], nil + case 1: + return parts[0], "", "", fmt.Errorf("provider name missing in model name: %s", compositeName) + } + n := len(parts) + return strings.Join(parts[:n-2], "@"), parts[n-2], parts[n-1], nil +} + +func newModelDriverForBaseURLLocal(driver modelModule.ModelDriver, providerName, region, baseURL string) (modelModule.ModelDriver, error) { + if driver == nil { + return nil, fmt.Errorf("provider %s driver not found", providerName) + } + if strings.TrimSpace(baseURL) == "" { + return driver, nil + } + baseURLByRegion := map[string]string{region: baseURL} + if region == "" { + baseURLByRegion["default"] = baseURL + } + newDriver := driver.NewInstance(baseURLByRegion) + if newDriver == nil { + return nil, fmt.Errorf("provider %s does not support custom base_url", providerName) + } + return newDriver, nil +} + +func errorsIsRecordNotFound(err error) bool { + return err != nil && (err == gorm.ErrRecordNotFound || strings.Contains(err.Error(), gorm.ErrRecordNotFound.Error())) +} diff --git a/internal/ingestion/component/docx_vision_dispatch.go b/internal/ingestion/component/docx_vision_dispatch.go new file mode 100644 index 0000000000..3b6a389464 --- /dev/null +++ b/internal/ingestion/component/docx_vision_dispatch.go @@ -0,0 +1,300 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// DOCX vision figure dispatch: enriches the parse result with +// LLM-generated descriptions of embedded images, mirroring +// Python's vision_figure_parser_docx_wrapper_naive in +// deepdoc/parser/figure_parser.py. +// +// Unlike the PDF vision path (which replaces dispatchParse +// entirely), DOCX vision is a post-processing step: it takes +// the already-parsed markdown + extracted figures and augments +// the markdown text with vision model descriptions. + +package component + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "ragflow/internal/entity" + modelModule "ragflow/internal/entity/models" + "ragflow/internal/ingestion/component/schema" + "ragflow/internal/utility" +) + +var ( + docxVisionPromptBuilder = buildDOCXVisionPrompt + visionChatInvoker = defaultVisionChatInvoker + docxVisionConcurrency uint = 10 +) + +const ( + docxVisionPromptFile = "vision_llm_figure_describe_prompt.md" + docxVisionPromptWithContextFile = "vision_llm_figure_describe_prompt_with_context.md" +) + +var ( + docxVisionPromptsBase string + docxVisionPromptsOnce sync.Once + docxVisionPromptCache = make(map[string]string) + docxVisionPromptMu sync.RWMutex +) + +// maybeDispatchDOCXVision checks whether the dispatch result for a +// DOCX file contains embedded image figures and, when a vision +// model is available, enriches the markdown with AI-generated +// figure descriptions. It mirrors the Python flow: +// +// 1. naive_merge_docx → chunks (text + images + context) +// 2. vision_figure_parser_docx_wrapper_naive → LLM descriptions +// +// The function is called AFTER dispatchParse so the normal parse +// path produces figures in dispatched.File["figures"]. +// It returns (result, handled, error). handled is true when the +// dispatched result was modified. +func maybeDispatchDOCXVision( + fileType utility.FileType, + dispatched parserDispatchResult, + inputs map[string]any, + setups map[string]schema.ParserSetup, +) (parserDispatchResult, bool, error) { + if fileType != utility.FileTypeDOCX { + return dispatched, false, nil + } + if dispatched.Err != nil || dispatched.OutputFormat != "markdown" { + return dispatched, false, nil + } + figs, hasFigures := extractDOCXFiguresFromDispatch(dispatched) + if !hasFigures { + return dispatched, false, nil + } + + tenantID := getStringOr(inputs, "tenant_id", "") + if tenantID == "" { + return dispatched, false, nil + } + + // Resolve the tenant's IMAGE2TEXT model. + driver, modelName, apiConfig, _, err := resolveTenantModelByType(tenantID, entity.ModelTypeImage2Text) + if err != nil { + // Model not available — skip vision enhancement silently, + // matching Python's try/except pass behaviour. + return dispatched, false, nil + } + + descriptions := make([]string, len(figs)) + var wg sync.WaitGroup + sem := make(chan struct{}, docxVisionConcurrency) + + for i, fig := range figs { + wg.Add(1) + go func(idx int, f map[string]any) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + imageB64, _ := f["image"].(string) + ctxAbove, _ := f["context_above"].(string) + ctxBelow, _ := f["context_below"].(string) + + if imageB64 == "" { + return + } + + prompt, err := docxVisionPromptBuilder(ctxAbove, ctxBelow) + if err != nil { + return + } + + messages := buildVisionMessages(prompt, imageB64) + resp, err := visionChatInvoker(driver, modelName, messages, apiConfig) + if err != nil { + return + } + descriptions[idx] = extractDOCXVisionAnswer(resp) + }(i, fig) + } + wg.Wait() + + // Insert each description at the figure's position in the markdown, + // matching Python's `chunks[idx]["text"] += description`. + // Figures carry a "marker" (text immediately before the image) to + // locate the insertion point. Process in reverse order so earlier + // insertions don't shift later markers. + md := dispatched.Markdown + type indexedDesc struct { + idx int + desc string + } + var inserts []indexedDesc + for i, d := range descriptions { + if d = strings.TrimSpace(d); d == "" { + continue + } + if i >= len(figs) { + continue + } + marker, _ := figs[i]["marker"].(string) + if marker != "" { + if pos := strings.LastIndex(md, marker); pos >= 0 { + inserts = append(inserts, indexedDesc{idx: pos + len(marker), desc: d}) + continue + } + } + // Fallback: try context_above as a search anchor. + if ctx, _ := figs[i]["context_above"].(string); ctx != "" { + if pos := strings.LastIndex(md, ctx); pos >= 0 { + inserts = append(inserts, indexedDesc{idx: pos + len(ctx), desc: d}) + continue + } + } + // No anchor found — append to end. + inserts = append(inserts, indexedDesc{idx: len(md), desc: "\n\n" + d}) + } + // Sort descending by position for stable insertion. + sort.Slice(inserts, func(a, b int) bool { return inserts[a].idx > inserts[b].idx }) + for _, ins := range inserts { + desc := ins.desc + if !strings.HasPrefix(desc, "\n") { + desc = "\n\n" + desc + } + md = md[:ins.idx] + desc + md[ins.idx:] + } + dispatched.Markdown = md + + return dispatched, true, nil +} + +func extractDOCXFiguresFromDispatch(dispatched parserDispatchResult) ([]map[string]any, bool) { + if dispatched.File == nil { + return nil, false + } + raw, ok := dispatched.File["figures"] + if !ok { + return nil, false + } + list, ok := raw.([]map[string]any) + if !ok { + return nil, false + } + if len(list) == 0 { + return nil, false + } + return list, true +} + +// buildDOCXVisionPrompt loads the figure-describe prompt template +// and, when context text is available, renders it with the +// with-context variant. Mirrors Python: +// +// if context_above or context_below: +// prompt = vision_llm_figure_describe_prompt_with_context(context_above, context_below) +// else: +// prompt = vision_llm_figure_describe_prompt() +func buildDOCXVisionPrompt(contextAbove, contextBelow string) (string, error) { + hasContext := strings.TrimSpace(contextAbove) != "" || strings.TrimSpace(contextBelow) != "" + + var templateName string + if hasContext { + templateName = docxVisionPromptWithContextFile + } else { + templateName = docxVisionPromptFile + } + + template, err := loadDOCXVisionPromptFile(templateName) + if err != nil { + return "", err + } + + if hasContext { + template = strings.ReplaceAll(template, "{{ context_above }}", contextAbove) + template = strings.ReplaceAll(template, "{{ context_below }}", contextBelow) + } + return template, nil +} + +func loadDOCXVisionPromptFile(filename string) (string, error) { + docxVisionPromptMu.RLock() + if cached, ok := docxVisionPromptCache[filename]; ok { + docxVisionPromptMu.RUnlock() + return cached, nil + } + docxVisionPromptMu.RUnlock() + + baseDir, err := docxVisionPromptsBaseDir() + if err != nil { + return "", err + } + promptPath := filepath.Join(baseDir, "rag", "prompts", filename) + content, err := os.ReadFile(promptPath) + if err != nil { + return "", fmt.Errorf("docx vision prompt %q: %w", filename, err) + } + cached := strings.TrimSpace(string(content)) + docxVisionPromptMu.Lock() + docxVisionPromptCache[filename] = cached + docxVisionPromptMu.Unlock() + return cached, nil +} + +func docxVisionPromptsBaseDir() (string, error) { + var initErr error + docxVisionPromptsOnce.Do(func() { + root := utility.GetProjectRoot() + if _, statErr := os.Stat(filepath.Join(root, "rag", "prompts")); statErr == nil { + docxVisionPromptsBase = root + return + } + initErr = fmt.Errorf("rag/prompts not found under project root %q", root) + }) + if initErr != nil { + return "", initErr + } + return docxVisionPromptsBase, nil +} + +func buildVisionMessages(prompt, imageBase64 string) []modelModule.Message { + dataURI := "data:image/png;base64," + imageBase64 + return []modelModule.Message{{ + Role: "user", + Content: []interface{}{ + map[string]any{"type": "text", "text": prompt}, + map[string]any{"type": "image_url", "image_url": map[string]any{"url": dataURI}}, + }, + }} +} + +func extractDOCXVisionAnswer(resp *modelModule.ChatResponse) string { + if resp == nil || resp.Answer == nil { + return "" + } + return strings.TrimSpace(*resp.Answer) +} + +func defaultVisionChatInvoker( + driver modelModule.ModelDriver, + modelName string, + messages []modelModule.Message, + apiConfig *modelModule.APIConfig, +) (*modelModule.ChatResponse, error) { + vision := true + return driver.ChatWithMessages(modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}) +} diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go index bd49264454..2473ae0a8c 100644 --- a/internal/ingestion/component/extractor.go +++ b/internal/ingestion/component/extractor.go @@ -31,10 +31,9 @@ // response. We DO NOT panic: errors are surfaced as a clean // "no driver for %q" wrap that callers can log and route. // -// - LLM CALL SHAPE: one chat call per chunk (no batching). Plan -// §AD-5a locks Parallelism at 1 because "LLM call is inherently -// serial"; sequential per-chunk processing keeps test ordering -// deterministic under -race. +// - LLM CALL SHAPE: one chat call per chunk (no batching). LLM +// calls are inherently serial; sequential per-chunk processing +// keeps test ordering deterministic under -race. // // - TIMEOUT / ELAPSED: the call is wrapped in // runtime.WithTimeout(60s) and runtime.TrackElapsed so the @@ -170,12 +169,6 @@ func (c *ExtractorComponent) Outputs() map[string]string { } } -// Parallelism is locked at 1 (plan §AD-5a: "Extractor: 1 (LLM call -// is inherently serial)"). The pipeline runner uses this to decide -// fan-out degree; sequential per-chunk processing keeps test -// ordering deterministic under -race. -func (c *ExtractorComponent) Parallelism() int { return 1 } - // extractorChatInvoker is the seam the Extractor uses to dispatch // its chat call. The production implementation // (einoExtractorChatInvoker below) mirrors diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.go index c5d5181104..fc05d0a1fa 100644 --- a/internal/ingestion/component/extractor_test.go +++ b/internal/ingestion/component/extractor_test.go @@ -33,7 +33,7 @@ import ( // stubExtractorChatInvoker is the test seam for the package-level // extractorChatInvoker. It records every call (for assertions) and // returns canned responses configured per-test. Concurrent-safe so -// it can backstop future Parallelism>1 cases without rewriting. +// it can backstop concurrent test cases without rewriting. type stubExtractorChatInvoker struct { mu sync.Mutex @@ -526,16 +526,6 @@ func TestExtractorComponent_InputsOutputs_NonEmpty(t *testing.T) { } } -// TestExtractorComponent_Parallelism asserts the fan-out is -// locked to 1 per plan §AD-5a ("Extractor: 1 (LLM call is -// inherently serial)"). -func TestExtractorComponent_Parallelism(t *testing.T) { - c := &ExtractorComponent{} - if got := c.Parallelism(); got != 1 { - t.Errorf("Parallelism() = %d, want 1", got) - } -} - // TestSplitExtractorLLID covers the composite-id parser in // isolation — keeps the matrix of edge cases at one call site // so a regression is easy to attribute. The "@" separator is diff --git a/internal/ingestion/component/file.go b/internal/ingestion/component/file.go index 4009835244..a3073d10d6 100644 --- a/internal/ingestion/component/file.go +++ b/internal/ingestion/component/file.go @@ -120,9 +120,6 @@ func (c *FileComponent) Outputs() map[string]string { } } -// Parallelism is fixed at 1 — File is metadata-only. -func (c *FileComponent) Parallelism() int { return 1 } - // Invoke resolves document/file metadata for downstream Parser use. // // The implementation mirrors the python flow's two paths: @@ -158,9 +155,7 @@ func (c *FileComponent) Invoke(ctx context.Context, inputs map[string]any) (map[ // re-emitting it. The Go runtime forwards only this explicit output // to the next node, so shared fields must live in Globals. globals.PublishGlobals(ctx, out) - return runtime.TrackElapsed("File", func() (map[string]any, error) { - return out, nil - }) + return out, nil } // fileInputs is the post-Validation view of the upstream input diff --git a/internal/ingestion/component/file_test.go b/internal/ingestion/component/file_test.go index 4813646b7d..70ce40d191 100644 --- a/internal/ingestion/component/file_test.go +++ b/internal/ingestion/component/file_test.go @@ -110,12 +110,6 @@ func TestFileComponent_Invoke_HappyPath(t *testing.T) { if _, ok := out["binary"]; ok { t.Fatalf("binary should not be emitted by File: %v", out["binary"]) } - if out["_elapsed_time"] == nil { - t.Error("_elapsed_time missing") - } - if out["_created_time"] == nil { - t.Error("_created_time missing") - } } func TestFileComponent_Invoke_ResolvesDocIDViaDocumentLocation(t *testing.T) { @@ -298,12 +292,3 @@ func TestFileComponent_InputsOutputs_NonEmpty(t *testing.T) { } } } - -// TestFileComponent_Parallelism asserts the fan-out is locked to -// 1 — File is metadata-only and intentionally non-fanned-out. -func TestFileComponent_Parallelism(t *testing.T) { - c := &FileComponent{} - if got := c.Parallelism(); got != 1 { - t.Errorf("Parallelism() = %d, want 1", got) - } -} diff --git a/internal/ingestion/component/markdown_vision_dispatch.go b/internal/ingestion/component/markdown_vision_dispatch.go new file mode 100644 index 0000000000..88a3294da8 --- /dev/null +++ b/internal/ingestion/component/markdown_vision_dispatch.go @@ -0,0 +1,162 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Markdown vision figure dispatch: enriches parsed markdown JSON +// items with LLM-generated descriptions of embedded images, +// mirroring Python's enhance_media_sections_with_vision in +// rag/flow/parser/utils.py, called from the _markdown path. +// +// Unlike the DOCX vision path (which processes a separate figures +// array), markdown vision iterates over the JSON items produced by +// MarkdownParser.ParseWithResult and enhances items whose +// doc_type_kwd == "image" and whose "image" field contains a +// base64-encoded image. + +package component + +import ( + "fmt" + "strings" + "sync" + + "ragflow/internal/entity" + "ragflow/internal/utility" +) + +var ( + markdownVisionConcurrency uint = 10 +) + +// maybeDispatchMarkdownVision checks whether the markdown parse result +// contains JSON items with embedded images and, when a vision model is +// available, enriches those items with AI-generated figure descriptions. +// +// Mirrors the Python flow: +// +// 1. _markdown → sections + section_images (parser.py:1005) +// 2. enhance_media_sections_with_vision (parser.py:1054) +// +// The function is called AFTER dispatchParse so the normal parse +// path produces JSON items with doc_type_kwd == "image" and an +// "image" base64 field. It returns (result, handled, error). +func maybeDispatchMarkdownVision( + fileType utility.FileType, + dispatched parserDispatchResult, + inputs map[string]any, +) (parserDispatchResult, bool, error) { + if fileType != utility.FileTypeMarkdown { + return dispatched, false, nil + } + if dispatched.Err != nil || dispatched.OutputFormat != "json" { + return dispatched, false, nil + } + if len(dispatched.JSON) == 0 { + return dispatched, false, nil + } + + // Collect indices of image items. + type imgItem struct { + idx int + imageB64 string + text string + } + var images []imgItem + for i, item := range dispatched.JSON { + kd, _ := item["doc_type_kwd"].(string) + if kd != "image" { + continue + } + img, _ := item["image"].(string) + if img == "" { + continue + } + text, _ := item["text"].(string) + images = append(images, imgItem{idx: i, imageB64: img, text: text}) + } + if len(images) == 0 { + return dispatched, false, nil + } + + tenantID := getStringOr(inputs, "tenant_id", "") + if tenantID == "" { + return dispatched, false, nil + } + + // Resolve the tenant's IMAGE2TEXT model. + driver, modelName, apiConfig, _, err := resolveTenantModelByType(tenantID, entity.ModelTypeImage2Text) + if err != nil { + // Model not available — skip vision enhancement silently, + // matching Python's try/except pass behaviour. + return dispatched, false, nil + } + + descriptions := make([]string, len(images)) + var wg sync.WaitGroup + sem := make(chan struct{}, markdownVisionConcurrency) + + for i, img := range images { + wg.Add(1) + go func(pos int, item imgItem) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + // Markdown images have no context — use the + // default (no-context) prompt template. + prompt, err := buildMarkdownVisionPrompt() + if err != nil { + return + } + + messages := buildVisionMessages(prompt, item.imageB64) + resp, err := visionChatInvoker(driver, modelName, messages, apiConfig) + if err != nil { + return + } + descriptions[pos] = extractDOCXVisionAnswer(resp) + }(i, img) + } + wg.Wait() + + // Append vision descriptions to each image item's text field, + // matching Python's `item["text"] = f"{text}\n{parsed_text}"`. + for pos, img := range images { + desc := strings.TrimSpace(descriptions[pos]) + if desc == "" { + continue + } + item := dispatched.JSON[img.idx] + existing, _ := item["text"].(string) + if existing != "" { + item["text"] = existing + "\n\n" + desc + } else { + item["text"] = desc + } + } + + return dispatched, true, nil +} + +// buildMarkdownVisionPrompt loads the default (no-context) figure +// describe prompt template, mirroring Python's +// vision_llm_figure_describe_prompt(). +func buildMarkdownVisionPrompt() (string, error) { + template, err := loadDOCXVisionPromptFile(docxVisionPromptFile) + if err != nil { + return "", fmt.Errorf("markdown vision prompt: %w", err) + } + return template, nil +} diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go new file mode 100644 index 0000000000..027b874dd3 --- /dev/null +++ b/internal/ingestion/component/media_dispatch.go @@ -0,0 +1,601 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Media dispatch: image, audio, video parser branches that require +// model access (OCR, IMAGE2TEXT, SPEECH2TEXT) at the component +// layer. Mirrors Python's _image / _audio / _video methods in +// rag/flow/parser/parser.py and rag/app/picture.py. +// +// These follow the maybeDispatchPDFVision pattern: they bypass +// dispatchParse and call the model directly from the component +// layer, returning a parserDispatchResult. + +package component + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "image" + // Import image decoders for common formats. + _ "image/gif" + _ "image/jpeg" + _ "image/png" + "os" + "path/filepath" + "sort" + "strings" + + "ragflow/internal/common" + inference "ragflow/internal/deepdoc/parser/pdf/inference" + "ragflow/internal/entity" + modelModule "ragflow/internal/entity/models" + "ragflow/internal/ingestion/component/schema" + "ragflow/internal/parser/parser" + "ragflow/internal/utility" +) + +// Video dispatch: IMAGE2TEXT vision chat --- + +func maybeDispatchVideo( + fileType utility.FileType, + filename string, + binary []byte, + inputs map[string]any, + setups map[string]schema.ParserSetup, +) (parserDispatchResult, bool, error) { + if fileType != utility.FileTypeVIDEO { + return parserDispatchResult{}, false, nil + } + setup, ok := setups["video"] + if !ok { + return parserDispatchResult{}, false, nil + } + tenantID := getStringOr(inputs, "tenant_id", "") + if tenantID == "" { + return parserDispatchResult{}, true, + fmt.Errorf("Parser: video requires tenant_id") + } + + // Resolve the tenant's IMAGE2TEXT model. + driver, modelName, apiConfig, _, err := resolveTenantModelByType(tenantID, entity.ModelTypeImage2Text) + if err != nil { + return parserDispatchResult{}, true, + fmt.Errorf("Parser: video image2text model: %w", err) + } + + videoPrompt, _ := setup["prompt"].(string) + videoB64 := base64.StdEncoding.EncodeToString(binary) + + // Build a multimodal message with the video payload. + // Python uses cv_mdl.async_chat(video_bytes=blob, ...); + // Go ChatWithMessages is synchronous and uses a data URI. + mimeType := videoMIME(filename) + dataURI := "data:" + mimeType + ";base64," + videoB64 + messages := []modelModule.Message{{ + Role: "user", + Content: []interface{}{ + map[string]any{"type": "text", "text": videoPrompt}, + map[string]any{"type": "video_url", "video_url": map[string]any{"url": dataURI}}, + }, + }} + vision := true + resp, err := driver.ChatWithMessages(modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}) + if err != nil { + return parserDispatchResult{}, true, + fmt.Errorf("Parser: video describe: %w", err) + } + txt := "" + if resp != nil && resp.Answer != nil { + txt = strings.TrimSpace(*resp.Answer) + } + + outputFormat, _ := setup["output_format"].(string) + if outputFormat == "" { + outputFormat = "text" + } + return parserDispatchResult{ + OutputFormat: outputFormat, + DocType: "video", + Text: txt, + }, true, nil +} + +// Image dispatch: OCR + IMAGE2TEXT vision describe --- +// Mirrors Python's rag/app/picture.py:chunk() image branch: +// 1. Try PaddleOCR if layout_recognize is "@PaddleOCR" +// 2. Fallback to local ONNX OCR (DeepDoc /predict/ocr endpoint) +// 3. If OCR text is short (≤32 chars or ≤32 English words), +// also call IMAGE2TEXT VLM describe() +// 4. Returns combined text + +func maybeDispatchImage( + fileType utility.FileType, + filename string, + binary []byte, + inputs map[string]any, + setups map[string]schema.ParserSetup, +) (parserDispatchResult, bool, error) { + if fileType != utility.FileTypeVISUAL { + return parserDispatchResult{}, false, nil + } + setup, ok := setups["image"] + if !ok { + return parserDispatchResult{}, false, nil + } + tenantID := getStringOr(inputs, "tenant_id", "") + if tenantID == "" { + return parserDispatchResult{}, true, + fmt.Errorf("Parser: image requires tenant_id") + } + + // --- Phase 1: OCR --- + var ocrText string + + // Step 1a: Try PaddleOCR if layout_recognize is set to PaddleOCR. + // Mirrors Python's picture.py:_try_paddleocr_image(). + layoutRecognize := getStringOr(setup, "layout_recognize", "") + if layoutRecognize != "" { + recognizer, _ := normalizeLayoutRecognizer(layoutRecognize) + if recognizer == "PaddleOCR" { + if txt, err := runPaddleOCRImage(binary, filename); err == nil && txt != "" { + ocrText = txt + } + } + } + + // Step 1b: Fallback to local ONNX OCR (DeepDoc /predict/ocr). + // Mirrors Python's picture.py:ocr(np.array(img)) from deepdoc.vision. + if ocrText == "" { + if txt, err := runLocalImageOCR(binary); err == nil && txt != "" { + ocrText = txt + } + } + + outputFormat, _ := setup["output_format"].(string) + if outputFormat == "" { + outputFormat = "text" + } + + // --- Phase 2: VLM description (when OCR text is short) --- + // Mirrors Python's check: if (eng and len(txt.split()) > 32) or len(txt) > 32 + // then use OCR text only; otherwise call cv_mdl.describe(). + lang := getStringOr(setup, "lang", "") + eng := strings.EqualFold(lang, "english") + + if ocrText != "" { + wordCount := len(strings.Fields(ocrText)) + charCount := len(ocrText) + if (eng && wordCount > 32) || charCount > 32 { + // OCR returned substantial text — skip VLM. + return parserDispatchResult{ + OutputFormat: outputFormat, + DocType: "image", + Text: ocrText, + }, true, nil + } + } + + // Short OCR text (or no text): supplement with VLM describe. + driver, modelName, apiConfig, _, err := resolveTenantModelByType(tenantID, entity.ModelTypeImage2Text) + if err != nil { + // If VLM is unavailable but we have OCR text, return it. + if ocrText != "" { + return parserDispatchResult{ + OutputFormat: outputFormat, + DocType: "image", + Text: ocrText, + }, true, nil + } + return parserDispatchResult{}, true, + fmt.Errorf("Parser: picture image2text model: %w", err) + } + + imageB64 := base64.StdEncoding.EncodeToString(binary) + mimeType := imageMIME(filename) + dataURI := "data:" + mimeType + ";base64," + imageB64 + + prompt := "Describe this image in detail." + if v, ok := setup["prompt"].(string); ok && v != "" { + prompt = v + } + messages := []modelModule.Message{{ + Role: "user", + Content: []interface{}{ + map[string]any{"type": "text", "text": prompt}, + map[string]any{"type": "image_url", "image_url": map[string]any{"url": dataURI}}, + }, + }} + vision := true + resp, err := driver.ChatWithMessages(modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}) + if err != nil { + if ocrText != "" { + return parserDispatchResult{ + OutputFormat: outputFormat, + DocType: "image", + Text: ocrText, + }, true, nil + } + return parserDispatchResult{}, true, + fmt.Errorf("Parser: picture describe: %w", err) + } + vlmText := "" + if resp != nil && resp.Answer != nil { + vlmText = strings.TrimSpace(*resp.Answer) + } + + // Combine OCR + VLM text. + // Mirrors Python: txt += "\n" + ans + combined := ocrText + if vlmText != "" { + if combined != "" { + combined += "\n" + vlmText + } else { + combined = vlmText + } + } + return parserDispatchResult{ + OutputFormat: outputFormat, + DocType: "image", + Text: combined, + }, true, nil +} + +// Audio dispatch: SPEECH2TEXT transcription --- +// Mirrors Python's rag/app/audio.py:chunk(): +// - Writes the audio binary to a temp file (extension-preserving) +// - Calls the tenant's SPEECH2TEXT model via TranscribeAudio() +// - Returns the transcription as text + +func maybeDispatchAudio( + fileType utility.FileType, + filename string, + binary []byte, + inputs map[string]any, + setups map[string]schema.ParserSetup, +) (parserDispatchResult, bool, error) { + if fileType != utility.FileTypeAURAL { + return parserDispatchResult{}, false, nil + } + setup, ok := setups["audio"] + if !ok { + return parserDispatchResult{}, false, nil + } + tenantID := getStringOr(inputs, "tenant_id", "") + if tenantID == "" { + return parserDispatchResult{}, true, + fmt.Errorf("Parser: audio requires tenant_id") + } + + driver, modelName, apiConfig, _, err := resolveTenantModelByType(tenantID, entity.ModelTypeSpeech2Text) + if err != nil { + return parserDispatchResult{}, true, + fmt.Errorf("Parser: audio speech2text model: %w", err) + } + + tmpFile, err := writeTempAudioFile(filename, binary) + if err != nil { + return parserDispatchResult{}, true, + fmt.Errorf("Parser: audio temp file: %w", err) + } + defer os.Remove(tmpFile) + + resp, err := driver.TranscribeAudio(&modelName, &tmpFile, apiConfig, nil) + if err != nil { + return parserDispatchResult{}, true, + fmt.Errorf("Parser: audio transcription: %w", err) + } + + transcription := "" + if resp != nil { + transcription = resp.Text + } + + outputFormat, _ := setup["output_format"].(string) + if outputFormat == "" { + outputFormat = "text" + } + return parserDispatchResult{ + OutputFormat: outputFormat, + DocType: "audio", + Text: transcription, + }, true, nil +} + +// writeTempAudioFile writes binary to a temp file preserving the +// original extension so the ASR provider can detect the format. +func writeTempAudioFile(filename string, binary []byte) (string, error) { + ext := filepath.Ext(filename) + tmp, err := os.CreateTemp("", "ragflow_audio_*"+ext) + if err != nil { + return "", err + } + defer tmp.Close() + if _, err := tmp.Write(binary); err != nil { + os.Remove(tmp.Name()) + return "", err + } + return tmp.Name(), nil +} + +// normalizeLayoutRecognizer parses layout_recognize strings like +// "model@PaddleOCR" → ("PaddleOCR", "model@PaddleOCR"). +// Mirrors Python's common/parser_config_utils.py:normalize_layout_recognizer(). +func normalizeLayoutRecognizer(raw string) (recognizer, modelName string) { + lowered := strings.ToLower(raw) + if strings.HasSuffix(lowered, "@paddleocr") { + return "PaddleOCR", raw + } + if strings.HasSuffix(lowered, "@mineru") { + return "MinerU", raw + } + if strings.HasSuffix(lowered, "@somark") { + return "SoMark", raw + } + if strings.HasSuffix(lowered, "@opendataloader") { + return "OpenDataLoader", raw + } + return raw, "" +} + +// imageMIME maps common image filename extensions to MIME types +// for constructing base64 data URIs. +func imageMIME(filename string) string { + dot := strings.LastIndex(filename, ".") + if dot == -1 { + return "image/png" + } + switch strings.ToLower(filename[dot+1:]) { + case "jpg", "jpeg": + return "image/jpeg" + case "png": + return "image/png" + case "gif": + return "image/gif" + case "bmp": + return "image/bmp" + case "webp": + return "image/webp" + case "svg": + return "image/svg+xml" + case "tiff", "tif": + return "image/tiff" + case "ico": + return "image/x-icon" + case "avif": + return "image/avif" + case "heic": + return "image/heic" + default: + return "image/png" + } +} + +// videoMIME maps common video filename extensions to MIME types +// for constructing base64 data URIs. +func videoMIME(filename string) string { + dot := strings.LastIndex(filename, ".") + if dot == -1 { + return "video/mp4" + } + switch strings.ToLower(filename[dot+1:]) { + case "mp4": + return "video/mp4" + case "avi": + return "video/x-msvideo" + case "mkv": + return "video/x-matroska" + case "mov": + return "video/quicktime" + case "wmv": + return "video/x-ms-wmv" + case "flv": + return "video/x-flv" + case "webm": + return "video/webm" + case "mpeg", "mpg": + return "video/mpeg" + case "3gp": + return "video/3gpp" + default: + return "video/mp4" + } +} + +// --- OCR helpers for picture dispatch --- + +// runPaddleOCRImage tries PaddleOCR remote API for image text extraction. +// Mirrors Python's picture.py:_try_paddleocr_image() which creates a +// PaddleOCRParser and calls parse_image(). +func runPaddleOCRImage(binary []byte, filename string) (string, error) { + client := parser.NewPaddleOCRClientFromEnv() + if !client.Enabled() { + return "", fmt.Errorf("paddleocr: not configured (set PADDLEOCR_ACCESS_TOKEN)") + } + return client.ParseImage(binary, filename) +} + +// runLocalImageOCR uses the DeepDoc inference service (/predict/ocr) to +// detect and recognize text in an image. Mirrors Python's +// deepdoc.vision.OCR (local ONNX pipeline), but routed through the +// DeepDoc HTTP service which wraps the same ONNX models. +// +// Pipeline: +// 1. Decode image bytes → image.Image +// 2. OCRDetect → find text region boxes +// 3. For each box: crop → OCRRecognize → text +// 4. Sort boxes by Y, then X (reading order) +// 5. Join all recognized text with newlines +func runLocalImageOCR(binary []byte) (string, error) { + deepdocURL := common.GetEnv(common.EnvDeepDocURL) + if deepdocURL == "" { + deepdocURL = common.GetEnv(common.EnvTensorrtDLAServer) + } + if deepdocURL == "" { + return "", fmt.Errorf("local OCR: DEEPDOC_URL not configured") + } + + client, err := inference.NewClient(deepdocURL) + if err != nil { + return "", fmt.Errorf("local OCR: %w", err) + } + + img, _, err := image.Decode(bytes.NewReader(binary)) + if err != nil { + return "", fmt.Errorf("local OCR: decode image: %w", err) + } + + // Step 1: Detect text regions. + ctx := context.Background() + boxes, err := client.OCRDetect(ctx, img) + if err != nil { + return "", fmt.Errorf("local OCR: detect: %w", err) + } + if len(boxes) == 0 { + return "", nil + } + + // Step 2: Sort boxes by Y (top to bottom), then X (left to right) + // for reading-order text assembly. + sort.Slice(boxes, func(i, j int) bool { + yi := (boxes[i].Y0 + boxes[i].Y2) / 2 + yj := (boxes[j].Y0 + boxes[j].Y2) / 2 + if yi < yj { + return true + } + if yi > yj { + return false + } + return boxes[i].X0 < boxes[j].X0 + }) + + // Step 3: Recognize text per box. + var texts []string + bounds := img.Bounds() + for _, box := range boxes { + // Convert quad box to axis-aligned crop rect. + x0 := int(min4(box.X0, box.X1, box.X2, box.X3)) + y0 := int(min4(box.Y0, box.Y1, box.Y2, box.Y3)) + x1 := int(max4(box.X0, box.X1, box.X2, box.X3)) + y1 := int(max4(box.Y0, box.Y1, box.Y2, box.Y3)) + + // Clamp to image bounds. + if x0 < bounds.Min.X { + x0 = bounds.Min.X + } + if y0 < bounds.Min.Y { + y0 = bounds.Min.Y + } + if x1 > bounds.Max.X { + x1 = bounds.Max.X + } + if y1 > bounds.Max.Y { + y1 = bounds.Max.Y + } + if x1 <= x0 || y1 <= y0 { + continue + } + + // Crop the region. This requires an image type that supports + // cropping; for simplicity we recode through a sub-image. + crop := cropImage(img, x0, y0, x1, y1) + if crop == nil { + continue + } + + recTexts, err := client.OCRRecognize(ctx, crop) + if err != nil { + continue // skip boxes that fail recognition + } + for _, t := range recTexts { + s := strings.TrimSpace(t.Text) + if s != "" { + texts = append(texts, s) + } + } + } + + if len(texts) == 0 { + return "", nil + } + return strings.Join(texts, "\n"), nil +} + +// cropImage extracts a sub-rectangle from img. Works with any image.Image +// by converting to RGBA if needed, then cropping. +func cropImage(img image.Image, x0, y0, x1, y1 int) image.Image { + bounds := img.Bounds() + cropRect := image.Rect( + bounds.Min.X+x0, bounds.Min.Y+y0, + bounds.Min.X+x1, bounds.Min.Y+y1, + ) + switch src := img.(type) { + case *image.RGBA: + return src.SubImage(cropRect) + case *image.NRGBA: + return src.SubImage(cropRect) + case *image.RGBA64: + return src.SubImage(cropRect) + case *image.NRGBA64: + return src.SubImage(cropRect) + case *image.Gray: + return src.SubImage(cropRect) + case *image.Gray16: + return src.SubImage(cropRect) + case *image.YCbCr: + return src.SubImage(cropRect) + case *image.Paletted: + return src.SubImage(cropRect) + default: + // Convert to RGBA for cropping. + rgba := image.NewRGBA(cropRect) + for y := cropRect.Min.Y; y < cropRect.Max.Y; y++ { + for x := cropRect.Min.X; x < cropRect.Max.X; x++ { + rgba.Set(x, y, img.At(x, y)) + } + } + return rgba + } +} + +func min4(a, b, c, d float64) float64 { + m := a + if b < m { + m = b + } + if c < m { + m = c + } + if d < m { + m = d + } + return m +} + +func max4(a, b, c, d float64) float64 { + m := a + if b > m { + m = b + } + if c > m { + m = c + } + if d > m { + m = d + } + return m +} diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go index f380310912..29453a4f00 100644 --- a/internal/ingestion/component/parser.go +++ b/internal/ingestion/component/parser.go @@ -143,7 +143,16 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) { if p.Setups == nil { p.Setups = make(map[string]schema.ParserSetup) } - p.Setups[fileType] = schema.ParserSetup(setupMap) + // Normalise to the canonical family name ("picture", + // "video", …) so downstream lookups via + // resolveParserFamily, maybeDispatchImage, etc. find + // the entry regardless of what key the template DSL + // used (e.g. "image", "visual", "picture"). + family := pythonFamilyName(fileType) + if family == "" { + family = fileType + } + p.Setups[family] = schema.ParserSetup(setupMap) } } if rawAllowed, ok := params["allowed_output_format"].(map[string]any); ok { @@ -273,9 +282,48 @@ func (c *ParserComponent) Invoke(ctx context.Context, inputs map[string]any) (ma if visionErr != nil { return nil, visionErr } + + var handledMedia bool if !handledVision { + // Video dispatch: IMAGE2TEXT vision chat. + // Mirrors Python's _video(). + dispatched, handledMedia, visionErr = maybeDispatchVideo(fileTypeExt, filename, binary, inputs, c.Param.Setups) + if visionErr != nil { + return nil, visionErr + } + } + var handledImage bool + if !handledVision && !handledMedia { + // Image/Picture dispatch: OCR + IMAGE2TEXT vision describe. + // Mirrors Python's rag/app/picture.py:chunk() image branch. + dispatched, handledImage, visionErr = maybeDispatchImage(fileTypeExt, filename, binary, inputs, c.Param.Setups) + if visionErr != nil { + return nil, visionErr + } + } + var handledAudio bool + if !handledVision && !handledMedia && !handledImage { + // Audio dispatch: SPEECH2TEXT transcription. + // Mirrors Python's rag/app/audio.py:chunk(). + dispatched, handledAudio, visionErr = maybeDispatchAudio(fileTypeExt, filename, binary, inputs, c.Param.Setups) + if visionErr != nil { + return nil, visionErr + } + } + if !handledVision && !handledMedia && !handledImage && !handledAudio { dispatched = dispatchParse(fileTypeExt, filename, binary, c.Param.Setups) dispatched = hydrateEmptyDispatchPayload(dispatched, binary) + + // DOCX vision figure enhancement: enrich the markdown + // with LLM-generated descriptions of embedded images. + // Mirrors Python's vision_figure_parser_docx_wrapper_naive. + dispatched, _, _ = maybeDispatchDOCXVision(fileTypeExt, dispatched, inputs, c.Param.Setups) + + // Markdown vision figure enhancement: enrich parsed + // markdown JSON items with LLM-generated descriptions of + // referenced images (![alt](url)). Mirrors Python's + // enhance_media_sections_with_vision in _markdown. + dispatched, _, _ = maybeDispatchMarkdownVision(fileTypeExt, dispatched, inputs) } // Known/supported families must fail loudly when dispatch or // parsing breaks. Only unknown families keep the raw-text fallback. @@ -320,7 +368,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, inputs map[string]any) (ma // this component only reshapes the parser output. The DETERMINISTIC // MERGE (plan §8 R8) keeps pages sorted by PageNumber so the // downstream chunker / tokenizer get stable chunk IDs. - parsed, err := buildPagesFromBytes(ctx, pages) + parsed, err := buildPagesFromBytes(ctx, pages, dispatched.DocType) if err != nil { return nil, fmt.Errorf("Parser: %w", err) } @@ -359,9 +407,13 @@ func (c *ParserComponent) Invoke(ctx context.Context, inputs map[string]any) (ma // // The function is format-agnostic: it does not resolve parsers or // inspect file families — it only carries the raw bytes under the -// "text" key with doc_type_kwd "text", matching the shape downstream -// readers expect from the raw-text fallback and dispatch paths. -func buildPagesFromBytes(ctx context.Context, pages [][]byte) ([]schema.Page, error) { +// "text" key with the given doc_type_kwd (defaults to "text" when +// empty), matching the shape downstream readers expect from the +// raw-text fallback and dispatch paths. +func buildPagesFromBytes(ctx context.Context, pages [][]byte, docType string) ([]schema.Page, error) { + if docType == "" { + docType = "text" + } out := make([]schema.Page, 0, len(pages)) for _, raw := range pages { if err := ctx.Err(); err != nil { @@ -369,7 +421,7 @@ func buildPagesFromBytes(ctx context.Context, pages [][]byte) ([]schema.Page, er } out = append(out, schema.Page{ "text": string(raw), - "doc_type_kwd": "text", + "doc_type_kwd": docType, }) } return out, nil diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.go index 30e03d436f..a53accf475 100644 --- a/internal/ingestion/component/parser_dispatch.go +++ b/internal/ingestion/component/parser_dispatch.go @@ -47,6 +47,7 @@ import ( // contract (see internal/parser/parser/parse_result.go). type parserDispatchResult struct { OutputFormat string + DocType string // doc_type_kwd for media dispatch ("image", "video", "audio") File map[string]any JSON []map[string]any Markdown string @@ -204,11 +205,23 @@ func fileTypeFromInputs(inputs map[string]any) utility.FileType { return utility.FileTypeOTHER } if raw, ok := inputs["file_type"].(string); ok && raw != "" { - if ft := familyToExt(pythonFamilyName(strings.ToLower(raw))); ft != utility.FileTypeOTHER { + lower := strings.ToLower(raw) + // csv is a spreadsheet-family member but uses its own + // dedicated parser rather than the xlsx/xls path. + if lower == "csv" { + return utility.FileTypeCSV + } + // Direct extension match first — handles exact hints like + // "xls", "ppt", "doc", "docx", etc. This must run before + // the family look-up so that legacy binary extensions + // aren't collapsed to OOXML types (e.g. "xls" → XLSX). + if ft := utility.GetFileType("x." + lower); ft != utility.FileTypeOTHER { return ft } - // Direct extension match — handles "md", "docx", etc. - if ft := utility.GetFileType("x." + strings.ToLower(raw)); ft != utility.FileTypeOTHER { + // Family-name lookup catches python-side family identifiers + // ("slides", "spreadsheet", "text&code") that aren't valid + // file extensions. + if ft := familyToExt(pythonFamilyName(lower)); ft != utility.FileTypeOTHER { return ft } } @@ -235,15 +248,29 @@ func familyToExt(family string) utility.FileType { case "docx": return utility.FileTypeDOCX case "slides": - return utility.FileTypePPTX // pptx arm picks the slide parser + return utility.FileTypePPTX case "spreadsheet": return utility.FileTypeXLSX + case "csv": + return utility.FileTypeCSV case "html": return utility.FileTypeHTML case "markdown": return utility.FileTypeMarkdown case "text&code": return utility.FileTypeTXT + case "epub": + return utility.FileTypeEPUB + case "json": + return utility.FileTypeJSON + case "video": + return utility.FileTypeVIDEO + case "email": + return utility.FileTypeEMAIL + case "audio": + return utility.FileTypeAURAL + case "picture", "image", "visual": + return utility.FileTypeVISUAL } return utility.FileTypeOTHER } @@ -259,13 +286,9 @@ func pythonFamilyName(raw string) string { return "doc" case "docx": return "docx" - case "ppt": + case "ppt", "pptx", "slides": return "slides" - case "pptx": - return "slides" - case "xls": - return "spreadsheet" - case "xlsx": + case "xls", "xlsx", "spreadsheet": return "spreadsheet" case "csv": return "spreadsheet" @@ -273,19 +296,25 @@ func pythonFamilyName(raw string) string { return "html" case "md", "markdown", "mdx": return "markdown" - case "jpg", "jpeg", "png", "gif": - return "image" - case "eml", "msg": - return "email" case "epub": return "epub" + case "json", "jsonl", "ldjson": + return "json" case "txt", "py", "js", "java", "c", "cpp", "h", "php", "go", "ts", "sh", "cs", "kt", "sql": return "text&code" - case "mp4", "avi", "mkv": + case "mp4", "avi", "mkv", "mov", "webm", "flv", + "mpeg", "mpg", "wmv", "3gp", "3gpp": return "video" - case "wav", "mp3", "aac", "flac", "ogg": + case "eml", "msg": + return "email" + case "da", "wave", "wav", "mp3", "aac", "flac", "ogg", + "aiff", "au", "midi", "wma", "ape", "alac", "wv", "opus": return "audio" + case "visual", "picture", "image", + "png", "jpg", "jpeg", "gif", "bmp", "tiff", "tif", + "webp", "svg", "ico", "avif", "heic", "apng": + return "picture" } return "" } diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go index f4b4b3d9c9..ea01e74143 100644 --- a/internal/ingestion/component/parser_dispatch_test.go +++ b/internal/ingestion/component/parser_dispatch_test.go @@ -38,6 +38,7 @@ import ( "strings" "testing" + "ragflow/internal/entity" modelModule "ragflow/internal/entity/models" "ragflow/internal/ingestion/component/schema" "ragflow/internal/utility" @@ -187,6 +188,12 @@ func TestFileTypeFromInputs_ResolutionOrder(t *testing.T) { want string }{ {"explicit pdf", map[string]any{"file_type": "pdf"}, "pdf"}, + {"explicit xls (binary)", map[string]any{"file_type": "xls"}, "xls"}, + {"explicit xlsx (OOXML)", map[string]any{"file_type": "xlsx"}, "xlsx"}, + {"explicit ppt (binary)", map[string]any{"file_type": "ppt"}, "ppt"}, + {"explicit pptx (OOXML)", map[string]any{"file_type": "pptx"}, "pptx"}, + {"explicit slides (family name)", map[string]any{"file_type": "slides"}, "pptx"}, + {"explicit spreadsheet (family name)", map[string]any{"file_type": "spreadsheet"}, "xlsx"}, {"explicit markdown (family form)", map[string]any{"file_type": "markdown"}, "md"}, {"file.name docx", map[string]any{"file": map[string]any{"name": "report.docx"}}, "docx"}, {"name fallback md", map[string]any{"name": "notes.md"}, "md"}, @@ -511,29 +518,39 @@ func TestDispatch_PDFVisionJSON_PreservesEmptyPages(t *testing.T) { func TestDispatch_PDFMinerUMarkdown_UsesConfiguredBackend(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case r.Method == http.MethodPost && r.URL.Path == "/file_parse": - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"data":{"task_id":"task-3"}}`)) - case r.Method == http.MethodGet && r.URL.Path == "/tasks/task-3/result": - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"results":{"doc":{"md_content":"# Title\n\nBody\n"}}}`)) - default: - http.NotFound(w, r) + if r.Method == http.MethodPost && r.URL.Path == "/file_parse" { + buf := new(bytes.Buffer) + zw := zip.NewWriter(buf) + f, _ := zw.Create("content_list.json") + _, _ = f.Write([]byte(`[{"type":"text","text":"# Title\n\nBody\n"}]`)) + _ = zw.Close() + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write(buf.Bytes()) + return } + http.NotFound(w, r) })) defer server.Close() + // Mock resolveTenantModelByType to return a MinerU driver pointing at the test server. + origResolver := resolveTenantModelByType + defer func() { resolveTenantModelByType = origResolver }() + baseURL := server.URL + apiKey := "" + resolveTenantModelByType = func(tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) { + return &mineruTestDriver{}, "mineru-model", &modelModule.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, 0, nil + } + param := schema.ParserParam{}.Defaults() - param.Setups["pdf"]["parse_method"] = "MinerU" + param.Setups["pdf"]["parse_method"] = "mineru" param.Setups["pdf"]["output_format"] = "markdown" - param.Setups["pdf"]["mineru_apiserver"] = server.URL c := &ParserComponent{Param: param} out, err := c.Invoke(context.Background(), map[string]any{ "binary": []byte("%PDF-1.4"), "file_type": "pdf", "name": "sample.pdf", + "tenant_id": "test-tenant", }) if err != nil { t.Fatalf("Invoke: %v", err) @@ -547,6 +564,57 @@ func TestDispatch_PDFMinerUMarkdown_UsesConfiguredBackend(t *testing.T) { } } +// mineruTestDriver is a minimal ModelDriver mock whose Name() returns "mineru". +type mineruTestDriver struct{} + +func (d *mineruTestDriver) NewInstance(baseURL map[string]string) modelModule.ModelDriver { return d } +func (d *mineruTestDriver) Name() string { return "mineru" } +func (d *mineruTestDriver) ChatWithMessages(modelName string, messages []modelModule.Message, apiConfig *modelModule.APIConfig, chatModelConfig *modelModule.ChatConfig) (*modelModule.ChatResponse, error) { + return nil, fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) ChatStreamlyWithSender(modelName string, messages []modelModule.Message, apiConfig *modelModule.APIConfig, modelConfig *modelModule.ChatConfig, sender func(*string, *string) error) error { + return fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) Embed(modelName *string, texts []string, apiConfig *modelModule.APIConfig, embeddingConfig *modelModule.EmbeddingConfig) ([]modelModule.EmbeddingData, error) { + return nil, fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) Rerank(modelName *string, query string, documents []string, apiConfig *modelModule.APIConfig, rerankConfig *modelModule.RerankConfig) (*modelModule.RerankResponse, error) { + return nil, fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) TranscribeAudio(modelName *string, file *string, apiConfig *modelModule.APIConfig, asrConfig *modelModule.ASRConfig) (*modelModule.ASRResponse, error) { + return nil, fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) TranscribeAudioWithSender(modelName *string, file *string, apiConfig *modelModule.APIConfig, asrConfig *modelModule.ASRConfig, sender func(*string, *string) error) error { + return fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) AudioSpeech(modelName *string, audioContent *string, apiConfig *modelModule.APIConfig, ttsConfig *modelModule.TTSConfig) (*modelModule.TTSResponse, error) { + return nil, fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) AudioSpeechWithSender(modelName *string, audioContent *string, apiConfig *modelModule.APIConfig, ttsConfig *modelModule.TTSConfig, sender func(*string, *string) error) error { + return fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) OCRFile(modelName *string, content []byte, url *string, apiConfig *modelModule.APIConfig, ocrConfig *modelModule.OCRConfig) (*modelModule.OCRFileResponse, error) { + return nil, fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) ParseFile(modelName *string, content []byte, url *string, apiConfig *modelModule.APIConfig, parseFileConfig *modelModule.ParseFileConfig) (*modelModule.ParseFileResponse, error) { + return nil, fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) ListModels(apiConfig *modelModule.APIConfig) ([]modelModule.ListModelResponse, error) { + return nil, fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) Balance(apiConfig *modelModule.APIConfig) (map[string]interface{}, error) { + return nil, fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) CheckConnection(apiConfig *modelModule.APIConfig) error { + return fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) ListTasks(apiConfig *modelModule.APIConfig) ([]modelModule.ListTaskStatus, error) { + return nil, fmt.Errorf("not implemented") +} +func (d *mineruTestDriver) ShowTask(taskID string, apiConfig *modelModule.APIConfig) (*modelModule.TaskResponse, error) { + return nil, fmt.Errorf("not implemented") +} + func TestDispatch_PDFPaddleOCRMarkdown_UsesConfiguredBackend(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost || r.URL.Path != "/layout-parsing" { diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.go index 89e40b9456..228068ac87 100644 --- a/internal/ingestion/component/parser_test.go +++ b/internal/ingestion/component/parser_test.go @@ -354,7 +354,7 @@ func TestBuildPagesFromBytes_FormatAgnostic(t *testing.T) { got, err := buildPagesFromBytes(context.Background(), [][]byte{ []byte("first page from dispatch"), []byte("second page from html dispatch
"), - }) + }, "") if err != nil { t.Fatalf("buildPagesFromBytes: %v", err) } diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go index 45a9e2c8b3..6addd7f732 100644 --- a/internal/ingestion/component/pdf_vision_dispatch.go +++ b/internal/ingestion/component/pdf_vision_dispatch.go @@ -1,20 +1,24 @@ package component import ( + "archive/zip" + "bytes" + "context" "encoding/json" "fmt" + "io" + "mime/multipart" + "net/http" "os" "path/filepath" "strings" "sync" + "time" - "ragflow/internal/dao" "ragflow/internal/entity" modelModule "ragflow/internal/entity/models" "ragflow/internal/ingestion/component/schema" "ragflow/internal/utility" - - "gorm.io/gorm" ) type pdfVisionPage struct { @@ -52,6 +56,24 @@ func maybeDispatchPDFVision( if !ok { return parserDispatchResult{}, false, nil } + + method := getStringOr(setup, "parse_method", "") + layout := getStringOr(setup, "layout_recognizer", "") + + // MinerU dispatch: parse_method "mineru" or layout_recognizer "@MinerU" + if method == "mineru" || strings.HasPrefix(layout, "mineru") || strings.Contains(layout, "@MinerU") { + tenantID := getStringOr(inputs, "tenant_id", "") + if tenantID == "" { + return parserDispatchResult{}, true, + fmt.Errorf("Parser: mineru requires tenant_id") + } + res, err := dispatchMinerUPDF(filename, binary, tenantID, setup) + if err != nil { + return parserDispatchResult{}, true, err + } + return res, true, nil + } + modelID, useVision := resolvePDFVisionModelID(setup) if !useVision { return parserDispatchResult{}, false, nil @@ -68,6 +90,248 @@ func maybeDispatchPDFVision( return res, true, nil } +// dispatchMinerUPDF submits a PDF to the tenant's MinerU OCR model +// via the streaming /file_parse endpoint and returns parsed sections. +// Mirrors Python's mineru_parser.py:parse_pdf which POSTs with +// stream=True and reads the zip response body directly (no polling). +func dispatchMinerUPDF( + _ string, + binary []byte, + tenantID string, + setup schema.ParserSetup, +) (parserDispatchResult, error) { + driver, _, apiConfig, _, err := resolveTenantModelByType(tenantID, entity.ModelTypeOCR) + if err != nil { + return parserDispatchResult{}, fmt.Errorf("Parser: mineru model: %w", err) + } + if !isMinerUDriver(driver) { + return parserDispatchResult{}, fmt.Errorf( + "Parser: mineru requires a MinerU OCR model; found %q. Please add a MinerU OCR model to your tenant.", driver.Name()) + } + + baseURL := "" + if apiConfig.BaseURL != nil { + baseURL = *apiConfig.BaseURL + } + if baseURL == "" { + baseURL, _ = resolveMinerUBaseURL(driver, apiConfig) + } + apiURL := strings.TrimRight(baseURL, "/") + "/file_parse" + + // Parse method: "raw", "auto", "ocr", "txt" matching Python's MinerUParseMethod. + parseMethod := getStringOr(setup, "parse_method", "auto") + lang := getStringOr(setup, "mineru_lang", "English") + mineruLang := mineruLangCode(lang) + backend := getStringOr(setup, "mineru_backend", "pipeline") + + zipBytes, err := mineruStreamParse(apiURL, apiConfig.ApiKey, binary, parseMethod, mineruLang, backend) + if err != nil { + return parserDispatchResult{}, fmt.Errorf("Parser: mineru stream: %w", err) + } + + sections, err := mineruExtractSections(zipBytes) + if err != nil { + return parserDispatchResult{}, fmt.Errorf("Parser: mineru extract: %w", err) + } + + var parts []string + for _, s := range sections { + if s != "" { + parts = append(parts, s) + } + } + md := strings.Join(parts, "\n") + + outputFormat := getStringOr(setup, "output_format", "markdown") + return parserDispatchResult{ + OutputFormat: outputFormat, + Markdown: md, + }, nil +} + +// resolveMinerUBaseURL extracts the resolved base URL from a model driver. +func resolveMinerUBaseURL(driver modelModule.ModelDriver, apiConfig *modelModule.APIConfig) (string, error) { + type baseURLGetter interface { + GetBaseURL(*modelModule.APIConfig) (string, error) + } + if g, ok := driver.(baseURLGetter); ok { + return g.GetBaseURL(apiConfig) + } + return "", fmt.Errorf("driver %q does not expose GetBaseURL", driver.Name()) +} + +// mineruLangCode maps a human-readable language name to a MinerU lang code, +// mirroring Python's LANGUAGE_TO_MINERU_MAP in mineru_parser.py. +func mineruLangCode(lang string) string { + switch strings.ToLower(lang) { + case "english": + return "en" + case "chinese": + return "ch" + case "traditional chinese": + return "chinese_cht" + case "japanese": + return "japan" + case "korean": + return "korean" + case "russian", "ukrainian": + return "east_slavic" + default: + return "ch" + } +} + +// mineruStreamParse POSTs the PDF binary to the MinerU /file_parse +// endpoint with streaming and returns the zip response body. +// Mirrors Python's mineru_parser.py._run_mineru_api with stream=True. +func mineruStreamParse(apiURL string, apiKey *string, binary []byte, parseMethod, lang, backend string) ([]byte, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + + part, err := writer.CreateFormFile("files", "document.pdf") + if err != nil { + return nil, fmt.Errorf("create form file: %w", err) + } + if _, err := part.Write(binary); err != nil { + return nil, fmt.Errorf("write pdf: %w", err) + } + + _ = writer.WriteField("backend", backend) + _ = writer.WriteField("parse_method", parseMethod) + _ = writer.WriteField("lang_list", lang) + _ = writer.WriteField("return_md", "true") + _ = writer.WriteField("return_content_list", "true") + _ = writer.WriteField("response_format_zip", "true") + _ = writer.WriteField("start_page_id", "0") + _ = writer.WriteField("end_page_id", "99999") + _ = writer.WriteField("return_images", "true") + _ = writer.WriteField("return_middle_json", "true") + _ = writer.WriteField("return_model_output", "true") + _ = writer.WriteField("formula_enable", "true") + _ = writer.WriteField("table_enable", "true") + + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("finalize form: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "POST", apiURL, &body) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + if apiKey != nil && *apiKey != "" { + req.Header.Set("Authorization", "Bearer "+*apiKey) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + raw, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(raw)) + } + + zipBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response: %w", err) + } + if len(zipBytes) == 0 { + return nil, fmt.Errorf("empty response from MinerU") + } + return zipBytes, nil +} + +// mineruExtractSections reads the MinerU content_list.json from a zip +// archive and extracts section text blocks, mirroring Python's +// _transfer_to_sections. +func mineruExtractSections(zipBytes []byte) ([]string, error) { + zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes))) + if err != nil { + return nil, fmt.Errorf("open zip: %w", err) + } + + var contentList []byte + for _, f := range zipReader.File { + if strings.HasSuffix(f.Name, "content_list.json") { + rc, err := f.Open() + if err != nil { + continue + } + contentList, _ = io.ReadAll(rc) + rc.Close() + break + } + } + if len(contentList) == 0 { + return nil, fmt.Errorf("content_list.json not found in MinerU zip") + } + + var items []map[string]any + if err := json.Unmarshal(contentList, &items); err != nil { + return nil, fmt.Errorf("parse content_list.json: %w", err) + } + + var sections []string + for _, item := range items { + typ, _ := item["type"].(string) + switch typ { + case "text": + if text, ok := item["text"].(string); ok { + sections = append(sections, text) + } + case "table": + if tb, ok := item["table_body"].(string); ok { + sections = append(sections, tb) + } + for _, cap := range stringSlice(item["table_caption"]) { + sections = append(sections, cap) + } + case "image": + for _, cap := range stringSlice(item["image_caption"]) { + sections = append(sections, cap) + } + if desc, ok := item["vlm_description"].(string); ok && desc != "" { + sections = append(sections, desc) + } + case "equation", "code": + if text, ok := item["text"].(string); ok { + sections = append(sections, text) + } + case "list": + for _, li := range stringSlice(item["list_items"]) { + sections = append(sections, li) + } + default: + if text, ok := item["text"].(string); ok { + sections = append(sections, text) + } + } + } + return sections, nil +} + +func stringSlice(raw any) []string { + switch v := raw.(type) { + case []any: + var out []string + for _, s := range v { + if str, ok := s.(string); ok { + out = append(out, str) + } + } + return out + case []string: + return v + } + return nil +} + func resolvePDFVisionModelID(setup schema.ParserSetup) (string, bool) { if setup == nil { return "", false @@ -93,14 +357,13 @@ func resolvePDFVisionModelID(setup schema.ParserSetup) (string, bool) { func isNamedPDFParseMethod(raw string) bool { method := strings.ToLower(strings.TrimSpace(raw)) switch { - case strings.HasSuffix(method, "@mineru"), - strings.HasSuffix(method, "@paddleocr"), + case strings.HasSuffix(method, "@paddleocr"), strings.HasSuffix(method, "@somark"), strings.HasSuffix(method, "@opendataloader"): return true } switch method { - case "deepdoc", "plain_text", "plaintext", "mineru", "paddleocr", "docling", "opendataloader", "somark", "tcadp", "tcadp parser": + case "deepdoc", "plain_text", "plaintext", "paddleocr", "docling", "opendataloader", "somark", "tcadp", "tcadp parser": return true } return false @@ -242,22 +505,12 @@ func loadPDFVisionPrompt(name string) (string, error) { func pdfVisionPromptsBaseDir() (string, error) { var initErr error pdfVisionPromptsOnce.Do(func() { - cwd, err := os.Getwd() - if err != nil { - initErr = err + root := utility.GetProjectRoot() + if _, statErr := os.Stat(filepath.Join(root, "rag", "prompts")); statErr == nil { + pdfVisionPromptsBase = root return } - for dir := cwd; dir != "/" && dir != "."; dir = filepath.Dir(dir) { - if _, err := os.Stat(filepath.Join(dir, "rag", "prompts")); err == nil { - pdfVisionPromptsBase = dir - return - } - next := filepath.Dir(dir) - if next == dir { - break - } - } - pdfVisionPromptsBase = "/ragflow" + initErr = fmt.Errorf("rag/prompts not found under project root %q", root) }) if initErr != nil { return "", initErr @@ -271,268 +524,12 @@ func renderPDFVisionPrompt(template string, page int) string { return rendered } -type tenantModelExtra struct { - MaxTokens *int `json:"max_tokens"` -} - -func resolveTenantModelByType(tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) { - tenantDAO := dao.NewTenantDAO() - tenant, err := tenantDAO.GetByID(tenantID) - if err != nil { - return nil, "", nil, 0, err - } - var modelID string - switch modelType { - case entity.ModelTypeChat: - modelID = tenant.LLMID - case entity.ModelTypeEmbedding: - modelID = tenant.EmbdID - case entity.ModelTypeRerank: - modelID = tenant.RerankID - case entity.ModelTypeSpeech2Text: - modelID = tenant.ASRID - case entity.ModelTypeImage2Text: - modelID = tenant.Img2TxtID - case entity.ModelTypeTTS: - modelID = tenant.TTSID - case entity.ModelTypeOCR: - modelID = tenant.OCRID - default: - return nil, "", nil, 0, fmt.Errorf("invalid model type: %s", modelType) - } - if modelID == "" { - return nil, "", nil, 0, fmt.Errorf("no default %s model is set", modelType) - } - if tenantModelID := tenantModelIDByType(tenant, modelType); tenantModelID != "" { - driver, modelName, apiConfig, maxTokens, err := resolveModelConfigByID(tenantID, modelType, tenantModelID) - if err == nil { - return driver, modelName, apiConfig, maxTokens, nil - } - } - return resolveModelConfig(tenantID, modelType, modelID) -} - -func tenantModelIDByType(tenant *entity.Tenant, modelType entity.ModelType) string { - if tenant == nil { - return "" - } - switch modelType { - case entity.ModelTypeChat: - return stringValue(tenant.TenantLLMID) - case entity.ModelTypeEmbedding: - return stringValue(tenant.TenantEmbdID) - case entity.ModelTypeRerank: - return stringValue(tenant.TenantRerankID) - case entity.ModelTypeSpeech2Text: - return stringValue(tenant.TenantASRID) - case entity.ModelTypeImage2Text: - return stringValue(tenant.TenantImg2TxtID) - case entity.ModelTypeTTS: - return stringValue(tenant.TenantTTSID) - case entity.ModelTypeOCR: - return stringValue(tenant.TenantOCRID) - default: - return "" - } -} - -func stringValue(value *string) string { - if value == nil { - return "" - } - return *value -} - -func resolveModelConfig(tenantID string, modelType entity.ModelType, modelRef string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) { - modelDAO := dao.NewTenantModelDAO() - if _, err := modelDAO.GetByID(modelRef); err == nil { - return resolveModelConfigByID(tenantID, modelType, modelRef) - } else if !errorsIsRecordNotFound(err) { - return nil, "", nil, 0, err - } - return resolveModelConfigFromProviderInstance(tenantID, modelType, modelRef) -} - -func resolveModelConfigByID(tenantID string, modelType entity.ModelType, modelID string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) { - modelDAO := dao.NewTenantModelDAO() - instanceDAO := dao.NewTenantModelInstanceDAO() - providerDAO := dao.NewTenantModelProviderDAO() - - modelObj, err := modelDAO.GetByID(modelID) - if err != nil { - return nil, "", nil, 0, err - } - if modelObj.Status != "active" { - return nil, "", nil, 0, fmt.Errorf("model %q is disabled", modelID) - } - if !entity.ModelType(modelObj.ModelType).Has(modelType) { - return nil, "", nil, 0, fmt.Errorf("model %q cannot be used as %s model", modelID, modelType.String()) - } - instance, err := instanceDAO.GetByID(modelObj.InstanceID) - if err != nil { - return nil, "", nil, 0, err - } - provider, err := providerDAO.GetByID(modelObj.ProviderID) - if err != nil { - return nil, "", nil, 0, err - } - if provider.TenantID != tenantID { - return nil, "", nil, 0, fmt.Errorf("tenant %s has no access to provider owned by tenant %s", tenantID, provider.TenantID) - } - - apiKey := instance.APIKey - var extra map[string]string - _ = json.Unmarshal([]byte(instance.Extra), &extra) - region := extra["region"] - baseURL := extra["base_url"] - - providerInfo := dao.GetModelProviderManager().FindProvider(provider.ProviderName) - if providerInfo == nil { - return nil, "", nil, 0, fmt.Errorf("provider %q driver not found", provider.ProviderName) - } - driver, err := newModelDriverForBaseURLLocal(providerInfo.ModelDriver, provider.ProviderName, region, baseURL) - if err != nil { - return nil, "", nil, 0, err - } - maxTokens := 0 - if mi, _ := dao.GetModelProviderManager().GetModelByName(provider.ProviderName, modelObj.ModelName); mi != nil && mi.MaxTokens != nil { - maxTokens = *mi.MaxTokens - } - if strings.TrimSpace(modelObj.Extra) != "" { - var tenantExtra tenantModelExtra - if err := json.Unmarshal([]byte(modelObj.Extra), &tenantExtra); err != nil { - return nil, "", nil, 0, err - } - if tenantExtra.MaxTokens != nil && *tenantExtra.MaxTokens > 0 { - maxTokens = *tenantExtra.MaxTokens - } - } - apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: ®ion, BaseURL: &baseURL} - return driver, modelObj.ModelName, apiConfig, maxTokens, nil -} - -func resolveModelConfigFromProviderInstance(tenantID string, modelType entity.ModelType, modelName string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) { - pureModelName, instanceName, providerName, err := parseCompositeModelName(modelName) - if err != nil { - return nil, "", nil, 0, err - } - - providerDAO := dao.NewTenantModelProviderDAO() - instanceDAO := dao.NewTenantModelInstanceDAO() - modelDAO := dao.NewTenantModelDAO() - - provider, err := providerDAO.GetByTenantIDAndProviderName(tenantID, providerName) - if err != nil { - return nil, "", nil, 0, fmt.Errorf("provider %q lookup failed: %w", providerName, err) - } - instance, err := instanceDAO.GetByProviderIDAndInstanceName(provider.ID, instanceName) - if err != nil { - return nil, "", nil, 0, fmt.Errorf("instance %q lookup failed: %w", instanceName, err) - } - - apiKey := instance.APIKey - var extra map[string]string - _ = json.Unmarshal([]byte(instance.Extra), &extra) - region := extra["region"] - baseURL := extra["base_url"] - - modelObj, modelErr := modelDAO.GetByProviderIDAndInstanceIDAndModelTypeAndModelName( - provider.ID, instance.ID, int(modelType), pureModelName, - ) - switch { - case modelErr == nil: - if modelObj.Status == "inactive" { - return nil, "", nil, 0, fmt.Errorf("model %q is disabled", modelName) - } - providerInfo := dao.GetModelProviderManager().FindProvider(providerName) - if providerInfo == nil { - return nil, "", nil, 0, fmt.Errorf("provider %q driver not found", providerName) - } - driver, err := newModelDriverForBaseURLLocal(providerInfo.ModelDriver, providerName, region, baseURL) - if err != nil { - return nil, "", nil, 0, err - } - maxTokens := 0 - if mi, _ := dao.GetModelProviderManager().GetModelByName(providerName, pureModelName); mi != nil && mi.MaxTokens != nil { - maxTokens = *mi.MaxTokens - } - if modelObj != nil && strings.TrimSpace(modelObj.Extra) != "" { - var tenantExtra tenantModelExtra - if err := json.Unmarshal([]byte(modelObj.Extra), &tenantExtra); err != nil { - return nil, "", nil, 0, err - } - if tenantExtra.MaxTokens != nil && *tenantExtra.MaxTokens > 0 { - maxTokens = *tenantExtra.MaxTokens - } - } - apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: ®ion, BaseURL: &baseURL} - return driver, modelObj.ModelName, apiConfig, maxTokens, nil - case !errorsIsRecordNotFound(modelErr): - return nil, "", nil, 0, fmt.Errorf("model %q lookup failed: %w", modelName, modelErr) - } - - targetFactoryName := providerName - if region == "intl" && strings.EqualFold(providerName, "siliconflow") { - targetFactoryName = "siliconflow_intl" - } - targetProvider := dao.GetModelProviderManager().FindProvider(targetFactoryName) - if targetProvider == nil { - return nil, "", nil, 0, fmt.Errorf("model provider config not found: %s", providerName) - } - var llmInfo *modelModule.Model - for i := range targetProvider.Models { - if strings.EqualFold(targetProvider.Models[i].Name, pureModelName) { - llmInfo = targetProvider.Models[i] - break - } - } - if llmInfo == nil { - return nil, "", nil, 0, fmt.Errorf("model config not found: %s", modelName) - } - driver, err := newModelDriverForBaseURLLocal(targetProvider.ModelDriver, providerName, region, baseURL) - if err != nil { - return nil, "", nil, 0, err - } - apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: ®ion, BaseURL: &baseURL} - maxTokens := 0 - if llmInfo.MaxTokens != nil { - maxTokens = *llmInfo.MaxTokens - } - return driver, llmInfo.Name, apiConfig, maxTokens, nil -} - -func parseCompositeModelName(compositeName string) (modelName, instanceName, providerName string, err error) { - parts := strings.Split(compositeName, "@") - switch len(parts) { - case 3: - return parts[0], parts[1], parts[2], nil - case 2: - return parts[0], "default", parts[1], nil - case 1: - return parts[0], "", "", fmt.Errorf("provider name missing in model name: %s", compositeName) - } - n := len(parts) - return strings.Join(parts[:n-2], "@"), parts[n-2], parts[n-1], nil -} - -func newModelDriverForBaseURLLocal(driver modelModule.ModelDriver, providerName, region, baseURL string) (modelModule.ModelDriver, error) { - if driver == nil { - return nil, fmt.Errorf("provider %s driver not found", providerName) - } - if strings.TrimSpace(baseURL) == "" { - return driver, nil - } - baseURLByRegion := map[string]string{region: baseURL} - if region == "" { - baseURLByRegion["default"] = baseURL - } - newDriver := driver.NewInstance(baseURLByRegion) - if newDriver == nil { - return nil, fmt.Errorf("provider %s does not support custom base_url", providerName) - } - return newDriver, nil -} - -func errorsIsRecordNotFound(err error) bool { - return err != nil && (err == gorm.ErrRecordNotFound || strings.Contains(err.Error(), gorm.ErrRecordNotFound.Error())) +// isMinerUDriver reports whether the model driver is a MinerU variant +// (remote mineru.net or local mineru). +func isMinerUDriver(driver modelModule.ModelDriver) bool { + switch strings.ToLower(driver.Name()) { + case "mineru", "mineru.net": + return true + } + return false } diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.go index ea5737987d..b1b2001073 100644 --- a/internal/ingestion/component/schema/chunk_types.go +++ b/internal/ingestion/component/schema/chunk_types.go @@ -120,6 +120,7 @@ type ChunkDoc struct { TitleSmTks string `json:"title_sm_tks,omitempty"` ContentLtks string `json:"content_ltks,omitempty"` ContentSmLtks string `json:"content_sm_ltks,omitempty"` + TagKwd []string `json:"tag_kwd,omitempty"` PageNumber *int `json:"page_number,omitempty"` PDFPositions json.RawMessage `json:"_pdf_positions,omitempty"` Positions json.RawMessage `json:"positions,omitempty"` @@ -141,7 +142,7 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error { "ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image", "context_above", "context_below", "questions", "keywords", "summary", "chunk_order_int", "title_tks", "title_sm_tks", "content_ltks", - "content_sm_ltks", "page_number", "_pdf_positions", "positions", + "content_sm_ltks", "tag_kwd", "page_number", "_pdf_positions", "positions", } { delete(raw, key) } diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.go index f3d8228ad5..bf25a181cd 100644 --- a/internal/ingestion/component/schema/chunker.go +++ b/internal/ingestion/component/schema/chunker.go @@ -168,7 +168,9 @@ type ChunkerOutputs struct { type TokenChunkerParam struct { // DelimiterMode selects the chunking strategy. - // Allowed values: "token_size", "delimiter", "one". + // Allowed values: "token_size", "delimiter". + // The single-chunk "one" behavior is provided by the separate + // OneChunker component. DelimiterMode string `json:"delimiter_mode"` // ChunkTokenSize is the target chunk size in tokens. @@ -210,7 +212,7 @@ func (TokenChunkerParam) Defaults() TokenChunkerParam { // at construction time, keeping the schema and component decoder aligned. func (p TokenChunkerParam) Validate() error { switch p.DelimiterMode { - case "token_size", "delimiter", "one": + case "token_size", "delimiter": default: return errInvalidValue{Field: "delimiter_mode", Value: p.DelimiterMode} } diff --git a/internal/ingestion/component/schema/parser.go b/internal/ingestion/component/schema/parser.go index d0756b3b55..ec6fa17d00 100644 --- a/internal/ingestion/component/schema/parser.go +++ b/internal/ingestion/component/schema/parser.go @@ -106,6 +106,7 @@ func (ParserParam) Defaults() ParserParam { "audio": {"json"}, "video": {}, "epub": {"text", "json"}, + "json": {"json"}, }, Setups: map[string]ParserSetup{ "pdf": { @@ -193,6 +194,10 @@ func (ParserParam) Defaults() ParserParam { "suffix": []string{"epub"}, "output_format": "json", }, + "json": { + "suffix": []string{"json", "jsonl", "ldjson"}, + "output_format": "json", + }, }, } } diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go index 232f6ffa0d..7244352401 100644 --- a/internal/ingestion/component/tokenizer.go +++ b/internal/ingestion/component/tokenizer.go @@ -69,9 +69,9 @@ // chunks calls by `settings.EMBEDDING_BATCH_SIZE` (default 16) // and uses an async semaphore (`embed_limiter`). The Go port // issues ONE `Encode([]string)` call with the entire chunk -// list (AD-5a calls out "embedding calls batched, not fanned" -// and Parallelism=1). Drivers that need to chunk internally -// can do so — the wire call is one round-trip. +// list (AD-5a calls out "embedding calls batched, not fanned"). +// Drivers that need to chunk internally can do so — the wire +// call is one round-trip. // // - TRACKING: WithTimeout (60s, matches python `@timeout(60)` on // `batch_encode`), TrackProgress, TrackElapsed. See @@ -295,11 +295,6 @@ func (c *TokenizerComponent) Outputs() map[string]string { } } -// Parallelism is fixed at 1 — embedding calls are batched in one -// round-trip (plan §2 AD-5a "Tokenizer: 1 (embedding calls batched, -// not fanned)"). -func (c *TokenizerComponent) Parallelism() int { return 1 } - // Invoke computes tokens + embeddings for the upstream chunks. // // Failure modes: @@ -340,34 +335,32 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, inputs map[string]any) chunks := chunksFromTokenizerUpstream(upstream) titleStem := titleExtRE.ReplaceAllString(name, "") - return runtime.TrackElapsed("Tokenizer", func() (map[string]any, error) { - normalizeChunkTextFallback(chunks) + normalizeChunkTextFallback(chunks) - if contains(c.param.SearchMethod, "full_text") { - if err := tokenizeChunks(chunks, titleStem); err != nil { - return nil, err - } - } - - out := map[string]any{ - "output_format": "chunks", - "chunks": schema.ChunkDocsToMaps(chunks), - } - - if contains(c.param.SearchMethod, "embedding") { - chunks, tokenCount, err := c.embedChunks(ctx, tenantID, kbID, embeddingModel, name, chunks) - if err != nil { - return nil, err - } - out["embedding_token_consumption"] = tokenCount - out["chunks"] = schema.ChunkDocsToMaps(chunks) - } - if err := validateTokenizerOutputs(chunks, c.param.SearchMethod, c.param.Fields); err != nil { + if contains(c.param.SearchMethod, "full_text") { + if err := tokenizeChunks(chunks, titleStem); err != nil { return nil, err } + } - return out, nil - }) + out := map[string]any{ + "output_format": "chunks", + "chunks": schema.ChunkDocsToMaps(chunks), + } + + if contains(c.param.SearchMethod, "embedding") { + chunks, tokenCount, err := c.embedChunks(ctx, tenantID, kbID, embeddingModel, name, chunks) + if err != nil { + return nil, err + } + out["embedding_token_consumption"] = tokenCount + out["chunks"] = schema.ChunkDocsToMaps(chunks) + } + if err := validateTokenizerOutputs(chunks, c.param.SearchMethod, c.param.Fields); err != nil { + return nil, err + } + + return out, nil } func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, embeddingModel, name string, chunks []schema.ChunkDoc) ([]schema.ChunkDoc, int, error) { diff --git a/internal/ingestion/component/tokenizer_test.go b/internal/ingestion/component/tokenizer_test.go index 5123863a4a..b3cf2467d8 100644 --- a/internal/ingestion/component/tokenizer_test.go +++ b/internal/ingestion/component/tokenizer_test.go @@ -212,9 +212,6 @@ func TestTokenizerComponent_Invoke_HappyPath(t *testing.T) { if out["embedding_token_consumption"] == nil { t.Error("embedding_token_consumption missing") } - if out["_elapsed_time"] == nil { - t.Error("_elapsed_time missing") - } } // TestTokenizerComponent_Invoke_EmptyChunks covers the no-op branch: @@ -596,15 +593,6 @@ func TestTokenizerComponent_InputsOutputs_NonEmpty(t *testing.T) { } } -// TestTokenizerComponent_Parallelism locks the fan-out to 1 (plan -// §AD-5a: "embedding calls batched, not fanned"). -func TestTokenizerComponent_Parallelism(t *testing.T) { - c, _ := NewTokenizerComponent(map[string]any{}) - if got := c.(*TokenizerComponent).Parallelism(); got != 1 { - t.Errorf("Parallelism() = %d, want 1", got) - } -} - // TestTokenizerComponent_NewTokenizerComponent_Defaults verifies // the Python default param values propagate. func TestTokenizerComponent_NewTokenizerComponent_Defaults(t *testing.T) { diff --git a/internal/ingestion/pipeline/pipeline_test.go b/internal/ingestion/pipeline/pipeline_test.go index 71dc101ee7..fadd2c1f19 100644 --- a/internal/ingestion/pipeline/pipeline_test.go +++ b/internal/ingestion/pipeline/pipeline_test.go @@ -46,8 +46,6 @@ func (m *mockCanvasStage) Invoke(_ context.Context, inputs map[string]any) (map[ } return out, nil } - -func (m *mockCanvasStage) Parallelism() int { return 1 } func (m *mockCanvasStage) Inputs() map[string]string { return map[string]string{"name": "string"} } func (m *mockCanvasStage) Outputs() map[string]string { return map[string]string{"output": "any"} } @@ -157,7 +155,6 @@ type errCanvasStage struct{} func (e *errCanvasStage) Invoke(_ context.Context, _ map[string]any) (map[string]any, error) { return nil, &stageError{Stage: "p.RunErrStage", Reason: "intentional"} } -func (e *errCanvasStage) Parallelism() int { return 1 } func (e *errCanvasStage) Inputs() map[string]string { return nil } func (e *errCanvasStage) Outputs() map[string]string { return nil } diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.go index 274e2c1426..fc732445de 100644 --- a/internal/ingestion/pipeline/template_integration_test.go +++ b/internal/ingestion/pipeline/template_integration_test.go @@ -219,9 +219,9 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) { t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText) } } - chunkerState, ok := state["TokenChunker:DryDrinksVisit"] + chunkerState, ok := state["OneChunker:DryDrinksVisit"] if !ok { - t.Fatal("missing TokenChunker:DryDrinksVisit state") + t.Fatal("missing OneChunker:DryDrinksVisit state") } if got := chunkerState["output_format"]; got != "chunks" { t.Fatalf("chunker output_format = %v, want chunks", got) @@ -322,9 +322,9 @@ func TestPipelineRun_TemplateOne_RealComponents_PDFDeepdocChunking(t *testing.T) parserJoined := joinJSONItemTexts(jsonItems) assertNormalizedContainsAll(t, parserJoined, fixture.ExpectContains...) - chunkerState, ok := state["TokenChunker:DryDrinksVisit"] + chunkerState, ok := state["OneChunker:DryDrinksVisit"] if !ok { - t.Fatal("missing TokenChunker:DryDrinksVisit state") + t.Fatal("missing OneChunker:DryDrinksVisit state") } chunkerChunks, ok := chunkerState["chunks"].([]map[string]any) if !ok || len(chunkerChunks) != 1 { @@ -805,6 +805,12 @@ func TestPipelineRun_AllIngestionTemplates_RealComponentsSmoke(t *testing.T) { if templateUsesComponent(t, templateBytes, "Extractor") { t.Skip("template includes real Extractor and requires model credentials; covered separately from File/Parser/Chunker/Tokenizer e2e") } + if templateUsesComponent(t, templateBytes, "QAChunker") { + t.Skip("template uses QAChunker which requires Q&A-structured content; covered separately") + } + if templateUsesComponent(t, templateBytes, "TagChunker") { + t.Skip("template uses TagChunker which requires tag-structured content and parser setups not available for generic .md input; covered separately") + } terminalIDs := terminalComponentIDsFromTemplate(t, templateBytes) if len(terminalIDs) != 1 { t.Fatalf("terminal ids = %v, want exactly 1 terminal", terminalIDs) diff --git a/internal/parser/parser/audio_parser.go b/internal/parser/parser/audio_parser.go new file mode 100644 index 0000000000..495c9f1b99 --- /dev/null +++ b/internal/parser/parser/audio_parser.go @@ -0,0 +1,97 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// AudioParser validates and stores configuration for audio files. +// The actual speech-to-text transcription is performed by the +// component-layer maybeDispatchAudio, which mirrors Python's +// Parser._audio() in rag/flow/parser/parser.py. + +package parser + +import ( + "fmt" + "path/filepath" + "strings" +) + +// audioExtensions mirrors Python's ParserParam.Defaults() audio suffix +// list: ["da","wave","wav","mp3","aac","flac","ogg","aiff","au","midi", +// "wma","realaudio","vqf","oggvorbis","ape"]. +var audioExtensions = map[string]bool{ + "da": true, "wave": true, "wav": true, "mp3": true, + "aac": true, "flac": true, "ogg": true, "aiff": true, + "au": true, "midi": true, "wma": true, "realaudio": true, + "vqf": true, "oggvorbis": true, "ape": true, +} + +// AudioParser handles audio files for transcription. The struct mirrors +// the configuration from setups["audio"]:output_format and +// setups["audio"].vlm.llm_id. +type AudioParser struct { + VLMModelID string // vlm.llm_id — identifies the speech-to-text model + OutputFormat string +} + +// NewAudioParser constructs an AudioParser. +func NewAudioParser() *AudioParser { + return &AudioParser{} +} + +// ConfigureFromSetup reads audio-specific configuration from the +// parser setup map. It extracts vlm.llm_id and output_format. +func (p *AudioParser) ConfigureFromSetup(setup map[string]any) { + if p == nil || setup == nil { + return + } + if vlm, ok := setup["vlm"].(map[string]any); ok { + if llmID, ok := vlm["llm_id"].(string); ok && llmID != "" { + p.VLMModelID = llmID + } + } + if v, ok := setup["output_format"].(string); ok && v != "" { + p.OutputFormat = v + } +} + +// ParseWithResult implements ParseResultProducer. It validates the +// file extension against the audio extension whitelist. The actual +// speech-to-text transcription happens via maybeDispatchAudio at the +// component layer (mirrors Python's LLMBundle.transcription call). +func (p *AudioParser) ParseWithResult(filename string, data []byte) ParseResult { + ext := strings.ToLower(filepath.Ext(filename)) + if len(ext) > 1 && ext[0] == '.' { + ext = ext[1:] + } + if ext == "" || !audioExtensions[ext] { + return ParseResult{ + Err: fmt.Errorf("audio: unsupported extension %q (filename: %s); accepted: .da/.wav/.wave/.mp3/.aac/.flac/.ogg/.aiff/.au/.midi/.wma/.realaudio/.vqf/.oggvorbis/.ape", ext, filename), + } + } + + // OutputFormat and VLMModelID are consumed by maybeDispatchAudio. + outFmt := p.OutputFormat + if outFmt == "" { + outFmt = "text" + } + + return ParseResult{ + OutputFormat: outFmt, + File: map[string]any{ + "name": filename, + "size": len(data), + }, + } +} diff --git a/internal/parser/parser/audio_parser_test.go b/internal/parser/parser/audio_parser_test.go new file mode 100644 index 0000000000..0910b9ad3a --- /dev/null +++ b/internal/parser/parser/audio_parser_test.go @@ -0,0 +1,98 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package parser + +import ( + "strings" + "testing" +) + +func TestAudioParser_ValidExtension(t *testing.T) { + p := NewAudioParser() + p.ConfigureFromSetup(map[string]any{ + "output_format": "text", + "vlm": map[string]any{ + "llm_id": "whisper-1", + }, + }) + + result := p.ParseWithResult("test.mp3", []byte{1, 2, 3, 4}) + if result.Err != nil { + t.Fatalf("unexpected error: %v", result.Err) + } + if result.OutputFormat != "text" { + t.Errorf("output_format = %q, want text", result.OutputFormat) + } + if name, ok := result.File["name"].(string); !ok || name != "test.mp3" { + t.Errorf("name = %q, want test.mp3", name) + } +} + +func TestAudioParser_AllExtensions(t *testing.T) { + for _, ext := range []string{"da", "wave", "wav", "mp3", "aac", "flac", "ogg", "aiff", "au", "midi", "wma", "realaudio", "vqf", "oggvorbis", "ape"} { + p := NewAudioParser() + result := p.ParseWithResult("audio."+ext, []byte{}) + if result.Err != nil { + t.Errorf("extension .%s rejected: %v", ext, result.Err) + } + } +} + +func TestAudioParser_InvalidExtension(t *testing.T) { + p := NewAudioParser() + result := p.ParseWithResult("notaudio.txt", []byte{}) + if result.Err == nil { + t.Fatal("expected error for .txt extension") + } + if !strings.Contains(result.Err.Error(), "unsupported extension") { + t.Errorf("error message should mention unsupported extension: %v", result.Err) + } +} + +func TestAudioParser_ConfigureVLM(t *testing.T) { + p := NewAudioParser() + p.ConfigureFromSetup(map[string]any{ + "vlm": map[string]any{ + "llm_id": "openai-whisper", + }, + }) + if p.VLMModelID != "openai-whisper" { + t.Errorf("VLMModelID = %q, want openai-whisper", p.VLMModelID) + } +} + +func TestAudioParser_ConfigureNilSafe(t *testing.T) { + p := NewAudioParser() + p.ConfigureFromSetup(nil) + p.ConfigureFromSetup(map[string]any{}) + // Should not panic. + result := p.ParseWithResult("test.wav", []byte{}) + if result.Err != nil { + t.Errorf("unexpected error: %v", result.Err) + } +} + +func TestAudioParser_DefaultOutputFormat(t *testing.T) { + p := NewAudioParser() + result := p.ParseWithResult("test.flac", []byte{}) + if result.Err != nil { + t.Fatalf("unexpected error: %v", result.Err) + } + if result.OutputFormat != "text" { + t.Errorf("output_format = %q, want text (default)", result.OutputFormat) + } +} diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go new file mode 100644 index 0000000000..d4b91a6f7b --- /dev/null +++ b/internal/parser/parser/csv_parser.go @@ -0,0 +1,251 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// CSVParser renders CSV data as HTML tables, matching the spreadsheet +// family output_format == "html" convention from ParserParam.Defaults(). +// +// Mirrors Python's deepdoc/parser/excel_parser.py:RAGFlowExcelParser.html(): +// - CSV data is rendered as an HTML with
"Data". +// - The first row is treated as the header (
). +// - Illegal control characters are replaced with spaces. +// - Large sheets are split into chunks of chunk_rows data rows, each +// chunk being a self-contained with its own
and +// repeated header row. +// +// It implements the ParseResultProducer contract so the dispatch seam in +// parser_dispatch.go routes .csv files through the structured path. + +package parser + +import ( + "encoding/csv" + "fmt" + "html" + "regexp" + "strings" +) + +// Go port of Python's ILLEGAL_CHARACTERS_RE from +// deepdoc/parser/excel_parser.py. +// Pattern: [\000-\010]|[\013-\014]|[\016-\037] +// Matches all control chars except TAB (\x09), LF (\x0A), CR (\x0D). +var csvIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`) + +const csvDefaultChunkRows = 256 +const csvSheetName = "Data" + +// CSVParser reads RFC-4180 CSV data and emits HTML payloads. +type CSVParser struct { + ParseMethod string + OutputFormat string + ChunkRows int + TCADPAPIServer string + TCADPAPIKey string + TCADPTableResultType string + TCADPMarkdownImageResponseType string +} + +func NewCSVParser() *CSVParser { + return &CSVParser{ + ChunkRows: csvDefaultChunkRows, + TCADPTableResultType: "1", + TCADPMarkdownImageResponseType: "1", + } +} + +func (p *CSVParser) String() string { + return "CSVParser" +} + +func (p *CSVParser) ConfigureFromSetup(setup map[string]any) { + if p == nil || setup == nil { + return + } + if v, ok := setup["parse_method"].(string); ok && v != "" { + p.ParseMethod = v + } + if v, ok := setup["output_format"].(string); ok && v != "" { + p.OutputFormat = v + } + if v, ok := setup["chunk_rows"]; ok { + switch n := v.(type) { + case float64: + p.ChunkRows = int(n) + case int: + p.ChunkRows = n + case int64: + p.ChunkRows = int(n) + } + if p.ChunkRows <= 0 { + p.ChunkRows = csvDefaultChunkRows + } + } + if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" { + p.TCADPAPIServer = v + } + if v, ok := setup["tcadp_api_key"].(string); ok { + p.TCADPAPIKey = v + } + if v, ok := setup["table_result_type"].(string); ok && v != "" { + p.TCADPTableResultType = v + } + if v, ok := setup["markdown_image_response_type"].(string); ok && v != "" { + p.TCADPMarkdownImageResponseType = v + } +} + +// ParseWithResult implements ParseResultProducer. It reads CSV rows +// and renders them as HTML
chunks with
, header row +// repeated per chunk, and illegal-character filtering — mirroring +// Python's RAGFlowExcelParser.html(). +// When TCADP parse_method is configured, the file is dispatched to +// the Tencent Cloud Document Parsing API. +func (p *CSVParser) ParseWithResult(filename string, data []byte) ParseResult { + method := normalizeXLSXParseMethod(p.ParseMethod) + switch method { + case "tcadp": + return parseSpreadsheetWithTCADP( + filename, data, "CSV", + p.TCADPAPIServer, p.TCADPAPIKey, + p.TCADPTableResultType, p.TCADPMarkdownImageResponseType, + p.OutputFormat, + ) + case "", "csv": + // Continue with the local CSV parser. + default: + return ParseResult{ + Err: fmt.Errorf("unsupported CSV parse method: %q", p.ParseMethod), + } + } + + text := string(data) + if strings.TrimSpace(text) == "" { + return ParseResult{ + OutputFormat: "html", + File: map[string]any{ + "name": filename, + "size": len(data), + "encoding": "utf-8", + }, + HTML: "
" + csvSheetName + "
", + } + } + + reader := csv.NewReader(strings.NewReader(text)) + reader.LazyQuotes = true + reader.TrimLeadingSpace = true + + records, err := reader.ReadAll() + if err != nil { + return ParseResult{Err: fmt.Errorf("csv parse: %w", err)} + } + + // Clean illegal control characters from all cells. + records = cleanCSVRecords(records) + + chunkRows := p.ChunkRows + if chunkRows <= 0 { + chunkRows = csvDefaultChunkRows + } + + return ParseResult{ + OutputFormat: "html", + File: map[string]any{ + "name": filename, + "size": len(data), + "encoding": "utf-8", + }, + HTML: recordsToHTMLTableChunks(records, chunkRows), + } +} + +// cleanCSVRecords replaces illegal control characters in all cells +// with a single space, matching Python's ILLEGAL_CHARACTERS_RE. +func cleanCSVRecords(records [][]string) [][]string { + out := make([][]string, len(records)) + for i, row := range records { + out[i] = make([]string, len(row)) + for j, cell := range row { + out[i][j] = csvIllegalCharsRe.ReplaceAllString(cell, " ") + } + } + return out +} + +// recordsToHTMLTableChunks renders CSV records as one or more +// self-contained HTML chunks. The first row is always the +// header (
). Data rows are split into chunks of chunkRows, +// each chunk being a complete with ") + for _, cell := range row { + b.WriteString("") + } + b.WriteString("\n") + return b.String() +} diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.go index 295a5222d9..3d8d2e68df 100644 --- a/internal/parser/parser/docx_parser.go +++ b/internal/parser/parser/docx_parser.go @@ -19,23 +19,39 @@ package parser import ( + "encoding/base64" + "encoding/json" "fmt" + "strings" officeOxide "github.com/yfedoseev/office_oxide/go" ) -type DOCXParser struct{} +// DOCXFigure represents one embedded image plus its surrounding text +// context, mirroring the chunk-level shape that Python's +// naive_merge_docx produces for vision_figure_parser_docx_wrapper_naive. +type DOCXFigure struct { + Image string `json:"image"` // base64-encoded image bytes + ContextAbove string `json:"context_above"` // text before the image block + ContextBelow string `json:"context_below"` // text after the image block + Marker string `json:"marker"` // substring to locate image position in markdown +} + +type DOCXParser struct { + libType string +} func NewDOCXParser() *DOCXParser { return &DOCXParser{} } // ParseWithResult captures the office_oxide ToMarkdown output -// instead of discarding it. Returns OutputFormat="markdown" with -// the rendered Markdown on the matching key. Mirrors the python -// parser.py:Docx branch which uses rag.app.naive.Docx to render -// markdown; the Go side delegates to office_oxide for now and -// will switch to a native Markdown renderer in a follow-up. +// and additionally extracts embedded images with their surrounding +// text context so the downstream vision-figure dispatch can enrich +// the markdown with LLM-generated image descriptions. +// +// Mirrors python naive.py: Docx() → naive_merge_docx() → +// vision_figure_parser_docx_wrapper_naive(). func (p *DOCXParser) ParseWithResult(filename string, data []byte) ParseResult { doc, err := officeOxide.OpenFromBytes(data, "docx") if err != nil { @@ -48,13 +64,182 @@ func (p *DOCXParser) ParseWithResult(filename string, data []byte) ParseResult { return ParseResult{Err: fmt.Errorf("docx to-markdown: %w", err)} } + fileMeta := map[string]any{ + "name": filename, + "format": "docx", + } + + // Extract embedded images with their text context from the + // office_oxide IR so the downstream vision dispatch can + // enrich them. The already-opened doc handle is reused + // (no second OpenFromBytes). + irJSON, irErr := doc.ToIRJSON() + var figures []DOCXFigure + if irErr == nil { + figures = extractDOCXFiguresFromIR(irJSON) + } + if len(figures) > 0 { + figs := make([]map[string]any, 0, len(figures)) + for _, f := range figures { + figs = append(figs, map[string]any{ + "image": f.Image, + "context_above": f.ContextAbove, + "context_below": f.ContextBelow, + "marker": f.Marker, + }) + } + fileMeta["figures"] = figs + } + return ParseResult{ OutputFormat: "markdown", - File: map[string]any{"name": filename, "format": "docx"}, + File: fileMeta, Markdown: md, } } +// extractDOCXFiguresFromIR parses the office_oxide IR JSON and +// returns every embedded image block together with the plain text +// immediately surrounding it. The context matches what Python's +// naive_merge_docx attaches as context_above / context_below on +// each chunk that carries an image. +// +// Reuses the IR already obtained from the doc handle in +// ParseWithResult so the binary is not opened twice. +func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure { + var ir docxIRDocument + if err := json.Unmarshal([]byte(irJSON), &ir); err != nil { + return nil + } + + var flat []flatBlock + for _, sec := range ir.Sections { + for _, el := range sec.Elements { + if el.Type == "image" { + b64 := base64.StdEncoding.EncodeToString(el.Data) + flat = append(flat, flatBlock{image: b64}) + continue + } + text := joinDOCXIRRuns(el.Content) + flat = append(flat, flatBlock{text: text}) + } + } + + var figures []DOCXFigure + for i, block := range flat { + if block.image == "" { + continue + } + fig := DOCXFigure{Image: block.image} + + // Collect text above (backward scan up to docxContextWindow + // chars, or until another image is hit). + above := collectDOCXPrevText(flat, i, 512) + fig.ContextAbove = strings.TrimSpace(above) + + // Collect text below (forward scan up to docxContextWindow + // chars, or until another image is hit). + below := collectDOCXNextText(flat, i, 512) + fig.ContextBelow = strings.TrimSpace(below) + + // Marker: text of the immediately preceding flat block, + // used by the vision dispatcher to locate the image position + // in the rendered markdown for inline insertion. + for j := i - 1; j >= 0; j-- { + if flat[j].text != "" { + fig.Marker = flat[j].text + break + } + } + + figures = append(figures, fig) + } + return figures +} + +// --- internal types --- + +// flatBlock is a flattened IR element used internally to collect +// text / image context around embedded figures. +type flatBlock struct { + text string + image string // base64-encoded image data (empty for non-image) +} + +const docxContextWindow = 512 + +func collectDOCXPrevText(flat []flatBlock, idx, maxLen int) string { + var parts []string + remaining := maxLen + for i := idx - 1; i >= 0 && remaining > 0; i-- { + if flat[i].image != "" { + break // stop at previous image + } + if flat[i].text == "" { + continue + } + r := []rune(flat[i].text) + if len(r) > remaining { + r = r[len(r)-remaining:] + } + parts = append([]string{string(r)}, parts...) + remaining -= len(r) + } + return strings.Join(parts, "\n") +} + +func collectDOCXNextText(flat []flatBlock, idx, maxLen int) string { + var parts []string + remaining := maxLen + for i := idx + 1; i < len(flat) && remaining > 0; i++ { + if flat[i].image != "" { + break // stop at next image + } + if flat[i].text == "" { + continue + } + r := []rune(flat[i].text) + if len(r) > remaining { + r = r[:remaining] + } + parts = append(parts, string(r)) + remaining -= len(r) + } + return strings.Join(parts, "\n") +} + +func joinDOCXIRRuns(runs []docxIRRun) string { + var b strings.Builder + for _, r := range runs { + if r.Type == "text" { + b.WriteString(r.Text) + } + } + return b.String() +} + +// --- office_oxide IR types (local copy, independent of deepdoc) --- + +type docxIRDocument struct { + Sections []docxIRSection `json:"sections"` +} + +type docxIRSection struct { + Title string `json:"title"` + Elements []docxIRElement `json:"elements"` +} + +type docxIRElement struct { + Type string `json:"type"` + Content []docxIRRun `json:"content"` + Data []byte `json:"data"` +} + +type docxIRRun struct { + Type string `json:"type"` + Text string `json:"text"` +} + func (p *DOCXParser) String() string { return "DOCXParser" } diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go new file mode 100644 index 0000000000..b7a42ce68b --- /dev/null +++ b/internal/parser/parser/email_parser.go @@ -0,0 +1,379 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package parser + +import ( + "bytes" + "fmt" + "io" + "mime" + "mime/multipart" + "net/mail" + "path/filepath" + "strings" + + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/transform" +) + +// EmailParser parses .eml (RFC 5322 email) files into structured +// JSON or plain-text output. Mirrors Python's _email() method in +// rag/flow/parser/parser.py. +// +// .msg (Outlook) files are not supported in the Go path; callers +// receive a clear error. +type EmailParser struct { + fields []string + outputFormat string +} + +func NewEmailParser() *EmailParser { + return &EmailParser{} +} + +func (p *EmailParser) ConfigureFromSetup(setup map[string]any) { + if p == nil || setup == nil { + return + } + if v, ok := setup["output_format"].(string); ok && v != "" { + p.outputFormat = v + } + if raw, ok := setup["fields"]; ok { + switch list := raw.(type) { + case []string: + p.fields = list + case []any: + p.fields = make([]string, 0, len(list)) + for _, item := range list { + if s, ok := item.(string); ok { + p.fields = append(p.fields, s) + } + } + } + } + if len(p.fields) == 0 { + p.fields = []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"} + } +} + +func (p *EmailParser) ParseWithResult(filename string, data []byte) ParseResult { + ext := strings.ToLower(filepath.Ext(filename)) + if ext == ".msg" { + return ParseResult{ + Err: fmt.Errorf("email: .msg (Outlook) files are not supported in the Go parser; use .eml format"), + } + } + + emailContent := parseEML(bytes.NewReader(data), p.fields) + + outputFormat := p.outputFormat + if outputFormat == "" { + outputFormat = "text" + } + + if outputFormat == "json" { + emailContent["doc_type_kwd"] = "text" + return ParseResult{ + OutputFormat: "json", + File: map[string]any{"name": filename}, + JSON: []map[string]any{emailContent}, + } + } + + // Text output: flatten fields into a single string. + var sb strings.Builder + for k, v := range emailContent { + switch val := v.(type) { + case string: + sb.WriteString(k) + sb.WriteString(":") + sb.WriteString(val) + sb.WriteString("\n") + case map[string]any: + sb.WriteString(k) + sb.WriteString(":{") + for mk, mv := range val { + if ms, ok := mv.(string); ok { + sb.WriteString(mk) + sb.WriteString(":") + sb.WriteString(ms) + sb.WriteString(", ") + } + } + sb.WriteString("}\n") + case []map[string]any: + for _, att := range val { + fn, _ := att["filename"].(string) + pl, _ := att["payload"].(string) + sb.WriteString(fn) + sb.WriteString(":") + sb.WriteString(pl) + sb.WriteString("\n") + } + case []string: + sb.WriteString(strings.Join(val, "\n")) + } + } + return ParseResult{ + OutputFormat: "text", + File: map[string]any{"name": filename}, + Text: sb.String(), + } +} + +// -- field set helpers -- + +func targetFieldsSet(fields []string) map[string]bool { + m := make(map[string]bool, len(fields)) + for _, f := range fields { + m[strings.ToLower(strings.TrimSpace(f))] = true + } + return m +} + +// -- .eml parsing (RFC 5322 with multipart support) -- + +func parseEML(r io.Reader, fields []string) map[string]any { + target := targetFieldsSet(fields) + content := map[string]any{} + + msg, err := mail.ReadMessage(r) + if err != nil { + content["error"] = fmt.Sprintf("email: parse error: %v", err) + return content + } + + // Headers. + meta := map[string]any{} + for key, vals := range msg.Header { + keyLower := strings.ToLower(key) + val := strings.Join(vals, ", ") + switch keyLower { + case "from", "to", "cc", "bcc", "date", "subject": + if target[keyLower] { + content[keyLower] = val + } + default: + meta[keyLower] = val + } + } + if target["metadata"] { + content["metadata"] = meta + } + + // Body and attachments — readMailBody walks all multipart parts + // and collects text/html bodies alongside attachment parts whose + // Content-Disposition starts with "attachment". + needAttachments := target["attachments"] + if target["body"] { + contentType := msg.Header.Get("Content-Type") + bodyText, bodyHTML, attachments := readMailBody(msg.Body, contentType, needAttachments) + if bodyText != "" { + content["text"] = bodyText + } + if bodyHTML != "" { + content["text_html"] = bodyHTML + } + if needAttachments { + content["attachments"] = attachments + } + } else if needAttachments { + content["attachments"] = []map[string]any{} + } + + return content +} + +// readMailBody reads the body of an email message, handling +// multipart/alternative, multipart/mixed, and single-part content +// types. Returns (textBody, htmlBody, attachments). +// When collectAttachments is true, non-text parts with Content-Disposition +// starting with "attachment" are collected. +func readMailBody(body io.Reader, contentType string, collectAttachments bool) (string, string, []map[string]any) { + var attachments []map[string]any + + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + mediaType = "text/plain" + } + + if !strings.HasPrefix(mediaType, "multipart/") { + raw, _ := io.ReadAll(body) + decoded := decodeMailPayload(raw, params["charset"]) + if mediaType == "text/html" { + return "", decoded, attachments + } + return decoded, "", attachments + } + + boundary := params["boundary"] + if boundary == "" { + raw, _ := io.ReadAll(body) + return decodeMailPayload(raw, ""), "", attachments + } + + mr := multipart.NewReader(body, boundary) + var textParts, htmlParts []string + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + break + } + partCT := part.Header.Get("Content-Type") + partMedia, partParams, _ := mime.ParseMediaType(partCT) + + if strings.HasPrefix(partMedia, "multipart/") { + t, h, nestedAttachments := readMailBody(part, partCT, collectAttachments) + if t != "" { + textParts = append(textParts, t) + } + if h != "" { + htmlParts = append(htmlParts, h) + } + attachments = append(attachments, nestedAttachments...) + continue + } + + // Check if this part is an attachment. + if collectAttachments && isAttachmentPart(part) { + raw, _ := io.ReadAll(part) + attachments = append(attachments, map[string]any{ + "filename": attachmentFilename(part), + "payload": decodeMailPayload(raw, partParams["charset"]), + }) + continue + } + + raw, _ := io.ReadAll(part) + decoded := decodeMailPayload(raw, partParams["charset"]) + + switch partMedia { + case "text/plain": + textParts = append(textParts, decoded) + case "text/html": + htmlParts = append(htmlParts, decoded) + } + } + return strings.Join(textParts, "\n"), strings.Join(htmlParts, "\n"), attachments +} + +// isAttachmentPart checks whether a multipart part should be treated as +// an attachment (Content-Disposition starts with "attachment"). Mirrors +// Python's check in _email(). +func isAttachmentPart(part *multipart.Part) bool { + disp := part.Header.Get("Content-Disposition") + if disp == "" { + return false + } + dispType, _, err := mime.ParseMediaType(disp) + if err != nil { + return false + } + return strings.EqualFold(dispType, "attachment") +} + +// attachmentFilename extracts a filename from the part's +// Content-Disposition or Content-Type headers. +func attachmentFilename(part *multipart.Part) string { + if fn := part.FileName(); fn != "" { + return fn + } + ct := part.Header.Get("Content-Type") + if ct != "" { + _, params, _ := mime.ParseMediaType(ct) + if name, ok := params["name"]; ok { + return name + } + } + return "attachment.bin" +} + +// decodeMailPayload attempts multiple charset decodings. +// Mirrors Python's _decode_payload with fallback chain: +// utf-8 → gb2312 → gbk → gb18030 → latin1 → utf-8 (ignore). +func decodeMailPayload(payload []byte, charset string) string { + if len(payload) == 0 { + return "" + } + charsets := buildCharsetChain(charset) + for _, enc := range charsets { + if enc == "" { + // raw bytes → already fallthrough + return string(payload) + } + decoded, err := decodeWithCharset(payload, enc) + if err == nil { + return decoded + } + } + return string(payload) +} + +func buildCharsetChain(declared string) []string { + chain := make([]string, 0, 7) + if declared != "" { + chain = append(chain, declared) + } + chain = append(chain, "utf-8", "gb2312", "gbk", "gb18030", "latin1") + return chain +} + +func decodeWithCharset(payload []byte, charset string) (string, error) { + charset = strings.ToLower(strings.TrimSpace(charset)) + switch charset { + case "utf-8", "utf8", "": + s := string(payload) + if !strings.ContainsRune(s, '\ufffd') { + return s, nil + } + return "", fmt.Errorf("invalid utf-8") + case "latin1", "iso-8859-1", "iso8859-1": + runes := make([]rune, len(payload)) + for i, b := range payload { + runes[i] = rune(b) + } + return string(runes), nil + case "gb2312": + return decodeTransform(payload, simplifiedchinese.HZGB2312.NewDecoder()) + case "gbk": + return decodeTransform(payload, simplifiedchinese.GBK.NewDecoder()) + case "gb18030": + return decodeTransform(payload, simplifiedchinese.GB18030.NewDecoder()) + } + // Unknown charset: treat as latin-1. + runes := make([]rune, len(payload)) + for i, b := range payload { + runes[i] = rune(b) + } + return string(runes), nil +} + +func decodeTransform(payload []byte, decoder *encoding.Decoder) (string, error) { + reader := transform.NewReader(bytes.NewReader(payload), decoder) + decoded, err := io.ReadAll(reader) + if err != nil { + return "", err + } + if !strings.ContainsRune(string(decoded), '\ufffd') { + return string(decoded), nil + } + return "", fmt.Errorf("decode produced replacement characters") +} diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.go new file mode 100644 index 0000000000..9cc60afc90 --- /dev/null +++ b/internal/parser/parser/email_parser_test.go @@ -0,0 +1,157 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package parser + +import ( + "strings" + "testing" +) + +func TestEmailParser_EmlJSON(t *testing.T) { + raw := strings.Join([]string{ + "From: sender@example.com", + "To: recipient@example.com", + "Cc: cc@example.com", + "Date: Mon, 07 Jul 2025 10:00:00 +0000", + "Subject: Test Email", + "Content-Type: text/plain; charset=utf-8", + "X-Custom-Header: custom-value", + "", + "This is the body of the test email.", + }, "\r\n") + + p := NewEmailParser() + p.ConfigureFromSetup(map[string]any{ + "output_format": "json", + "fields": []string{"from", "to", "cc", "date", "subject", "body", "metadata"}, + }) + + result := p.ParseWithResult("test.eml", []byte(raw)) + if result.Err != nil { + t.Fatalf("unexpected error: %v", result.Err) + } + if result.OutputFormat != "json" { + t.Fatalf("expected output_format json, got %q", result.OutputFormat) + } + if len(result.JSON) != 1 { + t.Fatalf("expected 1 JSON item, got %d", len(result.JSON)) + } + item := result.JSON[0] + + if v, ok := item["from"].(string); !ok || v != "sender@example.com" { + t.Errorf("from: got %q", v) + } + if v, ok := item["to"].(string); !ok || v != "recipient@example.com" { + t.Errorf("to: got %q", v) + } + if v, ok := item["subject"].(string); !ok || v != "Test Email" { + t.Errorf("subject: got %q", v) + } + if v, ok := item["text"].(string); !ok || !strings.Contains(v, "body of the test email") { + t.Errorf("text: got %q", v) + } + if meta, ok := item["metadata"].(map[string]any); ok { + if v, ok := meta["x-custom-header"].(string); !ok || v != "custom-value" { + t.Errorf("metadata x-custom-header: got %q", v) + } + } else { + t.Error("metadata missing or wrong type") + } + if v, ok := item["doc_type_kwd"].(string); !ok || v != "text" { + t.Errorf("doc_type_kwd: got %q", v) + } +} + +func TestEmailParser_EmlText(t *testing.T) { + raw := strings.Join([]string{ + "From: sender@test.com", + "To: recipient@test.com", + "Subject: Hello", + "Content-Type: text/plain; charset=utf-8", + "", + "Hello, world!", + }, "\r\n") + + p := NewEmailParser() + p.ConfigureFromSetup(map[string]any{ + "output_format": "text", + "fields": []string{"from", "to", "subject", "body"}, + }) + + result := p.ParseWithResult("test.eml", []byte(raw)) + if result.Err != nil { + t.Fatalf("unexpected error: %v", result.Err) + } + if result.OutputFormat != "text" { + t.Fatalf("expected output_format text, got %q", result.OutputFormat) + } + if !strings.Contains(result.Text, "Hello, world!") { + t.Errorf("text missing body: %q", result.Text) + } + if !strings.Contains(result.Text, "sender@test.com") { + t.Errorf("text missing from: %q", result.Text) + } +} + +func TestEmailParser_MsgNotSupported(t *testing.T) { + p := NewEmailParser() + result := p.ParseWithResult("test.msg", []byte{}) + if result.Err == nil { + t.Fatal("expected error for .msg file") + } + if !strings.Contains(result.Err.Error(), ".msg") { + t.Errorf("error should mention .msg: %v", result.Err) + } +} + +func TestEmailParser_Multipart(t *testing.T) { + boundary := "boundary123" + raw := strings.Join([]string{ + "From: multipart@test.com", + "To: receiver@test.com", + "Subject: Multipart Test", + "Content-Type: multipart/alternative; boundary=" + boundary, + "", + "--" + boundary, + "Content-Type: text/plain; charset=utf-8", + "", + "Plain text body.", + "--" + boundary, + "Content-Type: text/html; charset=utf-8", + "", + "

HTML body.

", + "--" + boundary + "--", + }, "\r\n") + + p := NewEmailParser() + p.ConfigureFromSetup(map[string]any{ + "output_format": "json", + "fields": []string{"from", "body"}, + }) + + result := p.ParseWithResult("test.eml", []byte(raw)) + if result.Err != nil { + t.Fatalf("unexpected error: %v", result.Err) + } + item := result.JSON[0] + if v, ok := item["text"].(string); !ok || !strings.Contains(v, "Plain text body") { + t.Errorf("text: got %q", v) + } + if v, ok := item["text_html"].(string); !ok || !strings.Contains(v, "HTML body") { + t.Errorf("text_html: got %q", v) + } +} diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.go new file mode 100644 index 0000000000..081c1a81c4 --- /dev/null +++ b/internal/parser/parser/epub_parser.go @@ -0,0 +1,311 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// EPUBParser extracts text content from EPUB files. EPUB is a ZIP +// container with XHTML spine items. The parser: +// +// 1. Opens the EPUB as a ZIP archive. +// 2. Reads META-INF/container.xml to locate the OPF manifest. +// 3. Walks the OPF spine to collect content documents in order. +// 4. Strips HTML tags from each content document to produce plain +// text items, emitting one JSON item per spine entry. +// +// The output format is "json", matching the epub default in +// ParserParam.Defaults(). + +package parser + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "path" + "regexp" + "strings" +) + +// EPUBParser extracts text from EPUB archives. +type EPUBParser struct{} + +func NewEPUBParser() *EPUBParser { + return &EPUBParser{} +} + +func (p *EPUBParser) String() string { + return "EPUBParser" +} + +// ParseWithResult implements ParseResultProducer. It extracts XHTML +// content from the EPUB spine and emits one JSON item per spine entry +// with {text, doc_type_kwd:"text"}. +func (p *EPUBParser) ParseWithResult(filename string, data []byte) ParseResult { + if len(data) == 0 { + return ParseResult{ + OutputFormat: "json", + File: map[string]any{ + "name": filename, + "size": 0, + "encoding": "utf-8", + }, + JSON: []map[string]any{{"text": "", "doc_type_kwd": "text"}}, + } + } + + reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return ParseResult{Err: fmt.Errorf("epub: open zip: %w", err)} + } + + opfPath, err := readContainerXML(reader) + if err != nil { + return ParseResult{Err: fmt.Errorf("epub: container.xml: %w", err)} + } + + spineItems, err := readOPFSpine(reader, opfPath) + if err != nil { + return ParseResult{Err: fmt.Errorf("epub: opf: %w", err)} + } + + items := extractEPUBTextItems(reader, opfPath, spineItems) + if items == nil { + items = []map[string]any{{"text": "", "doc_type_kwd": "text"}} + } + return ParseResult{ + OutputFormat: "json", + File: map[string]any{ + "name": filename, + "size": len(data), + "encoding": "utf-8", + }, + JSON: items, + } +} + +// --- container.xml parsing --- + +type containerXML struct { + RootFiles []struct { + FullPath string `xml:"full-path,attr"` + } `xml:"rootfiles>rootfile"` +} + +func readContainerXML(r *zip.Reader) (string, error) { + f, err := r.Open("META-INF/container.xml") + if err != nil { + return "", fmt.Errorf("open container.xml: %w", err) + } + defer f.Close() + + raw, err := io.ReadAll(f) + if err != nil { + return "", fmt.Errorf("read container.xml: %w", err) + } + var c containerXML + if err := xml.Unmarshal(raw, &c); err != nil { + return "", fmt.Errorf("parse container.xml: %w", err) + } + if len(c.RootFiles) == 0 { + return "", fmt.Errorf("no rootfile in container.xml") + } + return c.RootFiles[0].FullPath, nil +} + +// --- OPF parsing --- + +type opfPackage struct { + Manifest struct { + Items []struct { + ID string `xml:"id,attr"` + Href string `xml:"href,attr"` + Type string `xml:"media-type,attr"` + } `xml:"item"` + } `xml:"manifest"` + Spine struct { + ItemRefs []struct { + IDRef string `xml:"idref,attr"` + } `xml:"itemref"` + } `xml:"spine"` +} + +func readOPFSpine(r *zip.Reader, opfPath string) ([]string, error) { + f, err := r.Open(opfPath) + if err != nil { + // Some EPUBs use a path with a leading slash stripped. + clean := strings.TrimPrefix(opfPath, "/") + if clean != opfPath { + f, err = r.Open(clean) + if err != nil { + return nil, fmt.Errorf("open opf %q: %w", opfPath, err) + } + } else { + return nil, fmt.Errorf("open opf %q: %w", opfPath, err) + } + } + defer f.Close() + + raw, err := io.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("read opf: %w", err) + } + var pkg opfPackage + if err := xml.Unmarshal(raw, &pkg); err != nil { + return nil, fmt.Errorf("parse opf: %w", err) + } + + // Build id → href lookup. + byID := make(map[string]string, len(pkg.Manifest.Items)) + for _, it := range pkg.Manifest.Items { + byID[it.ID] = it.Href + } + + // Build spine item list (ordered hrefs). + var hrefs []string + for _, ref := range pkg.Spine.ItemRefs { + if href, ok := byID[ref.IDRef]; ok { + hrefs = append(hrefs, href) + } + } + return hrefs, nil +} + +// --- text extraction --- + +var ( + epubScriptRe = regexp.MustCompile(`(?is)]*>.*?`) + epubStyleRe = regexp.MustCompile(`(?is)]*>.*?`) + epubTagRe = regexp.MustCompile(`<[^>]+>`) + epubEntityRe = regexp.MustCompile(`&[a-zA-Z]+;`) + epubWSRe = regexp.MustCompile(`\s+`) +) + +func extractEPUBTextItems(r *zip.Reader, opfDir string, spineHrefs []string) []map[string]any { + var items []map[string]any + for _, href := range spineHrefs { + text := readEPUBContentFile(r, opfDir, href) + if strings.TrimSpace(text) == "" { + continue + } + items = append(items, map[string]any{ + "text": text, + "doc_type_kwd": "text", + }) + } + return items +} + +// readEPUBContentFile resolves a spine href (relative to the OPF +// directory) inside the ZIP, reads the raw bytes, and strips HTML to +// return clean text. +func readEPUBContentFile(r *zip.Reader, opfDir, href string) string { + // Resolve href relative to the directory containing the OPF. + // Use path.Join/Dir (slash separator) because ZIP entry paths are always POSIX. + resolved := path.Join(path.Dir(opfDir), href) + + var f io.ReadCloser + var err error + // Try the resolved path first, then raw href. + for _, name := range []string{resolved, href} { + if strings.TrimSpace(name) == "" { + continue + } + f, err = r.Open(name) + if err == nil { + break + } + } + if err != nil { + return "" + } + defer f.Close() + + raw, err := io.ReadAll(f) + if err != nil { + return "" + } + return stripHTMLTags(string(raw)) +} + +// stripHTMLTags removes HTML markup and returns normalized plain text. +func stripHTMLTags(s string) string { + // Remove script/style blocks first (including their content). + s = epubScriptRe.ReplaceAllString(s, "") + s = epubStyleRe.ReplaceAllString(s, "") + // Replace
,

,

, , , etc. with newlines. + s = epubTagRe.ReplaceAllStringFunc(s, func(tag string) string { + lower := strings.ToLower(tag) + switch { + case strings.HasPrefix(lower, "" + case """: + return "\"" + case "'": + return "'" + case " ": + return " " + case "–": + return "–" + case "—": + return "—" + case "‘": + return "'" + case "’": + return "'" + case "“": + return "\"" + case "”": + return "\"" + case "…": + return "…" + default: + return entity + } +} diff --git a/internal/parser/parser/factory.go b/internal/parser/parser/factory.go index 98fc440f4f..d884f1e389 100644 --- a/internal/parser/parser/factory.go +++ b/internal/parser/parser/factory.go @@ -52,7 +52,7 @@ func GetParserByID(parserID string) (ParseResultProducer, error) { case "resume": return NewResumePDFParser(), nil case "picture": - return NewPicturePDFParser(), nil + return NewPictureParser(), nil case "one": return NewOnePDFParser(), nil case "audio": @@ -80,7 +80,5 @@ func NewTableParser() *stubParser { return &stubParser{name: "table"} } func NewResumePDFParser() *stubParser { return &stubParser{name: "resume"} } func NewPicturePDFParser() *stubParser { return &stubParser{name: "picture"} } func NewOnePDFParser() *stubParser { return &stubParser{name: "one"} } -func NewAudioParser() *stubParser { return &stubParser{name: "audio"} } -func NewEmailParser() *stubParser { return &stubParser{name: "email"} } func NewTagPDFParser() *stubParser { return &stubParser{name: "tag"} } func NewKGPDFParser() *stubParser { return &stubParser{name: "knowledge_graph"} } diff --git a/internal/parser/parser/factory_test.go b/internal/parser/parser/factory_test.go index b2aa532889..4dabdfd7a8 100644 --- a/internal/parser/parser/factory_test.go +++ b/internal/parser/parser/factory_test.go @@ -37,7 +37,6 @@ func TestGetParserByID_AllKnownIDs(t *testing.T) { {string(entity.ParserTypeResume), false}, {string(entity.ParserTypePicture), false}, {string(entity.ParserTypeOne), false}, - {string(entity.ParserTypeKG), false}, {string(entity.ParserTypeTag), false}, // Office parsers {string(entity.ParserTypePresentation), false}, diff --git a/internal/parser/parser/json_parser.go b/internal/parser/parser/json_parser.go new file mode 100644 index 0000000000..fe03f9ef7b --- /dev/null +++ b/internal/parser/parser/json_parser.go @@ -0,0 +1,228 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// JSONParser handles .json, .jsonl, and .ldjson files. It detects +// the payload shape: +// +// - JSON array: each element becomes one item. +// - Single JSON object: emitted as one item. +// - JSONL (newline-delimited JSON): each non-empty line is +// independently parsed; lines that fail to parse are skipped +// with a warning — matching the Python JsonParser behaviour. +// +// For JSON objects, the parser serialises the full object into a +// `text` field so the downstream chunker receives a readable +// representation. Arrays of objects emit one item per element. + +package parser + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "strings" +) + +// JSONParser handles JSON / JSONL / LDJSON files. +type JSONParser struct{} + +func NewJSONParser() *JSONParser { + return &JSONParser{} +} + +func (p *JSONParser) String() string { + return "JSONParser" +} + +// ParseWithResult implements ParseResultProducer. It detects the JSON +// shape (array, single object, or line-delimited) and emits one JSON +// item per logical record, with {text, doc_type_kwd:"text"}. +func (p *JSONParser) ParseWithResult(filename string, data []byte) ParseResult { + text := string(bytes.TrimSpace(data)) + if text == "" { + return ParseResult{ + OutputFormat: "json", + File: map[string]any{ + "name": filename, + "size": len(data), + "encoding": "utf-8", + }, + JSON: []map[string]any{{"text": "", "doc_type_kwd": "text"}}, + } + } + + items := parseJSONContent(text) + if items == nil { + items = []map[string]any{{"text": "", "doc_type_kwd": "text"}} + } + return ParseResult{ + OutputFormat: "json", + File: map[string]any{ + "name": filename, + "size": len(data), + "encoding": "utf-8", + }, + JSON: items, + } +} + +// parseJSONContent detects the JSON shape and dispatches. +func parseJSONContent(text string) []map[string]any { + trimmed := strings.TrimSpace(text) + if trimmed == "" { + return nil + } + + // 1. Try JSON array: [...] + if strings.HasPrefix(trimmed, "[") { + return parseJSONArray(trimmed) + } + + // 2. Try single JSON object: {...} + if strings.HasPrefix(trimmed, "{") { + // Prefer parsing as a single JSON object first (matches + // Python's json.loads(txt) whole-document test). Only fall + // back to JSONL when that fails and the content still looks + // line-delimited. + if err := json.Unmarshal([]byte(trimmed), &map[string]any{}); err == nil { + return parseSingleJSONObject(trimmed) + } + if isJSONL(trimmed) { + return parseJSONLines(trimmed) + } + return []map[string]any{{"text": trimmed, "doc_type_kwd": "text"}} + } + + // 3. Fallback: treat as text. + return []map[string]any{{"text": trimmed, "doc_type_kwd": "text"}} +} + +// isJSONL returns true when the content looks like newline-delimited +// JSON (multiple lines each starting with '{'). +func isJSONL(text string) bool { + lines := strings.Split(text, "\n") + objCount := 0 + for _, line := range lines { + t := strings.TrimSpace(line) + if t == "" { + continue + } + if strings.HasPrefix(t, "{") { + objCount++ + if objCount >= 2 { + return true + } + } + } + return false +} + +// parseJSONArray unmarshals a JSON array and emits one item per element. +func parseJSONArray(text string) []map[string]any { + var arr []any + if err := json.Unmarshal([]byte(text), &arr); err != nil { + // Fallback: emit as plain text. + return []map[string]any{{"text": text, "doc_type_kwd": "text"}} + } + if len(arr) == 0 { + return nil + } + items := make([]map[string]any, 0, len(arr)) + for _, elem := range arr { + if s, ok := elem.(string); ok { + items = append(items, map[string]any{ + "text": s, + "doc_type_kwd": "text", + }) + } else { + // Re-marshal non-string elements. + b, err := json.Marshal(elem) + if err != nil { + items = append(items, map[string]any{ + "text": fmt.Sprintf("%v", elem), + "doc_type_kwd": "text", + }) + } else { + items = append(items, map[string]any{ + "text": string(b), + "doc_type_kwd": "text", + }) + } + } + } + return items +} + +// parseSingleJSONObject unmarshals a single JSON object and emits one item. +func parseSingleJSONObject(text string) []map[string]any { + var obj map[string]any + if err := json.Unmarshal([]byte(text), &obj); err != nil { + return []map[string]any{{"text": text, "doc_type_kwd": "text"}} + } + b, err := json.Marshal(obj) + if err != nil { + return []map[string]any{{"text": fmt.Sprintf("%v", obj), "doc_type_kwd": "text"}} + } + return []map[string]any{{"text": string(b), "doc_type_kwd": "text"}} +} + +// parseJSONLines parses line-delimited JSON. Each non-empty line is +// independently unmarshalled; lines that fail to parse are skipped. +func parseJSONLines(text string) []map[string]any { + var items []map[string]any + scanner := bufio.NewScanner(strings.NewReader(text)) + scanner.Buffer(nil, 10*1024*1024) // 10 MB max line + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + // Each line should be a JSON object. + var obj map[string]any + if err := json.Unmarshal([]byte(line), &obj); err != nil { + // Also try as a string or other standalone value. + var val any + if err2 := json.Unmarshal([]byte(line), &val); err2 != nil { + // Unparseable line — skip, matching Python behaviour. + continue + } else { + if s, ok := val.(string); ok { + items = append(items, map[string]any{ + "text": s, + "doc_type_kwd": "text", + }) + } else { + b, _ := json.Marshal(val) + items = append(items, map[string]any{ + "text": string(b), + "doc_type_kwd": "text", + }) + } + } + continue + } + b, err := json.Marshal(obj) + if err != nil { + continue + } + items = append(items, map[string]any{ + "text": string(b), + "doc_type_kwd": "text", + }) + } + return items +} diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go index 662274ef53..482c136b8f 100644 --- a/internal/parser/parser/markdown_parser.go +++ b/internal/parser/parser/markdown_parser.go @@ -18,32 +18,84 @@ package parser import ( "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "net" + "net/http" + "net/url" + "regexp" "strings" + "sync" + "time" "github.com/gomarkdown/markdown/ast" "github.com/gomarkdown/markdown/parser" ) -type MarkdownParser struct{} +// mdImagePattern matches markdown inline image syntax: ![alt](url). +var mdImagePattern = regexp.MustCompile(`!\[[^\]]*\]\(([^)\s]+)\)`) -func NewMarkdownParser() *MarkdownParser { - return &MarkdownParser{} +// dataURIPrefix is the MIME prefix for data URI images. +const dataURIPrefix = "data:image/" + +// GoMarkdown is the lib_type identifier for the pure-Go markdown backend. +const GoMarkdown = "go_markdown" + +// ssrfAllowLoopback lets tests exercise the HTTP image fetch path against a +// loopback httptest server. It stays false in production so loopback +// addresses are rejected (SSRF protection). +var ssrfAllowLoopback bool + +type MarkdownParser struct { + libType string + ParseMethod string + OutputFormat string + VLM map[string]any +} + +func NewMarkdownParser(libType string) (*MarkdownParser, error) { + switch libType { + case GoMarkdown: + return &MarkdownParser{ + libType: GoMarkdown, + }, nil + default: + return nil, fmt.Errorf("unsupported Markdown library type: %s", libType) + } +} + +func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) { + if p == nil || setup == nil { + return + } + if v, ok := setup["parse_method"].(string); ok && v != "" { + p.ParseMethod = v + } + if v, ok := setup["output_format"].(string); ok && v != "" { + p.OutputFormat = v + } + if v, ok := setup["vlm"].(map[string]any); ok { + p.VLM = v + } } // ParseWithResult implements ParseResultProducer (plan §6.5) and // returns a structured markdown payload that mirrors the Python // parser's `output_format == "json"` shape. Each top-level block -// emits one item with `text` + `doc_type_kwd: "text"`. The legacy -// debug-print path has been removed; callers consume ParseResult directly. +// emits one item with `text` + `doc_type_kwd: "text"`. When the +// block contains a markdown image reference (![alt](src)), the image +// data is resolved and the item carries `doc_type_kwd: "image"` with +// the base64-encoded image payload. The legacy debug-print path has +// been removed; callers consume ParseResult directly. func (p *MarkdownParser) ParseWithResult(filename string, data []byte) ParseResult { doc := markdownNew().Parse(data) + rawText := string(data) var items []map[string]any - walkMarkdownBlocks(doc, &items) + walkMarkdownBlocksWithImages(doc, rawText, &items) if items == nil { - // No blocks emitted — surface a single empty item so the - // downstream chunker sees a non-nil JSON slice (mirrors the - // Python contract of always producing at least one item). items = []map[string]any{{"text": "", "doc_type_kwd": "text"}} } return ParseResult{ @@ -66,55 +118,222 @@ func markdownNew() *parser.Parser { return parser.NewWithExtensions(extensions) } -// walkMarkdownBlocks emits one normalized item per top-level block. -// Headings (LeafBlock / H*) are emitted with the heading text so a -// downstream title chunker can find them; paragraphs (Paragraph) -// emit the leaf-node text. The walker is intentionally a -// best-effort pass — full TOC / outline handling lands with the -// deepdoc/parser port — but it satisfies the per-block "emit -// text+doc_type_kwd" contract enough for a Phase-1a migration -// test to verify wire-shape parity. -func walkMarkdownBlocks(doc ast.Node, out *[]map[string]any) { +// walkMarkdownBlocksWithImages emits one normalized item per +// top-level block. Headings, paragraphs, lists, and code blocks are +// emitted with their text. When a block contains a markdown image +// reference (![alt](src)), the image data is resolved via +// resolveMarkdownImage and the item carries `doc_type_kwd: "image"` +// together with the base64-encoded image payload. +func walkMarkdownBlocksWithImages(doc ast.Node, rawText string, out *[]map[string]any) { for _, child := range doc.GetChildren() { + var ckType string + var docTypeKwd string + var txt string + switch n := child.(type) { case *ast.Heading: - *out = append(*out, map[string]any{ - "text": headingText(n), - "doc_type_kwd": "text", - "ck_type": "heading", - }) + txt = headingText(n) + ckType = "heading" + docTypeKwd = "text" case *ast.Paragraph: - *out = append(*out, map[string]any{ - "text": leafText(n), - "doc_type_kwd": "text", - "ck_type": "text", - }) + txt = leafText(n) + ckType = "text" + docTypeKwd = "text" case *ast.List: - *out = append(*out, map[string]any{ - "text": leafText(n), - "doc_type_kwd": "text", - "ck_type": "list", - }) + txt = leafText(n) + ckType = "list" + docTypeKwd = "text" case *ast.CodeBlock: - *out = append(*out, map[string]any{ - "text": leafText(n), - "doc_type_kwd": "text", - "ck_type": "code", - }) + txt = leafText(n) + ckType = "code" + docTypeKwd = "text" default: - // Block types we don't yet normalize (HTML, tables, - // images, definitions) are best-effort: emit the leaf - // text without a ck_type so downstream components can - // still treat them as text chunks. - txt := leafText(n) - if strings.TrimSpace(txt) != "" { - *out = append(*out, map[string]any{ - "text": txt, - "doc_type_kwd": "text", - }) + txt = leafText(n) + if strings.TrimSpace(txt) == "" { + continue } + docTypeKwd = "text" + } + + item := map[string]any{ + "text": txt, + "doc_type_kwd": docTypeKwd, + } + if ckType != "" { + item["ck_type"] = ckType + } + + // Detect markdown image references in the raw source text + // that corresponds to this block. When found, resolve the + // image data so downstream vision enhancement can describe it. + if imgData, imgFound := resolveMarkdownImage(txt, rawText); imgFound && imgData != "" { + item["doc_type_kwd"] = "image" + item["image"] = imgData + } + + *out = append(*out, item) + } +} + +// resolveMarkdownImage extracts the first markdown image reference +// from the given text and returns its base64-encoded data. Supports: +// - data:image/... URIs → decoded directly +// - http:// / https:// URLs → fetched (with basic SSRF filtering) +// +// Returns (base64String, true) on success, ("", false) when no image +// is found or resolution fails. +func resolveMarkdownImage(leafText, rawFullText string) (string, bool) { + // Prefer matching against the raw full text to catch images + // whose alt-text was split across markdown rendering. + searchIn := rawFullText + if strings.TrimSpace(searchIn) == "" { + searchIn = leafText + } + matches := mdImagePattern.FindStringSubmatch(searchIn) + if len(matches) < 2 { + return "", false + } + url := matches[1] + + if strings.HasPrefix(url, dataURIPrefix) { + // data:image/png;base64,xxxx + idx := strings.Index(url, "base64,") + if idx < 0 { + return "", false + } + return url[idx+len("base64,"):], true + } + if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { + b64, err := fetchImageAsBase64(url) + if err != nil { + return "", false + } + return b64, true + } + // Local / relative paths — not fetched for security. + return "", false +} + +// fetchImageAsBase64 fetches an HTTP(S) image URL and returns its +// content as a base64-encoded string. Local/private addresses and +// redirects to them are rejected (SSRF guard). Hostnames are resolved +// once and the validated IP is pinned in a custom DialContext to +// prevent DNS-rebinding TOCTOU attacks. +func fetchImageAsBase64(rawURL string) (string, error) { + rawURL = strings.TrimSpace(rawURL) + + parsed, err := url.Parse(rawURL) + if err != nil { + return "", fmt.Errorf("markdown: invalid image URL: %w", err) + } + + // pinned maps hostname (without port) → validated IP. The hostname + // is resolved once per host, and the transport dials the pinned IP + // directly instead of re-resolving DNS. + var pinnedMu sync.Mutex + pinned := make(map[string]net.IP) + + pinHost := func(host string) error { + ip, err := resolveAndValidateHost(host) + if err != nil { + return err + } + h, _, _ := net.SplitHostPort(host) + if h == "" { + h = host + } + pinnedMu.Lock() + pinned[h] = ip + pinnedMu.Unlock() + return nil + } + + if err := pinHost(parsed.Host); err != nil { + return "", err + } + + transport := &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + pinnedMu.Lock() + ip, ok := pinned[host] + pinnedMu.Unlock() + if ok { + return (&net.Dialer{}).DialContext(ctx, network, net.JoinHostPort(ip.String(), port)) + } + return (&net.Dialer{}).DialContext(ctx, network, addr) + }, + } + + client := &http.Client{ + Transport: transport, + Timeout: 30 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 5 { + return fmt.Errorf("markdown: too many redirects") + } + return pinHost(req.URL.Host) + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return "", fmt.Errorf("markdown: create image request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("markdown: fetch image %s: %w", rawURL, err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", fmt.Errorf("markdown: fetch image %s: HTTP %d", rawURL, resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024)) // 32 MiB cap + if err != nil { + return "", fmt.Errorf("markdown: read image %s: %w", rawURL, err) + } + return base64.StdEncoding.EncodeToString(body), nil +} + +// resolveAndValidateHost resolves a host (which may include a port), +// validates none of its IPs are internal/private, and returns the +// first public IP for connection pinning. +func resolveAndValidateHost(host string) (net.IP, error) { + hostname := host + if h, _, err := net.SplitHostPort(host); err == nil { + hostname = h + } + + ip := net.ParseIP(hostname) + if ip != nil { + if (ip.IsLoopback() && !ssrfAllowLoopback) || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || + ip.IsPrivate() || ip.IsUnspecified() { + return nil, fmt.Errorf("markdown: rejected image URL to internal address: %s", host) + } + return ip, nil + } + + addrs, err := net.DefaultResolver.LookupIPAddr(context.Background(), hostname) + if err != nil { + return nil, fmt.Errorf("markdown: cannot resolve image host: %s", hostname) + } + for _, addr := range addrs { + ip := addr.IP + if (ip.IsLoopback() && !ssrfAllowLoopback) || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || + ip.IsPrivate() || ip.IsUnspecified() { + return nil, fmt.Errorf("markdown: rejected image URL resolving to internal address: %s (%s)", host, ip) } } + if len(addrs) == 0 { + return nil, fmt.Errorf("markdown: no addresses resolved for host: %s", hostname) + } + return addrs[0].IP, nil } // headingText returns the inline-text of a heading node by diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go new file mode 100644 index 0000000000..d0db085a97 --- /dev/null +++ b/internal/parser/parser/markdown_parser_test.go @@ -0,0 +1,179 @@ +package parser + +import ( + "encoding/base64" + "net/http" + "net/http/httptest" + "testing" +) + +func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) { + p, err := NewMarkdownParser(GoMarkdown) + if err != nil { + t.Fatalf("NewMarkdownParser: %v", err) + } + md := "# Hello\n\nThis is a paragraph.\n\n* List item 1\n* List item 2\n\n```go\nfunc main() {}\n```\n" + res := p.ParseWithResult("test.md", []byte(md)) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if got, want := res.OutputFormat, "json"; got != want { + t.Fatalf("OutputFormat = %q, want %q", got, want) + } + if len(res.JSON) == 0 { + t.Fatal("JSON is empty; want at least one item") + } + // Verify heading + if got, _ := res.JSON[0]["text"].(string); got != "Hello" { + t.Fatalf("first item text = %q, want %q", got, "Hello") + } + if got, _ := res.JSON[0]["ck_type"].(string); got != "heading" { + t.Fatalf("first item ck_type = %q, want %q", got, "heading") + } +} + +func TestMarkdownParser_ParseWithResult_EmptyInput(t *testing.T) { + p, _ := NewMarkdownParser(GoMarkdown) + res := p.ParseWithResult("empty.md", []byte("")) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if len(res.JSON) != 1 { + t.Fatalf("len(JSON) = %d, want 1", len(res.JSON)) + } +} + +func TestMarkdownParser_ParseWithResult_ImageDataURI(t *testing.T) { + p, _ := NewMarkdownParser(GoMarkdown) + // 1×1 pixel transparent PNG encoded as data URI + pixelPNG := make([]byte, 68) // minimal 1x1 PNG header + pixelB64 := base64.StdEncoding.EncodeToString([]byte("fake-png-data")) + md := "Some text with an image\n![test](data:image/png;base64," + pixelB64 + ")\n" + _ = pixelPNG + res := p.ParseWithResult("test.md", []byte(md)) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + // Find the image item + found := false + for _, item := range res.JSON { + if kd, _ := item["doc_type_kwd"].(string); kd == "image" { + found = true + if img, ok := item["image"].(string); !ok || img != pixelB64 { + t.Fatalf("image data = %q, want %q", img, pixelB64) + } + break + } + } + if !found { + t.Fatal("no item with doc_type_kwd == 'image' found") + } +} + +func TestMarkdownParser_ParseWithResult_NoImage(t *testing.T) { + p, _ := NewMarkdownParser(GoMarkdown) + md := "# Title\n\nJust some text, no images here.\n\nMore text." + res := p.ParseWithResult("test.md", []byte(md)) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + for _, item := range res.JSON { + if kd, _ := item["doc_type_kwd"].(string); kd == "image" { + t.Fatal("unexpected image item in text-only markdown") + } + } +} + +func TestMarkdownParser_ConfigureFromSetup(t *testing.T) { + p, _ := NewMarkdownParser(GoMarkdown) + p.ConfigureFromSetup(map[string]any{ + "parse_method": "deepdoc", + "output_format": "json", + "vlm": map[string]any{"llm_id": "gpt-4-vision"}, + "flatten_media_to_text": false, + }) + if p.ParseMethod != "deepdoc" { + t.Fatalf("ParseMethod = %q, want %q", p.ParseMethod, "deepdoc") + } + if p.OutputFormat != "json" { + t.Fatalf("OutputFormat = %q, want %q", p.OutputFormat, "json") + } + if p.VLM == nil { + t.Fatal("VLM is nil; want map") + } + if id, _ := p.VLM["llm_id"].(string); id != "gpt-4-vision" { + t.Fatalf("VLM[llm_id] = %q, want %q", id, "gpt-4-vision") + } +} + +func TestMarkdownParser_ConfigureFromSetup_NilSafe(t *testing.T) { + p, _ := NewMarkdownParser(GoMarkdown) + p.ConfigureFromSetup(nil) // should not panic + if p.ParseMethod != "" { + t.Fatalf("ParseMethod should be empty after nil setup, got %q", p.ParseMethod) + } +} + +func TestResolveMarkdownImage_DataURI(t *testing.T) { + b64 := base64.StdEncoding.EncodeToString([]byte("fakeimage")) + md := "![alt](data:image/png;base64," + b64 + ")" + result, found := resolveMarkdownImage("", md) + if !found { + t.Fatal("expected image found for data URI") + } + if result != b64 { + t.Fatalf("got %q, want %q", result, b64) + } +} + +func TestResolveMarkdownImage_NoImage(t *testing.T) { + _, found := resolveMarkdownImage("", "# Hello\nNo image here") + if found { + t.Fatal("expected no image found") + } +} + +func TestResolveMarkdownImage_HTTPImage(t *testing.T) { + // httptest servers bind loopback, which the SSRF guard rejects by + // default. Allow loopback for this test so the HTTP fetch path is + // exercised (production keeps ssrfAllowLoopback == false). + prev := ssrfAllowLoopback + ssrfAllowLoopback = true + defer func() { ssrfAllowLoopback = prev }() + + // Start a test HTTP server serving a fake PNG + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "image/png") + w.Write([]byte("fake-png-bytes")) + })) + defer ts.Close() + + md := "![alt](" + ts.URL + "/image.png)" + result, found := resolveMarkdownImage("", md) + if !found { + t.Fatal("expected image found for HTTP URL") + } + expectedB64 := base64.StdEncoding.EncodeToString([]byte("fake-png-bytes")) + if result != expectedB64 { + t.Fatalf("got %q, want %q", result, expectedB64) + } +} + +func TestFetchImageAsBase64_RejectsCredentials(t *testing.T) { + _, err := fetchImageAsBase64("https://user:pass@example.com/img.png") + if err == nil { + t.Fatal("expected error for URL with credentials") + } +} + +func TestFetchImageAsBase64_InvalidURL(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer ts.Close() + + _, err := fetchImageAsBase64(ts.URL + "/nonexistent.png") + if err == nil { + t.Fatal("expected error for 404 response") + } +} diff --git a/internal/parser/parser/office_parsers_no_cgo.go b/internal/parser/parser/office_parsers_no_cgo.go index abb6257a78..25a1f2ab6c 100644 --- a/internal/parser/parser/office_parsers_no_cgo.go +++ b/internal/parser/parser/office_parsers_no_cgo.go @@ -8,7 +8,7 @@ import ( ) // ErrOfficeCGORequired is returned by ParseWithResult on every -// office-parser family (DOC / DOCX / PPT / PPTX / XLS / XLSX) +// office-parser family (DOC / DOCX / PPT / PPTX) // when the build is not CGO-enabled. The CGO build's // implementation captures the office_oxide PlainText / ToMarkdown // output; this stub mirrors that surface so the package compiles @@ -31,20 +31,6 @@ func (p *DOCParser) ParseWithResult(filename string, _ []byte) ParseResult { } } -func (p *XLSParser) ParseWithResult(filename string, _ []byte) ParseResult { - return ParseResult{ - File: map[string]any{"name": filename}, - Err: fmt.Errorf("%w: xls", ErrOfficeCGORequired), - } -} - -func (p *XLSXParser) ParseWithResult(filename string, _ []byte) ParseResult { - return ParseResult{ - File: map[string]any{"name": filename}, - Err: fmt.Errorf("%w: xlsx", ErrOfficeCGORequired), - } -} - func (p *PPTParser) ParseWithResult(filename string, _ []byte) ParseResult { return ParseResult{ File: map[string]any{"name": filename}, @@ -79,26 +65,6 @@ func (p *DOCXParser) String() string { return "DOCXParser(no-cgo)" } -type XLSParser struct{} - -func NewXLSParser() *XLSParser { - return &XLSParser{} -} - -func (p *XLSParser) String() string { - return "XLSParser(no-cgo)" -} - -type XLSXParser struct{} - -func NewXLSXParser() *XLSXParser { - return &XLSXParser{} -} - -func (p *XLSXParser) String() string { - return "XLSXParser(no-cgo)" -} - type PPTParser struct{} func NewPPTParser() *PPTParser { diff --git a/internal/parser/parser/office_parsers_nocgo_test.go b/internal/parser/parser/office_parsers_nocgo_test.go index a706909290..3da84b6ce1 100644 --- a/internal/parser/parser/office_parsers_nocgo_test.go +++ b/internal/parser/parser/office_parsers_nocgo_test.go @@ -14,8 +14,6 @@ func TestOfficeParsers_ParseWithResult_NoCGO(t *testing.T) { }{ {name: "docx", res: (&DOCXParser{}).ParseWithResult("a.docx", nil)}, {name: "doc", res: (&DOCParser{}).ParseWithResult("a.doc", nil)}, - {name: "xlsx", res: (&XLSXParser{}).ParseWithResult("a.xlsx", nil)}, - {name: "xls", res: (&XLSParser{}).ParseWithResult("a.xls", nil)}, {name: "pptx", res: (&PPTXParser{}).ParseWithResult("a.pptx", nil)}, {name: "ppt", res: (&PPTParser{}).ParseWithResult("a.ppt", nil)}, } diff --git a/internal/parser/parser/office_parsers_shared.go b/internal/parser/parser/office_parsers_shared.go new file mode 100644 index 0000000000..e9411f3af5 --- /dev/null +++ b/internal/parser/parser/office_parsers_shared.go @@ -0,0 +1,12 @@ +//go:build !cgo + +package parser + +import "html" + +// OfficeOxide is the lib_type identifier for office_oxide backend. +const OfficeOxide = "office_oxide" + +func htmlEscape(s string) string { + return html.EscapeString(s) +} diff --git a/internal/parser/parser/paddleocr.go b/internal/parser/parser/paddleocr.go new file mode 100644 index 0000000000..7ec242a231 --- /dev/null +++ b/internal/parser/parser/paddleocr.go @@ -0,0 +1,433 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// PaddleOCR client for the async Job API. +// Mirrors Python's deepdoc/parser/paddleocr_parser.py: +// - Submit job via POST /api/v2/ocr/jobs +// - Poll with exponential backoff until "done" or "failed" +// - Fetch result JSONL and extract text +// +// For image parsing, only parse_image() semantics are needed; +// parse_pdf() is handled by the PDF vision dispatch path. + +package parser + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "mime/multipart" + "net/http" + "regexp" + "strings" + "time" + + "ragflow/internal/common" +) + +// imgTagPattern matches HTML tags and
wrappers, mirroring +// Python's _MARKDOWN_IMAGE_PATTERN in deepdoc/parser/paddleocr_parser.py. +var imgTagPattern = regexp.MustCompile( + `(?is)]*>\s*]*/>\s*
|]*/>`, +) + +const ( + defaultPaddleOCRBaseURL = "https://paddleocr.aistudio-app.com" + defaultPaddleOCRTimeout = 600 * time.Second + defaultPaddleOCRAlgo = "PaddleOCR-VL" + paddleOCRSubmitPath = "/api/v2/ocr/jobs" + paddleOCRPollInterval = 3 * time.Second + paddleOCRPollMultiplier = 1.5 + paddleOCRPollMaxInterval = 15 * time.Second +) + +// PaddleOCRClient talks to the PaddleOCR async Job API. +type PaddleOCRClient struct { + BaseURL string + AccessToken string + Algorithm string + Timeout time.Duration + httpClient *http.Client +} + +// NewPaddleOCRClientFromEnv creates a client from environment variables: +// +// PADDLEOCR_BASE_URL – base URL (default https://paddleocr.aistudio-app.com) +// PADDLEOCR_ACCESS_TOKEN – bearer token +// PADDLEOCR_ALGORITHM – algorithm name (default PaddleOCR-VL) +func NewPaddleOCRClientFromEnv() *PaddleOCRClient { + baseURL := common.GetEnv(common.EnvPaddleOCRBaseUrl) + if baseURL == "" { + baseURL = defaultPaddleOCRBaseURL + } + return &PaddleOCRClient{ + BaseURL: strings.TrimRight(baseURL, "/"), + AccessToken: common.GetEnv(common.EnvPaddleOCRAccessToken), + Algorithm: firstNonEmpty(common.GetEnv(common.EnvPaddleOCRAlgorithm), defaultPaddleOCRAlgo), + Timeout: defaultPaddleOCRTimeout, + httpClient: &http.Client{ + Timeout: 30 * time.Second, // per-request timeout; polling uses deadline + }, + } +} + +// Enabled reports whether the client has a usable access token. +func (c *PaddleOCRClient) Enabled() bool { + return c != nil && c.AccessToken != "" +} + +// ParseImage submits the image to PaddleOCR and returns extracted text. +// Mirrors Python's PaddleOCRParser.parse_image(). +func (c *PaddleOCRClient) ParseImage(binary []byte, filename string) (string, error) { + deadline := time.Now().Add(c.Timeout) + + // Step 1: Submit job + jobID, err := c.submitJob(binary, filename, deadline) + if err != nil { + return "", fmt.Errorf("paddleocr submit: %w", err) + } + + // Step 2: Poll until done + resultData, err := c.pollJob(jobID, deadline) + if err != nil { + return "", fmt.Errorf("paddleocr poll: %w", err) + } + + // Step 3: Fetch result JSONL + raw, err := c.fetchResult(resultData, deadline) + if err != nil { + return "", fmt.Errorf("paddleocr fetch: %w", err) + } + + // Step 4: Parse result + return c.extractImageText(raw), nil +} + +// submitJob POSTs the image file to /api/v2/ocr/jobs and returns the job ID. +func (c *PaddleOCRClient) submitJob(binary []byte, filename string, deadline time.Time) (string, error) { + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + // Write the image file field + fw, err := w.CreateFormFile("file", filename) + if err != nil { + return "", err + } + if _, err := fw.Write(binary); err != nil { + return "", err + } + + // Write form fields + if err := w.WriteField("model", c.Algorithm); err != nil { + return "", err + } + optionalPayload := map[string]any{"formatBlockContent": true} + payloadBytes, _ := json.Marshal(optionalPayload) + if err := w.WriteField("optionalPayload", string(payloadBytes)); err != nil { + return "", err + } + w.Close() + + url := c.BaseURL + paddleOCRSubmitPath + req, err := http.NewRequest(http.MethodPost, url, &buf) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", w.FormDataContentType()) + req.Header.Set("Client-Platform", "ragflow") + if c.AccessToken != "" { + req.Header.Set("Authorization", "Bearer "+c.AccessToken) + } + req = req.WithContext(withDeadline(req.Context(), deadline)) + + resp, err := c.httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("POST %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) + } + + var submitResp struct { + Data struct { + JobID string `json:"jobId"` + } `json:"data"` + JobID string `json:"jobId"` + } + if err := json.NewDecoder(resp.Body).Decode(&submitResp); err != nil { + return "", fmt.Errorf("decode submit response: %w", err) + } + jobID := submitResp.Data.JobID + if jobID == "" { + jobID = submitResp.JobID + } + if jobID == "" { + return "", fmt.Errorf("no jobId in submit response") + } + slog.Info("paddleocr: job submitted", "jobId", jobID) + return jobID, nil +} + +// pollJob polls the job status until it reaches "done" or "failed". +// Uses exponential backoff: 3s initial, 1.5x multiplier, 15s max. +// Returns the final poll response data. +func (c *PaddleOCRClient) pollJob(jobID string, deadline time.Time) (map[string]any, error) { + pollURL := fmt.Sprintf("%s/%s", c.BaseURL+paddleOCRSubmitPath, jobID) + interval := paddleOCRPollInterval + + for { + if time.Now().After(deadline) { + return nil, fmt.Errorf("job %s timed out after %v", jobID, c.Timeout) + } + remaining := time.Until(deadline) + if remaining <= 0 { + return nil, fmt.Errorf("job %s timed out", jobID) + } + + req, err := http.NewRequest(http.MethodGet, pollURL, nil) + if err != nil { + return nil, err + } + if c.AccessToken != "" { + req.Header.Set("Authorization", "Bearer "+c.AccessToken) + } + req.Header.Set("Client-Platform", "ragflow") + req = req.WithContext(withDeadline(req.Context(), deadline)) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("poll %s: %w", jobID, err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, fmt.Errorf("poll HTTP %d: %s", resp.StatusCode, string(body)) + } + + var pollResp struct { + Data struct { + State string `json:"state"` + ErrorMsg string `json:"errorMsg"` + } `json:"data"` + State string `json:"state"` + ErrorMsg string `json:"errorMsg"` + } + bodyBytes, _ := io.ReadAll(resp.Body) + if err := json.Unmarshal(bodyBytes, &pollResp); err != nil { + return nil, fmt.Errorf("decode poll response: %w", err) + } + state := pollResp.Data.State + if state == "" { + state = pollResp.State + } + + switch state { + case "done": + slog.Info("paddleocr: job done", "jobId", jobID) + // Return full data so caller can extract resultJsonUrl + var fullData map[string]any + json.Unmarshal(bodyBytes, &fullData) + return fullData, nil + case "failed": + errMsg := pollResp.Data.ErrorMsg + if errMsg == "" { + errMsg = pollResp.ErrorMsg + } + return nil, fmt.Errorf("job %s failed: %s", jobID, errMsg) + } + + // Exponential backoff + sleepTime := interval + if remaining < sleepTime { + sleepTime = remaining + } + time.Sleep(sleepTime) + interval = time.Duration(float64(interval) * paddleOCRPollMultiplier) + if interval > paddleOCRPollMaxInterval { + interval = paddleOCRPollMaxInterval + } + } +} + +// fetchResult downloads and parses the JSONL result from resultJsonUrl. +func (c *PaddleOCRClient) fetchResult(pollData map[string]any, deadline time.Time) ([]map[string]any, error) { + data, _ := pollData["data"].(map[string]any) + if data == nil { + data = pollData + } + resultJSONURL, _ := data["resultJsonUrl"].(string) + if resultJSONURL == "" { + if resultURL, ok := data["resultUrl"].(map[string]any); ok { + resultJSONURL, _ = resultURL["jsonUrl"].(string) + } + } + if resultJSONURL == "" { + return nil, fmt.Errorf("no resultJsonUrl in completion response") + } + + req, err := http.NewRequest(http.MethodGet, resultJSONURL, nil) + if err != nil { + return nil, err + } + if c.AccessToken != "" { + req.Header.Set("Authorization", "Bearer "+c.AccessToken) + } + req = req.WithContext(withDeadline(req.Context(), deadline)) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch result: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, fmt.Errorf("fetch result HTTP %d: %s", resp.StatusCode, string(body)) + } + + content, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read result: %w", err) + } + + // Parse JSONL: one JSON object per line + var lines []map[string]any + for _, line := range strings.Split(strings.TrimSpace(string(content)), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + var obj map[string]any + if err := json.Unmarshal([]byte(line), &obj); err != nil { + return nil, fmt.Errorf("parse JSONL line: %w", err) + } + lines = append(lines, obj) + } + return lines, nil +} + +// extractImageText extracts text from PaddleOCR JSONL results. +// Mirrors Python's parse_image() text extraction: +// +// layoutParsingResults[].prunedResult.parsing_res_list[].block_content +// Fallback: ocrResults[].prunedResult.rec_texts[] +func (c *PaddleOCRClient) extractImageText(resultLines []map[string]any) string { + var texts []string + + for _, line := range resultLines { + result, _ := line["result"].(map[string]any) + if result == nil { + continue + } + // Primary: layoutParsingResults + if lpr, ok := result["layoutParsingResults"].([]any); ok { + for _, lr := range lpr { + lrMap, _ := lr.(map[string]any) + if lrMap == nil { + continue + } + pruned, _ := lrMap["prunedResult"].(map[string]any) + if pruned == nil { + continue + } + if prl, ok := pruned["parsing_res_list"].([]any); ok { + for _, block := range prl { + blockMap, _ := block.(map[string]any) + if blockMap == nil { + continue + } + content, _ := blockMap["block_content"].(string) + content = strings.TrimSpace(content) + // Remove markdown image blocks + content = removeMarkdownImages(content) + if content != "" { + texts = append(texts, content) + } + } + } + } + } + } + + // Fallback: ocrResults for text-only models (e.g. PP-OCRv6) + if len(texts) == 0 { + for _, line := range resultLines { + result, _ := line["result"].(map[string]any) + if result == nil { + continue + } + if ocr, ok := result["ocrResults"].([]any); ok { + for _, o := range ocr { + ocrMap, _ := o.(map[string]any) + if ocrMap == nil { + continue + } + pruned, _ := ocrMap["prunedResult"].(map[string]any) + if pruned == nil { + continue + } + if recTexts, ok := pruned["rec_texts"].([]any); ok { + for _, t := range recTexts { + if s, ok := t.(string); ok { + s = strings.TrimSpace(s) + if s != "" { + texts = append(texts, s) + } + } + } + } + } + } + } + } + + if len(texts) == 0 { + return "" + } + return strings.Join(texts, "\n") +} + +// removeMarkdownImages strips markdown image syntax (![](...)) and HTML +// img/div blocks, mirroring Python's _remove_images_from_markdown. +func removeMarkdownImages(md string) string { + // Strip HTML and
blocks + md = imgTagPattern.ReplaceAllString(md, "") + return strings.TrimSpace(md) +} + +// withDeadline applies a deadline to a context, respecting the existing +// deadline if it is sooner. +func withDeadline(ctx context.Context, deadline time.Time) context.Context { + if d, ok := ctx.Deadline(); ok && d.Before(deadline) { + return ctx + } + newCtx, _ := context.WithDeadline(ctx, deadline) + return newCtx +} + +func firstNonEmpty(a, b string) string { + if a != "" { + return a + } + return b +} diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.go index 45567cb599..36d39adf34 100644 --- a/internal/parser/parser/parse_result_test.go +++ b/internal/parser/parser/parse_result_test.go @@ -130,7 +130,7 @@ func (s sentinelErr) Error() string { return string(s) } // tiny: 1 heading, 1 paragraph, 1 unordered list item, no nested // formatting. func TestMarkdownParser_ParseWithResult(t *testing.T) { - p := NewMarkdownParser() + p, _ := NewMarkdownParser(GoMarkdown) src := []byte("# Title\n\nFirst paragraph.\n\n- Item one\n") res := p.ParseWithResult("doc.md", src) if res.Err != nil { diff --git a/internal/parser/parser/parser_type.go b/internal/parser/parser/parser_type.go index 35cfd25422..5136fe542d 100644 --- a/internal/parser/parser/parser_type.go +++ b/internal/parser/parser/parser_type.go @@ -35,9 +35,11 @@ func GetParser(fileType utility.FileType) (ParseResultProducer, error) { case utility.FileTypePPT: return NewPPTParser(), nil case utility.FileTypeXLSX: - return NewXLSXParser(), nil + return NewXLSXParser("") case utility.FileTypeXLS: - return NewXLSParser(), nil + return NewXLSParser("") + case utility.FileTypeCSV: + return NewCSVParser(), nil case utility.FileTypeDOCX: return NewDOCXParser(), nil case utility.FileTypeDOC: @@ -47,9 +49,19 @@ func GetParser(fileType utility.FileType) (ParseResultProducer, error) { case utility.FileTypeHTML: return NewHTMLParser(), nil case utility.FileTypeMarkdown: - return NewMarkdownParser(), nil + return NewMarkdownParser(GoMarkdown) case utility.FileTypeTXT: return NewTextParser(), nil + case utility.FileTypeEPUB: + return NewEPUBParser(), nil + case utility.FileTypeJSON: + return NewJSONParser(), nil + case utility.FileTypeEMAIL: + return NewEmailParser(), nil + case utility.FileTypeVISUAL: + return NewPictureParser(), nil + case utility.FileTypeAURAL: + return NewAudioParser(), nil default: return nil, fmt.Errorf("unsupported file type: %s", fileType) } diff --git a/internal/parser/parser/pdf_parser_mineru.go b/internal/parser/parser/pdf_parser_mineru.go index 095ecb00ac..8f954f2cb9 100644 --- a/internal/parser/parser/pdf_parser_mineru.go +++ b/internal/parser/parser/pdf_parser_mineru.go @@ -97,7 +97,7 @@ func parseMinerUMarkdownResult(filename, markdown, outputFormat string, pageCoun fileMeta := pdfFileMeta(filename, pageCount) switch strings.ToLower(strings.TrimSpace(outputFormat)) { case "", "json": - mp := NewMarkdownParser() + mp, _ := NewMarkdownParser(GoMarkdown) res := mp.ParseWithResult(filename, []byte(markdown)) if res.Err != nil { return res diff --git a/internal/parser/parser/pdf_parser_plaintext_nocgo.go b/internal/parser/parser/pdf_parser_plaintext_nocgo.go index bd9b64f5c8..efbc991469 100644 --- a/internal/parser/parser/pdf_parser_plaintext_nocgo.go +++ b/internal/parser/parser/pdf_parser_plaintext_nocgo.go @@ -2,11 +2,41 @@ package parser -import "fmt" +import ( + "bytes" + "fmt" + "strings" + + "github.com/ledongthuc/pdf" +) func parsePDFWithPlainText(filename string, data []byte, parser *PDFParser) ParseResult { if len(data) == 0 { return emptyPDFResult(filename) } - return ParseResult{Err: fmt.Errorf("%w: %s", ErrPDFEngineUnavailable, filename)} + + reader := bytes.NewReader(data) + pdfReader, err := pdf.NewReader(reader, int64(len(data))) + if err != nil { + return ParseResult{Err: fmt.Errorf("parser: plain_text open: %w", err)} + } + + pageCount := pdfReader.NumPage() + items := make([]map[string]any, 0, pageCount) + for pageNum := 1; pageNum <= pageCount; pageNum++ { + p := pdfReader.Page(pageNum) + if p.V.IsNull() { + continue + } + text, err := p.GetPlainText(nil) + if err != nil { + return ParseResult{Err: fmt.Errorf("parser: plain_text page %d: %w", pageNum, err)} + } + items = append(items, map[string]any{ + "text": strings.TrimRight(text, "\n"), + "doc_type_kwd": "text", + "page_number": pageNum, + }) + } + return pdfItemsToResult(filename, items, parser.OutputFormat, pageCount) } diff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.go new file mode 100644 index 0000000000..b2a9e005f7 --- /dev/null +++ b/internal/parser/parser/picture_parser.go @@ -0,0 +1,147 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// PictureParser validates and stores configuration for image files. +// The actual OCR and VLM description is performed by the +// component-layer maybeDispatchImage, which mirrors Python's +// rag/app/picture.py:chunk() image branch. +// +// Python reference: +// - Image is opened with PIL, converted to RGB +// - PaddleOCR tried first (if layout_recognize == "@PaddleOCR") +// - Falls back to local deepdoc.vision.OCR (ONNX text detection) +// - If OCR text is short (≤32 chars / ≤32 words for English), +// calls IMAGE2TEXT VLM describe() for a natural-language description +// - Returns tokenized text with media context attached + +package parser + +import ( + "fmt" + "path/filepath" + "strings" +) + +// imageExtensions mirrors Python's visual extensions used in +// utility.FilenameType and the picture.py:chunk() image branch. +// Video extensions (.mp4, .mov, ...) are excluded — those are +// handled by maybeDispatchVideo. +var imageExtensions = map[string]bool{ + "jpg": true, "jpeg": true, "png": true, "gif": true, + "bmp": true, "tiff": true, "tif": true, "webp": true, + "svg": true, "ico": true, "avif": true, "heic": true, + "apng": true, "icon": true, "pcx": true, "tga": true, + "exif": true, "fpx": true, "psd": true, "cdr": true, + "pcd": true, "dxf": true, "ufo": true, "eps": true, + "ai": true, "raw": true, "wmf": true, +} + +// PictureParser handles image files for OCR and VLM description. +// Mirrors the configuration from setups["picture"]: +// - vlm.llm_id → VLMModelID (IMAGE2TEXT model for describe) +// - output_format +// - image_context_size +// - layout_recognize → ("@PaddleOCR:model_name" or empty) +type PictureParser struct { + VLMModelID string // vlm.llm_id — the IMAGE2TEXT model + OutputFormat string + ImageContextSize int // default 0 + LayoutRecognize string // layout_recognize (e.g. "@PaddleOCR") + VideoPrompt string // for video-in-image detection (Python fallback) +} + +// NewPictureParser constructs a PictureParser. +func NewPictureParser() *PictureParser { + return &PictureParser{} +} + +// ConfigureFromSetup reads picture-specific configuration from the +// parser setup map. Extracts vlm.llm_id, output_format, +// image_context_size, layout_recognize, and video_prompt. +func (p *PictureParser) ConfigureFromSetup(setup map[string]any) { + if p == nil || setup == nil { + return + } + if vlm, ok := setup["vlm"].(map[string]any); ok { + if llmID, ok := vlm["llm_id"].(string); ok && llmID != "" { + p.VLMModelID = llmID + } + } + if v, ok := setup["output_format"].(string); ok && v != "" { + p.OutputFormat = v + } + if v, ok := setup["image_context_size"].(float64); ok { + p.ImageContextSize = int(v) + } + if v, ok := setup["layout_recognize"].(string); ok && v != "" { + p.LayoutRecognize = v + } + if v, ok := setup["video_prompt"].(string); ok && v != "" { + p.VideoPrompt = v + } +} + +// ParseWithResult implements ParseResultProducer. It validates the +// file extension against the image extension whitelist. The actual +// OCR and VLM description happens via maybeDispatchImage at the +// component layer (mirrors Python's picture.py:chunk()). +func (p *PictureParser) ParseWithResult(filename string, data []byte) ParseResult { + ext := strings.ToLower(filepath.Ext(filename)) + if len(ext) > 1 && ext[0] == '.' { + ext = ext[1:] + } + + // Video extensions: defer to maybeDispatchVideo. The picture.py + // Python path handles both, but Go routes video separately. + if ext != "" && isVideoExtension(ext) { + return ParseResult{ + Err: fmt.Errorf("picture: video file %q should be routed through video parser, not picture", filename), + } + } + + if ext == "" || !imageExtensions[ext] { + return ParseResult{ + Err: fmt.Errorf("picture: unsupported extension %q (filename: %s); accepted: .jpg/.jpeg/.png/.gif/.bmp/.tiff/.tif/.webp/.svg/.ico/.avif/.heic/...", ext, filename), + } + } + + // OutputFormat, VLMModelID, ImageContextSize, and LayoutRecognize + // are consumed by maybeDispatchImage at the component layer. + outFmt := p.OutputFormat + if outFmt == "" { + outFmt = "text" + } + + return ParseResult{ + OutputFormat: outFmt, + File: map[string]any{ + "name": filename, + "size": len(data), + "doc_type_kwd": "image", + }, + } +} + +// isVideoExtension returns true when the extension is a video format +// that Python's picture.py routes through the video branch. +func isVideoExtension(ext string) bool { + switch ext { + case "mp4", "mov", "avi", "flv", "mpeg", "mpg", + "webm", "wmv", "3gp", "3gpp", "mkv": + return true + } + return false +} diff --git a/internal/parser/parser/picture_parser_test.go b/internal/parser/parser/picture_parser_test.go new file mode 100644 index 0000000000..e5327b4f51 --- /dev/null +++ b/internal/parser/parser/picture_parser_test.go @@ -0,0 +1,145 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package parser + +import ( + "strings" + "testing" +) + +func TestNewPictureParser(t *testing.T) { + p := NewPictureParser() + if p == nil { + t.Fatal("NewPictureParser returned nil") + } + if p.OutputFormat != "" { + t.Errorf("OutputFormat = %q, want empty", p.OutputFormat) + } + if p.VLMModelID != "" { + t.Errorf("VLMModelID = %q, want empty", p.VLMModelID) + } + if p.ImageContextSize != 0 { + t.Errorf("ImageContextSize = %d, want 0", p.ImageContextSize) + } +} + +func TestPictureParser_ConfigureFromSetup(t *testing.T) { + p := NewPictureParser() + p.ConfigureFromSetup(map[string]any{ + "vlm": map[string]any{ + "llm_id": "gpt-4-vision", + }, + "output_format": "text", + "image_context_size": float64(3), + "layout_recognize": "@PaddleOCR", + "video_prompt": "summarize", + }) + if p.VLMModelID != "gpt-4-vision" { + t.Errorf("VLMModelID = %q, want gpt-4-vision", p.VLMModelID) + } + if p.OutputFormat != "text" { + t.Errorf("OutputFormat = %q, want text", p.OutputFormat) + } + if p.ImageContextSize != 3 { + t.Errorf("ImageContextSize = %d, want 3", p.ImageContextSize) + } + if p.LayoutRecognize != "@PaddleOCR" { + t.Errorf("LayoutRecognize = %q, want @PaddleOCR", p.LayoutRecognize) + } + if p.VideoPrompt != "summarize" { + t.Errorf("VideoPrompt = %q, want summarize", p.VideoPrompt) + } +} + +func TestPictureParser_ParseWithResult_NilSetup(t *testing.T) { + p := NewPictureParser() + p.ConfigureFromSetup(nil) + res := p.ParseWithResult("photo.png", []byte("\x89PNG")) + if res.Err != nil { + t.Errorf("unexpected error for nil setup: %v", res.Err) + } + if res.OutputFormat != "text" { + t.Errorf("OutputFormat = %q, want text", res.OutputFormat) + } + file, ok := res.File["doc_type_kwd"].(string) + if !ok || file != "image" { + t.Errorf("doc_type_kwd = %q, want image", file) + } +} + +func TestPictureParser_ParseWithResult_ValidExtensions(t *testing.T) { + exts := []string{"png", "jpg", "jpeg", "gif", "bmp", "tiff", "tif", "webp", "svg", "ico", "avif", "heic", "apng"} + p := NewPictureParser() + for _, ext := range exts { + fn := "img." + ext + res := p.ParseWithResult(fn, []byte{1, 2, 3}) + if res.Err != nil { + t.Errorf("unexpected error for .%s: %v", ext, res.Err) + } + if res.OutputFormat != "text" { + t.Errorf("OutputFormat = %q for .%s, want text", res.OutputFormat, ext) + } + } +} + +func TestPictureParser_ParseWithResult_InvalidExtension(t *testing.T) { + p := NewPictureParser() + res := p.ParseWithResult("sound.mp3", []byte{1}) + if res.Err == nil { + t.Error("expected error for .mp3, got nil") + } + if !strings.Contains(res.Err.Error(), "mp3") { + t.Errorf("error should mention unsupported extension, got: %v", res.Err) + } +} + +func TestPictureParser_ParseWithResult_VideoExtension(t *testing.T) { + p := NewPictureParser() + res := p.ParseWithResult("video.mp4", []byte{1}) + if res.Err == nil { + t.Error("expected error for video .mp4, got nil") + } + if !strings.Contains(strings.ToLower(res.Err.Error()), "video") { + t.Errorf("error should mention video, got: %v", res.Err) + } +} + +func TestPictureParser_ParseWithResult_OutputFormat(t *testing.T) { + p := NewPictureParser() + p.ConfigureFromSetup(map[string]any{ + "output_format": "json", + }) + res := p.ParseWithResult("photo.jpg", []byte{1, 2, 3}) + if res.Err != nil { + t.Fatalf("unexpected error: %v", res.Err) + } + if res.OutputFormat != "json" { + t.Errorf("OutputFormat = %q, want json", res.OutputFormat) + } +} + +func TestImageExtensions(t *testing.T) { + for ext := range imageExtensions { + if ext == "" { + t.Error("empty extension in imageExtensions set") + } + // All extensions should be lowercase (no uppercase keys). + if strings.ToLower(ext) != ext { + t.Errorf("non-lowercase extension key: %q", ext) + } + } +} diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go index 05e22442e8..5101108eca 100644 --- a/internal/parser/parser/xls_parser.go +++ b/internal/parser/parser/xls_parser.go @@ -1,5 +1,3 @@ -//go:build cgo - // // Copyright 2026 The InfiniFlow Authors. All Rights Reserved. // @@ -26,21 +24,73 @@ import ( "github.com/xuri/excelize/v2" ) -type XLSParser struct{} +type XLSParser struct { + libType string + ParseMethod string + OutputFormat string + TCADPAPIServer string + TCADPAPIKey string + TCADPTableResultType string + TCADPMarkdownImageResponseType string +} -func NewXLSParser() *XLSParser { - return &XLSParser{} +func NewXLSParser(libType string) (*XLSParser, error) { + if libType == "" { + libType = "excelize" + } + return &XLSParser{ + libType: libType, + TCADPTableResultType: "1", + TCADPMarkdownImageResponseType: "1", + }, nil } func (p *XLSParser) String() string { return "XLSParser" } -// ParseWithResult delegates to excelize which handles both .xls -// and .xlsx through the same API. The python ExcelParser falls -// back to a similar delegation; on the Go side excelize is the -// single library for both extensions. +func (p *XLSParser) ConfigureFromSetup(setup map[string]any) { + if p == nil || setup == nil { + return + } + if v, ok := setup["parse_method"].(string); ok && v != "" { + p.ParseMethod = v + } + if v, ok := setup["output_format"].(string); ok && v != "" { + p.OutputFormat = v + } + if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" { + p.TCADPAPIServer = v + } + if v, ok := setup["tcadp_api_key"].(string); ok { + p.TCADPAPIKey = v + } + if v, ok := setup["table_result_type"].(string); ok && v != "" { + p.TCADPTableResultType = v + } + if v, ok := setup["markdown_image_response_type"].(string); ok && v != "" { + p.TCADPMarkdownImageResponseType = v + } +} + func (p *XLSParser) ParseWithResult(filename string, data []byte) ParseResult { + method := normalizeXLSXParseMethod(p.ParseMethod) + switch method { + case "tcadp": + return parseSpreadsheetWithTCADP( + filename, data, "XLS", + p.TCADPAPIServer, p.TCADPAPIKey, + p.TCADPTableResultType, p.TCADPMarkdownImageResponseType, + p.OutputFormat, + ) + case "", "excelize": + // Continue with the local Excelize parser. + default: + return ParseResult{ + Err: fmt.Errorf("unsupported XLS parse method: %q", p.ParseMethod), + } + } + f, err := excelize.OpenReader(bytes.NewReader(data)) if err != nil { return ParseResult{Err: fmt.Errorf("xls open: %w", err)} diff --git a/internal/parser/parser/xls_tcadp.go b/internal/parser/parser/xls_tcadp.go new file mode 100644 index 0000000000..43321dfc28 --- /dev/null +++ b/internal/parser/parser/xls_tcadp.go @@ -0,0 +1,83 @@ +package parser + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + + models "ragflow/internal/entity/models" +) + +func parseSpreadsheetWithTCADP(filename string, data []byte, fileType string, tcadpAPIServer, tcadpAPIKey, tableResultType, markdownImageResponseType string, outputFormat string) ParseResult { + if len(data) == 0 { + return emptyPDFResult(filename) + } + baseURL := strings.TrimSpace(tcadpAPIServer) + if baseURL == "" { + baseURL = strings.TrimSpace(os.Getenv("TCADP_APISERVER")) + } + if baseURL == "" { + return ParseResult{Err: fmt.Errorf("parser: TCADP requires tcadp_apiserver or TCADP_APISERVER")} + } + apiKey := strings.TrimSpace(tcadpAPIKey) + if apiKey == "" { + apiKey = strings.TrimSpace(os.Getenv("TCADP_API_KEY")) + } + requestBody := map[string]any{ + "file_type": fileType, + "file_base64": base64.StdEncoding.EncodeToString(data), + "file_start_page_number": 1, + "file_end_page_number": 1000, + "config": map[string]any{ + "TableResultType": tableResultType, + "MarkdownImageResponseType": markdownImageResponseType, + }, + } + resp, err := models.PostJSONRequest(context.Background(), models.NewDriverHTTPClient(), strings.TrimRight(baseURL, "/")+"/reconstruct_document", bearer(apiKey), requestBody) + if err != nil { + return ParseResult{Err: fmt.Errorf("parser: TCADP submit: %w", err)} + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + return ParseResult{Err: fmt.Errorf("parser: TCADP read submit: %w", err)} + } + if resp.StatusCode >= 300 { + return ParseResult{Err: fmt.Errorf("parser: TCADP HTTP %d: %s", resp.StatusCode, string(raw))} + } + var payload struct { + DocumentRecognizeResultURL string `json:"DocumentRecognizeResultUrl"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + return ParseResult{Err: fmt.Errorf("parser: TCADP decode submit: %w", err)} + } + if payload.DocumentRecognizeResultURL == "" { + return ParseResult{Err: fmt.Errorf("parser: TCADP returned no DocumentRecognizeResultUrl")} + } + downloadReq, err := http.NewRequestWithContext(context.Background(), http.MethodGet, payload.DocumentRecognizeResultURL, nil) + if err != nil { + return ParseResult{Err: fmt.Errorf("parser: TCADP download request: %w", err)} + } + if auth := bearer(apiKey); auth != "" { + downloadReq.Header.Set("Authorization", auth) + } + downloadResp, err := models.NewDriverHTTPClient().Do(downloadReq) + if err != nil { + return ParseResult{Err: fmt.Errorf("parser: TCADP download: %w", err)} + } + defer downloadResp.Body.Close() + zipBytes, err := io.ReadAll(downloadResp.Body) + if err != nil { + return ParseResult{Err: fmt.Errorf("parser: TCADP read zip: %w", err)} + } + items, pageCount, err := tcadpItemsFromZip(zipBytes) + if err != nil { + return ParseResult{Err: err} + } + return pdfItemsToResult(filename, items, outputFormat, pageCount) +} diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.go new file mode 100644 index 0000000000..5f1b3c916d --- /dev/null +++ b/internal/parser/parser/xls_tcadp_test.go @@ -0,0 +1,241 @@ +package parser + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestXLSXParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) { + zipPayload := tcadpZipFixture(t) + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/reconstruct_document": + _, _ = w.Write([]byte(`{"DocumentRecognizeResultUrl":"` + server.URL + `/download.zip"}`)) + case "/download.zip": + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write(zipPayload) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + p, err := NewXLSXParser("") + if err != nil { + t.Fatalf("NewXLSXParser: %v", err) + } + p.ConfigureFromSetup(map[string]any{ + "parse_method": "TCADP parser", + "output_format": "json", + "tcadp_apiserver": server.URL, + "tcadp_api_key": "tcadp-secret", + "table_result_type": "1", + "markdown_image_response_type": "1", + }) + res := p.ParseWithResult("sample.xlsx", []byte("mock xlsx content")) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if len(res.JSON) < 2 { + t.Fatalf("JSON len = %d, want >=2", len(res.JSON)) + } +} + +func TestXLSParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) { + zipPayload := tcadpZipFixture(t) + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/reconstruct_document": + _, _ = w.Write([]byte(`{"DocumentRecognizeResultUrl":"` + server.URL + `/download.zip"}`)) + case "/download.zip": + _, _ = w.Write(zipPayload) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + p, err := NewXLSParser("") + if err != nil { + t.Fatalf("NewXLSParser: %v", err) + } + p.ConfigureFromSetup(map[string]any{ + "parse_method": "TCADP parser", + "output_format": "json", + "tcadp_apiserver": server.URL, + "tcadp_api_key": "tcadp-secret", + }) + res := p.ParseWithResult("sample.xls", []byte("mock xls content")) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if len(res.JSON) < 2 { + t.Fatalf("JSON len = %d, want >=2", len(res.JSON)) + } +} + +func TestCSVParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) { + zipPayload := tcadpZipFixture(t) + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/reconstruct_document": + _, _ = w.Write([]byte(`{"DocumentRecognizeResultUrl":"` + server.URL + `/download.zip"}`)) + case "/download.zip": + _, _ = w.Write(zipPayload) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + p := NewCSVParser() + p.ConfigureFromSetup(map[string]any{ + "parse_method": "TCADP parser", + "output_format": "json", + "tcadp_apiserver": server.URL, + "tcadp_api_key": "tcadp-secret", + }) + res := p.ParseWithResult("sample.csv", []byte("a,b,c\n1,2,3")) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if len(res.JSON) < 2 { + t.Fatalf("JSON len = %d, want >=2", len(res.JSON)) + } +} + +func TestXLSXParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) { + zipPayload := tcadpZipFixture(t) + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/reconstruct_document": + _, _ = w.Write([]byte(`{"DocumentRecognizeResultUrl":"` + server.URL + `/download.zip"}`)) + case "/download.zip": + _, _ = w.Write(zipPayload) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + p, err := NewXLSXParser("") + if err != nil { + t.Fatalf("NewXLSXParser: %v", err) + } + p.ConfigureFromSetup(map[string]any{ + "parse_method": "TCADP parser", + "output_format": "markdown", + "tcadp_apiserver": server.URL, + }) + res := p.ParseWithResult("sample.xlsx", []byte("mock xlsx content")) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if res.Markdown == "" { + t.Fatal("Markdown is empty; want rendered content") + } +} + +func TestXLSXParser_ParseWithResult_TCADPRequiresAPIServer(t *testing.T) { + p, err := NewXLSXParser("") + if err != nil { + t.Fatalf("NewXLSXParser: %v", err) + } + p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser"}) + res := p.ParseWithResult("sample.xlsx", []byte("mock xlsx content")) + if res.Err == nil { + t.Fatal("expected error about tcadp_apiserver, got nil") + } +} + +func TestXLSParser_ParseWithResult_TCADPRequiresAPIServer(t *testing.T) { + p, err := NewXLSParser("") + if err != nil { + t.Fatalf("NewXLSParser: %v", err) + } + p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser"}) + res := p.ParseWithResult("sample.xls", []byte("mock xls content")) + if res.Err == nil { + t.Fatal("expected error about tcadp_apiserver, got nil") + } +} + +func TestCSVParser_ParseWithResult_TCADPRequiresAPIServer(t *testing.T) { + p := NewCSVParser() + p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser"}) + res := p.ParseWithResult("sample.csv", []byte("mock csv content")) + if res.Err == nil { + t.Fatal("expected error about tcadp_apiserver, got nil") + } +} + +func TestXLSXParser_ParseWithResult_InvalidXLSXHandled(t *testing.T) { + p, err := NewXLSXParser("") + if err != nil { + t.Fatalf("NewXLSXParser: %v", err) + } + res := p.ParseWithResult("sample.xlsx", []byte("not a valid xlsx")) + if res.Err == nil { + t.Fatal("expected error for invalid xlsx, got nil") + } +} + +func TestXLSParser_ParseWithResult_InvalidXLSHandled(t *testing.T) { + p, err := NewXLSParser("") + if err != nil { + t.Fatalf("NewXLSParser: %v", err) + } + res := p.ParseWithResult("sample.xls", []byte("not a valid xls")) + if res.Err == nil { + t.Fatal("expected error for invalid xls, got nil") + } +} + +func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) { + p := NewCSVParser() + res := p.ParseWithResult("sample.csv", []byte("a,b\n1,2")) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if got, want := res.OutputFormat, "html"; got != want { + t.Fatalf("OutputFormat = %q, want %q", got, want) + } + if res.HTML == "" { + t.Fatal("HTML is empty; want rendered table") + } +} + +func TestXLSXParser_ConfigureFromSetup_TCADP(t *testing.T) { + p, err := NewXLSXParser("") + if err != nil { + t.Fatalf("NewXLSXParser: %v", err) + } + p.ConfigureFromSetup(map[string]any{ + "parse_method": "TCADP parser", + "output_format": "json", + "tcadp_apiserver": "https://tcadp.example.com", + "tcadp_api_key": "secret", + "table_result_type": "2", + "markdown_image_response_type": "2", + }) + if got, want := p.ParseMethod, "TCADP parser"; got != want { + t.Fatalf("ParseMethod = %q, want %q", got, want) + } + if got, want := p.OutputFormat, "json"; got != want { + t.Fatalf("OutputFormat = %q, want %q", got, want) + } + if got, want := p.TCADPAPIServer, "https://tcadp.example.com"; got != want { + t.Fatalf("TCADPAPIServer = %q, want %q", got, want) + } + if got, want := p.TCADPTableResultType, "2"; got != want { + t.Fatalf("TCADPTableResultType = %q, want %q", got, want) + } + if got, want := p.TCADPMarkdownImageResponseType, "2"; got != want { + t.Fatalf("TCADPMarkdownImageResponseType = %q, want %q", got, want) + } +} diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go index 8676cd27a7..a88cccbd07 100644 --- a/internal/parser/parser/xlsx_parser.go +++ b/internal/parser/parser/xlsx_parser.go @@ -1,5 +1,3 @@ -//go:build cgo - // // Copyright 2026 The InfiniFlow Authors. All Rights Reserved. // @@ -26,23 +24,81 @@ import ( "github.com/xuri/excelize/v2" ) -type XLSXParser struct{} +type XLSXParser struct { + libType string + ParseMethod string + OutputFormat string + TCADPAPIServer string + TCADPAPIKey string + TCADPTableResultType string + TCADPMarkdownImageResponseType string +} -func NewXLSXParser() *XLSXParser { - return &XLSXParser{} +func NewXLSXParser(libType string) (*XLSXParser, error) { + if libType == "" { + libType = "excelize" + } + return &XLSXParser{ + libType: libType, + TCADPTableResultType: "1", + TCADPMarkdownImageResponseType: "1", + }, nil } func (p *XLSXParser) String() string { return "XLSXParser" } -// ParseWithResult renders the spreadsheet as HTML — the python -// ExcelParser shape. Each sheet becomes a
and a repeated +// header row. Chunks are joined with newlines. +// +// Mirrors Python's RAGFlowExcelParser.html() chunking: +// +// chunks = (n_data_rows + chunk_rows - 1) // chunk_rows +func recordsToHTMLTableChunks(records [][]string, chunkRows int) string { + if len(records) == 0 { + return "
" + csvSheetName + "
" + } + + // Build the header row once — repeated in every chunk. + headerHTML := buildCSVHeaderRow(records[0]) + dataRows := records[1:] + nData := len(dataRows) + + if nData == 0 { + // Only a header row exists. + return "\n" + headerHTML + "\n
" + csvSheetName + "
" + } + + nChunks := (nData + chunkRows - 1) / chunkRows + var b strings.Builder + for ci := 0; ci < nChunks; ci++ { + start := ci * chunkRows + end := start + chunkRows + if end > nData { + end = nData + } + + b.WriteString("\n") + b.WriteString(headerHTML) + + for _, row := range dataRows[start:end] { + b.WriteString("") + for _, cell := range row { + b.WriteString("") + } + b.WriteString("\n") + } + b.WriteString("
") + b.WriteString(csvSheetName) + b.WriteString("
") + b.WriteString(html.EscapeString(strings.TrimSpace(cell))) + b.WriteString("
\n") + } + return b.String() +} + +// buildCSVHeaderRow renders the first row as an HTML
header row. +func buildCSVHeaderRow(row []string) string { + var b strings.Builder + b.WriteString("
") + b.WriteString(html.EscapeString(strings.TrimSpace(cell))) + b.WriteString("
with row / -// column structure preserved; sheet names become

headings -// so a downstream title chunker can pick them up. Implementation -// uses excelize (already in go.mod) instead of office_oxide's -// PlainText/ToMarkdown so cell boundaries survive the round-trip. +func (p *XLSXParser) ConfigureFromSetup(setup map[string]any) { + if p == nil || setup == nil { + return + } + if v, ok := setup["parse_method"].(string); ok && v != "" { + p.ParseMethod = v + } + if v, ok := setup["output_format"].(string); ok && v != "" { + p.OutputFormat = v + } + if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" { + p.TCADPAPIServer = v + } + if v, ok := setup["tcadp_api_key"].(string); ok { + p.TCADPAPIKey = v + } + if v, ok := setup["table_result_type"].(string); ok && v != "" { + p.TCADPTableResultType = v + } + if v, ok := setup["markdown_image_response_type"].(string); ok && v != "" { + p.TCADPMarkdownImageResponseType = v + } +} + +func normalizeXLSXParseMethod(raw string) string { + method := strings.ToLower(strings.TrimSpace(raw)) + if method == "tcadp parser" { + return "tcadp" + } + return method +} + func (p *XLSXParser) ParseWithResult(filename string, data []byte) ParseResult { + method := normalizeXLSXParseMethod(p.ParseMethod) + switch method { + case "tcadp": + return parseSpreadsheetWithTCADP( + filename, data, "XLSX", + p.TCADPAPIServer, p.TCADPAPIKey, + p.TCADPTableResultType, p.TCADPMarkdownImageResponseType, + p.OutputFormat, + ) + case "", "excelize": + // Continue with the local Excelize parser. + default: + return ParseResult{ + Err: fmt.Errorf("unsupported XLSX parse method: %q", p.ParseMethod), + } + } + f, err := excelize.OpenReader(bytes.NewReader(data)) if err != nil { return ParseResult{Err: fmt.Errorf("xlsx open: %w", err)} diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go index 833b692494..679496b8c9 100644 --- a/internal/service/chunk/chunk.go +++ b/internal/service/chunk/chunk.go @@ -817,7 +817,6 @@ func (s *ChunkService) pdfParseTaskRanges(doc *entity.Document, bucket, objectNa pageSize = int64(parserConfigInt(doc.ParserConfig, "task_page_size", 22)) } if doc.ParserID == string(entity.ParserTypeOne) || - doc.ParserID == string(entity.ParserTypeKG) || parserConfigString(doc.ParserConfig, "layout_recognize", "DeepDOC") != "DeepDOC" || parserConfigBool(doc.ParserConfig, "toc_extraction", false) { pageSize = maximumTaskPageNumber diff --git a/internal/service/components_service_test.go b/internal/service/components_service_test.go index ebe4095c06..1466fac979 100644 --- a/internal/service/components_service_test.go +++ b/internal/service/components_service_test.go @@ -63,7 +63,7 @@ func TestComponentsService_List_FilterIngestion(t *testing.T) { } wantNames := []string{ "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", - "parser", "titlechunker", "tokenchunker", "tokenizer", + "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer", } assertComponentNameSet(t, "ingestion", namesOf(got), wantNames) } @@ -86,7 +86,7 @@ func TestComponentsService_List_FilterIngestionAndShared(t *testing.T) { } wantNames := []string{ "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", - "parser", "titlechunker", "tokenchunker", "tokenizer", + "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer", } assertComponentNameSet(t, "ingestion+shared", namesOf(got), wantNames) } @@ -105,7 +105,7 @@ func TestComponentsService_List_FilterDuplicates(t *testing.T) { } wantNames := []string{ "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", - "parser", "titlechunker", "tokenchunker", "tokenizer", + "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer", } if len(got) != len(wantNames) { t.Errorf("duplicate category produced %d rows, want %d", len(got), len(wantNames)) diff --git a/internal/service/dataset.go b/internal/service/dataset.go index 967cc43b34..1945eba9f0 100644 --- a/internal/service/dataset.go +++ b/internal/service/dataset.go @@ -1382,7 +1382,6 @@ func datasetPDFParseTaskRanges(doc *entity.Document, bucket, objectName string) pageSize = int64(datasetParserConfigInt(doc.ParserConfig, "task_page_size", 22)) } if doc.ParserID == string(entity.ParserTypeOne) || - doc.ParserID == string(entity.ParserTypeKG) || datasetParserConfigString(doc.ParserConfig, "layout_recognize", "DeepDOC") != "DeepDOC" || datasetParserConfigBool(doc.ParserConfig, "toc_extraction", false) { pageSize = maximumTaskPageNumber diff --git a/internal/service/document.go b/internal/service/document.go index 27c3f31a0a..c8d2fd8039 100644 --- a/internal/service/document.go +++ b/internal/service/document.go @@ -1516,7 +1516,6 @@ func documentParseTaskRanges(doc *entity.Document, bucket, objectName string) ([ pageSize = int64(documentParserConfigInt(doc.ParserConfig, "task_page_size", 22)) } if doc.ParserID == string(entity.ParserTypeOne) || - doc.ParserID == string(entity.ParserTypeKG) || documentParserConfigBool(doc.ParserConfig, "toc_extraction", false) { pageSize = maximumTaskPageNumber } diff --git a/internal/service/load_prompt.go b/internal/service/load_prompt.go index 80a8e898e0..9da2e0e3c0 100644 --- a/internal/service/load_prompt.go +++ b/internal/service/load_prompt.go @@ -23,6 +23,8 @@ import ( "regexp" "strings" "sync" + + "ragflow/internal/utility" ) var ( @@ -55,38 +57,11 @@ var thinkBlockRE = regexp.MustCompile(`^[\s\S]*`) var jsonFenceRE = regexp.MustCompile("```json\\n|```\\n*$") func init() { - // Strategy 1: Check working directory first (most reliable during development/tests) - cwd, err := os.Getwd() - if err == nil { - // Check if CWD has rag/prompts directly - if _, err := os.Stat(filepath.Join(cwd, "rag", "prompts")); err == nil { - promptsBaseDir = cwd - return - } - // Walk up from CWD looking for rag/prompts - dir := cwd - for dir != "/" && dir != "" { - if _, err := os.Stat(filepath.Join(dir, "rag", "prompts")); err == nil { - promptsBaseDir = dir - return - } - dir = filepath.Dir(dir) - } + root := utility.GetProjectRoot() + if _, err := os.Stat(filepath.Join(root, "rag", "prompts")); err == nil { + promptsBaseDir = root + return } - - // Strategy 2: Walk up from executable (for production Docker where binary is in /ragflow/bin/) - exe, err := os.Executable() - if err == nil { - dir := filepath.Dir(exe) - for dir != "/" && dir != "" { - if _, err := os.Stat(filepath.Join(dir, "rag", "prompts")); err == nil { - promptsBaseDir = dir - return - } - dir = filepath.Dir(dir) - } - } - // Final fallback promptsBaseDir = "/ragflow" } diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.go index 60405bd26e..67a67e068e 100644 --- a/internal/tokenizer/tokenizer.go +++ b/internal/tokenizer/tokenizer.go @@ -519,17 +519,15 @@ func getCL100KEncoder() (*tiktoken.Tiktoken, error) { } // NumTokensFromString returns the number of tokens in s using the cl100k_base -// BPE encoding +// BPE encoding. Mirrors Python's num_tokens_from_string (common/token_utils.py): +// returns 0 on encoder error. func NumTokensFromString(s string) int { if s == "" { return 0 } enc, err := getCL100KEncoder() if err != nil { - // Fail closed: avoid dangerous undercounting when encoder is unavailable. - // A conservative byte-length estimate errs on the side of over-counting, - // which is safer for budget enforcement than returning zero. - return len([]byte(s)) + return 0 } return len(enc.Encode(s, nil, nil)) } diff --git a/internal/utility/file.go b/internal/utility/file.go index 2fb1cfaec3..2bd5fac94b 100644 --- a/internal/utility/file.go +++ b/internal/utility/file.go @@ -35,9 +35,14 @@ const ( FileTypePPTX FileType = "pptx" FileTypeXLS FileType = "xls" FileTypeXLSX FileType = "xlsx" + FileTypeCSV FileType = "csv" FileTypeHTML FileType = "html" FileTypeMarkdown FileType = "md" FileTypeTXT FileType = "txt" + FileTypeEPUB FileType = "epub" + FileTypeJSON FileType = "json" + FileTypeVIDEO FileType = "video" + FileTypeEMAIL FileType = "email" FileTypeVISUAL FileType = "visual" FileTypeAURAL FileType = "aural" FileTypeFOLDER FileType = "folder" @@ -80,6 +85,8 @@ func GetFileType(filename string) FileType { return FileTypeXLS case "xlsx": return FileTypeXLSX + case "csv": + return FileTypeCSV case "doc": return FileTypeDOC case "docx": @@ -90,10 +97,23 @@ func GetFileType(filename string) FileType { return FileTypePPTX case "html", "htm": return FileTypeHTML - case "md": + case "md", "markdown", "mdx": return FileTypeMarkdown - case "txt": + case "txt", "py", "js", "java", "c", "cpp", "h", "php", + "go", "ts", "sh", "cs", "kt", "sql": return FileTypeTXT + case "epub": + return FileTypeEPUB + case "json", "jsonl", "ldjson": + return FileTypeJSON + case "eml", "msg": + return FileTypeEMAIL + case "da", "wave", "wav", "mp3", "aac", "flac", "ogg", + "aiff", "au", "midi", "wma", "ape", "alac", "wv", "opus": + return FileTypeAURAL + case "mp4", "avi", "mkv", "mov", "webm", "flv", "mpeg", + "mpg", "wmv", "3gp", "3gpp": + return FileTypeVIDEO default: return FileTypeOTHER }