From dc8b6d767c9562109313ac77c8e9e39a7c7c7709 Mon Sep 17 00:00:00 2001 From: Rene Arredondo <120709323+Rene0422@users.noreply.github.com> Date: Tue, 30 Jun 2026 00:48:59 -0700 Subject: [PATCH] fix(agent): inject uploaded attachments into LLM context (#15215) (#15220) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes #15215 — attachments uploaded to an agent were not reaching the LLM. When a user uploads a file in an agent chat, `canvas.run` parses it into the `sys.files` global (text content for documents, `data:image/...` URIs for images — see `agent/canvas.py:752-768`). But the LLM/Agent component's `_prepare_prompt_variables` only substitutes variables the user's prompt template explicitly references via `{var}` placeholders. The default prompt is `[{"role": "user", "content": "{sys.query}"}]` with no `{sys.files}`, so the parsed attachment content never reaches the model. In the reporter's logs, this is why the agent saw only the bare query `附件 摘要 attachment summary` and went searching the dataset instead of reading the uploaded PDF. ## Fix `agent/component/llm.py` — added `_collect_sys_files()` and an auto-injection step in `_prepare_prompt_variables`: - If `sys.files` is non-empty **and** neither `sys_prompt` nor any entry in `prompts` already contains `{sys.files}` (no double-injection), split the entries into text vs. `data:image/...` URIs. - Image URIs are merged into `self.imgs`, which the existing logic uses to switch the chat model to `IMAGE2TEXT` and pass `images=...` to `async_chat`. - Text content is appended to the last `user` role message in `msg`, mirroring how `dialog_service.async_chat_solo` handles attachments for the non-agent chat path (`api/db/services/dialog_service.py:318-321`). Both `LLM._invoke_async` and `Agent._invoke_async` (tool-using) go through `_prepare_prompt_variables`, so plain LLM nodes and Agent nodes are fixed in both streaming and non-streaming paths. ## Test plan - [ ] Upload a PDF attachment to an agent with the default `{sys.query}` prompt and ask "summarize the attachment" — the model should answer from the file content rather than searching the knowledge base. - [ ] Upload an image attachment to an agent and ask about its contents — the model should switch to the vision-capable LLM and answer from the image. - [ ] Verify that an agent whose prompt **does** include `{sys.files}` still works and does **not** include the file content twice. - [ ] Verify that an agent run with no attachments behaves unchanged. - [ ] Run `uv run pytest` to make sure no existing tests regress. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [ ] New Feature (non-breaking change which adds functionality) - [ ] Documentation Update - [ ] Refactoring - [ ] Performance Improvement - [ ] Other (please describe): --------- Co-authored-by: yzc --- agent/component/llm.py | 142 ++++++++++++++++-- conf/all_models.json | 16 +- .../agent/component/test_llm_sys_files.py | 113 ++++++++++++++ web/src/components/ui/modal/modal.tsx | 1 + web/src/constants/agent.tsx | 1 + web/src/pages/agent/hooks/use-export-json.ts | 2 +- 6 files changed, 248 insertions(+), 27 deletions(-) create mode 100644 test/unit_test/agent/component/test_llm_sys_files.py diff --git a/agent/component/llm.py b/agent/component/llm.py index ebfe8f09c5..59fc60de72 100644 --- a/agent/component/llm.py +++ b/agent/component/llm.py @@ -23,7 +23,6 @@ from typing import Any, AsyncGenerator import json_repair from functools import partial from common.constants import LLMType -from api.db.services.dialog_service import _stream_with_think_delta from api.db.services.llm_service import LLMBundle from api.db.joint_services.tenant_model_service import get_model_config_from_provider_instance, get_model_type_by_name from agent.component.base import ComponentBase, ComponentParamBase @@ -49,7 +48,6 @@ class LLMParam(ComponentParamBase): self.output_structure = None self.cite = True self.visual_files_var = None - self.thinking = "" def check(self): self.check_decimal_float(float(self.temperature), "[Agent] Temperature") @@ -78,8 +76,6 @@ class LLMParam(ComponentParamBase): conf["presence_penalty"] = float(self.presence_penalty) if float(self.frequency_penalty) > 0 and get_attr("frequencyPenaltyEnabled"): conf["frequency_penalty"] = float(self.frequency_penalty) - if get_attr("thinking") in {"enabled", "disabled"}: - conf["thinking"] = get_attr("thinking") return conf @@ -229,6 +225,40 @@ class LLM(ComponentBase): return value + def _collect_sys_files(self) -> tuple[list[str], list[str]]: + files = self._canvas.globals.get("sys.files") or [] + if not files: + logging.debug("[LLM] sys.files empty; skipping attachment injection") + return [], [] + + logging.info("[LLM] sys.files present: count=%d", len(files)) + + explicit = "{sys.files}" in (self._param.sys_prompt or "") + if not explicit and isinstance(self._param.prompts, list): + for p in self._param.prompts: + if isinstance(p, dict) and "{sys.files}" in (p.get("content") or ""): + explicit = True + break + if explicit: + logging.info("[LLM] prompt template references {sys.files}; skipping auto-injection (explicit=%s)", explicit) + return [], [] + + text_parts: list[str] = [] + image_data_uris: list[str] = [] + for f in files: + if not isinstance(f, str): + logging.debug("[LLM] skipping non-str sys.files entry: type=%s", type(f).__name__) + continue + if f.startswith("data:image/"): + image_data_uris.append(f) + else: + text_parts.append(f) + logging.info( + "[LLM] sys.files split: text_parts=%d image_data_uris=%d (explicit=%s)", + len(text_parts), len(image_data_uris), explicit, + ) + return text_parts, image_data_uris + def _prepare_prompt_variables(self): self.imgs = [] if self._param.visual_files_var: @@ -251,7 +281,13 @@ class LLM(ComponentBase): args[k] = str(args[k]) self.set_input_value(k, args[k]) - self.imgs = self._uniq_images(self.imgs + extracted_imgs) + sys_file_texts, sys_file_imgs = self._collect_sys_files() + prev_img_count = len(self.imgs) + len(extracted_imgs) + self.imgs = self._uniq_images(self.imgs + extracted_imgs + sys_file_imgs) + logging.debug( + "[LLM] imgs rebuilt: total=%d sys_files_added=%d unique_dropped=%d", + len(self.imgs), len(sys_file_imgs), max(0, prev_img_count + len(sys_file_imgs) - len(self.imgs)), + ) model_types = get_model_type_by_name(self._canvas.get_tenant_id(), self._param.llm_id) if self.imgs and LLMType.IMAGE2TEXT.value in model_types: model_type = LLMType.IMAGE2TEXT.value @@ -266,6 +302,24 @@ class LLM(ComponentBase): ) msg, sys_prompt = self._sys_prompt_and_msg(self._canvas.get_history(self._param.message_history_window_size)[:-1], args) + + if sys_file_texts: + joined = "\n\n".join(sys_file_texts) + merged_idx = -1 + for i in range(len(msg) - 1, -1, -1): + if msg[i].get("role") == "user": + msg[i]["content"] = (msg[i].get("content") or "") + "\n\n" + joined + merged_idx = i + break + else: + msg.append({"role": "user", "content": joined}) + merged_idx = len(msg) - 1 + logging.info( + "[LLM] sys.files text merged into msg: parts=%d total_chars=%d msg_index=%d action=%s", + len(sys_file_texts), len(joined), merged_idx, + "merged_into_existing_user" if merged_idx < len(msg) - 1 or msg[merged_idx].get("content", "") != joined else "appended_new_user", + ) + user_defined_prompt, sys_prompt = self._extract_prompts(sys_prompt) if self._param.cite and self._canvas.get_reference()["chunks"]: sys_prompt += citation_prompt(user_defined_prompt) @@ -288,23 +342,84 @@ class LLM(ComponentBase): return await self.chat_mdl.async_chat(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs) async def _generate_streamly(self, msg: list[dict], **kwargs) -> AsyncGenerator[str, None]: - stream_kwargs = {"images": self.imgs} if self.imgs else {} - stream_kwargs.update(kwargs) - stream = self.chat_mdl.async_chat_streamly_delta(msg[0]["content"], msg[1:], self._param.gen_conf(), **stream_kwargs) - async for _, value, _ in _stream_with_think_delta(stream, min_tokens=0): - yield value + async def delta_wrapper(txt_iter): + ans = "" + last_idx = 0 + endswith_think = False + + def delta(txt): + nonlocal ans, last_idx, endswith_think + delta_ans = txt[last_idx:] + ans = txt + + if delta_ans.find("") == 0: + last_idx += len("") + return "" + elif delta_ans.find("") > 0: + delta_ans = txt[last_idx:last_idx + delta_ans.find("")] + last_idx += delta_ans.find("") + return delta_ans + elif delta_ans.endswith(""): + endswith_think = True + elif endswith_think: + endswith_think = False + return "" + + last_idx = len(ans) + if ans.endswith(""): + last_idx -= len("") + return re.sub(r"(|)", "", delta_ans) + + async for t in txt_iter: + yield delta(t) + + if not self.imgs: + async for t in delta_wrapper(self.chat_mdl.async_chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), **kwargs)): + yield t + return + + async for t in delta_wrapper(self.chat_mdl.async_chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs)): + yield t async def _stream_output_async(self, prompt, msg): _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(self.chat_mdl.max_length * 0.97)) answer = "" + last_idx = 0 + endswith_think = False + + def delta(txt): + nonlocal answer, last_idx, endswith_think + delta_ans = txt[last_idx:] + answer = txt + + if delta_ans.find("") == 0: + last_idx += len("") + return "" + elif delta_ans.find("") > 0: + delta_ans = txt[last_idx:last_idx + delta_ans.find("")] + last_idx += delta_ans.find("") + return delta_ans + elif delta_ans.endswith(""): + endswith_think = True + elif endswith_think: + endswith_think = False + return "" + + last_idx = len(answer) + if answer.endswith(""): + last_idx -= len("") + return re.sub(r"(|)", "", delta_ans) + stream_kwargs = {"images": self.imgs} if self.imgs else {} extra_chat_kwargs = self._get_chat_template_kwargs() stream_kwargs.update(extra_chat_kwargs) - stream = self.chat_mdl.async_chat_streamly_delta(msg[0]["content"], msg[1:], self._param.gen_conf(), **stream_kwargs) - async for _, ans, _ in _stream_with_think_delta(stream, min_tokens=0): + async for ans in self.chat_mdl.async_chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), **stream_kwargs): if self.check_if_canceled("LLM streaming"): return + if isinstance(ans, int): + continue + if ans.find("**ERROR**") >= 0: if self.get_exception_default_value(): self.set_output("content", self.get_exception_default_value()) @@ -313,8 +428,7 @@ class LLM(ComponentBase): self.set_output("_ERROR", ans) return - answer += ans - yield ans + yield delta(ans) self.set_output("content", answer) diff --git a/conf/all_models.json b/conf/all_models.json index ff6854bc5d..629486a747 100644 --- a/conf/all_models.json +++ b/conf/all_models.json @@ -24001,18 +24001,14 @@ }, { "name": "paddleocr/paddleocr-vl-0.9b", - "alias": [ - "paddleocr-vl-1.5" - ], + "alias": [], "model_types": [ "ocr" ] }, { "name": "paddleocr/pp-ocrv5", - "alias": [ - "PP-OCRv5" - ], + "alias": [], "model_types": [ "ocr" ], @@ -24020,9 +24016,7 @@ }, { "name": "paddleocr/pp-structurev3", - "alias": [ - "PP-StructureV3" - ], + "alias": [], "model_types": [ "ocr" ], @@ -24043,9 +24037,7 @@ }, { "name": "paddlepaddle/paddleocr-vl", - "alias": [ - "paddleocr-vl" - ], + "alias": [], "max_tokens": 16384, "max_completion_tokens": 16384, "model_types": [ diff --git a/test/unit_test/agent/component/test_llm_sys_files.py b/test/unit_test/agent/component/test_llm_sys_files.py new file mode 100644 index 0000000000..69fd9a3dec --- /dev/null +++ b/test/unit_test/agent/component/test_llm_sys_files.py @@ -0,0 +1,113 @@ +# +# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from types import SimpleNamespace + +import pytest + +from agent.component.llm import LLM + +pytestmark = pytest.mark.p1 + + +class _FakeCanvas: + def __init__(self, sys_files=None): + self.globals = {"sys.files": list(sys_files) if sys_files is not None else []} + + +def _build_component(sys_files=None, sys_prompt="", prompts=None): + component = LLM.__new__(LLM) + component._canvas = _FakeCanvas(sys_files=sys_files) + component._param = SimpleNamespace( + sys_prompt=sys_prompt, + prompts=prompts if prompts is not None else [{"role": "user", "content": "{sys.query}"}], + ) + return component + + +def test_collect_sys_files_empty_returns_empty(): + component = _build_component(sys_files=[]) + assert component._collect_sys_files() == ([], []) + + +def test_collect_sys_files_missing_key_returns_empty(): + component = _build_component() + component._canvas.globals.pop("sys.files", None) + assert component._collect_sys_files() == ([], []) + + +def test_collect_sys_files_text_only(): + files = ["File: a.pdf\ncontent A", "File: b.txt\ncontent B"] + component = _build_component(sys_files=files) + text_parts, image_data_uris = component._collect_sys_files() + assert text_parts == files + assert image_data_uris == [] + + +def test_collect_sys_files_images_only(): + files = ["data:image/png;base64,AAAA", "data:image/jpeg;base64,BBBB"] + component = _build_component(sys_files=files) + text_parts, image_data_uris = component._collect_sys_files() + assert text_parts == [] + assert image_data_uris == files + + +def test_collect_sys_files_mixed(): + files = [ + "File: a.pdf\ncontent A", + "data:image/png;base64,AAAA", + "File: b.txt\ncontent B", + ] + component = _build_component(sys_files=files) + text_parts, image_data_uris = component._collect_sys_files() + assert text_parts == ["File: a.pdf\ncontent A", "File: b.txt\ncontent B"] + assert image_data_uris == ["data:image/png;base64,AAAA"] + + +def test_collect_sys_files_skips_non_str_entries(): + files = ["File: a.pdf\ncontent A", None, 123, {"name": "x"}, "data:image/png;base64,AAAA"] + component = _build_component(sys_files=files) + text_parts, image_data_uris = component._collect_sys_files() + assert text_parts == ["File: a.pdf\ncontent A"] + assert image_data_uris == ["data:image/png;base64,AAAA"] + + +def test_collect_sys_files_explicit_in_sys_prompt_skips_injection(): + files = ["File: a.pdf\ncontent A", "data:image/png;base64,AAAA"] + component = _build_component( + sys_files=files, + sys_prompt="Answer using {sys.files} as context.", + ) + assert component._collect_sys_files() == ([], []) + + +def test_collect_sys_files_explicit_in_prompts_entry_skips_injection(): + files = ["File: a.pdf\ncontent A"] + component = _build_component( + sys_files=files, + prompts=[{"role": "user", "content": "{sys.query}\n\n{sys.files}"}], + ) + assert component._collect_sys_files() == ([], []) + + +def test_collect_sys_files_string_prompts_does_not_crash(): + # _param.prompts can be a string before normalization elsewhere; the explicit + # check must not raise in that case, it just falls through to splitting. + files = ["File: a.pdf\ncontent A"] + component = _build_component(sys_files=files, prompts="some raw template") + text_parts, image_data_uris = component._collect_sys_files() + assert text_parts == files + assert image_data_uris == [] diff --git a/web/src/components/ui/modal/modal.tsx b/web/src/components/ui/modal/modal.tsx index f5483455a6..f1fc7bc9b3 100644 --- a/web/src/components/ui/modal/modal.tsx +++ b/web/src/components/ui/modal/modal.tsx @@ -1,4 +1,5 @@ // src/components/ui/modal.tsx +import React from 'react'; import { cn } from '@/lib/utils'; import * as DialogPrimitive from '@radix-ui/react-dialog'; import { AlertCircle, CheckCircle, Info, Loader, X } from 'lucide-react'; diff --git a/web/src/constants/agent.tsx b/web/src/constants/agent.tsx index 6e08a34370..d2095f5b41 100644 --- a/web/src/constants/agent.tsx +++ b/web/src/constants/agent.tsx @@ -1,3 +1,4 @@ +import React from 'react'; import { setInitialChatVariableEnabledFieldValue } from '@/utils/chat'; import { Circle, diff --git a/web/src/pages/agent/hooks/use-export-json.ts b/web/src/pages/agent/hooks/use-export-json.ts index e8a2e4038d..706cbddffe 100644 --- a/web/src/pages/agent/hooks/use-export-json.ts +++ b/web/src/pages/agent/hooks/use-export-json.ts @@ -9,7 +9,7 @@ import { exportDsl } from '../utils/dsl-bridge'; /** * Recursively clear sensitive fields (api_key) from the DSL object */ -const clearSensitiveFields = (obj: T): T => +const clearSensitiveFields = (obj: T): T => cloneDeepWith(obj, (value) => { if ( isPlainObject(value) &&