mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-14 00:48:26 +08:00
## Summary Fixes #15215 — attachments uploaded to an agent were not reaching the LLM. When a user uploads a file in an agent chat, `canvas.run` parses it into the `sys.files` global (text content for documents, `data:image/...` URIs for images — see `agent/canvas.py:752-768`). But the LLM/Agent component's `_prepare_prompt_variables` only substitutes variables the user's prompt template explicitly references via `{var}` placeholders. The default prompt is `[{"role": "user", "content": "{sys.query}"}]` with no `{sys.files}`, so the parsed attachment content never reaches the model. In the reporter's logs, this is why the agent saw only the bare query `附件 摘要 attachment summary` and went searching the dataset instead of reading the uploaded PDF. ## Fix `agent/component/llm.py` — added `_collect_sys_files()` and an auto-injection step in `_prepare_prompt_variables`: - If `sys.files` is non-empty **and** neither `sys_prompt` nor any entry in `prompts` already contains `{sys.files}` (no double-injection), split the entries into text vs. `data:image/...` URIs. - Image URIs are merged into `self.imgs`, which the existing logic uses to switch the chat model to `IMAGE2TEXT` and pass `images=...` to `async_chat`. - Text content is appended to the last `user` role message in `msg`, mirroring how `dialog_service.async_chat_solo` handles attachments for the non-agent chat path (`api/db/services/dialog_service.py:318-321`). Both `LLM._invoke_async` and `Agent._invoke_async` (tool-using) go through `_prepare_prompt_variables`, so plain LLM nodes and Agent nodes are fixed in both streaming and non-streaming paths. ## Test plan - [ ] Upload a PDF attachment to an agent with the default `{sys.query}` prompt and ask "summarize the attachment" — the model should answer from the file content rather than searching the knowledge base. - [ ] Upload an image attachment to an agent and ask about its contents — the model should switch to the vision-capable LLM and answer from the image. - [ ] Verify that an agent whose prompt **does** include `{sys.files}` still works and does **not** include the file content twice. - [ ] Verify that an agent run with no attachments behaves unchanged. - [ ] Run `uv run pytest` to make sure no existing tests regress. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [ ] New Feature (non-breaking change which adds functionality) - [ ] Documentation Update - [ ] Refactoring - [ ] Performance Improvement - [ ] Other (please describe): --------- Co-authored-by: yzc <yuzhichang@gmail.com>
This commit is contained in:
Rene Arredondo
@@ -23,7 +23,6 @@ from typing import Any, AsyncGenerator
|
||||
import json_repair
|
||||
from functools import partial
|
||||
from common.constants import LLMType
|
||||
from api.db.services.dialog_service import _stream_with_think_delta
|
||||
from api.db.services.llm_service import LLMBundle
|
||||
from api.db.joint_services.tenant_model_service import get_model_config_from_provider_instance, get_model_type_by_name
|
||||
from agent.component.base import ComponentBase, ComponentParamBase
|
||||
@@ -49,7 +48,6 @@ class LLMParam(ComponentParamBase):
|
||||
self.output_structure = None
|
||||
self.cite = True
|
||||
self.visual_files_var = None
|
||||
self.thinking = ""
|
||||
|
||||
def check(self):
|
||||
self.check_decimal_float(float(self.temperature), "[Agent] Temperature")
|
||||
@@ -78,8 +76,6 @@ class LLMParam(ComponentParamBase):
|
||||
conf["presence_penalty"] = float(self.presence_penalty)
|
||||
if float(self.frequency_penalty) > 0 and get_attr("frequencyPenaltyEnabled"):
|
||||
conf["frequency_penalty"] = float(self.frequency_penalty)
|
||||
if get_attr("thinking") in {"enabled", "disabled"}:
|
||||
conf["thinking"] = get_attr("thinking")
|
||||
return conf
|
||||
|
||||
|
||||
@@ -229,6 +225,40 @@ class LLM(ComponentBase):
|
||||
|
||||
return value
|
||||
|
||||
def _collect_sys_files(self) -> tuple[list[str], list[str]]:
|
||||
files = self._canvas.globals.get("sys.files") or []
|
||||
if not files:
|
||||
logging.debug("[LLM] sys.files empty; skipping attachment injection")
|
||||
return [], []
|
||||
|
||||
logging.info("[LLM] sys.files present: count=%d", len(files))
|
||||
|
||||
explicit = "{sys.files}" in (self._param.sys_prompt or "")
|
||||
if not explicit and isinstance(self._param.prompts, list):
|
||||
for p in self._param.prompts:
|
||||
if isinstance(p, dict) and "{sys.files}" in (p.get("content") or ""):
|
||||
explicit = True
|
||||
break
|
||||
if explicit:
|
||||
logging.info("[LLM] prompt template references {sys.files}; skipping auto-injection (explicit=%s)", explicit)
|
||||
return [], []
|
||||
|
||||
text_parts: list[str] = []
|
||||
image_data_uris: list[str] = []
|
||||
for f in files:
|
||||
if not isinstance(f, str):
|
||||
logging.debug("[LLM] skipping non-str sys.files entry: type=%s", type(f).__name__)
|
||||
continue
|
||||
if f.startswith("data:image/"):
|
||||
image_data_uris.append(f)
|
||||
else:
|
||||
text_parts.append(f)
|
||||
logging.info(
|
||||
"[LLM] sys.files split: text_parts=%d image_data_uris=%d (explicit=%s)",
|
||||
len(text_parts), len(image_data_uris), explicit,
|
||||
)
|
||||
return text_parts, image_data_uris
|
||||
|
||||
def _prepare_prompt_variables(self):
|
||||
self.imgs = []
|
||||
if self._param.visual_files_var:
|
||||
@@ -251,7 +281,13 @@ class LLM(ComponentBase):
|
||||
args[k] = str(args[k])
|
||||
self.set_input_value(k, args[k])
|
||||
|
||||
self.imgs = self._uniq_images(self.imgs + extracted_imgs)
|
||||
sys_file_texts, sys_file_imgs = self._collect_sys_files()
|
||||
prev_img_count = len(self.imgs) + len(extracted_imgs)
|
||||
self.imgs = self._uniq_images(self.imgs + extracted_imgs + sys_file_imgs)
|
||||
logging.debug(
|
||||
"[LLM] imgs rebuilt: total=%d sys_files_added=%d unique_dropped=%d",
|
||||
len(self.imgs), len(sys_file_imgs), max(0, prev_img_count + len(sys_file_imgs) - len(self.imgs)),
|
||||
)
|
||||
model_types = get_model_type_by_name(self._canvas.get_tenant_id(), self._param.llm_id)
|
||||
if self.imgs and LLMType.IMAGE2TEXT.value in model_types:
|
||||
model_type = LLMType.IMAGE2TEXT.value
|
||||
@@ -266,6 +302,24 @@ class LLM(ComponentBase):
|
||||
)
|
||||
|
||||
msg, sys_prompt = self._sys_prompt_and_msg(self._canvas.get_history(self._param.message_history_window_size)[:-1], args)
|
||||
|
||||
if sys_file_texts:
|
||||
joined = "\n\n".join(sys_file_texts)
|
||||
merged_idx = -1
|
||||
for i in range(len(msg) - 1, -1, -1):
|
||||
if msg[i].get("role") == "user":
|
||||
msg[i]["content"] = (msg[i].get("content") or "") + "\n\n" + joined
|
||||
merged_idx = i
|
||||
break
|
||||
else:
|
||||
msg.append({"role": "user", "content": joined})
|
||||
merged_idx = len(msg) - 1
|
||||
logging.info(
|
||||
"[LLM] sys.files text merged into msg: parts=%d total_chars=%d msg_index=%d action=%s",
|
||||
len(sys_file_texts), len(joined), merged_idx,
|
||||
"merged_into_existing_user" if merged_idx < len(msg) - 1 or msg[merged_idx].get("content", "") != joined else "appended_new_user",
|
||||
)
|
||||
|
||||
user_defined_prompt, sys_prompt = self._extract_prompts(sys_prompt)
|
||||
if self._param.cite and self._canvas.get_reference()["chunks"]:
|
||||
sys_prompt += citation_prompt(user_defined_prompt)
|
||||
@@ -288,23 +342,84 @@ class LLM(ComponentBase):
|
||||
return await self.chat_mdl.async_chat(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs)
|
||||
|
||||
async def _generate_streamly(self, msg: list[dict], **kwargs) -> AsyncGenerator[str, None]:
|
||||
stream_kwargs = {"images": self.imgs} if self.imgs else {}
|
||||
stream_kwargs.update(kwargs)
|
||||
stream = self.chat_mdl.async_chat_streamly_delta(msg[0]["content"], msg[1:], self._param.gen_conf(), **stream_kwargs)
|
||||
async for _, value, _ in _stream_with_think_delta(stream, min_tokens=0):
|
||||
yield value
|
||||
async def delta_wrapper(txt_iter):
|
||||
ans = ""
|
||||
last_idx = 0
|
||||
endswith_think = False
|
||||
|
||||
def delta(txt):
|
||||
nonlocal ans, last_idx, endswith_think
|
||||
delta_ans = txt[last_idx:]
|
||||
ans = txt
|
||||
|
||||
if delta_ans.find("<think>") == 0:
|
||||
last_idx += len("<think>")
|
||||
return "<think>"
|
||||
elif delta_ans.find("<think>") > 0:
|
||||
delta_ans = txt[last_idx:last_idx + delta_ans.find("<think>")]
|
||||
last_idx += delta_ans.find("<think>")
|
||||
return delta_ans
|
||||
elif delta_ans.endswith("</think>"):
|
||||
endswith_think = True
|
||||
elif endswith_think:
|
||||
endswith_think = False
|
||||
return "</think>"
|
||||
|
||||
last_idx = len(ans)
|
||||
if ans.endswith("</think>"):
|
||||
last_idx -= len("</think>")
|
||||
return re.sub(r"(<think>|</think>)", "", delta_ans)
|
||||
|
||||
async for t in txt_iter:
|
||||
yield delta(t)
|
||||
|
||||
if not self.imgs:
|
||||
async for t in delta_wrapper(self.chat_mdl.async_chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), **kwargs)):
|
||||
yield t
|
||||
return
|
||||
|
||||
async for t in delta_wrapper(self.chat_mdl.async_chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs)):
|
||||
yield t
|
||||
|
||||
async def _stream_output_async(self, prompt, msg):
|
||||
_, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(self.chat_mdl.max_length * 0.97))
|
||||
answer = ""
|
||||
last_idx = 0
|
||||
endswith_think = False
|
||||
|
||||
def delta(txt):
|
||||
nonlocal answer, last_idx, endswith_think
|
||||
delta_ans = txt[last_idx:]
|
||||
answer = txt
|
||||
|
||||
if delta_ans.find("<think>") == 0:
|
||||
last_idx += len("<think>")
|
||||
return "<think>"
|
||||
elif delta_ans.find("<think>") > 0:
|
||||
delta_ans = txt[last_idx:last_idx + delta_ans.find("<think>")]
|
||||
last_idx += delta_ans.find("<think>")
|
||||
return delta_ans
|
||||
elif delta_ans.endswith("</think>"):
|
||||
endswith_think = True
|
||||
elif endswith_think:
|
||||
endswith_think = False
|
||||
return "</think>"
|
||||
|
||||
last_idx = len(answer)
|
||||
if answer.endswith("</think>"):
|
||||
last_idx -= len("</think>")
|
||||
return re.sub(r"(<think>|</think>)", "", delta_ans)
|
||||
|
||||
stream_kwargs = {"images": self.imgs} if self.imgs else {}
|
||||
extra_chat_kwargs = self._get_chat_template_kwargs()
|
||||
stream_kwargs.update(extra_chat_kwargs)
|
||||
stream = self.chat_mdl.async_chat_streamly_delta(msg[0]["content"], msg[1:], self._param.gen_conf(), **stream_kwargs)
|
||||
async for _, ans, _ in _stream_with_think_delta(stream, min_tokens=0):
|
||||
async for ans in self.chat_mdl.async_chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), **stream_kwargs):
|
||||
if self.check_if_canceled("LLM streaming"):
|
||||
return
|
||||
|
||||
if isinstance(ans, int):
|
||||
continue
|
||||
|
||||
if ans.find("**ERROR**") >= 0:
|
||||
if self.get_exception_default_value():
|
||||
self.set_output("content", self.get_exception_default_value())
|
||||
@@ -313,8 +428,7 @@ class LLM(ComponentBase):
|
||||
self.set_output("_ERROR", ans)
|
||||
return
|
||||
|
||||
answer += ans
|
||||
yield ans
|
||||
yield delta(ans)
|
||||
|
||||
self.set_output("content", answer)
|
||||
|
||||
|
||||
@@ -24001,18 +24001,14 @@
|
||||
},
|
||||
{
|
||||
"name": "paddleocr/paddleocr-vl-0.9b",
|
||||
"alias": [
|
||||
"paddleocr-vl-1.5"
|
||||
],
|
||||
"alias": [],
|
||||
"model_types": [
|
||||
"ocr"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "paddleocr/pp-ocrv5",
|
||||
"alias": [
|
||||
"PP-OCRv5"
|
||||
],
|
||||
"alias": [],
|
||||
"model_types": [
|
||||
"ocr"
|
||||
],
|
||||
@@ -24020,9 +24016,7 @@
|
||||
},
|
||||
{
|
||||
"name": "paddleocr/pp-structurev3",
|
||||
"alias": [
|
||||
"PP-StructureV3"
|
||||
],
|
||||
"alias": [],
|
||||
"model_types": [
|
||||
"ocr"
|
||||
],
|
||||
@@ -24043,9 +24037,7 @@
|
||||
},
|
||||
{
|
||||
"name": "paddlepaddle/paddleocr-vl",
|
||||
"alias": [
|
||||
"paddleocr-vl"
|
||||
],
|
||||
"alias": [],
|
||||
"max_tokens": 16384,
|
||||
"max_completion_tokens": 16384,
|
||||
"model_types": [
|
||||
|
||||
113
test/unit_test/agent/component/test_llm_sys_files.py
Normal file
113
test/unit_test/agent/component/test_llm_sys_files.py
Normal file
@@ -0,0 +1,113 @@
|
||||
#
|
||||
# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.component.llm import LLM
|
||||
|
||||
pytestmark = pytest.mark.p1
|
||||
|
||||
|
||||
class _FakeCanvas:
|
||||
def __init__(self, sys_files=None):
|
||||
self.globals = {"sys.files": list(sys_files) if sys_files is not None else []}
|
||||
|
||||
|
||||
def _build_component(sys_files=None, sys_prompt="", prompts=None):
|
||||
component = LLM.__new__(LLM)
|
||||
component._canvas = _FakeCanvas(sys_files=sys_files)
|
||||
component._param = SimpleNamespace(
|
||||
sys_prompt=sys_prompt,
|
||||
prompts=prompts if prompts is not None else [{"role": "user", "content": "{sys.query}"}],
|
||||
)
|
||||
return component
|
||||
|
||||
|
||||
def test_collect_sys_files_empty_returns_empty():
|
||||
component = _build_component(sys_files=[])
|
||||
assert component._collect_sys_files() == ([], [])
|
||||
|
||||
|
||||
def test_collect_sys_files_missing_key_returns_empty():
|
||||
component = _build_component()
|
||||
component._canvas.globals.pop("sys.files", None)
|
||||
assert component._collect_sys_files() == ([], [])
|
||||
|
||||
|
||||
def test_collect_sys_files_text_only():
|
||||
files = ["File: a.pdf\ncontent A", "File: b.txt\ncontent B"]
|
||||
component = _build_component(sys_files=files)
|
||||
text_parts, image_data_uris = component._collect_sys_files()
|
||||
assert text_parts == files
|
||||
assert image_data_uris == []
|
||||
|
||||
|
||||
def test_collect_sys_files_images_only():
|
||||
files = ["data:image/png;base64,AAAA", "data:image/jpeg;base64,BBBB"]
|
||||
component = _build_component(sys_files=files)
|
||||
text_parts, image_data_uris = component._collect_sys_files()
|
||||
assert text_parts == []
|
||||
assert image_data_uris == files
|
||||
|
||||
|
||||
def test_collect_sys_files_mixed():
|
||||
files = [
|
||||
"File: a.pdf\ncontent A",
|
||||
"data:image/png;base64,AAAA",
|
||||
"File: b.txt\ncontent B",
|
||||
]
|
||||
component = _build_component(sys_files=files)
|
||||
text_parts, image_data_uris = component._collect_sys_files()
|
||||
assert text_parts == ["File: a.pdf\ncontent A", "File: b.txt\ncontent B"]
|
||||
assert image_data_uris == ["data:image/png;base64,AAAA"]
|
||||
|
||||
|
||||
def test_collect_sys_files_skips_non_str_entries():
|
||||
files = ["File: a.pdf\ncontent A", None, 123, {"name": "x"}, "data:image/png;base64,AAAA"]
|
||||
component = _build_component(sys_files=files)
|
||||
text_parts, image_data_uris = component._collect_sys_files()
|
||||
assert text_parts == ["File: a.pdf\ncontent A"]
|
||||
assert image_data_uris == ["data:image/png;base64,AAAA"]
|
||||
|
||||
|
||||
def test_collect_sys_files_explicit_in_sys_prompt_skips_injection():
|
||||
files = ["File: a.pdf\ncontent A", "data:image/png;base64,AAAA"]
|
||||
component = _build_component(
|
||||
sys_files=files,
|
||||
sys_prompt="Answer using {sys.files} as context.",
|
||||
)
|
||||
assert component._collect_sys_files() == ([], [])
|
||||
|
||||
|
||||
def test_collect_sys_files_explicit_in_prompts_entry_skips_injection():
|
||||
files = ["File: a.pdf\ncontent A"]
|
||||
component = _build_component(
|
||||
sys_files=files,
|
||||
prompts=[{"role": "user", "content": "{sys.query}\n\n{sys.files}"}],
|
||||
)
|
||||
assert component._collect_sys_files() == ([], [])
|
||||
|
||||
|
||||
def test_collect_sys_files_string_prompts_does_not_crash():
|
||||
# _param.prompts can be a string before normalization elsewhere; the explicit
|
||||
# check must not raise in that case, it just falls through to splitting.
|
||||
files = ["File: a.pdf\ncontent A"]
|
||||
component = _build_component(sys_files=files, prompts="some raw template")
|
||||
text_parts, image_data_uris = component._collect_sys_files()
|
||||
assert text_parts == files
|
||||
assert image_data_uris == []
|
||||
@@ -1,4 +1,5 @@
|
||||
// src/components/ui/modal.tsx
|
||||
import React from 'react';
|
||||
import { cn } from '@/lib/utils';
|
||||
import * as DialogPrimitive from '@radix-ui/react-dialog';
|
||||
import { AlertCircle, CheckCircle, Info, Loader, X } from 'lucide-react';
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import React from 'react';
|
||||
import { setInitialChatVariableEnabledFieldValue } from '@/utils/chat';
|
||||
import {
|
||||
Circle,
|
||||
|
||||
@@ -9,7 +9,7 @@ import { exportDsl } from '../utils/dsl-bridge';
|
||||
/**
|
||||
* Recursively clear sensitive fields (api_key) from the DSL object
|
||||
*/
|
||||
const clearSensitiveFields = <T>(obj: T): T =>
|
||||
const clearSensitiveFields = <T,>(obj: T): T =>
|
||||
cloneDeepWith(obj, (value) => {
|
||||
if (
|
||||
isPlainObject(value) &&
|
||||
|
||||
Reference in New Issue
Block a user