mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-07 03:48:44 +08:00
dc8b6d767c
## Summary Fixes #15215 — attachments uploaded to an agent were not reaching the LLM. When a user uploads a file in an agent chat, `canvas.run` parses it into the `sys.files` global (text content for documents, `data:image/...` URIs for images — see `agent/canvas.py:752-768`). But the LLM/Agent component's `_prepare_prompt_variables` only substitutes variables the user's prompt template explicitly references via `{var}` placeholders. The default prompt is `[{"role": "user", "content": "{sys.query}"}]` with no `{sys.files}`, so the parsed attachment content never reaches the model. In the reporter's logs, this is why the agent saw only the bare query `附件 摘要 attachment summary` and went searching the dataset instead of reading the uploaded PDF. ## Fix `agent/component/llm.py` — added `_collect_sys_files()` and an auto-injection step in `_prepare_prompt_variables`: - If `sys.files` is non-empty **and** neither `sys_prompt` nor any entry in `prompts` already contains `{sys.files}` (no double-injection), split the entries into text vs. `data:image/...` URIs. - Image URIs are merged into `self.imgs`, which the existing logic uses to switch the chat model to `IMAGE2TEXT` and pass `images=...` to `async_chat`. - Text content is appended to the last `user` role message in `msg`, mirroring how `dialog_service.async_chat_solo` handles attachments for the non-agent chat path (`api/db/services/dialog_service.py:318-321`). Both `LLM._invoke_async` and `Agent._invoke_async` (tool-using) go through `_prepare_prompt_variables`, so plain LLM nodes and Agent nodes are fixed in both streaming and non-streaming paths. ## Test plan - [ ] Upload a PDF attachment to an agent with the default `{sys.query}` prompt and ask "summarize the attachment" — the model should answer from the file content rather than searching the knowledge base. - [ ] Upload an image attachment to an agent and ask about its contents — the model should switch to the vision-capable LLM and answer from the image. - [ ] Verify that an agent whose prompt **does** include `{sys.files}` still works and does **not** include the file content twice. - [ ] Verify that an agent run with no attachments behaves unchanged. - [ ] Run `uv run pytest` to make sure no existing tests regress. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [ ] New Feature (non-breaking change which adds functionality) - [ ] Documentation Update - [ ] Refactoring - [ ] Performance Improvement - [ ] Other (please describe): --------- Co-authored-by: yzc <yuzhichang@gmail.com>
114 lines
4.0 KiB
Python
114 lines
4.0 KiB
Python
#
|
|
# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from agent.component.llm import LLM
|
|
|
|
pytestmark = pytest.mark.p1
|
|
|
|
|
|
class _FakeCanvas:
|
|
def __init__(self, sys_files=None):
|
|
self.globals = {"sys.files": list(sys_files) if sys_files is not None else []}
|
|
|
|
|
|
def _build_component(sys_files=None, sys_prompt="", prompts=None):
|
|
component = LLM.__new__(LLM)
|
|
component._canvas = _FakeCanvas(sys_files=sys_files)
|
|
component._param = SimpleNamespace(
|
|
sys_prompt=sys_prompt,
|
|
prompts=prompts if prompts is not None else [{"role": "user", "content": "{sys.query}"}],
|
|
)
|
|
return component
|
|
|
|
|
|
def test_collect_sys_files_empty_returns_empty():
|
|
component = _build_component(sys_files=[])
|
|
assert component._collect_sys_files() == ([], [])
|
|
|
|
|
|
def test_collect_sys_files_missing_key_returns_empty():
|
|
component = _build_component()
|
|
component._canvas.globals.pop("sys.files", None)
|
|
assert component._collect_sys_files() == ([], [])
|
|
|
|
|
|
def test_collect_sys_files_text_only():
|
|
files = ["File: a.pdf\ncontent A", "File: b.txt\ncontent B"]
|
|
component = _build_component(sys_files=files)
|
|
text_parts, image_data_uris = component._collect_sys_files()
|
|
assert text_parts == files
|
|
assert image_data_uris == []
|
|
|
|
|
|
def test_collect_sys_files_images_only():
|
|
files = ["data:image/png;base64,AAAA", "data:image/jpeg;base64,BBBB"]
|
|
component = _build_component(sys_files=files)
|
|
text_parts, image_data_uris = component._collect_sys_files()
|
|
assert text_parts == []
|
|
assert image_data_uris == files
|
|
|
|
|
|
def test_collect_sys_files_mixed():
|
|
files = [
|
|
"File: a.pdf\ncontent A",
|
|
"data:image/png;base64,AAAA",
|
|
"File: b.txt\ncontent B",
|
|
]
|
|
component = _build_component(sys_files=files)
|
|
text_parts, image_data_uris = component._collect_sys_files()
|
|
assert text_parts == ["File: a.pdf\ncontent A", "File: b.txt\ncontent B"]
|
|
assert image_data_uris == ["data:image/png;base64,AAAA"]
|
|
|
|
|
|
def test_collect_sys_files_skips_non_str_entries():
|
|
files = ["File: a.pdf\ncontent A", None, 123, {"name": "x"}, "data:image/png;base64,AAAA"]
|
|
component = _build_component(sys_files=files)
|
|
text_parts, image_data_uris = component._collect_sys_files()
|
|
assert text_parts == ["File: a.pdf\ncontent A"]
|
|
assert image_data_uris == ["data:image/png;base64,AAAA"]
|
|
|
|
|
|
def test_collect_sys_files_explicit_in_sys_prompt_skips_injection():
|
|
files = ["File: a.pdf\ncontent A", "data:image/png;base64,AAAA"]
|
|
component = _build_component(
|
|
sys_files=files,
|
|
sys_prompt="Answer using {sys.files} as context.",
|
|
)
|
|
assert component._collect_sys_files() == ([], [])
|
|
|
|
|
|
def test_collect_sys_files_explicit_in_prompts_entry_skips_injection():
|
|
files = ["File: a.pdf\ncontent A"]
|
|
component = _build_component(
|
|
sys_files=files,
|
|
prompts=[{"role": "user", "content": "{sys.query}\n\n{sys.files}"}],
|
|
)
|
|
assert component._collect_sys_files() == ([], [])
|
|
|
|
|
|
def test_collect_sys_files_string_prompts_does_not_crash():
|
|
# _param.prompts can be a string before normalization elsewhere; the explicit
|
|
# check must not raise in that case, it just falls through to splitting.
|
|
files = ["File: a.pdf\ncontent A"]
|
|
component = _build_component(sys_files=files, prompts="some raw template")
|
|
text_parts, image_data_uris = component._collect_sys_files()
|
|
assert text_parts == files
|
|
assert image_data_uris == []
|