Files
ragflow/test/unit_test/agent/component/test_llm_sys_files.py

114 lines
4.0 KiB
Python
Raw Normal View History

fix(agent): inject uploaded attachments into LLM context (#15215) (#15220) ## Summary Fixes #15215 — attachments uploaded to an agent were not reaching the LLM. When a user uploads a file in an agent chat, `canvas.run` parses it into the `sys.files` global (text content for documents, `data:image/...` URIs for images — see `agent/canvas.py:752-768`). But the LLM/Agent component's `_prepare_prompt_variables` only substitutes variables the user's prompt template explicitly references via `{var}` placeholders. The default prompt is `[{"role": "user", "content": "{sys.query}"}]` with no `{sys.files}`, so the parsed attachment content never reaches the model. In the reporter's logs, this is why the agent saw only the bare query `附件 摘要 attachment summary` and went searching the dataset instead of reading the uploaded PDF. ## Fix `agent/component/llm.py` — added `_collect_sys_files()` and an auto-injection step in `_prepare_prompt_variables`: - If `sys.files` is non-empty **and** neither `sys_prompt` nor any entry in `prompts` already contains `{sys.files}` (no double-injection), split the entries into text vs. `data:image/...` URIs. - Image URIs are merged into `self.imgs`, which the existing logic uses to switch the chat model to `IMAGE2TEXT` and pass `images=...` to `async_chat`. - Text content is appended to the last `user` role message in `msg`, mirroring how `dialog_service.async_chat_solo` handles attachments for the non-agent chat path (`api/db/services/dialog_service.py:318-321`). Both `LLM._invoke_async` and `Agent._invoke_async` (tool-using) go through `_prepare_prompt_variables`, so plain LLM nodes and Agent nodes are fixed in both streaming and non-streaming paths. ## Test plan - [ ] Upload a PDF attachment to an agent with the default `{sys.query}` prompt and ask "summarize the attachment" — the model should answer from the file content rather than searching the knowledge base. - [ ] Upload an image attachment to an agent and ask about its contents — the model should switch to the vision-capable LLM and answer from the image. - [ ] Verify that an agent whose prompt **does** include `{sys.files}` still works and does **not** include the file content twice. - [ ] Verify that an agent run with no attachments behaves unchanged. - [ ] Run `uv run pytest` to make sure no existing tests regress. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [ ] New Feature (non-breaking change which adds functionality) - [ ] Documentation Update - [ ] Refactoring - [ ] Performance Improvement - [ ] Other (please describe): --------- Co-authored-by: yzc <yuzhichang@gmail.com>
2026-06-30 00:48:59 -07:00
#
# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from types import SimpleNamespace
import pytest
from agent.component.llm import LLM
pytestmark = pytest.mark.p1
class _FakeCanvas:
def __init__(self, sys_files=None):
self.globals = {"sys.files": list(sys_files) if sys_files is not None else []}
def _build_component(sys_files=None, sys_prompt="", prompts=None):
component = LLM.__new__(LLM)
component._canvas = _FakeCanvas(sys_files=sys_files)
component._param = SimpleNamespace(
sys_prompt=sys_prompt,
prompts=prompts if prompts is not None else [{"role": "user", "content": "{sys.query}"}],
)
return component
def test_collect_sys_files_empty_returns_empty():
component = _build_component(sys_files=[])
assert component._collect_sys_files() == ([], [])
def test_collect_sys_files_missing_key_returns_empty():
component = _build_component()
component._canvas.globals.pop("sys.files", None)
assert component._collect_sys_files() == ([], [])
def test_collect_sys_files_text_only():
files = ["File: a.pdf\ncontent A", "File: b.txt\ncontent B"]
component = _build_component(sys_files=files)
text_parts, image_data_uris = component._collect_sys_files()
assert text_parts == files
assert image_data_uris == []
def test_collect_sys_files_images_only():
files = ["data:image/png;base64,AAAA", "data:image/jpeg;base64,BBBB"]
component = _build_component(sys_files=files)
text_parts, image_data_uris = component._collect_sys_files()
assert text_parts == []
assert image_data_uris == files
def test_collect_sys_files_mixed():
files = [
"File: a.pdf\ncontent A",
"data:image/png;base64,AAAA",
"File: b.txt\ncontent B",
]
component = _build_component(sys_files=files)
text_parts, image_data_uris = component._collect_sys_files()
assert text_parts == ["File: a.pdf\ncontent A", "File: b.txt\ncontent B"]
assert image_data_uris == ["data:image/png;base64,AAAA"]
def test_collect_sys_files_skips_non_str_entries():
files = ["File: a.pdf\ncontent A", None, 123, {"name": "x"}, "data:image/png;base64,AAAA"]
component = _build_component(sys_files=files)
text_parts, image_data_uris = component._collect_sys_files()
assert text_parts == ["File: a.pdf\ncontent A"]
assert image_data_uris == ["data:image/png;base64,AAAA"]
def test_collect_sys_files_explicit_in_sys_prompt_skips_injection():
files = ["File: a.pdf\ncontent A", "data:image/png;base64,AAAA"]
component = _build_component(
sys_files=files,
sys_prompt="Answer using {sys.files} as context.",
)
assert component._collect_sys_files() == ([], [])
def test_collect_sys_files_explicit_in_prompts_entry_skips_injection():
files = ["File: a.pdf\ncontent A"]
component = _build_component(
sys_files=files,
prompts=[{"role": "user", "content": "{sys.query}\n\n{sys.files}"}],
)
assert component._collect_sys_files() == ([], [])
def test_collect_sys_files_string_prompts_does_not_crash():
# _param.prompts can be a string before normalization elsewhere; the explicit
# check must not raise in that case, it just falls through to splitting.
files = ["File: a.pdf\ncontent A"]
component = _build_component(sys_files=files, prompts="some raw template")
text_parts, image_data_uris = component._collect_sys_files()
assert text_parts == files
assert image_data_uris == []