From e961ef04bf55ffc505548d9fe9248701913bfc05 Mon Sep 17 00:00:00 2001 From: Zane Date: Thu, 16 Jul 2026 09:41:18 +0800 Subject: [PATCH] fix: avoid duplicating video descriptions during parsing (#16847) ## What this PR does Removes the self-concatenation of the vision model response in the video parsing path, so each generated video description is tokenized and indexed exactly once. A focused regression test exercises the public `picture.chunk` video path with a mocked vision model and asserts that the returned description is passed to `tokenize` once without duplication. ## Root cause The original video parsing implementation used: ```python ans += "\n" + ans tokenize(doc, ans, ...) ``` This duplicates the same model response. The adjacent image path combines two distinct values (`OCR text + vision description`); the video path has only the model response, so concatenating it with itself is an unintended copy/paste error from that image logic. ## Impact Before this fix, every successfully parsed video stored repeated text, increasing token and embedding input and potentially distorting indexed chunk content and retrieval scoring. ## Compatibility The change affects only the video branch in `rag/app/picture.py`. Image parsing, model invocation, prompts, callbacks, and error handling remain unchanged. ## Validation - `pytest --confcutdir=test/unit_test/rag/app test/unit_test/rag/app/test_picture_video.py -q`: 1 passed - Ruff check: passed - Ruff format check for the new test: passed - `git diff --check`: passed Closes #16846. --------- Co-authored-by: openhands --- rag/app/picture.py | 1 - test/unit_test/rag/app/test_picture_video.py | 108 +++++++++++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 test/unit_test/rag/app/test_picture_video.py diff --git a/rag/app/picture.py b/rag/app/picture.py index f50efdd027..ef7630e081 100644 --- a/rag/app/picture.py +++ b/rag/app/picture.py @@ -60,7 +60,6 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs): video_prompt = str(parser_config.get("video_prompt", "") or "") ans = asyncio.run(cv_mdl.async_chat(system="", history=[], gen_conf={}, video_bytes=binary, filename=filename, video_prompt=video_prompt)) callback(0.8, "CV LLM respond: %s ..." % ans[:32]) - ans += "\n" + ans tokenize(doc, ans, eng, language=lang) return [doc] except Exception as e: diff --git a/test/unit_test/rag/app/test_picture_video.py b/test/unit_test/rag/app/test_picture_video.py new file mode 100644 index 0000000000..151db2e542 --- /dev/null +++ b/test/unit_test/rag/app/test_picture_video.py @@ -0,0 +1,108 @@ +# +# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import importlib.util +import sys +from pathlib import Path +from types import ModuleType, SimpleNamespace +from unittest.mock import patch + + +def _load_picture_module(tokenized_texts): + """Load the picture parser with lightweight fakes for external services.""" + + class FakeLLMBundle: + def __init__(self, *args, **kwargs): + """Accept the same construction arguments as the real LLM bundle.""" + + pass + + async def async_chat(self, **kwargs): + """Return a deterministic video description for the regression test.""" + + return "A concise video description." + + llm_service = ModuleType("api.db.services.llm_service") + llm_service.LLMBundle = FakeLLMBundle + + tenant_model_service = ModuleType("api.db.joint_services.tenant_model_service") + tenant_model_service.get_tenant_default_model_by_type = lambda *args, **kwargs: {} + tenant_model_service.get_first_provider_model_name = lambda *args, **kwargs: None + tenant_model_service.resolve_model_config = lambda *args, **kwargs: {} + tenant_model_service.ensure_paddleocr_from_env = lambda *args, **kwargs: None + + constants = ModuleType("common.constants") + constants.LLMType = SimpleNamespace(VISION="vision", OCR="ocr") + + parser_config_utils = ModuleType("common.parser_config_utils") + parser_config_utils.normalize_layout_recognizer = lambda value: (value, "") + + string_utils = ModuleType("common.string_utils") + string_utils.clean_markdown_block = lambda value: value + + vision = ModuleType("deepdoc.vision") + vision.OCR = lambda: object() + + nlp = ModuleType("rag.nlp") + nlp.attach_media_context = lambda docs, *_args: docs + nlp.rag_tokenizer = SimpleNamespace(tokenize=lambda value: value) + + def fake_tokenize(doc, text, *_args, **_kwargs): + """Capture the exact text passed to tokenization.""" + + tokenized_texts.append(text) + doc["content_with_weight"] = text + + nlp.tokenize = fake_tokenize + + stubs = { + "api.db.services.llm_service": llm_service, + "api.db.joint_services.tenant_model_service": tenant_model_service, + "common.constants": constants, + "common.parser_config_utils": parser_config_utils, + "common.string_utils": string_utils, + "deepdoc.vision": vision, + "rag.nlp": nlp, + } + + module_path = Path(__file__).resolve().parents[4] / "rag" / "app" / "picture.py" + spec = importlib.util.spec_from_file_location("picture_video_under_test", module_path) + module = importlib.util.module_from_spec(spec) + with patch.dict(sys.modules, stubs): + spec.loader.exec_module(module) + return module + + +def test_video_description_is_tokenized_once(): + """Ensure one model response produces one tokenized video description.""" + + tokenized_texts = [] + picture = _load_picture_module(tokenized_texts) + + callback_calls = [] + chunks = picture.chunk( + "clip.mp4", + b"video bytes", + "tenant", + "English", + callback=lambda *args, **kwargs: callback_calls.append((args, kwargs)), + ) + + errors = [kwargs.get("msg") for args, kwargs in callback_calls if kwargs.get("prog") == -1] + assert not errors, f"chunk() reported an error instead of producing a chunk: {errors}" + assert len(chunks) == 1 + assert chunks[0]["doc_type_kwd"] == "video" + assert tokenized_texts == ["A concise video description."]