fix: avoid duplicating video descriptions during parsing (#16847)

## What this PR does

Removes the self-concatenation of the vision model response in the video
parsing path, so each generated video description is tokenized and
indexed exactly once.

A focused regression test exercises the public `picture.chunk` video
path with a mocked vision model and asserts that the returned
description is passed to `tokenize` once without duplication.

## Root cause

The original video parsing implementation used:

```python
ans += "\n" + ans
tokenize(doc, ans, ...)
```

This duplicates the same model response. The adjacent image path
combines two distinct values (`OCR text + vision description`); the
video path has only the model response, so concatenating it with itself
is an unintended copy/paste error from that image logic.

## Impact

Before this fix, every successfully parsed video stored repeated text,
increasing token and embedding input and potentially distorting indexed
chunk content and retrieval scoring.

## Compatibility

The change affects only the video branch in `rag/app/picture.py`. Image
parsing, model invocation, prompts, callbacks, and error handling remain
unchanged.

## Validation

- `pytest --confcutdir=test/unit_test/rag/app
test/unit_test/rag/app/test_picture_video.py -q`: 1 passed
- Ruff check: passed
- Ruff format check for the new test: passed
- `git diff --check`: passed

Closes #16846.

---------

Co-authored-by: openhands <openhands@all-hands.dev>

rag/app/picture.py

@@ -60,7 +60,6 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs):
             video_prompt = str(parser_config.get("video_prompt", "") or "")
             ans = asyncio.run(cv_mdl.async_chat(system="", history=[], gen_conf={}, video_bytes=binary, filename=filename, video_prompt=video_prompt))
             callback(0.8, "CV LLM respond: %s ..." % ans[:32])
-            ans += "\n" + ans
             tokenize(doc, ans, eng, language=lang)
             return [doc]
         except Exception as e:

test/unit_test/rag/app/test_picture_video.py

@@ -0,0 +1,108 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+from unittest.mock import patch
+
+
+def _load_picture_module(tokenized_texts):
+    """Load the picture parser with lightweight fakes for external services."""
+
+    class FakeLLMBundle:
+        def __init__(self, *args, **kwargs):
+            """Accept the same construction arguments as the real LLM bundle."""
+
+            pass
+
+        async def async_chat(self, **kwargs):
+            """Return a deterministic video description for the regression test."""
+
+            return "A concise video description."
+
+    llm_service = ModuleType("api.db.services.llm_service")
+    llm_service.LLMBundle = FakeLLMBundle
+
+    tenant_model_service = ModuleType("api.db.joint_services.tenant_model_service")
+    tenant_model_service.get_tenant_default_model_by_type = lambda *args, **kwargs: {}
+    tenant_model_service.get_first_provider_model_name = lambda *args, **kwargs: None
+    tenant_model_service.resolve_model_config = lambda *args, **kwargs: {}
+    tenant_model_service.ensure_paddleocr_from_env = lambda *args, **kwargs: None
+
+    constants = ModuleType("common.constants")
+    constants.LLMType = SimpleNamespace(VISION="vision", OCR="ocr")
+
+    parser_config_utils = ModuleType("common.parser_config_utils")
+    parser_config_utils.normalize_layout_recognizer = lambda value: (value, "")
+
+    string_utils = ModuleType("common.string_utils")
+    string_utils.clean_markdown_block = lambda value: value
+
+    vision = ModuleType("deepdoc.vision")
+    vision.OCR = lambda: object()
+
+    nlp = ModuleType("rag.nlp")
+    nlp.attach_media_context = lambda docs, *_args: docs
+    nlp.rag_tokenizer = SimpleNamespace(tokenize=lambda value: value)
+
+    def fake_tokenize(doc, text, *_args, **_kwargs):
+        """Capture the exact text passed to tokenization."""
+
+        tokenized_texts.append(text)
+        doc["content_with_weight"] = text
+
+    nlp.tokenize = fake_tokenize
+
+    stubs = {
+        "api.db.services.llm_service": llm_service,
+        "api.db.joint_services.tenant_model_service": tenant_model_service,
+        "common.constants": constants,
+        "common.parser_config_utils": parser_config_utils,
+        "common.string_utils": string_utils,
+        "deepdoc.vision": vision,
+        "rag.nlp": nlp,
+    }
+
+    module_path = Path(__file__).resolve().parents[4] / "rag" / "app" / "picture.py"
+    spec = importlib.util.spec_from_file_location("picture_video_under_test", module_path)
+    module = importlib.util.module_from_spec(spec)
+    with patch.dict(sys.modules, stubs):
+        spec.loader.exec_module(module)
+    return module
+
+
+def test_video_description_is_tokenized_once():
+    """Ensure one model response produces one tokenized video description."""
+
+    tokenized_texts = []
+    picture = _load_picture_module(tokenized_texts)
+
+    callback_calls = []
+    chunks = picture.chunk(
+        "clip.mp4",
+        b"video bytes",
+        "tenant",
+        "English",
+        callback=lambda *args, **kwargs: callback_calls.append((args, kwargs)),
+    )
+
+    errors = [kwargs.get("msg") for args, kwargs in callback_calls if kwargs.get("prog") == -1]
+    assert not errors, f"chunk() reported an error instead of producing a chunk: {errors}"
+    assert len(chunks) == 1
+    assert chunks[0]["doc_type_kwd"] == "video"
+    assert tokenized_texts == ["A concise video description."]