Refactor: migrate chunk retrieval_test and knowledge_graph to REST API endpoints (#14402)

### What problem does this PR solve?

## Summary

Migrate two web API endpoints to REST-style HTTP API endpoints,
following the pattern established in #14222:

| Old Endpoint | New Endpoint |
|---|---|
| `POST /v1/chunk/retrieval_test` | `POST
/api/v1/datasets/<dataset_id>/search` |
| `GET /v1/chunk/knowledge_graph` | `GET
/api/v1/datasets/<dataset_id>/graph` |

test/testcases/test_web_api/test_chunk_app/test_chunk_routes_unit.py

@@ -17,7 +17,6 @@
 import asyncio
 import inspect
 import importlib.util
-import json
 import sys
 from pathlib import Path
 from types import ModuleType, SimpleNamespace
@@ -491,13 +490,15 @@ def _load_chunk_module(monkeypatch):
     monkeypatch.setitem(sys.modules, "api.db.services.user_service", user_service_mod)
     services_pkg.user_service = user_service_mod
 
-    module_name = "test_chunk_routes_unit_module"
     module_path = repo_root / "api" / "apps" / "chunk_app.py"
-    spec = importlib.util.spec_from_file_location(module_name, module_path)
-    module = importlib.util.module_from_spec(spec)
-    module.manager = _DummyManager()
-    monkeypatch.setitem(sys.modules, module_name, module)
-    spec.loader.exec_module(module)
+    module = None
+    if module_path.exists():
+        module_name = "test_chunk_routes_unit_module"
+        spec = importlib.util.spec_from_file_location(module_name, module_path)
+        module = importlib.util.module_from_spec(spec)
+        module.manager = _DummyManager()
+        monkeypatch.setitem(sys.modules, module_name, module)
+        spec.loader.exec_module(module)
     return module
 
 
@@ -653,167 +654,3 @@ def test_restful_chunk_guard_branches_unit(monkeypatch):
     assert res["message"] == "`available_int` or `available` is required.", res
 
 
-@pytest.mark.p2
-def test_retrieval_test_branch_matrix_unit(monkeypatch):
-    module = _load_chunk_module(monkeypatch)
-    module.request = SimpleNamespace(headers={"X-Request-ID": "req-r"}, args={})
-
-    applied_filters = []
-    llm_calls = []
-    cross_calls = []
-    keyword_calls = []
-
-    async def _apply_filter(meta_data_filter, metas, question, chat_mdl, local_doc_ids):
-        applied_filters.append(
-            {
-                "meta_data_filter": meta_data_filter,
-                "metas": metas,
-                "question": question,
-                "chat_mdl": chat_mdl,
-                "local_doc_ids": list(local_doc_ids),
-            }
-        )
-        return ["doc-filtered"]
-
-    async def _cross_languages(_tenant_id, _dialog, question, langs):
-        cross_calls.append((question, tuple(langs)))
-        return f"{question}-xl"
-
-    async def _keyword_extraction(_chat_mdl, question):
-        keyword_calls.append(question)
-        return "-kw"
-
-    class _Retriever:
-        def __init__(self, mode="ok"):
-            self.mode = mode
-            self.retrieval_questions = []
-
-        async def retrieval(self, question, *_args, **_kwargs):
-            if self.mode == "not_found":
-                raise Exception("boom not_found boom")
-            if self.mode == "explode":
-                raise RuntimeError("retrieval boom")
-            self.retrieval_questions.append(question)
-            return {"chunks": [{"id": "c1", "vector": [0.1], "content_with_weight": "chunk-content"}]}
-
-        def retrieval_by_children(self, chunks, _tenant_ids):
-            return list(chunks)
-
-    class _KgRetriever:
-        async def retrieval(self, *_args, **_kwargs):
-            return {"id": "kg-1", "content_with_weight": "kg-content"}
-
-    class _NoContentKgRetriever:
-        async def retrieval(self, *_args, **_kwargs):
-            return {"id": "kg-2", "content_with_weight": ""}
-
-    monkeypatch.setattr(module, "LLMBundle", lambda *args, **kwargs: llm_calls.append((args, kwargs)) or SimpleNamespace())
-    monkeypatch.setattr(module, "get_model_config_by_type_and_name", lambda *_args, **_kwargs: {"llm_name": "stub-model", "model_type": "chat"})
-    monkeypatch.setattr(module, "get_tenant_default_model_by_type", lambda *_args, **_kwargs: {"llm_name": "stub-model", "model_type": "chat"})
-    monkeypatch.setattr(module, "get_model_config_by_id", lambda *_args, **_kwargs: {"llm_name": "stub-model", "model_type": "embedding"})
-    monkeypatch.setattr(module.DocMetadataService, "get_flatted_meta_by_kbs", lambda _kb_ids: [{"meta": "v"}], raising=False)
-    monkeypatch.setattr(module, "apply_meta_data_filter", _apply_filter)
-    monkeypatch.setattr(module.SearchService, "get_detail", lambda _sid: {"search_config": {"meta_data_filter": {"method": "auto"}, "chat_id": "chat-1"}}, raising=False)
-    monkeypatch.setattr(module, "cross_languages", _cross_languages)
-    monkeypatch.setattr(module, "keyword_extraction", _keyword_extraction)
-    monkeypatch.setattr(module, "label_question", lambda *_args, **_kwargs: ["lbl"])
-    monkeypatch.setattr(module.UserTenantService, "query", lambda **_kwargs: [_DummyTenant("tenant-1")])
-
-    monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: False, raising=False)
-    _set_request_json(monkeypatch, module, {"kb_id": "kb-1", "question": "q", "search_id": "search-1"})
-    res = _run(module.retrieval_test())
-    assert res["code"] == module.RetCode.OPERATING_ERROR, res
-    assert "Only owner of dataset authorized for this operation." in res["message"], res
-    assert applied_filters and applied_filters[-1]["meta_data_filter"]["method"] == "auto"
-    assert llm_calls, "search_id metadata auto branch should instantiate chat model"
-
-    _set_request_json(monkeypatch, module, {"kb_id": [], "question": "q"})
-    res = _run(module.retrieval_test())
-    assert res["code"] == module.RetCode.DATA_ERROR, res
-    assert "Please specify dataset firstly." in res["message"], res
-
-    monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: True, raising=False)
-    monkeypatch.setattr(module.KnowledgebaseService, "get_by_id", lambda _kb_id: (False, None), raising=False)
-    _set_request_json(
-        monkeypatch,
-        module,
-        {"kb_id": ["kb-1"], "question": "q", "meta_data_filter": {"method": "semi_auto"}},
-    )
-    res = _run(module.retrieval_test())
-    assert res["code"] == module.RetCode.DATA_ERROR, res
-    assert "Knowledgebase not found!" in res["message"], res
-
-    retriever = _Retriever(mode="ok")
-    monkeypatch.setattr(module.KnowledgebaseService, "get_by_id", lambda _kb_id: (True, SimpleNamespace(tenant_id="tenant-kb", embd_id="embd-1", tenant_embd_id=2)), raising=False)
-    monkeypatch.setattr(module.settings, "retriever", retriever)
-    monkeypatch.setattr(module.settings, "kg_retriever", _KgRetriever(), raising=False)
-    _set_request_json(
-        monkeypatch,
-        module,
-        {
-            "kb_id": ["kb-1"],
-            "question": "q",
-            "cross_languages": ["fr"],
-            "rerank_id": "rerank-1",
-            "keyword": True,
-            "use_kg": True,
-        },
-    )
-    res = _run(module.retrieval_test())
-    assert res["code"] == 0, res
-    assert cross_calls[-1] == ("q", ("fr",))
-    assert keyword_calls[-1] == "q-xl"
-    assert retriever.retrieval_questions[-1] == "q-xl-kw"
-    assert res["data"]["chunks"][0]["id"] == "kg-1", res
-    assert all("vector" not in chunk for chunk in res["data"]["chunks"])
-
-    monkeypatch.setattr(module.settings, "kg_retriever", _NoContentKgRetriever(), raising=False)
-    _set_request_json(monkeypatch, module, {"kb_id": ["kb-1"], "question": "q", "use_kg": True})
-    res = _run(module.retrieval_test())
-    assert res["code"] == 0, res
-    assert res["data"]["chunks"][0]["id"] == "c1", res
-
-    monkeypatch.setattr(module.settings, "retriever", _Retriever(mode="not_found"))
-    _set_request_json(monkeypatch, module, {"kb_id": ["kb-1"], "question": "q"})
-    res = _run(module.retrieval_test())
-    assert res["code"] == module.RetCode.DATA_ERROR, res
-    assert "No chunk found! Check the chunk status please!" in res["message"], res
-
-    monkeypatch.setattr(module.settings, "retriever", _Retriever(mode="explode"))
-    _set_request_json(monkeypatch, module, {"kb_id": ["kb-1"], "question": "q"})
-    res = _run(module.retrieval_test())
-    assert res["code"] == module.RetCode.EXCEPTION_ERROR, res
-    assert "retrieval boom" in res["message"], res
-
-
-@pytest.mark.p2
-def test_knowledge_graph_repeat_deal_matrix_unit(monkeypatch):
-    module = _load_chunk_module(monkeypatch)
-    module.request = SimpleNamespace(args={"doc_id": "doc-1"}, headers={})
-
-    payload = {
-        "id": "root",
-        "children": [
-            {"id": "dup"},
-            {"id": "dup", "children": [{"id": "dup"}]},
-        ],
-    }
-
-    class _SRes:
-        ids = ["bad-json", "mind-map"]
-        field = {
-            "bad-json": {"knowledge_graph_kwd": "graph", "content_with_weight": "{bad json"},
-            "mind-map": {"knowledge_graph_kwd": "mind_map", "content_with_weight": json.dumps(payload)},
-        }
-
-    async def _search(*_args, **_kwargs):
-        return _SRes()
-
-    monkeypatch.setattr(module.settings.retriever, "search", _search)
-    res = _run(module.knowledge_graph())
-    assert res["code"] == 0, res
-    assert res["data"]["graph"] == {}, res
-    mind_map = res["data"]["mind_map"]
-    assert mind_map["children"][0]["id"] == "dup", res
-    assert mind_map["children"][1]["id"] == "dup(1)", res
-    assert mind_map["children"][1]["children"][0]["id"] == "dup(2)", res

test/testcases/test_web_api/test_chunk_app/test_retrieval_chunks.py

@@ -1,308 +0,0 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
-
-import pytest
-from test_common import retrieval_chunks
-from configs import INVALID_API_TOKEN
-from libs.auth import RAGFlowWebApiAuth
-
-
-@pytest.mark.p2
-class TestAuthorization:
-    @pytest.mark.parametrize(
-        "invalid_auth, expected_code, expected_message",
-        [
-            (None, 401, "<Unauthorized '401: Unauthorized'>"),
-            (RAGFlowWebApiAuth(INVALID_API_TOKEN), 401, "<Unauthorized '401: Unauthorized'>"),
-        ],
-    )
-    def test_invalid_auth(self, invalid_auth, expected_code, expected_message):
-        res = retrieval_chunks(invalid_auth, {"kb_id": "dummy_kb_id", "question": "dummy question"})
-        assert res["code"] == expected_code, res
-        assert res["message"] == expected_message, res
-
-
-class TestChunksRetrieval:
-    @pytest.mark.p1
-    @pytest.mark.parametrize(
-        "payload, expected_code, expected_page_size, expected_message",
-        [
-            ({"question": "chunk", "kb_id": None}, 0, 4, ""),
-            ({"question": "chunk", "doc_ids": None}, 101, 0, "required argument are missing: kb_id; "),
-            ({"question": "chunk", "kb_id": None, "doc_ids": None}, 0, 4, ""),
-            ({"question": "chunk"}, 101, 0, "required argument are missing: kb_id; "),
-        ],
-    )
-    def test_basic_scenarios(self, WebApiAuth, add_chunks, payload, expected_code, expected_page_size, expected_message):
-        dataset_id, document_id, _ = add_chunks
-        if "kb_id" in payload:
-            payload["kb_id"] = [dataset_id]
-        if "doc_ids" in payload:
-            payload["doc_ids"] = [document_id]
-        res = retrieval_chunks(WebApiAuth, payload)
-        assert res["code"] == expected_code, res
-        if expected_code == 0:
-            assert len(res["data"]["chunks"]) == expected_page_size, res
-        else:
-            assert res["message"] == expected_message, res
-
-    @pytest.mark.p2
-    @pytest.mark.parametrize(
-        "payload, expected_code, expected_page_size, expected_message",
-        [
-            pytest.param(
-                {"page": None, "size": 2},
-                100,
-                0,
-                """TypeError("int() argument must be a string, a bytes-like object or a real number, not 'NoneType'")""",
-                marks=pytest.mark.skip,
-            ),
-            pytest.param(
-                {"page": 0, "size": 2},
-                100,
-                0,
-                "ValueError('Search does not support negative slicing.')",
-                marks=pytest.mark.skip,
-            ),
-            pytest.param({"page": 2, "size": 2}, 0, 2, "", marks=pytest.mark.skip(reason="issues/6646")),
-            ({"page": 3, "size": 2}, 0, 0, ""),
-            ({"page": "3", "size": 2}, 0, 0, ""),
-            pytest.param(
-                {"page": -1, "size": 2},
-                100,
-                0,
-                "ValueError('Search does not support negative slicing.')",
-                marks=pytest.mark.skip,
-            ),
-            pytest.param(
-                {"page": "a", "size": 2},
-                100,
-                0,
-                """ValueError("invalid literal for int() with base 10: 'a'")""",
-                marks=pytest.mark.skip,
-            ),
-        ],
-    )
-    def test_page(self, WebApiAuth, add_chunks, payload, expected_code, expected_page_size, expected_message):
-        dataset_id, _, _ = add_chunks
-        payload.update({"question": "chunk", "kb_id": [dataset_id]})
-        res = retrieval_chunks(WebApiAuth, payload)
-        assert res["code"] == expected_code, res
-        if expected_code == 0:
-            assert len(res["data"]["chunks"]) == expected_page_size, res
-        else:
-            assert res["message"] == expected_message, res
-
-    @pytest.mark.p3
-    @pytest.mark.parametrize(
-        "payload, expected_code, expected_page_size, expected_message",
-        [
-            pytest.param(
-                {"size": None},
-                100,
-                0,
-                """TypeError("int() argument must be a string, a bytes-like object or a real number, not 'NoneType'")""",
-                marks=pytest.mark.skip,
-            ),
-            # ({"size": 0}, 0, 0, ""),
-            ({"size": 1}, 0, 1, ""),
-            ({"size": 5}, 0, 4, ""),
-            ({"size": "1"}, 0, 1, ""),
-            # ({"size": -1}, 0, 0, ""),
-            pytest.param(
-                {"size": "a"},
-                100,
-                0,
-                """ValueError("invalid literal for int() with base 10: 'a'")""",
-                marks=pytest.mark.skip,
-            ),
-        ],
-    )
-    def test_page_size(self, WebApiAuth, add_chunks, payload, expected_code, expected_page_size, expected_message):
-        dataset_id, _, _ = add_chunks
-        payload.update({"question": "chunk", "kb_id": [dataset_id]})
-
-        res = retrieval_chunks(WebApiAuth, payload)
-        assert res["code"] == expected_code, res
-        if expected_code == 0:
-            assert len(res["data"]["chunks"]) == expected_page_size, res
-        else:
-            assert res["message"] == expected_message, res
-
-    @pytest.mark.p3
-    @pytest.mark.parametrize(
-        "payload, expected_code, expected_page_size, expected_message",
-        [
-            ({"vector_similarity_weight": 0}, 0, 4, ""),
-            ({"vector_similarity_weight": 0.5}, 0, 4, ""),
-            ({"vector_similarity_weight": 10}, 0, 4, ""),
-            pytest.param(
-                {"vector_similarity_weight": "a"},
-                100,
-                0,
-                """ValueError("could not convert string to float: 'a'")""",
-                marks=pytest.mark.skip,
-            ),
-        ],
-    )
-    def test_vector_similarity_weight(self, WebApiAuth, add_chunks, payload, expected_code, expected_page_size, expected_message):
-        dataset_id, _, _ = add_chunks
-        payload.update({"question": "chunk", "kb_id": [dataset_id]})
-        res = retrieval_chunks(WebApiAuth, payload)
-        assert res["code"] == expected_code, res
-        if expected_code == 0:
-            assert len(res["data"]["chunks"]) == expected_page_size, res
-        else:
-            assert res["message"] == expected_message, res
-
-    @pytest.mark.p2
-    @pytest.mark.parametrize(
-        "payload, expected_code, expected_page_size, expected_message",
-        [
-            ({"top_k": 10}, 0, 4, ""),
-            pytest.param(
-                {"top_k": 1},
-                0,
-                4,
-                "",
-                marks=pytest.mark.skipif(os.getenv("DOC_ENGINE") in ["infinity", "opensearch"], reason="Infinity"),
-            ),
-            pytest.param(
-                {"top_k": 1},
-                0,
-                1,
-                "",
-                marks=pytest.mark.skipif(os.getenv("DOC_ENGINE") in [None, "opensearch", "elasticsearch"], reason="elasticsearch"),
-            ),
-            pytest.param(
-                {"top_k": -1},
-                100,
-                4,
-                "must be greater than 0",
-                marks=pytest.mark.skip(reason="Web API does not validate top_k"),
-            ),
-            pytest.param(
-                {"top_k": -1},
-                100,
-                4,
-                "3014",
-                marks=pytest.mark.skip(reason="Web API does not validate top_k"),
-            ),
-            pytest.param(
-                {"top_k": "a"},
-                100,
-                0,
-                """ValueError("invalid literal for int() with base 10: 'a'")""",
-                marks=pytest.mark.skip,
-            ),
-        ],
-    )
-    def test_top_k(self, WebApiAuth, add_chunks, payload, expected_code, expected_page_size, expected_message):
-        dataset_id, _, _ = add_chunks
-        payload.update({"question": "chunk", "kb_id": [dataset_id]})
-        res = retrieval_chunks(WebApiAuth, payload)
-        assert res["code"] == expected_code, res
-        if expected_code == 0:
-            assert len(res["data"]["chunks"]) == expected_page_size, res
-        else:
-            assert expected_message in res["message"], res
-
-    @pytest.mark.skip
-    @pytest.mark.parametrize(
-        "payload, expected_code, expected_message",
-        [
-            ({"rerank_id": "BAAI/bge-reranker-v2-m3"}, 0, ""),
-            pytest.param({"rerank_id": "unknown"}, 100, "LookupError('Model(unknown) not authorized')", marks=pytest.mark.skip),
-        ],
-    )
-    def test_rerank_id(self, WebApiAuth, add_chunks, payload, expected_code, expected_message):
-        dataset_id, _, _ = add_chunks
-        payload.update({"question": "chunk", "kb_id": [dataset_id]})
-        res = retrieval_chunks(WebApiAuth, payload)
-        assert res["code"] == expected_code, res
-        if expected_code == 0:
-            assert len(res["data"]["chunks"]) > 0, res
-        else:
-            assert expected_message in res["message"], res
-
-    @pytest.mark.skip
-    @pytest.mark.parametrize(
-        "payload, expected_code, expected_page_size, expected_message",
-        [
-            ({"keyword": True}, 0, 5, ""),
-            ({"keyword": "True"}, 0, 5, ""),
-            ({"keyword": False}, 0, 5, ""),
-            ({"keyword": "False"}, 0, 5, ""),
-            ({"keyword": None}, 0, 5, ""),
-        ],
-    )
-    def test_keyword(self, WebApiAuth, add_chunks, payload, expected_code, expected_page_size, expected_message):
-        dataset_id, _, _ = add_chunks
-        payload.update({"question": "chunk test", "kb_id": [dataset_id]})
-        res = retrieval_chunks(WebApiAuth, payload)
-        assert res["code"] == expected_code, res
-        if expected_code == 0:
-            assert len(res["data"]["chunks"]) == expected_page_size, res
-        else:
-            assert res["message"] == expected_message, res
-
-    @pytest.mark.p3
-    @pytest.mark.parametrize(
-        "payload, expected_code, expected_highlight, expected_message",
-        [
-            pytest.param({"highlight": True}, 0, True, "", marks=pytest.mark.skip(reason="highlight not functionnal")),
-            pytest.param({"highlight": "True"}, 0, True, "", marks=pytest.mark.skip(reason="highlight not functionnal")),
-            ({"highlight": False}, 0, False, ""),
-            ({"highlight": "False"}, 0, False, ""),
-            ({"highlight": None}, 0, False, "")
-        ],
-    )
-    def test_highlight(self, WebApiAuth, add_chunks, payload, expected_code, expected_highlight, expected_message):
-        dataset_id, _, _ = add_chunks
-        payload.update({"question": "chunk", "kb_id": [dataset_id]})
-        res = retrieval_chunks(WebApiAuth, payload)
-        assert res["code"] == expected_code, res
-        if expected_highlight:
-            for chunk in res["data"]["chunks"]:
-                assert "highlight" in chunk, res
-        else:
-            for chunk in res["data"]["chunks"]:
-                assert "highlight" not in chunk, res
-
-        if expected_code != 0:
-            assert res["message"] == expected_message, res
-
-    @pytest.mark.p3
-    def test_invalid_params(self, WebApiAuth, add_chunks):
-        dataset_id, _, _ = add_chunks
-        payload = {"question": "chunk", "kb_id": [dataset_id], "a": "b"}
-        res = retrieval_chunks(WebApiAuth, payload)
-        assert res["code"] == 0, res
-        assert len(res["data"]["chunks"]) == 4, res
-
-    @pytest.mark.p3
-    def test_concurrent_retrieval(self, WebApiAuth, add_chunks):
-        dataset_id, _, _ = add_chunks
-        count = 100
-        payload = {"question": "chunk", "kb_id": [dataset_id]}
-
-        with ThreadPoolExecutor(max_workers=5) as executor:
-            futures = [executor.submit(retrieval_chunks, WebApiAuth, payload) for i in range(count)]
-        responses = list(as_completed(futures))
-        assert len(responses) == count, responses
-        assert all(future.result()["code"] == 0 for future in futures)

test/testcases/test_web_api/test_common.py

@@ -244,22 +244,6 @@ def kb_pipeline_log_detail(auth, dataset_id, log_id, *, headers=HEADERS):
     return res.json()
 
 
-# DATASET GRAPH AND TASKS
-def knowledge_graph(auth, dataset_id, params=None):
-    url = f"{HOST_ADDRESS}{DATASETS_URL}/{dataset_id}/knowledge_graph"
-    res = requests.get(url=url, headers=HEADERS, auth=auth, params=params)
-    return res.json()
-
-
-def delete_knowledge_graph(auth, dataset_id, payload=None):
-    url = f"{HOST_ADDRESS}{DATASETS_URL}/{dataset_id}/knowledge_graph"
-    if payload is None:
-        res = requests.delete(url=url, headers=HEADERS, auth=auth)
-    else:
-        res = requests.delete(url=url, headers=HEADERS, auth=auth, json=payload)
-    return res.json()
-
-
 def list_tags_from_kbs(auth, dataset_ids, *, headers=HEADERS):
     params = {"dataset_ids": dataset_ids}
     res = requests.get(url=f"{HOST_ADDRESS}{DATASETS_URL}/tags/aggregation", headers=headers, auth=auth, params=params)
@@ -518,11 +502,6 @@ def delete_chunks(auth, dataset_id, document_id, payload=None, *, headers=HEADER
     return res.json()
 
 
-def retrieval_chunks(auth, payload=None, *, headers=HEADERS):
-    res = requests.post(url=f"{HOST_ADDRESS}{CHUNK_APP_URL}/retrieval_test", headers=headers, auth=auth, json=payload)
-    return res.json()
-
-
 def batch_add_chunks(auth, dataset_id, document_id, num):
     chunk_ids = []
     for i in range(num):