Fix tag_feas code injection in retrieval ranking (#13923)

## Summary
- remove eval-based parsing from retrieval rank feature scoring
- validate `tag_feas` at write time in chunk APIs and SDK routes
- add regression tests for safe parsing and malicious payload rejection

## Details
`tag_feas` is intended to be structured rank-feature data, but the
retrieval ranking path was evaluating stored values as Python
expressions. This change treats `tag_feas` strictly as data.

### What changed
- replace `eval()` in `rag/nlp/search.py` with safe parsing via
`json.loads()` and optional `ast.literal_eval()` compatibility for
legacy Python-dict strings
- strictly filter parsed values down to `dict[str, finite number]`
- reject invalid `tag_feas` payloads at write time in web chunk routes
and SDK document chunk routes
- add focused regression tests to prove executable strings are ignored
and invalid payloads are rejected

## Validation
- `python -m pytest test/unit_test/common/test_tag_feature_utils.py
test/unit_test/rag/test_rank_feature_scores.py -q`

---------

Co-authored-by: unknown <zhenglinkai@CCN.Local>
Co-authored-by: Yingfeng Zhang <yingfeng.zhang@gmail.com>

test/testcases/test_web_api/test_chunk_app/test_chunk_routes_unit.py

@@ -584,6 +584,14 @@ def test_set_chunk_bytes_qa_image_and_guard_matrix_unit(monkeypatch):
         "get_by_id",
         lambda _doc_id: (True, _DummyDoc(doc_id="doc-1", parser_id=module.ParserType.NAIVE)),
     )
+    _set_request_json(
+        monkeypatch,
+        module,
+        {"doc_id": "doc-1", "chunk_id": "chunk-1", "content_with_weight": "abc", "tag_feas": [0.1]},
+    )
+    res = _run(module.set())
+    assert "`tag_feas` must be an object mapping string tags to finite numeric scores" in res["message"], res
+
     _set_request_json(
         monkeypatch,
         module,
@@ -594,7 +602,7 @@ def test_set_chunk_bytes_qa_image_and_guard_matrix_unit(monkeypatch):
             "important_kwd": ["important"],
             "question_kwd": ["question"],
             "tag_kwd": ["tag"],
-            "tag_feas": [0.1],
+            "tag_feas": {"tag": 0.1},
             "available_int": 0,
         },
     )
@@ -762,6 +770,14 @@ def test_create_chunk_guards_pagerank_and_success_unit(monkeypatch):
     assert res["message"] == "Knowledgebase not found!", res
 
     monkeypatch.setattr(module.KnowledgebaseService, "get_by_id", lambda _kb_id: (True, SimpleNamespace(pagerank=0.8)))
+    _set_request_json(
+        monkeypatch,
+        module,
+        {"doc_id": "doc-1", "content_with_weight": "chunk", "tag_feas": [0.2]},
+    )
+    res = _run(module.create())
+    assert "`tag_feas` must be an object mapping string tags to finite numeric scores" in res["message"], res
+
     _set_request_json(
         monkeypatch,
         module,
@@ -770,7 +786,7 @@ def test_create_chunk_guards_pagerank_and_success_unit(monkeypatch):
             "content_with_weight": "chunk",
             "important_kwd": ["i1"],
             "question_kwd": ["q1"],
-            "tag_feas": [0.2],
+            "tag_feas": {"tag": 0.2},
         },
     )
     res = _run(module.create())

test/testcases/test_web_api/test_chunk_app/test_create_chunk.py

@@ -166,7 +166,7 @@ class TestAddChunk:
         payload = {
             "doc_id": doc_id,
             "content_with_weight": "chunk with tags",
-            "tag_feas": [0.1, 0.2],
+            "tag_feas": {"tag1": 0.1, "tag2": 0.2},
             "important_kwd": ["tag"],
             "question_kwd": ["question"],
         }

test/unit_test/common/test_tag_feature_utils.py

@@ -0,0 +1,32 @@
+import pytest
+
+from common.tag_feature_utils import parse_tag_features, validate_tag_features
+
+
+def test_validate_tag_features_accepts_numeric_dict():
+    assert validate_tag_features({"apple": 1, "banana": 2.5}) == {
+        "apple": 1.0,
+        "banana": 2.5,
+    }
+
+
+def test_validate_tag_features_rejects_string_payload():
+    with pytest.raises(ValueError, match="object mapping string tags"):
+        validate_tag_features('{"apple": 1.0}')
+
+
+def test_validate_tag_features_rejects_non_finite_or_non_numeric_values():
+    with pytest.raises(ValueError, match="finite numbers"):
+        validate_tag_features({"apple": float("inf")})
+
+    with pytest.raises(ValueError, match="finite numbers"):
+        validate_tag_features({"apple": "1.0"})
+
+
+def test_parse_tag_features_supports_legacy_python_literal_strings():
+    assert parse_tag_features("{'apple': 2.0}", allow_python_literal=True) == {"apple": 2.0}
+
+
+def test_parse_tag_features_ignores_executable_strings():
+    payload = '{"apple": (__import__("time").sleep(1) or 1.0)}'
+    assert parse_tag_features(payload, allow_python_literal=True) == {}

test/unit_test/rag/test_rank_feature_scores.py

@@ -0,0 +1,97 @@
+import sys
+import types
+
+import numpy as np
+
+from common.constants import PAGERANK_FLD, TAG_FLD
+
+
+class _DummyTokenizer:
+    def tag(self, *args, **kwargs):
+        return []
+
+    def freq(self, *args, **kwargs):
+        return 0
+
+    def _tradi2simp(self, text):
+        return text
+
+    def _strQ2B(self, text):
+        return text
+
+
+fake_infinity = types.ModuleType("infinity")
+fake_infinity_tokenizer = types.ModuleType("infinity.rag_tokenizer")
+fake_infinity_tokenizer.RagTokenizer = _DummyTokenizer
+fake_infinity_tokenizer.is_chinese = lambda text: False
+fake_infinity_tokenizer.is_number = lambda text: False
+fake_infinity_tokenizer.is_alphabet = lambda text: True
+fake_infinity_tokenizer.naive_qie = lambda text: text.split()
+fake_infinity.rag_tokenizer = fake_infinity_tokenizer
+sys.modules.setdefault("infinity", fake_infinity)
+sys.modules.setdefault("infinity.rag_tokenizer", fake_infinity_tokenizer)
+
+fake_query = types.ModuleType("rag.nlp.query")
+
+
+class _DummyFulltextQueryer:
+    pass
+
+
+fake_query.FulltextQueryer = _DummyFulltextQueryer
+sys.modules.setdefault("rag.nlp.query", fake_query)
+
+fake_settings = types.ModuleType("common.settings")
+sys.modules.setdefault("common.settings", fake_settings)
+
+from rag.nlp.search import Dealer
+
+
+def _make_search_res(tag_feas):
+    return Dealer.SearchResult(
+        total=1,
+        ids=["c1"],
+        field={"c1": {TAG_FLD: tag_feas, PAGERANK_FLD: 0}},
+    )
+
+
+def test_rank_feature_scores_parses_python_dict_string():
+    dealer = Dealer.__new__(Dealer)
+    sres = _make_search_res("{'apple': 2.0}")
+    scores = dealer._rank_feature_scores({"apple": 1.0}, sres)
+    assert np.isclose(scores[0], 10.0)
+
+
+def test_rank_feature_scores_parses_json_string():
+    dealer = Dealer.__new__(Dealer)
+    sres = _make_search_res('{"apple": 2.0}')
+    scores = dealer._rank_feature_scores({"apple": 1.0}, sres)
+    assert np.isclose(scores[0], 10.0)
+
+
+def test_rank_feature_scores_handles_dict_value():
+    dealer = Dealer.__new__(Dealer)
+    sres = _make_search_res({"apple": 2.0})
+    scores = dealer._rank_feature_scores({"apple": 1.0}, sres)
+    assert np.isclose(scores[0], 10.0)
+
+
+def test_rank_feature_scores_ignores_invalid_tag_feas_string():
+    dealer = Dealer.__new__(Dealer)
+    sres = _make_search_res("not a dict")
+    scores = dealer._rank_feature_scores({"apple": 1.0}, sres)
+    assert np.isclose(scores[0], 0.0)
+
+
+def test_rank_feature_scores_ignores_executable_tag_feas_string():
+    dealer = Dealer.__new__(Dealer)
+    sres = _make_search_res('{"apple": (__import__("time").sleep(1) or 1.0)}')
+    scores = dealer._rank_feature_scores({"apple": 1.0}, sres)
+    assert np.isclose(scores[0], 0.0)
+
+
+def test_rank_feature_scores_returns_pagerank_when_no_tag_feature():
+    dealer = Dealer.__new__(Dealer)
+    sres = _make_search_res("{'apple': 2.0}")
+    scores = dealer._rank_feature_scores({PAGERANK_FLD: 10}, sres)
+    assert np.isclose(scores[0], 0.0)