mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-10 13:45:44 +08:00
Fix tag_feas code injection in retrieval ranking (#13923)
## Summary - remove eval-based parsing from retrieval rank feature scoring - validate `tag_feas` at write time in chunk APIs and SDK routes - add regression tests for safe parsing and malicious payload rejection ## Details `tag_feas` is intended to be structured rank-feature data, but the retrieval ranking path was evaluating stored values as Python expressions. This change treats `tag_feas` strictly as data. ### What changed - replace `eval()` in `rag/nlp/search.py` with safe parsing via `json.loads()` and optional `ast.literal_eval()` compatibility for legacy Python-dict strings - strictly filter parsed values down to `dict[str, finite number]` - reject invalid `tag_feas` payloads at write time in web chunk routes and SDK document chunk routes - add focused regression tests to prove executable strings are ignored and invalid payloads are rejected ## Validation - `python -m pytest test/unit_test/common/test_tag_feature_utils.py test/unit_test/rag/test_rank_feature_scores.py -q` --------- Co-authored-by: unknown <zhenglinkai@CCN.Local> Co-authored-by: Yingfeng Zhang <yingfeng.zhang@gmail.com>
This commit is contained in:
@@ -38,6 +38,7 @@ from api.utils.api_utils import (
|
||||
get_request_json,
|
||||
)
|
||||
from common.misc_utils import thread_pool_exec
|
||||
from common.tag_feature_utils import validate_tag_features
|
||||
from rag.app.qa import beAdoc, rmPrefix
|
||||
from rag.app.tag import label_question
|
||||
from rag.nlp import rag_tokenizer, search
|
||||
@@ -161,7 +162,10 @@ async def set():
|
||||
return get_data_error_result(message="`tag_kwd` must be a list of strings")
|
||||
d["tag_kwd"] = req["tag_kwd"]
|
||||
if "tag_feas" in req:
|
||||
d["tag_feas"] = req["tag_feas"]
|
||||
try:
|
||||
d["tag_feas"] = validate_tag_features(req["tag_feas"])
|
||||
except ValueError as exc:
|
||||
return get_data_error_result(message=f"`tag_feas` {exc}")
|
||||
if "available_int" in req:
|
||||
d["available_int"] = req["available_int"]
|
||||
|
||||
@@ -328,7 +332,10 @@ async def create():
|
||||
return get_data_error_result(message="`tag_kwd` must be a list of strings")
|
||||
d["tag_kwd"] = req["tag_kwd"]
|
||||
if "tag_feas" in req:
|
||||
d["tag_feas"] = req["tag_feas"]
|
||||
try:
|
||||
d["tag_feas"] = validate_tag_features(req["tag_feas"])
|
||||
except ValueError as exc:
|
||||
return get_data_error_result(message=f"`tag_feas` {exc}")
|
||||
image_base64 = req.get("image_base64", None)
|
||||
|
||||
try:
|
||||
|
||||
@@ -39,6 +39,7 @@ from common.constants import FileSource, LLMType, ParserType, RetCode, TaskStatu
|
||||
from common.metadata_utils import convert_conditions, meta_filter
|
||||
from common.misc_utils import thread_pool_exec
|
||||
from common.string_utils import remove_redundant_spaces
|
||||
from common.tag_feature_utils import validate_tag_features
|
||||
from rag.app.qa import beAdoc, rmPrefix
|
||||
from rag.app.tag import label_question
|
||||
from rag.nlp import rag_tokenizer, search
|
||||
@@ -963,7 +964,10 @@ async def add_chunk(tenant_id, dataset_id, document_id):
|
||||
return get_error_data_result("`tag_kwd` must be a list of strings")
|
||||
d["tag_kwd"] = req["tag_kwd"]
|
||||
if "tag_feas" in req:
|
||||
d["tag_feas"] = req["tag_feas"]
|
||||
try:
|
||||
d["tag_feas"] = validate_tag_features(req["tag_feas"])
|
||||
except ValueError as exc:
|
||||
return get_error_data_result(f"`tag_feas` {exc}")
|
||||
import base64
|
||||
|
||||
image_base64 = req.get("image_base64", None)
|
||||
@@ -1202,7 +1206,10 @@ async def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
|
||||
return get_error_data_result("`tag_kwd` must be a list of strings")
|
||||
d["tag_kwd"] = req["tag_kwd"]
|
||||
if "tag_feas" in req:
|
||||
d["tag_feas"] = req["tag_feas"]
|
||||
try:
|
||||
d["tag_feas"] = validate_tag_features(req["tag_feas"])
|
||||
except ValueError as exc:
|
||||
return get_error_data_result(f"`tag_feas` {exc}")
|
||||
tenant_embd_id = DocumentService.get_tenant_embd_id(document_id)
|
||||
if tenant_embd_id:
|
||||
model_config = get_model_config_by_id(tenant_embd_id)
|
||||
|
||||
Reference in New Issue
Block a user