zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-02 00:35:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove evaluation_app.py and kb_app.py (#14394)

Browse Source 
### What problem does this PR solve?

Delete not used APIs

### Type of change

- [x] Refactoring
This commit is contained in:
Wang Qi
2026-04-27 16:08:54 +08:00
committed by GitHub
parent 290f0294d6
commit d88f7ac8d2
3 changed files with 0 additions and 1500 deletions
Download Patch File Download Diff File Expand all files Collapse all files

479
api/apps/evaluation_app.py
View File

@@ -1,479 +0,0 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
RAG Evaluation API Endpoints
Provides REST API for RAG evaluation functionality including:
- Dataset management
- Test case management
- Evaluation execution
- Results retrieval
- Configuration recommendations
"""
from quart import request
from api.apps import login_required, current_user
from api.db.services.evaluation_service import EvaluationService
from api.utils.api_utils import (
get_data_error_result,
get_json_result,
get_request_json,
server_error_response,
validate_request
)
from common.constants import RetCode
# ==================== Dataset Management ====================
@manager.route('/dataset/create', methods=['POST']) # noqa: F821
@login_required
@validate_request("name", "kb_ids")
async def create_dataset():
"""
Create a new evaluation dataset.
Request body:
{
"name": "Dataset name",
"description": "Optional description",
"kb_ids": ["kb_id1", "kb_id2"]
}
"""
try:
req = await get_request_json()
name = req.get("name", "").strip()
description = req.get("description", "")
kb_ids = req.get("kb_ids", [])
if not name:
return get_data_error_result(message="Dataset name cannot be empty")
if not kb_ids or not isinstance(kb_ids, list):
return get_data_error_result(message="kb_ids must be a non-empty list")
success, result = EvaluationService.create_dataset(
name=name,
description=description,
kb_ids=kb_ids,
tenant_id=current_user.id,
user_id=current_user.id
)
if not success:
return get_data_error_result(message=result)
return get_json_result(data={"dataset_id": result})
except Exception as e:
return server_error_response(e)
@manager.route('/dataset/list', methods=['GET']) # noqa: F821
@login_required
async def list_datasets():
"""
List evaluation datasets for current tenant.
Query params:
- page: Page number (default: 1)
- page_size: Items per page (default: 20)
"""
try:
page = int(request.args.get("page", 1))
page_size = int(request.args.get("page_size", 20))
result = EvaluationService.list_datasets(
tenant_id=current_user.id,
user_id=current_user.id,
page=page,
page_size=page_size
)
return get_json_result(data=result)
except Exception as e:
return server_error_response(e)
@manager.route('/dataset/<dataset_id>', methods=['GET']) # noqa: F821
@login_required
async def get_dataset(dataset_id):
"""Get dataset details by ID"""
try:
dataset = EvaluationService.get_dataset(dataset_id)
if not dataset:
return get_data_error_result(
message="Dataset not found",
code=RetCode.DATA_ERROR
)
return get_json_result(data=dataset)
except Exception as e:
return server_error_response(e)
@manager.route('/dataset/<dataset_id>', methods=['PUT']) # noqa: F821
@login_required
async def update_dataset(dataset_id):
"""
Update dataset.
Request body:
{
"name": "New name",
"description": "New description",
"kb_ids": ["kb_id1", "kb_id2"]
}
"""
try:
req = await get_request_json()
# Remove fields that shouldn't be updated
req.pop("id", None)
req.pop("tenant_id", None)
req.pop("created_by", None)
req.pop("create_time", None)
success = EvaluationService.update_dataset(dataset_id, **req)
if not success:
return get_data_error_result(message="Failed to update dataset")
return get_json_result(data={"dataset_id": dataset_id})
except Exception as e:
return server_error_response(e)
@manager.route('/dataset/<dataset_id>', methods=['DELETE']) # noqa: F821
@login_required
async def delete_dataset(dataset_id):
"""Delete dataset (soft delete)"""
try:
success = EvaluationService.delete_dataset(dataset_id)
if not success:
return get_data_error_result(message="Failed to delete dataset")
return get_json_result(data={"dataset_id": dataset_id})
except Exception as e:
return server_error_response(e)
# ==================== Test Case Management ====================
@manager.route('/dataset/<dataset_id>/case/add', methods=['POST']) # noqa: F821
@login_required
@validate_request("question")
async def add_test_case(dataset_id):
"""
Add a test case to a dataset.
Request body:
{
"question": "Test question",
"reference_answer": "Optional ground truth answer",
"relevant_doc_ids": ["doc_id1", "doc_id2"],
"relevant_chunk_ids": ["chunk_id1", "chunk_id2"],
"metadata": {"key": "value"}
}
"""
try:
req = await get_request_json()
question = req.get("question", "").strip()
if not question:
return get_data_error_result(message="Question cannot be empty")
success, result = EvaluationService.add_test_case(
dataset_id=dataset_id,
question=question,
reference_answer=req.get("reference_answer"),
relevant_doc_ids=req.get("relevant_doc_ids"),
relevant_chunk_ids=req.get("relevant_chunk_ids"),
metadata=req.get("metadata")
)
if not success:
return get_data_error_result(message=result)
return get_json_result(data={"case_id": result})
except Exception as e:
return server_error_response(e)
@manager.route('/dataset/<dataset_id>/case/import', methods=['POST']) # noqa: F821
@login_required
@validate_request("cases")
async def import_test_cases(dataset_id):
"""
Bulk import test cases.
Request body:
{
"cases": [
{
"question": "Question 1",
"reference_answer": "Answer 1",
...
},
{
"question": "Question 2",
...
}
]
}
"""
try:
req = await get_request_json()
cases = req.get("cases", [])
if not cases or not isinstance(cases, list):
return get_data_error_result(message="cases must be a non-empty list")
success_count, failure_count = EvaluationService.import_test_cases(
dataset_id=dataset_id,
cases=cases
)
return get_json_result(data={
"success_count": success_count,
"failure_count": failure_count,
"total": len(cases)
})
except Exception as e:
return server_error_response(e)
@manager.route('/dataset/<dataset_id>/cases', methods=['GET']) # noqa: F821
@login_required
async def get_test_cases(dataset_id):
"""Get all test cases for a dataset"""
try:
cases = EvaluationService.get_test_cases(dataset_id)
return get_json_result(data={"cases": cases, "total": len(cases)})
except Exception as e:
return server_error_response(e)
@manager.route('/case/<case_id>', methods=['DELETE']) # noqa: F821
@login_required
async def delete_test_case(case_id):
"""Delete a test case"""
try:
success = EvaluationService.delete_test_case(case_id)
if not success:
return get_data_error_result(message="Failed to delete test case")
return get_json_result(data={"case_id": case_id})
except Exception as e:
return server_error_response(e)
# ==================== Evaluation Execution ====================
@manager.route('/run/start', methods=['POST']) # noqa: F821
@login_required
@validate_request("dataset_id", "dialog_id")
async def start_evaluation():
"""
Start an evaluation run.
Request body:
{
"dataset_id": "dataset_id",
"dialog_id": "dialog_id",
"name": "Optional run name"
}
"""
try:
req = await get_request_json()
dataset_id = req.get("dataset_id")
dialog_id = req.get("dialog_id")
name = req.get("name")
success, result = EvaluationService.start_evaluation(
dataset_id=dataset_id,
dialog_id=dialog_id,
user_id=current_user.id,
name=name
)
if not success:
return get_data_error_result(message=result)
return get_json_result(data={"run_id": result})
except Exception as e:
return server_error_response(e)
@manager.route('/run/<run_id>', methods=['GET']) # noqa: F821
@login_required
async def get_evaluation_run(run_id):
"""Get evaluation run details"""
try:
result = EvaluationService.get_run_results(run_id)
if not result:
return get_data_error_result(
message="Evaluation run not found",
code=RetCode.DATA_ERROR
)
return get_json_result(data=result)
except Exception as e:
return server_error_response(e)
@manager.route('/run/<run_id>/results', methods=['GET']) # noqa: F821
@login_required
async def get_run_results(run_id):
"""Get detailed results for an evaluation run"""
try:
result = EvaluationService.get_run_results(run_id)
if not result:
return get_data_error_result(
message="Evaluation run not found",
code=RetCode.DATA_ERROR
)
return get_json_result(data=result)
except Exception as e:
return server_error_response(e)
@manager.route('/run/list', methods=['GET']) # noqa: F821
@login_required
async def list_evaluation_runs():
"""
List evaluation runs.
Query params:
- dataset_id: Filter by dataset (optional)
- dialog_id: Filter by dialog (optional)
- page: Page number (default: 1)
- page_size: Items per page (default: 20)
"""
try:
# TODO: Implement list_runs in EvaluationService
return get_json_result(data={"runs": [], "total": 0})
except Exception as e:
return server_error_response(e)
@manager.route('/run/<run_id>', methods=['DELETE']) # noqa: F821
@login_required
async def delete_evaluation_run(run_id):
"""Delete an evaluation run"""
try:
# TODO: Implement delete_run in EvaluationService
return get_json_result(data={"run_id": run_id})
except Exception as e:
return server_error_response(e)
# ==================== Analysis & Recommendations ====================
@manager.route('/run/<run_id>/recommendations', methods=['GET']) # noqa: F821
@login_required
async def get_recommendations(run_id):
"""Get configuration recommendations based on evaluation results"""
try:
recommendations = EvaluationService.get_recommendations(run_id)
return get_json_result(data={"recommendations": recommendations})
except Exception as e:
return server_error_response(e)
@manager.route('/compare', methods=['POST']) # noqa: F821
@login_required
@validate_request("run_ids")
async def compare_runs():
"""
Compare multiple evaluation runs.
Request body:
{
"run_ids": ["run_id1", "run_id2", "run_id3"]
}
"""
try:
req = await get_request_json()
run_ids = req.get("run_ids", [])
if not run_ids or not isinstance(run_ids, list) or len(run_ids) < 2:
return get_data_error_result(
message="run_ids must be a list with at least 2 run IDs"
)
# TODO: Implement compare_runs in EvaluationService
return get_json_result(data={"comparison": {}})
except Exception as e:
return server_error_response(e)
@manager.route('/run/<run_id>/export', methods=['GET']) # noqa: F821
@login_required
async def export_results(run_id):
"""Export evaluation results as JSON/CSV"""
try:
# format_type = request.args.get("format", "json") # TODO: Use for CSV export
result = EvaluationService.get_run_results(run_id)
if not result:
return get_data_error_result(
message="Evaluation run not found",
code=RetCode.DATA_ERROR
)
# TODO: Implement CSV export
return get_json_result(data=result)
except Exception as e:
return server_error_response(e)
# ==================== Real-time Evaluation ====================
@manager.route('/evaluate_single', methods=['POST']) # noqa: F821
@login_required
@validate_request("question", "dialog_id")
async def evaluate_single():
"""
Evaluate a single question-answer pair in real-time.
Request body:
{
"question": "Test question",
"dialog_id": "dialog_id",
"reference_answer": "Optional ground truth",
"relevant_chunk_ids": ["chunk_id1", "chunk_id2"]
}
"""
try:
# req = await get_request_json() # TODO: Use for single evaluation implementation
# TODO: Implement single evaluation
# This would execute the RAG pipeline and return metrics immediately
return get_json_result(data={
"answer": "",
"metrics": {},
"retrieved_chunks": []
})
except Exception as e:
return server_error_response(e)

446
api/apps/kb_app.py
View File

@@ -1,446 +0,0 @@
#
# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Deprecated, todo delete
@manager.route('/create', methods=['post']) # noqa: F821
@login_required
@validate_request("name")
async def create():
req = await get_request_json()
create_dict = ensure_tenant_model_id_for_params(current_user.id, req)
e, res = KnowledgebaseService.create_with_name(
name = create_dict.pop("name", None),
tenant_id = current_user.id,
parser_id = create_dict.pop("parser_id", None),
**create_dict
)
if not e:
return res
try:
if not KnowledgebaseService.save(**res):
return get_data_error_result()
return get_json_result(data={"kb_id":res["id"]})
except Exception as e:
return server_error_response(e)
@manager.route('/update', methods=['post']) # noqa: F821
@login_required
@validate_request("kb_id", "name", "description", "parser_id")
@not_allowed_parameters("id", "tenant_id", "created_by", "create_time", "update_time", "create_date", "update_date", "created_by")
async def update():
req = await get_request_json()
update_dict = ensure_tenant_model_id_for_params(current_user.id, req)
if not isinstance(update_dict["name"], str):
return get_data_error_result(message="Dataset name must be string.")
if update_dict["name"].strip() == "":
return get_data_error_result(message="Dataset name can't be empty.")
if len(update_dict["name"].encode("utf-8")) > DATASET_NAME_LIMIT:
return get_data_error_result(
message=f"Dataset name length is {len(update_dict['name'])} which is large than {DATASET_NAME_LIMIT}")
update_dict["name"] = update_dict["name"].strip()
if settings.DOC_ENGINE_INFINITY:
parser_id = update_dict.get("parser_id")
if isinstance(parser_id, str) and parser_id.lower() == "tag":
return get_json_result(
code=RetCode.OPERATING_ERROR,
message="The chunking method Tag has not been supported by Infinity yet.",
data=False,
)
if "pagerank" in update_dict and update_dict["pagerank"] > 0:
return get_json_result(
code=RetCode.DATA_ERROR,
message="'pagerank' can only be set when doc_engine is elasticsearch",
data=False,
)
if not KnowledgebaseService.accessible4deletion(update_dict["kb_id"], current_user.id):
return get_json_result(
data=False,
message='No authorization.',
code=RetCode.AUTHENTICATION_ERROR
)
try:
if not KnowledgebaseService.query(
created_by=current_user.id, id=update_dict["kb_id"]):
return get_json_result(
data=False, message='Only owner of dataset authorized for this operation.',
code=RetCode.OPERATING_ERROR)
e, kb = KnowledgebaseService.get_by_id(update_dict["kb_id"])
# Rename folder in FileService
if e and update_dict["name"].lower() != kb.name.lower():
FileService.filter_update(
[
File.tenant_id == kb.tenant_id,
File.source_type == FileSource.KNOWLEDGEBASE,
File.type == "folder",
File.name == kb.name,
],
{"name": update_dict["name"]},
)
if not e:
return get_data_error_result(
message="Can't find this dataset!")
if update_dict["name"].lower() != kb.name.lower() \
and len(
KnowledgebaseService.query(name=update_dict["name"], tenant_id=current_user.id, status=StatusEnum.VALID.value)) >= 1:
return get_data_error_result(
message="Duplicated dataset name.")
del update_dict["kb_id"]
connectors = []
if "connectors" in update_dict:
connectors = update_dict["connectors"]
del update_dict["connectors"]
if not KnowledgebaseService.update_by_id(kb.id, update_dict):
return get_data_error_result()
if kb.pagerank != update_dict.get("pagerank", 0):
if update_dict.get("pagerank", 0) > 0:
await thread_pool_exec(
settings.docStoreConn.update,
{"kb_id": kb.id},
{PAGERANK_FLD: update_dict["pagerank"]},
search.index_name(kb.tenant_id),
kb.id,
)
else:
# Elasticsearch requires PAGERANK_FLD be non-zero!
await thread_pool_exec(
settings.docStoreConn.update,
{"exists": PAGERANK_FLD},
{"remove": PAGERANK_FLD},
search.index_name(kb.tenant_id),
kb.id,
)
e, kb = KnowledgebaseService.get_by_id(kb.id)
if not e:
return get_data_error_result(
message="Database error (Knowledgebase rename)!")
errors = Connector2KbService.link_connectors(kb.id, [conn for conn in connectors], current_user.id)
if errors:
logging.error("Link KB errors: ", errors)
kb = kb.to_dict()
kb.update(update_dict)
kb["connectors"] = connectors
return get_json_result(data=kb)
except Exception as e:
return server_error_response(e)
"""
"""
Deprecated, todo delete
@manager.route('/list', methods=['POST']) # noqa: F821
@login_required
async def list_kbs():
args = request.args
keywords = args.get("keywords", "")
page_number = int(args.get("page", 0))
items_per_page = int(args.get("page_size", 0))
parser_id = args.get("parser_id")
orderby = args.get("orderby", "create_time")
if args.get("desc", "true").lower() == "false":
desc = False
else:
desc = True
req = await get_request_json()
owner_ids = req.get("owner_ids", [])
try:
if not owner_ids:
tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
tenants = [m["tenant_id"] for m in tenants]
kbs, total = KnowledgebaseService.get_by_tenant_ids(
tenants, current_user.id, page_number,
items_per_page, orderby, desc, keywords, parser_id)
else:
tenants = owner_ids
kbs, total = KnowledgebaseService.get_by_tenant_ids(
tenants, current_user.id, 0,
0, orderby, desc, keywords, parser_id)
kbs = [kb for kb in kbs if kb["tenant_id"] in tenants]
total = len(kbs)
if page_number and items_per_page:
kbs = kbs[(page_number-1)*items_per_page:page_number*items_per_page]
return get_json_result(data={"kbs": kbs, "total": total})
except Exception as e:
return server_error_response(e)
@manager.route('/rm', methods=['post']) # noqa: F821
@login_required
@validate_request("kb_id")
async def rm():
req = await get_request_json()
uid = current_user.id
if not KnowledgebaseService.accessible4deletion(req["kb_id"], uid):
return get_json_result(
data=False,
message='No authorization.',
code=RetCode.AUTHENTICATION_ERROR
)
try:
kbs = KnowledgebaseService.query(
created_by=uid, id=req["kb_id"])
if not kbs:
return get_json_result(
data=False, message='Only owner of dataset authorized for this operation.',
code=RetCode.OPERATING_ERROR)
def _rm_sync():
for doc in DocumentService.query(kb_id=req["kb_id"]):
if not DocumentService.remove_document(doc, kbs[0].tenant_id):
return get_data_error_result(
message="Database error (Document removal)!")
f2d = File2DocumentService.get_by_document_id(doc.id)
if f2d:
FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
File2DocumentService.delete_by_document_id(doc.id)
FileService.filter_delete(
[
File.tenant_id == kbs[0].tenant_id,
File.source_type == FileSource.KNOWLEDGEBASE,
File.type == "folder",
File.name == kbs[0].name,
]
)
# Delete the table BEFORE deleting the database record
for kb in kbs:
try:
settings.docStoreConn.delete({"kb_id": kb.id}, search.index_name(kb.tenant_id), kb.id)
settings.docStoreConn.delete_idx(search.index_name(kb.tenant_id), kb.id)
logging.info(f"Dropped index for dataset {kb.id}")
except Exception as e:
logging.error(f"Failed to drop index for dataset {kb.id}: {e}")
if not KnowledgebaseService.delete_by_id(req["kb_id"]):
return get_data_error_result(
message="Database error (Knowledgebase removal)!")
for kb in kbs:
if hasattr(settings.STORAGE_IMPL, 'remove_bucket'):
settings.STORAGE_IMPL.remove_bucket(kb.id)
return get_json_result(data=True)
return await thread_pool_exec(_rm_sync)
except Exception as e:
return server_error_response(e)
"""
"""
Deprecated, todo delete
@manager.route('/<kb_id>/knowledge_graph', methods=['GET']) # noqa: F821
@login_required
async def knowledge_graph(kb_id):
if not KnowledgebaseService.accessible(kb_id, current_user.id):
return get_json_result(
data=False,
message='No authorization.',
code=RetCode.AUTHENTICATION_ERROR
)
_, kb = KnowledgebaseService.get_by_id(kb_id)
req = {
"kb_id": [kb_id],
"knowledge_graph_kwd": ["graph"]
}
obj = {"graph": {}, "mind_map": {}}
if not settings.docStoreConn.index_exist(search.index_name(kb.tenant_id), kb_id):
return get_json_result(data=obj)
sres = await settings.retriever.search(req, search.index_name(kb.tenant_id), [kb_id])
if not len(sres.ids):
return get_json_result(data=obj)
for id in sres.ids[:1]:
ty = sres.field[id]["knowledge_graph_kwd"]
try:
content_json = json.loads(sres.field[id]["content_with_weight"])
except Exception:
continue
obj[ty] = content_json
if "nodes" in obj["graph"]:
obj["graph"]["nodes"] = sorted(obj["graph"]["nodes"], key=lambda x: x.get("pagerank", 0), reverse=True)[:256]
if "edges" in obj["graph"]:
node_id_set = { o["id"] for o in obj["graph"]["nodes"] }
filtered_edges = [o for o in obj["graph"]["edges"] if o["source"] != o["target"] and o["source"] in node_id_set and o["target"] in node_id_set]
obj["graph"]["edges"] = sorted(filtered_edges, key=lambda x: x.get("weight", 0), reverse=True)[:128]
return get_json_result(data=obj)
@manager.route('/<kb_id>/knowledge_graph', methods=['DELETE']) # noqa: F821
@login_required
def delete_knowledge_graph(kb_id):
if not KnowledgebaseService.accessible(kb_id, current_user.id):
return get_json_result(
data=False,
message='No authorization.',
code=RetCode.AUTHENTICATION_ERROR
)
_, kb = KnowledgebaseService.get_by_id(kb_id)
settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]}, search.index_name(kb.tenant_id), kb_id)
return get_json_result(data=True)
"""
"""
Deprecated, todo delete
@manager.route("/run_graphrag", methods=["POST"]) # noqa: F821
@login_required
async def run_graphrag():
req = await get_request_json()
kb_id = req.get("kb_id", "")
if not kb_id:
return get_error_data_result(message='Lack of "KB ID"')
ok, kb = KnowledgebaseService.get_by_id(kb_id)
if not ok:
return get_error_data_result(message="Invalid Knowledgebase ID")
task_id = kb.graphrag_task_id
if task_id:
ok, task = TaskService.get_by_id(task_id)
if not ok:
logging.warning(f"A valid GraphRAG task id is expected for kb {kb_id}")
if task and task.progress not in [-1, 1]:
return get_error_data_result(message=f"Task {task_id} in progress with status {task.progress}. A Graph Task is already running.")
documents, _ = DocumentService.get_by_kb_id(
kb_id=kb_id,
page_number=0,
items_per_page=0,
orderby="create_time",
desc=False,
keywords="",
run_status=[],
types=[],
suffix=[],
)
if not documents:
return get_error_data_result(message=f"No documents in Knowledgebase {kb_id}")
sample_document = documents[0]
document_ids = [document["id"] for document in documents]
task_id = queue_raptor_o_graphrag_tasks(sample_doc_id=sample_document, ty="graphrag", priority=0, fake_doc_id=GRAPH_RAPTOR_FAKE_DOC_ID, doc_ids=list(document_ids))
if not KnowledgebaseService.update_by_id(kb.id, {"graphrag_task_id": task_id}):
logging.warning(f"Cannot save graphrag_task_id for kb {kb_id}")
return get_json_result(data={"graphrag_task_id": task_id})
@manager.route("/trace_graphrag", methods=["GET"]) # noqa: F821
@login_required
def trace_graphrag():
kb_id = request.args.get("kb_id", "")
if not kb_id:
return get_error_data_result(message='Lack of "KB ID"')
ok, kb = KnowledgebaseService.get_by_id(kb_id)
if not ok:
return get_error_data_result(message="Invalid Knowledgebase ID")
task_id = kb.graphrag_task_id
if not task_id:
return get_json_result(data={})
ok, task = TaskService.get_by_id(task_id)
if not ok:
return get_json_result(data={})
return get_json_result(data=task.to_dict())
@manager.route("/run_raptor", methods=["POST"]) # noqa: F821
@login_required
async def run_raptor():
req = await get_request_json()
kb_id = req.get("kb_id", "")
if not kb_id:
return get_error_data_result(message='Lack of "KB ID"')
ok, kb = KnowledgebaseService.get_by_id(kb_id)
if not ok:
return get_error_data_result(message="Invalid Knowledgebase ID")
task_id = kb.raptor_task_id
if task_id:
ok, task = TaskService.get_by_id(task_id)
if not ok:
logging.warning(f"A valid RAPTOR task id is expected for kb {kb_id}")
if task and task.progress not in [-1, 1]:
return get_error_data_result(message=f"Task {task_id} in progress with status {task.progress}. A RAPTOR Task is already running.")
documents, _ = DocumentService.get_by_kb_id(
kb_id=kb_id,
page_number=0,
items_per_page=0,
orderby="create_time",
desc=False,
keywords="",
run_status=[],
types=[],
suffix=[],
)
if not documents:
return get_error_data_result(message=f"No documents in Knowledgebase {kb_id}")
sample_document = documents[0]
document_ids = [document["id"] for document in documents]
task_id = queue_raptor_o_graphrag_tasks(sample_doc_id=sample_document, ty="raptor", priority=0, fake_doc_id=GRAPH_RAPTOR_FAKE_DOC_ID, doc_ids=list(document_ids))
if not KnowledgebaseService.update_by_id(kb.id, {"raptor_task_id": task_id}):
logging.warning(f"Cannot save raptor_task_id for kb {kb_id}")
return get_json_result(data={"raptor_task_id": task_id})
@manager.route("/trace_raptor", methods=["GET"]) # noqa: F821
@login_required
def trace_raptor():
kb_id = request.args.get("kb_id", "")
if not kb_id:
return get_error_data_result(message='Lack of "KB ID"')
ok, kb = KnowledgebaseService.get_by_id(kb_id)
if not ok:
return get_error_data_result(message="Invalid Knowledgebase ID")
task_id = kb.raptor_task_id
if not task_id:
return get_json_result(data={})
ok, task = TaskService.get_by_id(task_id)
if not ok:
return get_error_data_result(message="RAPTOR Task Not Found or Error Occurred")
return get_json_result(data=task.to_dict())
"""

575
test/testcases/test_web_api/test_evaluation_app/test_evaluation_routes_unit.py
View File

@@ -1,575 +0,0 @@
#
# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import asyncio
import importlib.util
import sys
from pathlib import Path
from types import ModuleType, SimpleNamespace
import pytest
class _DummyManager:
def route(self, *_args, **_kwargs):
def decorator(func):
return func
return decorator
class _Args(dict):
def get(self, key, default=None):
return super().get(key, default)
class _DummyRetCode:
SUCCESS = 0
EXCEPTION_ERROR = 100
ARGUMENT_ERROR = 101
DATA_ERROR = 102
OPERATING_ERROR = 103
AUTHENTICATION_ERROR = 109
def _run(coro):
return asyncio.run(coro)
def _set_request_json(monkeypatch, module, payload):
async def _request_json():
return payload
monkeypatch.setattr(module, "get_request_json", _request_json)
def _set_request_args(monkeypatch, module, args=None):
monkeypatch.setattr(module, "request", SimpleNamespace(args=_Args(args or {})))
@pytest.fixture(scope="session")
def auth():
return "unit-auth"
@pytest.fixture(scope="session", autouse=True)
def set_tenant_info():
return None
def _load_evaluation_app(monkeypatch):
repo_root = Path(__file__).resolve().parents[4]
quart_mod = ModuleType("quart")
quart_mod.request = SimpleNamespace(args=_Args())
monkeypatch.setitem(sys.modules, "quart", quart_mod)
common_pkg = ModuleType("common")
common_pkg.__path__ = [str(repo_root / "common")]
monkeypatch.setitem(sys.modules, "common", common_pkg)
constants_mod = ModuleType("common.constants")
constants_mod.RetCode = _DummyRetCode
monkeypatch.setitem(sys.modules, "common.constants", constants_mod)
common_pkg.constants = constants_mod
api_pkg = ModuleType("api")
api_pkg.__path__ = [str(repo_root / "api")]
monkeypatch.setitem(sys.modules, "api", api_pkg)
apps_mod = ModuleType("api.apps")
apps_mod.__path__ = [str(repo_root / "api" / "apps")]
apps_mod.current_user = SimpleNamespace(id="tenant-1")
apps_mod.login_required = lambda func: func
monkeypatch.setitem(sys.modules, "api.apps", apps_mod)
api_pkg.apps = apps_mod
db_pkg = ModuleType("api.db")
db_pkg.__path__ = []
monkeypatch.setitem(sys.modules, "api.db", db_pkg)
api_pkg.db = db_pkg
services_pkg = ModuleType("api.db.services")
services_pkg.__path__ = []
monkeypatch.setitem(sys.modules, "api.db.services", services_pkg)
evaluation_service_mod = ModuleType("api.db.services.evaluation_service")
class _EvaluationService:
@staticmethod
def create_dataset(**_kwargs):
return True, "dataset-1"
@staticmethod
def list_datasets(**_kwargs):
return {"datasets": [], "total": 0}
@staticmethod
def get_dataset(_dataset_id):
return {"id": _dataset_id}
@staticmethod
def update_dataset(_dataset_id, **_kwargs):
return True
@staticmethod
def delete_dataset(_dataset_id):
return True
@staticmethod
def add_test_case(**_kwargs):
return True, "case-1"
@staticmethod
def import_test_cases(**_kwargs):
return 0, 0
@staticmethod
def get_test_cases(_dataset_id):
return []
@staticmethod
def delete_test_case(_case_id):
return True
@staticmethod
def start_evaluation(**_kwargs):
return True, "run-1"
@staticmethod
def get_run_results(_run_id):
return {"id": _run_id}
@staticmethod
def get_recommendations(_run_id):
return []
evaluation_service_mod.EvaluationService = _EvaluationService
monkeypatch.setitem(sys.modules, "api.db.services.evaluation_service", evaluation_service_mod)
utils_pkg = ModuleType("api.utils")
utils_pkg.__path__ = []
monkeypatch.setitem(sys.modules, "api.utils", utils_pkg)
api_utils_mod = ModuleType("api.utils.api_utils")
async def _default_request_json():
return {}
def _get_data_error_result(code=_DummyRetCode.DATA_ERROR, message="Sorry! Data missing!"):
return {"code": code, "message": message}
def _get_json_result(code=_DummyRetCode.SUCCESS, message="success", data=None):
return {"code": code, "message": message, "data": data}
def _server_error_response(error):
return {"code": _DummyRetCode.EXCEPTION_ERROR, "message": repr(error)}
def _validate_request(*_args, **_kwargs):
def _decorator(func):
return func
return _decorator
api_utils_mod.get_data_error_result = _get_data_error_result
api_utils_mod.get_json_result = _get_json_result
api_utils_mod.get_request_json = _default_request_json
api_utils_mod.server_error_response = _server_error_response
api_utils_mod.validate_request = _validate_request
monkeypatch.setitem(sys.modules, "api.utils.api_utils", api_utils_mod)
utils_pkg.api_utils = api_utils_mod
module_name = "test_evaluation_routes_unit_module"
module_path = repo_root / "api" / "apps" / "evaluation_app.py"
spec = importlib.util.spec_from_file_location(module_name, module_path)
module = importlib.util.module_from_spec(spec)
module.manager = _DummyManager()
monkeypatch.setitem(sys.modules, module_name, module)
spec.loader.exec_module(module)
return module
@pytest.mark.p2
def test_dataset_routes_matrix_unit(monkeypatch):
module = _load_evaluation_app(monkeypatch)
_set_request_json(monkeypatch, module, {"name": " data-1 ", "description": "desc", "kb_ids": ["kb-1"]})
monkeypatch.setattr(module.EvaluationService, "create_dataset", lambda **_kwargs: (True, "dataset-ok"))
res = _run(module.create_dataset())
assert res["code"] == 0
assert res["data"]["dataset_id"] == "dataset-ok"
_set_request_json(monkeypatch, module, {"name": " ", "kb_ids": ["kb-1"]})
res = _run(module.create_dataset())
assert res["code"] == module.RetCode.DATA_ERROR
assert "empty" in res["message"].lower()
_set_request_json(monkeypatch, module, {"name": "data-2", "kb_ids": "kb-1"})
res = _run(module.create_dataset())
assert res["code"] == module.RetCode.DATA_ERROR
assert "kb_ids" in res["message"]
_set_request_json(monkeypatch, module, {"name": "data-3", "kb_ids": ["kb-1"]})
monkeypatch.setattr(module.EvaluationService, "create_dataset", lambda **_kwargs: (False, "create failed"))
res = _run(module.create_dataset())
assert res["code"] == module.RetCode.DATA_ERROR
assert res["message"] == "create failed"
def _raise_create(**_kwargs):
raise RuntimeError("create boom")
monkeypatch.setattr(module.EvaluationService, "create_dataset", _raise_create)
res = _run(module.create_dataset())
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "create boom" in res["message"]
_set_request_args(monkeypatch, module, {"page": "2", "page_size": "3"})
monkeypatch.setattr(module.EvaluationService, "list_datasets", lambda **_kwargs: {"datasets": [{"id": "a"}], "total": 1})
res = _run(module.list_datasets())
assert res["code"] == 0
assert res["data"]["total"] == 1
_set_request_args(monkeypatch, module, {"page": "x"})
res = _run(module.list_datasets())
assert res["code"] == module.RetCode.EXCEPTION_ERROR
monkeypatch.setattr(module.EvaluationService, "get_dataset", lambda _dataset_id: None)
res = _run(module.get_dataset("dataset-1"))
assert res["code"] == module.RetCode.DATA_ERROR
assert "not found" in res["message"].lower()
monkeypatch.setattr(module.EvaluationService, "get_dataset", lambda _dataset_id: {"id": _dataset_id})
res = _run(module.get_dataset("dataset-2"))
assert res["code"] == 0
assert res["data"]["id"] == "dataset-2"
def _raise_get(_dataset_id):
raise RuntimeError("get dataset boom")
monkeypatch.setattr(module.EvaluationService, "get_dataset", _raise_get)
res = _run(module.get_dataset("dataset-3"))
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "get dataset boom" in res["message"]
captured = {}
def _update(dataset_id, **kwargs):
captured["dataset_id"] = dataset_id
captured["kwargs"] = kwargs
return True
_set_request_json(
monkeypatch,
module,
{
"id": "forbidden",
"tenant_id": "forbidden",
"created_by": "forbidden",
"create_time": 123,
"name": "new-name",
},
)
monkeypatch.setattr(module.EvaluationService, "update_dataset", _update)
res = _run(module.update_dataset("dataset-4"))
assert res["code"] == 0
assert res["data"]["dataset_id"] == "dataset-4"
assert captured["dataset_id"] == "dataset-4"
assert "id" not in captured["kwargs"]
assert "tenant_id" not in captured["kwargs"]
assert "created_by" not in captured["kwargs"]
assert "create_time" not in captured["kwargs"]
_set_request_json(monkeypatch, module, {"name": "new-name"})
monkeypatch.setattr(module.EvaluationService, "update_dataset", lambda _dataset_id, **_kwargs: False)
res = _run(module.update_dataset("dataset-5"))
assert res["code"] == module.RetCode.DATA_ERROR
assert "failed" in res["message"].lower()
def _raise_update(_dataset_id, **_kwargs):
raise RuntimeError("update boom")
monkeypatch.setattr(module.EvaluationService, "update_dataset", _raise_update)
res = _run(module.update_dataset("dataset-6"))
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "update boom" in res["message"]
monkeypatch.setattr(module.EvaluationService, "delete_dataset", lambda _dataset_id: False)
res = _run(module.delete_dataset("dataset-7"))
assert res["code"] == module.RetCode.DATA_ERROR
assert "failed" in res["message"].lower()
monkeypatch.setattr(module.EvaluationService, "delete_dataset", lambda _dataset_id: True)
res = _run(module.delete_dataset("dataset-8"))
assert res["code"] == 0
assert res["data"]["dataset_id"] == "dataset-8"
def _raise_delete(_dataset_id):
raise RuntimeError("delete dataset boom")
monkeypatch.setattr(module.EvaluationService, "delete_dataset", _raise_delete)
res = _run(module.delete_dataset("dataset-9"))
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "delete dataset boom" in res["message"]
@pytest.mark.p2
def test_test_case_routes_matrix_unit(monkeypatch):
module = _load_evaluation_app(monkeypatch)
_set_request_json(monkeypatch, module, {"question": " "})
res = _run(module.add_test_case("dataset-1"))
assert res["code"] == module.RetCode.DATA_ERROR
assert "question" in res["message"].lower()
_set_request_json(monkeypatch, module, {"question": "q1"})
monkeypatch.setattr(module.EvaluationService, "add_test_case", lambda **_kwargs: (False, "add failed"))
res = _run(module.add_test_case("dataset-2"))
assert res["code"] == module.RetCode.DATA_ERROR
assert "add failed" in res["message"]
_set_request_json(
monkeypatch,
module,
{
"question": "q2",
"reference_answer": "a2",
"relevant_doc_ids": ["doc-1"],
"relevant_chunk_ids": ["chunk-1"],
"metadata": {"k": "v"},
},
)
monkeypatch.setattr(module.EvaluationService, "add_test_case", lambda **_kwargs: (True, "case-ok"))
res = _run(module.add_test_case("dataset-3"))
assert res["code"] == 0
assert res["data"]["case_id"] == "case-ok"
def _raise_add(**_kwargs):
raise RuntimeError("add case boom")
monkeypatch.setattr(module.EvaluationService, "add_test_case", _raise_add)
res = _run(module.add_test_case("dataset-4"))
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "add case boom" in res["message"]
_set_request_json(monkeypatch, module, {"cases": {}})
res = _run(module.import_test_cases("dataset-5"))
assert res["code"] == module.RetCode.DATA_ERROR
assert "cases" in res["message"]
_set_request_json(monkeypatch, module, {"cases": [{"question": "q1"}, {"question": "q2"}]})
monkeypatch.setattr(module.EvaluationService, "import_test_cases", lambda **_kwargs: (2, 0))
res = _run(module.import_test_cases("dataset-6"))
assert res["code"] == 0
assert res["data"]["success_count"] == 2
assert res["data"]["failure_count"] == 0
assert res["data"]["total"] == 2
def _raise_import(**_kwargs):
raise RuntimeError("import boom")
monkeypatch.setattr(module.EvaluationService, "import_test_cases", _raise_import)
res = _run(module.import_test_cases("dataset-7"))
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "import boom" in res["message"]
monkeypatch.setattr(module.EvaluationService, "get_test_cases", lambda _dataset_id: [{"id": "case-1"}])
res = _run(module.get_test_cases("dataset-8"))
assert res["code"] == 0
assert res["data"]["total"] == 1
assert res["data"]["cases"][0]["id"] == "case-1"
def _raise_get_cases(_dataset_id):
raise RuntimeError("get cases boom")
monkeypatch.setattr(module.EvaluationService, "get_test_cases", _raise_get_cases)
res = _run(module.get_test_cases("dataset-9"))
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "get cases boom" in res["message"]
monkeypatch.setattr(module.EvaluationService, "delete_test_case", lambda _case_id: False)
res = _run(module.delete_test_case("case-1"))
assert res["code"] == module.RetCode.DATA_ERROR
assert "failed" in res["message"].lower()
monkeypatch.setattr(module.EvaluationService, "delete_test_case", lambda _case_id: True)
res = _run(module.delete_test_case("case-2"))
assert res["code"] == 0
assert res["data"]["case_id"] == "case-2"
def _raise_delete_case(_case_id):
raise RuntimeError("delete case boom")
monkeypatch.setattr(module.EvaluationService, "delete_test_case", _raise_delete_case)
res = _run(module.delete_test_case("case-3"))
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "delete case boom" in res["message"]
@pytest.mark.p2
def test_run_and_recommendation_routes_matrix_unit(monkeypatch):
module = _load_evaluation_app(monkeypatch)
_set_request_json(monkeypatch, module, {"dataset_id": "d1", "dialog_id": "dialog-1", "name": "run 1"})
monkeypatch.setattr(module.EvaluationService, "start_evaluation", lambda **_kwargs: (False, "start failed"))
res = _run(module.start_evaluation())
assert res["code"] == module.RetCode.DATA_ERROR
assert "start failed" in res["message"]
monkeypatch.setattr(module.EvaluationService, "start_evaluation", lambda **_kwargs: (True, "run-ok"))
res = _run(module.start_evaluation())
assert res["code"] == 0
assert res["data"]["run_id"] == "run-ok"
def _raise_start(**_kwargs):
raise RuntimeError("start boom")
monkeypatch.setattr(module.EvaluationService, "start_evaluation", _raise_start)
res = _run(module.start_evaluation())
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "start boom" in res["message"]
monkeypatch.setattr(module.EvaluationService, "get_run_results", lambda _run_id: None)
res = _run(module.get_evaluation_run("run-1"))
assert res["code"] == module.RetCode.DATA_ERROR
assert "not found" in res["message"].lower()
monkeypatch.setattr(module.EvaluationService, "get_run_results", lambda _run_id: {"id": _run_id})
res = _run(module.get_evaluation_run("run-2"))
assert res["code"] == 0
assert res["data"]["id"] == "run-2"
def _raise_get_run(_run_id):
raise RuntimeError("get run boom")
monkeypatch.setattr(module.EvaluationService, "get_run_results", _raise_get_run)
res = _run(module.get_evaluation_run("run-3"))
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "get run boom" in res["message"]
monkeypatch.setattr(module.EvaluationService, "get_run_results", lambda _run_id: None)
res = _run(module.get_run_results("run-4"))
assert res["code"] == module.RetCode.DATA_ERROR
assert "not found" in res["message"].lower()
monkeypatch.setattr(module.EvaluationService, "get_run_results", lambda _run_id: {"id": _run_id, "score": 0.9})
res = _run(module.get_run_results("run-5"))
assert res["code"] == 0
assert res["data"]["id"] == "run-5"
def _raise_results(_run_id):
raise RuntimeError("get results boom")
monkeypatch.setattr(module.EvaluationService, "get_run_results", _raise_results)
res = _run(module.get_run_results("run-6"))
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "get results boom" in res["message"]
res = _run(module.list_evaluation_runs())
assert res["code"] == 0
assert res["data"]["total"] == 0
def _raise_json_list(*_args, **_kwargs):
raise RuntimeError("list runs boom")
monkeypatch.setattr(module, "get_json_result", _raise_json_list)
res = _run(module.list_evaluation_runs())
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "list runs boom" in res["message"]
monkeypatch.setattr(module, "get_json_result", lambda code=0, message="success", data=None: {"code": code, "message": message, "data": data})
res = _run(module.delete_evaluation_run("run-7"))
assert res["code"] == 0
assert res["data"]["run_id"] == "run-7"
def _raise_json_delete(*_args, **_kwargs):
raise RuntimeError("delete run boom")
monkeypatch.setattr(module, "get_json_result", _raise_json_delete)
res = _run(module.delete_evaluation_run("run-8"))
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "delete run boom" in res["message"]
monkeypatch.setattr(module, "get_json_result", lambda code=0, message="success", data=None: {"code": code, "message": message, "data": data})
monkeypatch.setattr(module.EvaluationService, "get_recommendations", lambda _run_id: [{"name": "cfg-1"}])
res = _run(module.get_recommendations("run-9"))
assert res["code"] == 0
assert res["data"]["recommendations"][0]["name"] == "cfg-1"
def _raise_recommend(_run_id):
raise RuntimeError("recommend boom")
monkeypatch.setattr(module.EvaluationService, "get_recommendations", _raise_recommend)
res = _run(module.get_recommendations("run-10"))
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "recommend boom" in res["message"]
@pytest.mark.p2
def test_compare_export_and_evaluate_single_matrix_unit(monkeypatch):
module = _load_evaluation_app(monkeypatch)
_set_request_json(monkeypatch, module, {"run_ids": ["run-1"]})
res = _run(module.compare_runs())
assert res["code"] == module.RetCode.DATA_ERROR
assert "at least 2" in res["message"]
_set_request_json(monkeypatch, module, {"run_ids": ["run-1", "run-2"]})
res = _run(module.compare_runs())
assert res["code"] == 0
assert res["data"]["comparison"] == {}
def _raise_json_compare(*_args, **_kwargs):
raise RuntimeError("compare boom")
monkeypatch.setattr(module, "get_json_result", _raise_json_compare)
_set_request_json(monkeypatch, module, {"run_ids": ["run-1", "run-2", "run-3"]})
res = _run(module.compare_runs())
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "compare boom" in res["message"]
monkeypatch.setattr(module, "get_json_result", lambda code=0, message="success", data=None: {"code": code, "message": message, "data": data})
monkeypatch.setattr(module.EvaluationService, "get_run_results", lambda _run_id: None)
res = _run(module.export_results("run-11"))
assert res["code"] == module.RetCode.DATA_ERROR
assert "not found" in res["message"].lower()
monkeypatch.setattr(module.EvaluationService, "get_run_results", lambda _run_id: {"id": _run_id, "rows": []})
res = _run(module.export_results("run-12"))
assert res["code"] == 0
assert res["data"]["id"] == "run-12"
def _raise_export(_run_id):
raise RuntimeError("export boom")
monkeypatch.setattr(module.EvaluationService, "get_run_results", _raise_export)
res = _run(module.export_results("run-13"))
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "export boom" in res["message"]
monkeypatch.setattr(module, "get_json_result", lambda code=0, message="success", data=None: {"code": code, "message": message, "data": data})
res = _run(module.evaluate_single())
assert res["code"] == 0
assert res["data"]["answer"] == ""
assert res["data"]["metrics"] == {}
assert res["data"]["retrieved_chunks"] == []
def _raise_json_single(*_args, **_kwargs):
raise RuntimeError("single boom")
monkeypatch.setattr(module, "get_json_result", _raise_json_single)
res = _run(module.evaluate_single())
assert res["code"] == module.RetCode.EXCEPTION_ERROR
assert "single boom" in res["message"]