feat(api): add unified index API and dataset management endpoints (#14222)

### What problem does this PR solve?

## Summary

Refactor the dataset API layer into a clean service/REST separation
pattern, add a unified `/index` API for graph/raptor/mindmap operations,
and introduce several new dataset management endpoints with full test
coverage.

## Changes

### Service Layer (`dataset_api_service.py`)

- Added `trace_index(dataset_id, tenant_id, index_type)` — unified trace
function for all index types
- Added `run_index`, `delete_index` service functions
- Added `get_dataset`, `get_ingestion_summary`, `list_ingestion_logs`,
`get_ingestion_log`
- Added `run_embedding`, `list_tags`, `aggregate_tags`, `delete_tags`,
`rename_tag`
- Added `get_flattened_metadata`, `get_auto_metadata`,
`update_auto_metadata`

### REST API Layer (`dataset_api.py`)

**New unified routes:**

| Method | Route | Description |
|--------|-------|-------------|
| POST | `/datasets/<id>/index?type=graph\|raptor\|mindmap` | Run index
task |
| GET | `/datasets/<id>/index?type=graph\|raptor\|mindmap` | Trace index
task |
| DELETE | `/datasets/<id>/<index_type>` | Delete index |
| GET | `/datasets/<id>` | Get dataset details |
| GET | `/datasets/<id>/ingestions/summary` | Ingestion summary |
| GET | `/datasets/<id>/ingestions` | List ingestion logs |
| GET | `/datasets/<id>/ingestions/<log_id>` | Get single ingestion log
|
| POST | `/datasets/<id>/embedding` | Run embedding |
| GET | `/datasets/<id>/tags` | List tags |
| GET | `/datasets/tags/aggregation` | Aggregate tags across datasets |
| DELETE | `/datasets/<id>/tags` | Delete tags |
| PUT | `/datasets/<id>/tags` | Rename tag |
| GET | `/datasets/metadata/flattened` | Get flattened metadata |
| GET/PUT | `/datasets/<id>/metadata/config` | New metadata config path
|

**Removed routes (replaced by unified `/index`):**

- `POST /datasets/<id>/mindmap`
- `GET /datasets/<id>/mindmap`

**Preserved legacy routes (backward compatibility):**

- `/run_graphrag`, `/trace_graphrag`, `/run_raptor`, `/trace_raptor`
- `/auto_metadata` GET/PUT

### Test Suite

- Updated `common.py` helpers: added `trace_index`, removed
`run_mindmap`/`trace_mindmap`
- Added 7 new test files with 39 test cases total:

| Test File | Cases |
|-----------|-------|
| `test_get_dataset.py` | 4 |
| `test_ingestion_summary.py` | 2 |
| `test_ingestion_logs.py` | 5 |
| `test_index_api.py` | 14 |
| `test_embedding.py` | 2 |
| `test_tags.py` | 8 |
| `test_flattened_metadata.py` | 4 |

- Deleted `test_mindmap_tasks.py` (covered by unified index tests)

## Design Decisions

1. **Unified `/index?type=...`** — single endpoint replaces 3 separate
route pairs for graph/raptor/mindmap
2. **Backward compatibility** — old routes (`/run_graphrag`,
`/run_raptor`, `/auto_metadata`) preserved alongside new paths
3. **`_VALID_INDEX_TYPES = {"graph", "raptor", "mindmap"}`** — input
validation via constant set
4. **`_INDEX_TYPE_TO_TASK_ID_FIELD`** — maps index type to KB model task
ID field for clean dispatch

## Files Changed

- `api/apps/restful_apis/dataset_api.py`
- `api/apps/services/dataset_api_service.py`
- `sdk/python/ragflow_sdk/modules/dataset.py`
- `test/testcases/test_http_api/common.py`
- `test/testcases/test_http_api/test_dataset_management/` (7 new files)
### Type of change

- [x] New Feature (non-breaking change which adds functionality)
- [x] Refactoring

---------

Signed-off-by: noob <yixiao121314@outlook.com>
This commit is contained in:
euvre
2026-04-27 01:38:01 +00:00
committed by GitHub
parent fb95136f39
commit 4dcc42e0e1
51 changed files with 1765 additions and 4381 deletions

View File

@@ -23,7 +23,8 @@ from utils.file_utils import create_txt_file
HEADERS = {"Content-Type": "application/json"}
DATASETS_API_URL = f"/api/{VERSION}/datasets"
FILE_API_URL = f"/api/{VERSION}/datasets/{{dataset_id}}/documents"
FILE_CHUNK_API_URL = f"/api/{VERSION}/datasets/{{dataset_id}}/chunks"
FILE_PARSE_API_URL = f"/api/{VERSION}/datasets/{{dataset_id}}/documents/parse"
FILE_STOP_PARSE_API_URL = f"/api/{VERSION}/datasets/{{dataset_id}}/documents/stop"
CHUNK_API_URL = f"/api/{VERSION}/datasets/{{dataset_id}}/documents/{{document_id}}/chunks"
CHAT_ASSISTANT_API_URL = f"/api/{VERSION}/chats"
SESSION_WITH_CHAT_ASSISTANT_API_URL = f"/api/{VERSION}/chats/{{chat_id}}/sessions"
@@ -136,15 +137,15 @@ def delete_all_documents(auth, dataset_id, *, page_size=1000):
return delete_documents(auth, dataset_id, {"ids": None, "delete_all": True})
def parse_documents(auth, dataset_id, payload=None):
url = f"{HOST_ADDRESS}{FILE_CHUNK_API_URL}".format(dataset_id=dataset_id)
res = requests.post(url=url, headers=HEADERS, auth=auth, json=payload)
def parse_documents(auth, dataset_id, payload=None, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{FILE_PARSE_API_URL}".format(dataset_id=dataset_id)
res = requests.post(url=url, headers=headers, auth=auth, json=payload)
return res.json()
def stop_parse_documents(auth, dataset_id, payload=None):
url = f"{HOST_ADDRESS}{FILE_CHUNK_API_URL}".format(dataset_id=dataset_id)
res = requests.delete(url=url, headers=HEADERS, auth=auth, json=payload)
url = f"{HOST_ADDRESS}{FILE_STOP_PARSE_API_URL}".format(dataset_id=dataset_id)
res = requests.post(url=url, headers=HEADERS, auth=auth, json=payload)
return res.json()
@@ -161,9 +162,9 @@ def bulk_upload_documents(auth, dataset_id, num, tmp_path):
# CHUNK MANAGEMENT WITHIN DATASET
def add_chunk(auth, dataset_id, document_id, payload=None):
def add_chunk(auth, dataset_id, document_id, payload=None, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{CHUNK_API_URL}".format(dataset_id=dataset_id, document_id=document_id)
res = requests.post(url=url, headers=HEADERS, auth=auth, json=payload)
res = requests.post(url=url, headers=headers, auth=auth, json=payload)
return res.json()
@@ -195,9 +196,9 @@ def delete_all_chunks(auth, dataset_id, document_id, *, page_size=1000):
return delete_chunks(auth, dataset_id, document_id, {"chunk_ids": None, "delete_all": True})
def retrieval_chunks(auth, payload=None):
def retrieval_chunks(auth, payload=None, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{RETRIEVAL_API_URL}"
res = requests.post(url=url, headers=HEADERS, auth=auth, json=payload)
res = requests.post(url=url, headers=headers, auth=auth, json=payload)
return res.json()
@@ -210,9 +211,9 @@ def batch_add_chunks(auth, dataset_id, document_id, num):
# CHAT ASSISTANT MANAGEMENT
def create_chat_assistant(auth, payload=None):
def create_chat_assistant(auth, payload=None, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{CHAT_ASSISTANT_API_URL}"
res = requests.post(url=url, headers=HEADERS, auth=auth, json=payload)
res = requests.post(url=url, headers=headers, auth=auth, json=payload)
return res.json()
@@ -259,9 +260,9 @@ def batch_create_chat_assistants(auth, num):
# SESSION MANAGEMENT
def create_session_with_chat_assistant(auth, chat_assistant_id, payload=None):
def create_session_with_chat_assistant(auth, chat_assistant_id, payload=None, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{SESSION_WITH_CHAT_ASSISTANT_API_URL}".format(chat_id=chat_assistant_id)
res = requests.post(url=url, headers=HEADERS, auth=auth, json=payload)
res = requests.post(url=url, headers=headers, auth=auth, json=payload)
return res.json()
@@ -297,13 +298,13 @@ def batch_add_sessions_with_chat_assistant(auth, chat_assistant_id, num):
# DATASET GRAPH AND TASKS
def knowledge_graph(auth, dataset_id, params=None):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/knowledge_graph"
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/graph/search"
res = requests.get(url=url, headers=HEADERS, auth=auth, params=params)
return res.json()
def delete_knowledge_graph(auth, dataset_id, payload=None):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/knowledge_graph"
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/graph"
if payload is None:
res = requests.delete(url=url, headers=HEADERS, auth=auth)
else:
@@ -311,39 +312,15 @@ def delete_knowledge_graph(auth, dataset_id, payload=None):
return res.json()
def run_graphrag(auth, dataset_id, payload=None):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/run_graphrag"
res = requests.post(url=url, headers=HEADERS, auth=auth, json=payload)
return res.json()
def trace_graphrag(auth, dataset_id, params=None):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/trace_graphrag"
res = requests.get(url=url, headers=HEADERS, auth=auth, params=params)
return res.json()
def run_raptor(auth, dataset_id, payload=None):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/run_raptor"
res = requests.post(url=url, headers=HEADERS, auth=auth, json=payload)
return res.json()
def trace_raptor(auth, dataset_id, params=None):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/trace_raptor"
res = requests.get(url=url, headers=HEADERS, auth=auth, params=params)
return res.json()
def metadata_summary(auth, dataset_id, params=None):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/metadata/summary"
res = requests.get(url=url, headers=HEADERS, auth=auth, params=params)
return res.json()
def metadata_batch_update(auth, dataset_id, payload=None):
def metadata_batch_update(auth, dataset_id, payload=None, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/metadata/update"
res = requests.post(url=url, headers=HEADERS, auth=auth, json=payload)
res = requests.post(url=url, headers=headers, auth=auth, json=payload)
return res.json()
@@ -358,16 +335,16 @@ def update_documents_metadata(auth, dataset_id, payload=None):
# CHAT COMPLETIONS AND RELATED QUESTIONS
def related_questions(auth, payload=None):
def related_questions(auth, payload=None, *, headers=HEADERS):
url = f"{HOST_ADDRESS}/api/{VERSION}/sessions/related_questions"
res = requests.post(url=url, headers=HEADERS, auth=auth, json=payload)
res = requests.post(url=url, headers=headers, auth=auth, json=payload)
return res.json()
# AGENT MANAGEMENT AND SESSIONS
def create_agent(auth, payload=None):
def create_agent(auth, payload=None, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{AGENT_API_URL}"
res = requests.post(url=url, headers=HEADERS, auth=auth, json=payload)
res = requests.post(url=url, headers=headers, auth=auth, json=payload)
return res.json()
@@ -439,7 +416,7 @@ def chat_completions(auth, chat_id=None, payload=None):
return res.json()
def chat_completions_openai(auth, chat_id, payload=None):
def chat_completions_openai(auth, chat_id, payload=None, *, headers=HEADERS):
"""
Send a request to the OpenAI-compatible chat completions endpoint.
@@ -454,5 +431,88 @@ def chat_completions_openai(auth, chat_id, payload=None):
Response JSON in OpenAI chat completions format with usage information
"""
url = f"{HOST_ADDRESS}/api/{VERSION}/chats_openai/{chat_id}/chat/completions"
res = requests.post(url=url, headers=HEADERS, auth=auth, json=payload)
res = requests.post(url=url, headers=headers, auth=auth, json=payload)
return res.json()
# NEW DATASET ENDPOINTS
def get_dataset(auth, dataset_id, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}"
res = requests.get(url=url, headers=headers, auth=auth)
return res.json()
def get_ingestion_summary(auth, dataset_id, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/ingestions/summary"
res = requests.get(url=url, headers=headers, auth=auth)
return res.json()
def list_ingestion_logs(auth, dataset_id, params=None, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/ingestions"
res = requests.get(url=url, headers=headers, auth=auth, params=params)
return res.json()
def get_ingestion_log(auth, dataset_id, log_id, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/ingestions/{log_id}"
res = requests.get(url=url, headers=headers, auth=auth)
return res.json()
def run_index(auth, dataset_id, index_type, payload=None, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/index"
params = {"type": index_type}
res = requests.post(url=url, headers=headers, auth=auth, json=payload, params=params)
return res.json()
def trace_index(auth, dataset_id, index_type, params=None, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/index"
all_params = {"type": index_type}
if params:
all_params.update(params)
res = requests.get(url=url, headers=headers, auth=auth, params=all_params)
return res.json()
def delete_index(auth, dataset_id, index_type, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/{index_type}"
res = requests.delete(url=url, headers=headers, auth=auth)
return res.json()
def run_embedding(auth, dataset_id, payload=None, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/embedding"
res = requests.post(url=url, headers=headers, auth=auth, json=payload)
return res.json()
def list_tags(auth, dataset_id, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/tags"
res = requests.get(url=url, headers=headers, auth=auth)
return res.json()
def aggregate_tags(auth, dataset_ids, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/tags/aggregation"
res = requests.get(url=url, headers=headers, auth=auth, params={"dataset_ids": ",".join(dataset_ids)})
return res.json()
def delete_tags(auth, dataset_id, tags, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/tags"
res = requests.delete(url=url, headers=headers, auth=auth, json={"tags": tags})
return res.json()
def rename_tag(auth, dataset_id, from_tag, to_tag, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/{dataset_id}/tags"
res = requests.put(url=url, headers=headers, auth=auth, json={"from_tag": from_tag, "to_tag": to_tag})
return res.json()
def get_flattened_metadata(auth, dataset_ids, *, headers=HEADERS):
url = f"{HOST_ADDRESS}{DATASETS_API_URL}/metadata/flattened"
res = requests.get(url=url, headers=headers, auth=auth, params={"dataset_ids": ",".join(dataset_ids)})
return res.json()

View File

@@ -43,7 +43,7 @@ from utils.file_utils import (
)
@wait_for(30, 1, "Document parsing timeout")
@wait_for(200, 1, "Document parsing timeout")
def condition(_auth, _dataset_id):
res = list_documents(_auth, _dataset_id)
for doc in res["data"]["docs"]:

View File

@@ -0,0 +1,32 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from common import run_embedding
@pytest.mark.usefixtures("clear_datasets")
class TestRunEmbedding:
@pytest.mark.p2
def test_run_embedding_no_documents(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = run_embedding(HttpApiAuth, dataset_id)
assert res["code"] == 102, res
assert "No documents in Dataset" in res.get("message", ""), res
@pytest.mark.p2
def test_run_embedding_invalid_id(self, HttpApiAuth):
res = run_embedding(HttpApiAuth, "invalid_id")
assert res["code"] != 0, res

View File

@@ -0,0 +1,42 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from common import get_flattened_metadata
@pytest.mark.usefixtures("clear_datasets")
class TestFlattenedMetadata:
@pytest.mark.p2
def test_get_flattened_metadata_success(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = get_flattened_metadata(HttpApiAuth, [dataset_id])
assert res["code"] == 0, res
@pytest.mark.p2
def test_get_flattened_metadata_multiple_datasets(self, HttpApiAuth, add_datasets_func):
dataset_ids = add_datasets_func
res = get_flattened_metadata(HttpApiAuth, dataset_ids)
assert res["code"] == 0, res
@pytest.mark.p2
def test_get_flattened_metadata_empty_ids(self, HttpApiAuth):
res = get_flattened_metadata(HttpApiAuth, [])
assert res["code"] != 0, res
@pytest.mark.p2
def test_get_flattened_metadata_invalid_id(self, HttpApiAuth):
res = get_flattened_metadata(HttpApiAuth, ["invalid_id"])
assert res["code"] != 0, res

View File

@@ -0,0 +1,45 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from common import get_dataset
from libs.auth import RAGFlowHttpApiAuth
from configs import INVALID_API_TOKEN
@pytest.mark.usefixtures("clear_datasets")
class TestGetDataset:
@pytest.mark.p2
def test_get_dataset_success(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = get_dataset(HttpApiAuth, dataset_id)
assert res["code"] == 0, res
assert res["data"]["id"] == dataset_id, res
@pytest.mark.p2
def test_get_dataset_invalid_id(self, HttpApiAuth):
res = get_dataset(HttpApiAuth, "invalid_dataset_id")
assert res["code"] != 0, res
@pytest.mark.p2
def test_get_dataset_unauthorized(self, add_dataset_func):
dataset_id = add_dataset_func
res = get_dataset(RAGFlowHttpApiAuth(INVALID_API_TOKEN), dataset_id)
assert res["code"] != 0, res
@pytest.mark.p2
def test_get_dataset_nonexistent(self, HttpApiAuth):
res = get_dataset(HttpApiAuth, "0" * 32)
assert res["code"] != 0, res

View File

@@ -1,89 +0,0 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from common import bulk_upload_documents, list_documents, parse_documents, run_graphrag, trace_graphrag
from utils import wait_for
@wait_for(200, 1, "Document parsing timeout")
def _parse_done(auth, dataset_id, document_ids=None):
res = list_documents(auth, dataset_id)
target_docs = res["data"]["docs"]
if document_ids is None:
return all(doc.get("run") == "DONE" for doc in target_docs)
target_ids = set(document_ids)
for doc in target_docs:
if doc.get("id") in target_ids and doc.get("run") != "DONE":
return False
return True
class TestGraphRAGTasks:
@pytest.mark.p2
def test_trace_graphrag_before_run(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = trace_graphrag(HttpApiAuth, dataset_id)
assert res["code"] == 0, res
assert res["data"] == {}, res
@pytest.mark.p2
def test_run_graphrag_no_documents(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = run_graphrag(HttpApiAuth, dataset_id)
assert res["code"] == 102, res
assert "No documents in Dataset" in res.get("message", ""), res
@pytest.mark.p3
def test_run_graphrag_returns_task_id(self, HttpApiAuth, add_dataset_func, tmp_path):
dataset_id = add_dataset_func
bulk_upload_documents(HttpApiAuth, dataset_id, 1, tmp_path)
res = run_graphrag(HttpApiAuth, dataset_id)
assert res["code"] == 0, res
assert res["data"].get("graphrag_task_id"), res
@pytest.mark.p3
def test_trace_graphrag_until_complete(self, HttpApiAuth, add_dataset_func, tmp_path):
dataset_id = add_dataset_func
document_ids = bulk_upload_documents(HttpApiAuth, dataset_id, 1, tmp_path)
res = parse_documents(HttpApiAuth, dataset_id, {"document_ids": document_ids})
assert res["code"] == 0, res
_parse_done(HttpApiAuth, dataset_id, document_ids)
res = run_graphrag(HttpApiAuth, dataset_id)
assert res["code"] == 0, res
last_res = {}
@wait_for(200, 1, "GraphRAG task timeout")
def condition():
res = trace_graphrag(HttpApiAuth, dataset_id)
if res["code"] != 0:
return False
data = res.get("data") or {}
if not data:
return False
if data.get("task_type") != "graphrag":
return False
progress = data.get("progress")
if progress in (-1, 1, -1.0, 1.0):
last_res["res"] = res
return True
return False
condition()
res = last_res["res"]
assert res["data"]["task_type"] == "graphrag", res
assert res["data"].get("progress") in (-1, 1, -1.0, 1.0), res

View File

@@ -0,0 +1,166 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from common import (
bulk_upload_documents,
list_documents,
run_index,
trace_index,
delete_index,
)
from utils import wait_for
@wait_for(200, 1, "Document parsing timeout")
def _parse_done(auth, dataset_id, document_ids=None):
res = list_documents(auth, dataset_id)
if res.get("code") != 0:
return False
target_docs = res.get("data", {}).get("docs", [])
if not target_docs:
return False
if document_ids is None:
return all(doc.get("run") == "DONE" for doc in target_docs)
target_ids = set(document_ids)
seen_ids = set()
for doc in target_docs:
doc_id = doc.get("id")
if doc_id in target_ids:
seen_ids.add(doc_id)
if doc.get("run") != "DONE":
return False
return seen_ids == target_ids
@wait_for(60, 1, "Index task creation timeout")
def _index_task_created(auth, dataset_id, index_type):
res = trace_index(auth, dataset_id, index_type)
if res.get("code") != 0:
return False
return bool(res.get("data", {}).get("id"))
@pytest.mark.usefixtures("clear_datasets")
class TestRunIndex:
@pytest.mark.p2
def test_run_index_graph(self, HttpApiAuth, add_dataset_func, tmp_path):
dataset_id = add_dataset_func
bulk_upload_documents(HttpApiAuth, dataset_id, 1, tmp_path)
res = run_index(HttpApiAuth, dataset_id, "graph")
assert res["code"] == 0, res
assert res["data"].get("task_id"), res
@pytest.mark.p2
def test_run_index_raptor(self, HttpApiAuth, add_dataset_func, tmp_path):
dataset_id = add_dataset_func
bulk_upload_documents(HttpApiAuth, dataset_id, 1, tmp_path)
res = run_index(HttpApiAuth, dataset_id, "raptor")
assert res["code"] == 0, res
assert res["data"].get("task_id"), res
@pytest.mark.p2
def test_run_index_mindmap(self, HttpApiAuth, add_dataset_func, tmp_path):
dataset_id = add_dataset_func
bulk_upload_documents(HttpApiAuth, dataset_id, 1, tmp_path)
res = run_index(HttpApiAuth, dataset_id, "mindmap")
assert res["code"] == 0, res
assert res["data"].get("task_id"), res
@pytest.mark.p2
def test_run_index_invalid_type(self, HttpApiAuth, add_dataset_func, tmp_path):
dataset_id = add_dataset_func
bulk_upload_documents(HttpApiAuth, dataset_id, 1, tmp_path)
res = run_index(HttpApiAuth, dataset_id, "invalid_type")
assert res["code"] != 0, res
@pytest.mark.p2
def test_run_index_no_documents(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = run_index(HttpApiAuth, dataset_id, "raptor")
assert res["code"] == 102, res
@pytest.mark.usefixtures("clear_datasets")
class TestDeleteIndex:
@pytest.mark.p2
def test_delete_graph(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = delete_index(HttpApiAuth, dataset_id, "graph")
assert res["code"] == 0, res
@pytest.mark.p2
def test_delete_raptor(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = delete_index(HttpApiAuth, dataset_id, "raptor")
assert res["code"] == 0, res
@pytest.mark.p2
def test_delete_mindmap(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = delete_index(HttpApiAuth, dataset_id, "mindmap")
assert res["code"] == 0, res
@pytest.mark.p2
def test_delete_invalid_type(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = delete_index(HttpApiAuth, dataset_id, "invalid_type")
assert res["code"] != 0, res
@pytest.mark.usefixtures("clear_datasets")
class TestTraceIndex:
@pytest.mark.p2
def test_trace_index_graph(self, HttpApiAuth, add_dataset_func, tmp_path):
dataset_id = add_dataset_func
bulk_upload_documents(HttpApiAuth, dataset_id, 1, tmp_path)
res = run_index(HttpApiAuth, dataset_id, "graph")
assert res["code"] == 0, res
_index_task_created(HttpApiAuth, dataset_id, "graph")
res = trace_index(HttpApiAuth, dataset_id, "graph")
assert res["code"] == 0, res
@pytest.mark.p2
def test_trace_index_raptor(self, HttpApiAuth, add_dataset_func, tmp_path):
dataset_id = add_dataset_func
bulk_upload_documents(HttpApiAuth, dataset_id, 1, tmp_path)
res = run_index(HttpApiAuth, dataset_id, "raptor")
assert res["code"] == 0, res
_index_task_created(HttpApiAuth, dataset_id, "raptor")
res = trace_index(HttpApiAuth, dataset_id, "raptor")
assert res["code"] == 0, res
@pytest.mark.p2
def test_trace_index_mindmap(self, HttpApiAuth, add_dataset_func, tmp_path):
dataset_id = add_dataset_func
bulk_upload_documents(HttpApiAuth, dataset_id, 1, tmp_path)
res = run_index(HttpApiAuth, dataset_id, "mindmap")
assert res["code"] == 0, res
_index_task_created(HttpApiAuth, dataset_id, "mindmap")
res = trace_index(HttpApiAuth, dataset_id, "mindmap")
assert res["code"] == 0, res
@pytest.mark.p2
def test_trace_index_invalid_type(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = trace_index(HttpApiAuth, dataset_id, "invalid_type")
assert res["code"] != 0, res
@pytest.mark.p2
def test_trace_index_no_task(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = trace_index(HttpApiAuth, dataset_id, "graph")
assert res["code"] == 0, res
assert res["data"] == {}

View File

@@ -0,0 +1,53 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from common import list_ingestion_logs, get_ingestion_log
@pytest.mark.usefixtures("clear_datasets")
class TestListIngestionLogs:
@pytest.mark.p2
def test_list_ingestion_logs_success(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = list_ingestion_logs(HttpApiAuth, dataset_id)
assert res["code"] == 0, res
assert "total" in res["data"], res
assert "logs" in res["data"], res
@pytest.mark.p2
def test_list_ingestion_logs_with_pagination(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = list_ingestion_logs(HttpApiAuth, dataset_id, params={"page": 1, "page_size": 10})
assert res["code"] == 0, res
@pytest.mark.p2
def test_list_ingestion_logs_invalid_id(self, HttpApiAuth):
res = list_ingestion_logs(HttpApiAuth, "invalid_id")
assert res["code"] != 0, res
@pytest.mark.usefixtures("clear_datasets")
class TestGetIngestionLog:
@pytest.mark.p2
def test_get_ingestion_log_not_found(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = get_ingestion_log(HttpApiAuth, dataset_id, "nonexistent_log_id")
assert res["code"] != 0, res
@pytest.mark.p2
def test_get_ingestion_log_invalid_dataset(self, HttpApiAuth):
res = get_ingestion_log(HttpApiAuth, "invalid_id", "some_log_id")
assert res["code"] != 0, res

View File

@@ -0,0 +1,35 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from common import get_ingestion_summary
@pytest.mark.usefixtures("clear_datasets")
class TestIngestionSummary:
@pytest.mark.p2
def test_ingestion_summary_success(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = get_ingestion_summary(HttpApiAuth, dataset_id)
assert res["code"] == 0, res
assert "doc_num" in res["data"], res
assert "chunk_num" in res["data"], res
assert "token_num" in res["data"], res
assert "status" in res["data"], res
@pytest.mark.p2
def test_ingestion_summary_invalid_id(self, HttpApiAuth):
res = get_ingestion_summary(HttpApiAuth, "invalid_id")
assert res["code"] != 0, res

View File

@@ -1,89 +0,0 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from common import bulk_upload_documents, list_documents, parse_documents, run_raptor, trace_raptor
from utils import wait_for
@wait_for(200, 1, "Document parsing timeout")
def _parse_done(auth, dataset_id, document_ids=None):
res = list_documents(auth, dataset_id)
target_docs = res["data"]["docs"]
if document_ids is None:
return all(doc.get("run") == "DONE" for doc in target_docs)
target_ids = set(document_ids)
for doc in target_docs:
if doc.get("id") in target_ids and doc.get("run") != "DONE":
return False
return True
class TestRaptorTasks:
@pytest.mark.p2
def test_trace_raptor_before_run(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = trace_raptor(HttpApiAuth, dataset_id)
assert res["code"] == 0, res
assert res["data"] == {}, res
@pytest.mark.p2
def test_run_raptor_no_documents(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = run_raptor(HttpApiAuth, dataset_id)
assert res["code"] == 102, res
assert "No documents in Dataset" in res.get("message", ""), res
@pytest.mark.p3
def test_run_raptor_returns_task_id(self, HttpApiAuth, add_dataset_func, tmp_path):
dataset_id = add_dataset_func
bulk_upload_documents(HttpApiAuth, dataset_id, 1, tmp_path)
res = run_raptor(HttpApiAuth, dataset_id)
assert res["code"] == 0, res
assert res["data"].get("raptor_task_id"), res
@pytest.mark.p3
def test_trace_raptor_until_complete(self, HttpApiAuth, add_dataset_func, tmp_path):
dataset_id = add_dataset_func
document_ids = bulk_upload_documents(HttpApiAuth, dataset_id, 1, tmp_path)
res = parse_documents(HttpApiAuth, dataset_id, {"document_ids": document_ids})
assert res["code"] == 0, res
_parse_done(HttpApiAuth, dataset_id, document_ids)
res = run_raptor(HttpApiAuth, dataset_id)
assert res["code"] == 0, res
last_res = {}
@wait_for(200, 1, "RAPTOR task timeout")
def condition():
res = trace_raptor(HttpApiAuth, dataset_id)
if res["code"] != 0:
return False
data = res.get("data") or {}
if not data:
return False
if data.get("task_type") != "raptor":
return False
progress = data.get("progress")
if progress in (-1, 1, -1.0, 1.0):
last_res["res"] = res
return True
return False
condition()
res = last_res["res"]
assert res["data"]["task_type"] == "raptor", res
assert res["data"].get("progress") in (-1, 1, -1.0, 1.0), res

View File

@@ -0,0 +1,84 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from common import (
list_tags,
aggregate_tags,
delete_tags,
rename_tag,
)
@pytest.mark.usefixtures("clear_datasets")
class TestListTags:
@pytest.mark.p2
def test_list_tags_success(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = list_tags(HttpApiAuth, dataset_id)
assert res["code"] == 0, res
@pytest.mark.p2
def test_list_tags_invalid_id(self, HttpApiAuth):
res = list_tags(HttpApiAuth, "invalid_id")
assert res["code"] != 0, res
@pytest.mark.usefixtures("clear_datasets")
class TestAggregateTags:
@pytest.mark.p2
def test_aggregate_tags_success(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = aggregate_tags(HttpApiAuth, [dataset_id])
assert res["code"] == 0, res
@pytest.mark.p2
def test_aggregate_tags_multiple_datasets(self, HttpApiAuth, add_datasets_func):
dataset_ids = add_datasets_func
res = aggregate_tags(HttpApiAuth, dataset_ids)
assert res["code"] == 0, res
@pytest.mark.p2
def test_aggregate_tags_empty_ids(self, HttpApiAuth):
res = aggregate_tags(HttpApiAuth, [])
assert res["code"] != 0, res
@pytest.mark.usefixtures("clear_datasets")
class TestDeleteTags:
@pytest.mark.p2
def test_delete_tags_missing_body(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = delete_tags(HttpApiAuth, dataset_id, [])
assert res["code"] == 0, res
@pytest.mark.p2
def test_delete_tags_invalid_id(self, HttpApiAuth):
res = delete_tags(HttpApiAuth, "invalid_id", ["tag1"])
assert res["code"] != 0, res
@pytest.mark.usefixtures("clear_datasets")
class TestRenameTag:
@pytest.mark.p2
def test_rename_tag_empty_names(self, HttpApiAuth, add_dataset_func):
dataset_id = add_dataset_func
res = rename_tag(HttpApiAuth, dataset_id, "", "")
assert res["code"] != 0, res
@pytest.mark.p2
def test_rename_tag_invalid_id(self, HttpApiAuth):
res = rename_tag(HttpApiAuth, "invalid_id", "old_tag", "new_tag")
assert res["code"] != 0, res

View File

@@ -27,11 +27,14 @@ from common import (
delete_datasets,
list_documents,
update_document,
upload_documents,
parse_documents,
retrieval_chunks,
)
from utils import wait_for
@wait_for(30, 1, "Document parsing timeout")
@wait_for(120, 1, "Document parsing timeout")
def _condition_parsing_complete(_auth, dataset_id):
res = list_documents(_auth, dataset_id)
if res["code"] != 0:
@@ -39,7 +42,7 @@ def _condition_parsing_complete(_auth, dataset_id):
for doc in res["data"]["docs"]:
status = doc.get("run", "UNKNOWN")
if status == "FAILED":
if status in ("FAIL", "FAILED"):
pytest.fail(f"Document parsing failed: {doc}")
return False
if status != "DONE":
@@ -62,35 +65,17 @@ def add_dataset_with_metadata(HttpApiAuth):
import requests
from configs import HOST_ADDRESS, VERSION
metadata_config = {
"type": "object",
"properties": {
"character": {
"description": "Historical figure name",
"type": "string"
},
"era": {
"description": "Historical era",
"type": "string"
},
"achievements": {
"description": "Major achievements",
"type": "array",
"items": {
"type": "string"
}
}
}
}
res = requests.post(
url=f"{HOST_ADDRESS}/{VERSION}/kb/update_metadata_setting",
res = requests.put(
url=f"{HOST_ADDRESS}/api/{VERSION}/datasets/{dataset_id}/metadata/config",
headers={"Content-Type": "application/json"},
auth=HttpApiAuth,
json={
"kb_id": dataset_id,
"metadata": metadata_config,
"enable_metadata": False
"enabled": False,
"fields": [
{"name": "character", "type": "string", "description": "Historical figure name"},
{"name": "era", "type": "string", "description": "Historical era"},
{"name": "achievements", "type": "list", "description": "Major achievements"},
]
}
).json()
@@ -112,8 +97,6 @@ class TestMetadataWithRetrieval:
Verifies that chunks are only retrieved from documents matching the metadata condition.
"""
from common import upload_documents, parse_documents, retrieval_chunks
dataset_id = add_dataset_with_metadata
# Create two documents with different metadata

View File

@@ -28,16 +28,12 @@ def _summary_to_counts(summary):
class TestMetadataSummary:
@pytest.mark.p2
def test_metadata_summary_missing_kb_id(self, HttpApiAuth, add_document_func):
def test_metadata_summary_nonexistent_kb_id(self, HttpApiAuth, add_document_func):
"""
Call with non-existent dataset
:param HttpApiAuth:
:param add_document_func:
:return:
"""
res = metadata_summary(HttpApiAuth, "")
assert res["code"] == 404, res
assert res["message"] == "Not Found: /api/v1/datasets//metadata/summary", res
res = metadata_summary(HttpApiAuth, "0" * 32)
assert res["code"] == 102, res
@pytest.mark.p2
def test_metadata_summary_invalid_kb_id(self, HttpApiAuth, add_document_func):

View File

@@ -58,11 +58,11 @@ class TestAuthorization:
@pytest.mark.parametrize(
"invalid_auth, expected_code, expected_message",
[
(None, 0, "`Authorization` can't be empty"),
(None, 401, "<Unauthorized '401: Unauthorized'>"),
(
RAGFlowHttpApiAuth(INVALID_API_TOKEN),
109,
"Authentication error: API key is invalid!",
401,
"<Unauthorized '401: Unauthorized'>",
),
],
)
@@ -101,7 +101,7 @@ class TestDocumentsParse:
@pytest.mark.parametrize(
"dataset_id, expected_code, expected_message",
[
("", 100, "<MethodNotAllowed '405: Method Not Allowed'>"),
("", 102, "You don't own the dataset ."),
(
"invalid_dataset_id",
102,

View File

@@ -48,11 +48,11 @@ class TestAuthorization:
@pytest.mark.parametrize(
"invalid_auth, expected_code, expected_message",
[
(None, 0, "`Authorization` can't be empty"),
(None, 401, "<Unauthorized '401: Unauthorized'>"),
(
RAGFlowHttpApiAuth(INVALID_API_TOKEN),
109,
"Authentication error: API key is invalid!",
401,
"<Unauthorized '401: Unauthorized'>",
),
],
)
@@ -105,7 +105,7 @@ class TestDocumentsParseStop:
@pytest.mark.parametrize(
"invalid_dataset_id, expected_code, expected_message",
[
("", 100, "<MethodNotAllowed '405: Method Not Allowed'>"),
("", 102, "You don't own the dataset ."),
(
"invalid_dataset_id",
102,