Feature: Allow page_size max value 100 (#15292)

Feature: Allow page_size max value 100

test/playwright/conftest.py

@@ -1189,7 +1189,7 @@ def _ensure_dataset_ready_via_api(
     base_url: str, auth_header: str, dataset_name: str
 ) -> dict:
     headers = {"Authorization": auth_header}
-    list_url = _build_url(base_url, "/api/v1/datasets?page=1&page_size=200")
+    list_url = _build_url(base_url, "/api/v1/datasets?page=1&page_size=100")
 
     _, list_payload = _api_request_json(list_url, method="GET", headers=headers)
     existing = _find_dataset_by_name(list_payload, dataset_name)

test/testcases/restful_api/test_chunks.py

@@ -499,7 +499,7 @@ def test_chunk_delete_concurrent_and_bulk_contract(rest_client, create_document)
     for index in range(40):
         payload = rest_client.post(base_path, json={"content": f"bulk chunk {index}"}).json()
         assert payload["code"] == 0, payload
-    bulk_ids_payload = rest_client.get(base_path, params={"page_size": 200}).json()
+    bulk_ids_payload = rest_client.get(base_path, params={"page_size": 100}).json()
     assert bulk_ids_payload["code"] == 0, bulk_ids_payload
     bulk_ids = [chunk["id"] for chunk in bulk_ids_payload["data"]["chunks"]]
     bulk_res = rest_client.delete(base_path, json={"chunk_ids": bulk_ids})

test/testcases/test_http_api/common.py

@@ -59,7 +59,7 @@ def delete_datasets(auth, payload=None, *, headers=HEADERS, data=None):
     return res.json()
 
 
-def delete_all_datasets(auth, *, page_size=1000):
+def delete_all_datasets(auth, *, page_size=100):
     return delete_datasets(auth, {"ids": None, "delete_all": True})
 
 
@@ -133,7 +133,7 @@ def delete_documents(auth, dataset_id, payload=None):
     return res.json()
 
 
-def delete_all_documents(auth, dataset_id, *, page_size=1000):
+def delete_all_documents(auth, dataset_id, *, page_size=100):
     return delete_documents(auth, dataset_id, {"ids": None, "delete_all": True})
 
 
@@ -192,7 +192,7 @@ def delete_chunks(auth, dataset_id, document_id, payload=None):
     return res.json()
 
 
-def delete_all_chunks(auth, dataset_id, document_id, *, page_size=1000):
+def delete_all_chunks(auth, dataset_id, document_id, *, page_size=100):
     return delete_chunks(auth, dataset_id, document_id, {"chunk_ids": None, "delete_all": True})
 
 
@@ -247,7 +247,7 @@ def delete_chat_assistants(auth, payload=None):
     return res.json()
 
 
-def delete_all_chat_assistants(auth, *, page_size=1000):
+def delete_all_chat_assistants(auth, *, page_size=100):
     return delete_chat_assistants(auth, {"ids": None, "delete_all": True})
 
 
@@ -284,7 +284,7 @@ def delete_session_with_chat_assistants(auth, chat_assistant_id, payload=None):
     return res.json()
 
 
-def delete_all_sessions_with_chat_assistant(auth, chat_assistant_id, *, page_size=1000):
+def delete_all_sessions_with_chat_assistant(auth, chat_assistant_id, *, page_size=100):
     return delete_session_with_chat_assistants(auth, chat_assistant_id, {"ids": None, "delete_all": True})
 
 
@@ -378,7 +378,7 @@ def delete_agent_sessions(auth, agent_id, payload=None):
     return res.json()
 
 
-def delete_all_agent_sessions(auth, agent_id, *, page_size=1000):
+def delete_all_agent_sessions(auth, agent_id, *, page_size=100):
     return delete_agent_sessions(auth, agent_id, {"ids": None, "delete_all": True})
 
 
@@ -525,4 +525,3 @@ def search_dataset(auth, dataset_id, payload=None, *, headers=HEADERS):
     return res.json()
 
 
-

test/testcases/test_http_api/test_session_management/test_agent_completions.py

@@ -58,7 +58,7 @@ def _agent_items(res):
 
 @pytest.fixture(scope="function")
 def agent_id(HttpApiAuth, request):
-    res = list_agents(HttpApiAuth, {"page_size": 1000})
+    res = list_agents(HttpApiAuth, {"page_size": 100})
     assert res["code"] == 0, res
     for agent in _agent_items(res):
         if agent.get("title") == AGENT_TITLE:

test/testcases/test_http_api/test_session_management/test_agent_sessions.py

@@ -58,7 +58,7 @@ def _agent_items(res):
 
 @pytest.fixture(scope="function")
 def agent_id(HttpApiAuth, request):
-    res = list_agents(HttpApiAuth, {"page_size": 1000})
+    res = list_agents(HttpApiAuth, {"page_size": 100})
     assert res["code"] == 0, res
     for agent in _agent_items(res):
         if agent.get("title") == AGENT_TITLE:

test/testcases/test_sdk_api/common.py

@@ -25,11 +25,11 @@ def batch_create_datasets(client: RAGFlow, num: int) -> list[DataSet]:
     return [client.create_dataset(name=f"dataset_{i}") for i in range(num)]
 
 
-def delete_all_datasets(client: RAGFlow, *, page_size: int = 1000) -> None:
+def delete_all_datasets(client: RAGFlow, *, page_size: int = 100) -> None:
     client.delete_datasets(delete_all=True)
 
 
-def delete_all_chats(client: RAGFlow, *, page_size: int = 1000) -> None:
+def delete_all_chats(client: RAGFlow, *, page_size: int = 100) -> None:
     client.delete_chats(delete_all=True)
 
 
@@ -45,15 +45,15 @@ def bulk_upload_documents(dataset: DataSet, num: int, tmp_path: Path) -> list[Do
     return dataset.upload_documents(document_infos)
 
 
-def delete_all_documents(dataset: DataSet, *, page_size: int = 1000) -> None:
+def delete_all_documents(dataset: DataSet, *, page_size: int = 100) -> None:
     dataset.delete_documents(delete_all=True)
 
 
-def delete_all_sessions(chat_assistant: Chat, *, page_size: int = 1000) -> None:
+def delete_all_sessions(chat_assistant: Chat, *, page_size: int = 100) -> None:
     chat_assistant.delete_sessions(delete_all=True)
 
 
-def delete_all_chunks(document: Document, *, page_size: int = 1000) -> None:
+def delete_all_chunks(document: Document, *, page_size: int = 100) -> None:
     document.delete_chunks(delete_all=True)
 
 

test/testcases/test_sdk_api/conftest.py

@@ -48,7 +48,7 @@ from utils.file_utils import (
 
 @wait_for(200, 1, "Document parsing timeout")
 def condition(_dataset: DataSet):
-    documents = _dataset.list_documents(page_size=1000)
+    documents = _dataset.list_documents(page_size=100)
     for document in documents:
         if document.run != "DONE":
             return False

test/testcases/test_sdk_api/test_chat_assistant_management/conftest.py

@@ -22,7 +22,7 @@ from utils import wait_for
 
 @wait_for(200, 1, "Document parsing timeout")
 def condition(_dataset: DataSet):
-    documents = _dataset.list_documents(page_size=1000)
+    documents = _dataset.list_documents(page_size=100)
     for document in documents:
         if document.run != "DONE":
             return False

test/testcases/test_sdk_api/test_chunk_management_within_dataset/conftest.py

@@ -25,7 +25,7 @@ from utils import wait_for
 
 @wait_for(30, 1, "Document parsing timeout")
 def condition(_dataset: DataSet):
-    documents = _dataset.list_documents(page_size=1000)
+    documents = _dataset.list_documents(page_size=100)
     for document in documents:
         if document.run != "DONE":
             return False
@@ -33,7 +33,7 @@ def condition(_dataset: DataSet):
 
 @wait_for(30, 1, "Chunk indexing timeout")
 def chunks_visible(_document: Document, _chunk_ids: list[str]):
-    visible_ids = {chunk.id for chunk in _document.list_chunks(page_size=1000)}
+    visible_ids = {chunk.id for chunk in _document.list_chunks(page_size=100)}
     return set(_chunk_ids).issubset(visible_ids)
 
 @pytest.fixture(scope="function")

test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py

@@ -51,7 +51,10 @@ class TestCapability:
         for i in range(count):
             payload = {"name": f"dataset_{i}"}
             client.create_dataset(**payload)
-        assert len(client.list_datasets(page_size=2000)) == count
+        datasets = []
+        for page in range(1, (count // 100) + 1):
+            datasets.extend(client.list_datasets(page=page, page_size=100))
+        assert len(datasets) == count
 
     @pytest.mark.p3
     def test_create_dataset_concurrent(self, client):

test/testcases/test_sdk_api/test_file_management_within_dataset/test_parse_documents.py

@@ -23,7 +23,7 @@ from utils import wait_for
 
 @wait_for(30, 1, "Document parsing timeout")
 def condition(_dataset: DataSet, _document_ids: list[str] = None):
-    documents = _dataset.list_documents(page_size=1000)
+    documents = _dataset.list_documents(page_size=100)
 
     if _document_ids is None:
         for document in documents:
@@ -40,7 +40,7 @@ def condition(_dataset: DataSet, _document_ids: list[str] = None):
 
 
 def validate_document_details(dataset, document_ids):
-    documents = dataset.list_documents(page_size=1000)
+    documents = dataset.list_documents(page_size=100)
     for document in documents:
         if document.id in document_ids:
             assert document.run == "DONE"

test/testcases/test_sdk_api/test_file_management_within_dataset/test_stop_parse_documents.py

@@ -18,7 +18,7 @@ import pytest
 
 
 def validate_document_parse_done(dataset, document_ids):
-    documents = dataset.list_documents(page_size=1000)
+    documents = dataset.list_documents(page_size=100)
     for document in documents:
         if document.id in document_ids:
             assert document.run == "DONE"
@@ -29,7 +29,7 @@ def validate_document_parse_done(dataset, document_ids):
 
 
 def validate_document_parse_cancel(dataset, document_ids):
-    documents = dataset.list_documents(page_size=1000)
+    documents = dataset.list_documents(page_size=100)
     for document in documents:
         assert document.run == "CANCEL"
         assert len(document.process_begin_at) > 0

test/testcases/test_web_api/conftest.py

@@ -104,7 +104,7 @@ def require_env_flag():
 @pytest.fixture(scope="function")
 def clear_datasets(request: FixtureRequest, WebApiAuth: RAGFlowWebApiAuth):
     def cleanup():
-        res = list_datasets(WebApiAuth, params={"page_size": 1000})
+        res = list_datasets(WebApiAuth, params={"page_size": 100})
         kb_ids = [kb["id"] for kb in res["data"]]
         delete_datasets(WebApiAuth, {"ids": kb_ids})
 
@@ -122,7 +122,7 @@ def clear_chats(request, WebApiAuth):
 @pytest.fixture(scope="class")
 def add_dataset(request: FixtureRequest, WebApiAuth: RAGFlowWebApiAuth) -> str:
     def cleanup():
-        res = list_datasets(WebApiAuth, params={"page_size": 1000})
+        res = list_datasets(WebApiAuth, params={"page_size": 100})
         kb_ids = [kb["id"] for kb in res["data"]]
         delete_datasets(WebApiAuth, {"ids": kb_ids})
 
@@ -133,7 +133,7 @@ def add_dataset(request: FixtureRequest, WebApiAuth: RAGFlowWebApiAuth) -> str:
 @pytest.fixture(scope="function")
 def add_dataset_func(request: FixtureRequest, WebApiAuth: RAGFlowWebApiAuth) -> str:
     def cleanup():
-        res = list_datasets(WebApiAuth, params={"page_size": 1000})
+        res = list_datasets(WebApiAuth, params={"page_size": 100})
         kb_ids = [kb["id"] for kb in res["data"]]
         delete_datasets(WebApiAuth, {"ids": kb_ids})
 

test/unit_test/api/utils/test_doc_validation.py

@@ -17,7 +17,14 @@
 """Unit tests for api.apps.sdk.doc_validation module."""
 
 from unittest.mock import Mock
+
+import pytest
+from pydantic import ValidationError
+
+from api.utils.pagination_utils import REST_API_MAX_PAGE_SIZE, validate_rest_api_page_size
 from api.utils.validation_utils import (
+    ListDatasetReq,
+    ListFileReq,
     ParserConfig,
     UpdateDocumentReq,
     validate_chunk_method,
@@ -29,6 +36,16 @@ from api.db import FileType
 from common.constants import RetCode
 
 
+def test_rest_api_page_size_rejects_values_above_100():
+    assert validate_rest_api_page_size(REST_API_MAX_PAGE_SIZE) == REST_API_MAX_PAGE_SIZE
+    with pytest.raises(ValueError, match="page_size must be less than or equal to 100"):
+        validate_rest_api_page_size(REST_API_MAX_PAGE_SIZE + 1)
+    with pytest.raises(ValidationError, match="page_size must be less than or equal to 100"):
+        ListDatasetReq(page_size=REST_API_MAX_PAGE_SIZE + 1)
+    with pytest.raises(ValidationError, match="page_size must be less than or equal to 100"):
+        ListFileReq(page_size=REST_API_MAX_PAGE_SIZE + 1)
+
+
 def test_validate_immutable_fields_no_changes():
     """Test when no immutable fields are present in request."""
     update_doc_req = UpdateDocumentReq()
@@ -311,4 +328,4 @@ def test_parser_config_normalizes_legacy_vectorize_table_column_role():
         "title": "indexing",
         "country": "metadata",
         "x": "both",
-    }
+    }