From 5355edefe366bc6dce831a38a8b191b4a17bd8a1 Mon Sep 17 00:00:00 2001 From: Zhu Lei <123629497+kabishou11@users.noreply.github.com> Date: Fri, 4 Sep 2026 07:19:33 +0000 Subject: [PATCH] fix(api): correct dataset and document list has_more pagination (#41784) --- .../service_api/dataset/dataset.py | 7 +- .../service_api/dataset/document.py | 7 +- .../service_api/dataset/test_dataset_apis.py | 60 ++++++++++++++++ .../service_api/dataset/test_document.py | 70 +++++++++++++++++++ 4 files changed, 138 insertions(+), 6 deletions(-) diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py index 09a404b9eff..32035f539b1 100644 --- a/api/controllers/service_api/dataset/dataset.py +++ b/api/controllers/service_api/dataset/dataset.py @@ -430,10 +430,11 @@ class DatasetListApi(DatasetApiResource): query_params["tag_ids"] = request.args.getlist("tag_ids") query = DatasetListQuery.model_validate(query_params) # provider = request.args.get("provider", default="vendor") + effective_limit = min(query.limit, 100) datasets, total = DatasetService.get_datasets( query.page, - query.limit, + effective_limit, session, tenant_id, current_user, @@ -467,8 +468,8 @@ class DatasetListApi(DatasetApiResource): item["embedding_available"] = True response = { "data": data, - "has_more": len(datasets) == query.limit, - "limit": query.limit, + "has_more": query.page * effective_limit < total, + "limit": effective_limit, "total": total, "page": query.page, } diff --git a/api/controllers/service_api/dataset/document.py b/api/controllers/service_api/dataset/document.py index 5049c2a3782..a2288a99c43 100644 --- a/api/controllers/service_api/dataset/document.py +++ b/api/controllers/service_api/dataset/document.py @@ -1015,8 +1015,9 @@ class DocumentListApi(DatasetApiResource): query = query.order_by(desc(Document.created_at), desc(Document.position)) + effective_limit = min(query_params.limit, 100) paginated_documents = paginate_query( - query, session=session, page=query_params.page, per_page=query_params.limit, max_per_page=100 + query, session=session, page=query_params.page, per_page=effective_limit, max_per_page=100 ) documents = paginated_documents.items @@ -1029,8 +1030,8 @@ class DocumentListApi(DatasetApiResource): response = { "data": document_responses(documents, session=session), - "has_more": len(documents) == query_params.limit, - "limit": query_params.limit, + "has_more": query_params.page * effective_limit < paginated_documents.total, + "limit": effective_limit, "total": paginated_documents.total, "page": query_params.page, } diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py index 581f95d8ce5..8b3c7d42c79 100644 --- a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py +++ b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py @@ -257,6 +257,66 @@ class TestDatasetListApiGet: False, ) + @patch("controllers.service_api.dataset.dataset.create_plugin_provider_manager") + @patch("controllers.service_api.dataset.dataset.DatasetService") + def test_list_datasets_has_more_false_on_last_page_exact_limit( + self, + mock_dataset_svc: MagicMock, + mock_provider_mgr: MagicMock, + app: Flask, + account: Account, + tenant: Tenant, + controller_session: Session, + ) -> None: + """A full last page must set has_more false instead of forcing another fetch.""" + from controllers.service_api.dataset.dataset import DatasetListApi + + page_size = 20 + dataset = make_dataset(controller_session, tenant, account) + mock_dataset_svc.get_datasets.return_value = ([dataset] * page_size, page_size) + mock_provider_mgr.return_value.get_configurations.return_value.get_models.return_value = list[object]() + + with app.test_request_context(f"/datasets?page=1&limit={page_size}", method="GET"): + api = DatasetListApi() + response, status = unwrap(api.get)(api, controller_session, tenant_id=tenant.id) + + assert status == 200 + assert response["has_more"] is False + assert response["limit"] == page_size + assert response["total"] == page_size + assert response["page"] == 1 + + @patch("controllers.service_api.dataset.dataset.create_plugin_provider_manager") + @patch("controllers.service_api.dataset.dataset.DatasetService") + def test_list_datasets_has_more_true_when_limit_exceeds_cap( + self, + mock_dataset_svc: MagicMock, + mock_provider_mgr: MagicMock, + app: Flask, + account: Account, + tenant: Tenant, + controller_session: Session, + ) -> None: + """limit>100 still reports remaining rows after the server cap of 100.""" + from controllers.service_api.dataset.dataset import DatasetListApi + + returned_count = 100 + total = 150 + dataset = make_dataset(controller_session, tenant, account) + mock_dataset_svc.get_datasets.return_value = ([dataset] * returned_count, total) + mock_provider_mgr.return_value.get_configurations.return_value.get_models.return_value = list[object]() + + with app.test_request_context("/datasets?page=1&limit=200", method="GET"): + api = DatasetListApi() + response, status = unwrap(api.get)(api, controller_session, tenant_id=tenant.id) + + assert status == 200 + assert response["has_more"] is True + assert response["limit"] == 100 + assert response["total"] == total + assert response["page"] == 1 + assert mock_dataset_svc.get_datasets.call_args.args[1] == 100 + class TestDatasetListApiPost: """Test suite for DatasetListApi.post() endpoint.""" diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py index a2e8d760feb..c297b95987e 100644 --- a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py +++ b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py @@ -1189,6 +1189,76 @@ class TestDocumentListApi(SQLiteControllerTest): assert "data_source_info_dict" not in response["data"][0] assert "doc_metadata_details" not in response["data"][0] + @patch("controllers.service_api.dataset.document.paginate_query") + @patch("controllers.service_api.dataset.document.DocumentService") + def test_list_documents_has_more_false_on_last_page_exact_limit( + self, mock_doc_svc, mock_paginate, app: Flask, mock_tenant, mock_dataset + ): + """A full last page must set has_more false instead of forcing another fetch.""" + self._persist_dataset(mock_dataset) + page_size = 20 + documents = [ + make_serializable_document( + id=f"doc-{index}", + name=f"Document {index}", + tenant_id=mock_tenant, + dataset_id=mock_dataset.id, + ) + for index in range(page_size) + ] + mock_paginate.return_value = _PaginationRecord(items=documents, total=page_size) + mock_doc_svc.enrich_documents_with_summary_index_status.return_value = None + + with app.test_request_context( + f"/datasets/{mock_dataset.id}/documents?page=1&limit={page_size}", + method="GET", + ): + api = DocumentListApi() + response = inspect.unwrap(type(api).get)( + api, self.session, tenant_id=mock_tenant, dataset_id=mock_dataset.id + ) + + assert response["has_more"] is False + assert response["limit"] == page_size + assert response["total"] == page_size + assert response["page"] == 1 + + @patch("controllers.service_api.dataset.document.paginate_query") + @patch("controllers.service_api.dataset.document.DocumentService") + def test_list_documents_has_more_true_when_limit_exceeds_cap( + self, mock_doc_svc, mock_paginate, app: Flask, mock_tenant, mock_dataset + ): + """limit>100 still reports remaining rows after the server cap of 100.""" + self._persist_dataset(mock_dataset) + returned_count = 100 + total = 150 + documents = [ + make_serializable_document( + id=f"doc-{index}", + name=f"Document {index}", + tenant_id=mock_tenant, + dataset_id=mock_dataset.id, + ) + for index in range(returned_count) + ] + mock_paginate.return_value = _PaginationRecord(items=documents, total=total) + mock_doc_svc.enrich_documents_with_summary_index_status.return_value = None + + with app.test_request_context( + f"/datasets/{mock_dataset.id}/documents?page=1&limit=200", + method="GET", + ): + api = DocumentListApi() + response = inspect.unwrap(type(api).get)( + api, self.session, tenant_id=mock_tenant, dataset_id=mock_dataset.id + ) + + assert response["has_more"] is True + assert response["limit"] == 100 + assert response["total"] == total + assert response["page"] == 1 + assert mock_paginate.call_args.kwargs["per_page"] == 100 + def test_list_documents_dataset_not_found(self, app: Flask, mock_tenant, mock_dataset): """Test 404 when dataset not found.""" # Arrange