fix(api): correct dataset and document list has_more pagination (#41784)

api/controllers/service_api/dataset/dataset.py

@@ -430,10 +430,11 @@ class DatasetListApi(DatasetApiResource):
             query_params["tag_ids"] = request.args.getlist("tag_ids")
         query = DatasetListQuery.model_validate(query_params)
         # provider = request.args.get("provider", default="vendor")
+        effective_limit = min(query.limit, 100)
 
         datasets, total = DatasetService.get_datasets(
             query.page,
-            query.limit,
+            effective_limit,
             session,
             tenant_id,
             current_user,
@@ -467,8 +468,8 @@ class DatasetListApi(DatasetApiResource):
                 item["embedding_available"] = True
         response = {
             "data": data,
-            "has_more": len(datasets) == query.limit,
-            "limit": query.limit,
+            "has_more": query.page * effective_limit < total,
+            "limit": effective_limit,
             "total": total,
             "page": query.page,
         }

api/controllers/service_api/dataset/document.py

@@ -1015,8 +1015,9 @@ class DocumentListApi(DatasetApiResource):
 
         query = query.order_by(desc(Document.created_at), desc(Document.position))
 
+        effective_limit = min(query_params.limit, 100)
         paginated_documents = paginate_query(
-            query, session=session, page=query_params.page, per_page=query_params.limit, max_per_page=100
+            query, session=session, page=query_params.page, per_page=effective_limit, max_per_page=100
         )
         documents = paginated_documents.items
 
@@ -1029,8 +1030,8 @@ class DocumentListApi(DatasetApiResource):
 
         response = {
             "data": document_responses(documents, session=session),
-            "has_more": len(documents) == query_params.limit,
-            "limit": query_params.limit,
+            "has_more": query_params.page * effective_limit < paginated_documents.total,
+            "limit": effective_limit,
             "total": paginated_documents.total,
             "page": query_params.page,
         }

api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py

@@ -257,6 +257,66 @@ class TestDatasetListApiGet:
             False,
         )
 
+    @patch("controllers.service_api.dataset.dataset.create_plugin_provider_manager")
+    @patch("controllers.service_api.dataset.dataset.DatasetService")
+    def test_list_datasets_has_more_false_on_last_page_exact_limit(
+        self,
+        mock_dataset_svc: MagicMock,
+        mock_provider_mgr: MagicMock,
+        app: Flask,
+        account: Account,
+        tenant: Tenant,
+        controller_session: Session,
+    ) -> None:
+        """A full last page must set has_more false instead of forcing another fetch."""
+        from controllers.service_api.dataset.dataset import DatasetListApi
+
+        page_size = 20
+        dataset = make_dataset(controller_session, tenant, account)
+        mock_dataset_svc.get_datasets.return_value = ([dataset] * page_size, page_size)
+        mock_provider_mgr.return_value.get_configurations.return_value.get_models.return_value = list[object]()
+
+        with app.test_request_context(f"/datasets?page=1&limit={page_size}", method="GET"):
+            api = DatasetListApi()
+            response, status = unwrap(api.get)(api, controller_session, tenant_id=tenant.id)
+
+        assert status == 200
+        assert response["has_more"] is False
+        assert response["limit"] == page_size
+        assert response["total"] == page_size
+        assert response["page"] == 1
+
+    @patch("controllers.service_api.dataset.dataset.create_plugin_provider_manager")
+    @patch("controllers.service_api.dataset.dataset.DatasetService")
+    def test_list_datasets_has_more_true_when_limit_exceeds_cap(
+        self,
+        mock_dataset_svc: MagicMock,
+        mock_provider_mgr: MagicMock,
+        app: Flask,
+        account: Account,
+        tenant: Tenant,
+        controller_session: Session,
+    ) -> None:
+        """limit>100 still reports remaining rows after the server cap of 100."""
+        from controllers.service_api.dataset.dataset import DatasetListApi
+
+        returned_count = 100
+        total = 150
+        dataset = make_dataset(controller_session, tenant, account)
+        mock_dataset_svc.get_datasets.return_value = ([dataset] * returned_count, total)
+        mock_provider_mgr.return_value.get_configurations.return_value.get_models.return_value = list[object]()
+
+        with app.test_request_context("/datasets?page=1&limit=200", method="GET"):
+            api = DatasetListApi()
+            response, status = unwrap(api.get)(api, controller_session, tenant_id=tenant.id)
+
+        assert status == 200
+        assert response["has_more"] is True
+        assert response["limit"] == 100
+        assert response["total"] == total
+        assert response["page"] == 1
+        assert mock_dataset_svc.get_datasets.call_args.args[1] == 100
+
 
 class TestDatasetListApiPost:
     """Test suite for DatasetListApi.post() endpoint."""

api/tests/unit_tests/controllers/service_api/dataset/test_document.py

@@ -1189,6 +1189,76 @@ class TestDocumentListApi(SQLiteControllerTest):
         assert "data_source_info_dict" not in response["data"][0]
         assert "doc_metadata_details" not in response["data"][0]
 
+    @patch("controllers.service_api.dataset.document.paginate_query")
+    @patch("controllers.service_api.dataset.document.DocumentService")
+    def test_list_documents_has_more_false_on_last_page_exact_limit(
+        self, mock_doc_svc, mock_paginate, app: Flask, mock_tenant, mock_dataset
+    ):
+        """A full last page must set has_more false instead of forcing another fetch."""
+        self._persist_dataset(mock_dataset)
+        page_size = 20
+        documents = [
+            make_serializable_document(
+                id=f"doc-{index}",
+                name=f"Document {index}",
+                tenant_id=mock_tenant,
+                dataset_id=mock_dataset.id,
+            )
+            for index in range(page_size)
+        ]
+        mock_paginate.return_value = _PaginationRecord(items=documents, total=page_size)
+        mock_doc_svc.enrich_documents_with_summary_index_status.return_value = None
+
+        with app.test_request_context(
+            f"/datasets/{mock_dataset.id}/documents?page=1&limit={page_size}",
+            method="GET",
+        ):
+            api = DocumentListApi()
+            response = inspect.unwrap(type(api).get)(
+                api, self.session, tenant_id=mock_tenant, dataset_id=mock_dataset.id
+            )
+
+        assert response["has_more"] is False
+        assert response["limit"] == page_size
+        assert response["total"] == page_size
+        assert response["page"] == 1
+
+    @patch("controllers.service_api.dataset.document.paginate_query")
+    @patch("controllers.service_api.dataset.document.DocumentService")
+    def test_list_documents_has_more_true_when_limit_exceeds_cap(
+        self, mock_doc_svc, mock_paginate, app: Flask, mock_tenant, mock_dataset
+    ):
+        """limit>100 still reports remaining rows after the server cap of 100."""
+        self._persist_dataset(mock_dataset)
+        returned_count = 100
+        total = 150
+        documents = [
+            make_serializable_document(
+                id=f"doc-{index}",
+                name=f"Document {index}",
+                tenant_id=mock_tenant,
+                dataset_id=mock_dataset.id,
+            )
+            for index in range(returned_count)
+        ]
+        mock_paginate.return_value = _PaginationRecord(items=documents, total=total)
+        mock_doc_svc.enrich_documents_with_summary_index_status.return_value = None
+
+        with app.test_request_context(
+            f"/datasets/{mock_dataset.id}/documents?page=1&limit=200",
+            method="GET",
+        ):
+            api = DocumentListApi()
+            response = inspect.unwrap(type(api).get)(
+                api, self.session, tenant_id=mock_tenant, dataset_id=mock_dataset.id
+            )
+
+        assert response["has_more"] is True
+        assert response["limit"] == 100
+        assert response["total"] == total
+        assert response["page"] == 1
+        assert mock_paginate.call_args.kwargs["per_page"] == 100
+
     def test_list_documents_dataset_not_found(self, app: Flask, mock_tenant, mock_dataset):
         """Test 404 when dataset not found."""
         # Arrange