refactor(api): migrate dataset endpoints to BaseModel (#37957)

Co-authored-by: Asuka Minato <i@asukaminato.eu.org>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

api/controllers/console/datasets/datasets.py

@@ -30,6 +30,7 @@ from controllers.console.wraps import (
     with_current_tenant_id,
     with_current_user,
 )
+from core.entities.knowledge_entities import IndexingEstimate
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.indexing_runner import IndexingRunner
 from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
@@ -268,21 +269,10 @@ class ErrorDocsResponse(DocumentStatusListResponse):
     total: int
 
 
-class IndexingEstimatePreviewItemResponse(ResponseModel):
+class IndexingEstimateResponse(IndexingEstimate):
-    content: str
+    tokens: int
-    child_chunks: list[str] | None = None
+    total_price: float | int
-    summary: str | None = None
+    currency: str
-
-
-class IndexingEstimateQaPreviewItemResponse(ResponseModel):
-    question: str
-    answer: str
-
-
-class IndexingEstimateResponse(ResponseModel):
-    total_segments: int
-    preview: list[IndexingEstimatePreviewItemResponse]
-    qa_preview: list[IndexingEstimateQaPreviewItemResponse] | None = None
 
 
 class RetrievalSettingResponse(ResponseModel):
@@ -647,7 +637,7 @@ class DatasetApi(Resource):
         else:
             data["embedding_available"] = True
 
-        return data, 200
+        return dump_response(DatasetDetailWithPartialMembersResponse, data), 200
 
     @console_ns.doc("update_dataset")
     @console_ns.doc(description="Update dataset details")
@@ -717,7 +707,7 @@ class DatasetApi(Resource):
         partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session())
         result_data.update({"partial_member_list": partial_member_list})
 
-        return result_data, 200
+        return dump_response(DatasetDetailWithPartialMembersResponse, result_data), 200
 
     @setup_required
     @login_required
@@ -760,7 +750,7 @@ class DatasetUseCheckApi(Resource):
         dataset_id_str = str(dataset_id)
 
         dataset_is_using = DatasetService.dataset_use_check(dataset_id_str, db.session())
-        return {"is_using": dataset_is_using}, 200
+        return UsageCheckResponse(is_using=dataset_is_using).model_dump(mode="json"), 200
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/queries")
@@ -901,7 +891,17 @@ class DatasetIndexingEstimateApi(Resource):
         except Exception as e:
             raise IndexingEstimateError(str(e))
 
-        return response.model_dump(), 200
+        return (
+            IndexingEstimateResponse(
+                tokens=0,
+                total_price=0,
+                currency="USD",
+                total_segments=response.total_segments,
+                preview=response.preview,
+                qa_preview=response.qa_preview,
+            ).model_dump(mode="json", exclude_none=True),
+            200,
+        )
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/related-apps")
@@ -1018,7 +1018,7 @@ class DatasetApiKeyApi(Resource):
         keys = db.session.scalars(
             select(ApiToken).where(ApiToken.type == self.resource_type, ApiToken.tenant_id == current_tenant_id)
         ).all()
-        return ApiKeyList.model_validate({"data": keys}, from_attributes=True).model_dump(mode="json")
+        return dump_response(ApiKeyList, {"data": keys})
 
     @console_ns.response(200, "API key created successfully", console_ns.models[ApiKeyItem.__name__])
     @console_ns.response(400, "Maximum keys exceeded")
@@ -1052,7 +1052,7 @@ class DatasetApiKeyApi(Resource):
         api_token.type = self.resource_type
         db.session.add(api_token)
         db.session.commit()
-        return ApiKeyItem.model_validate(api_token, from_attributes=True).model_dump(mode="json"), 200
+        return dump_response(ApiKeyItem, api_token), 200
 
 
 @console_ns.route("/datasets/api-keys/<uuid:api_key_id>")
@@ -1107,7 +1107,7 @@ class DatasetEnableApiApi(Resource):
 
         DatasetService.update_dataset_api_status(dataset_id_str, status == "enable", db.session())
 
-        return {"result": "success"}, 200
+        return SimpleResultResponse(result="success").model_dump(mode="json"), 200
 
 
 @console_ns.route("/datasets/api-base-info")
@@ -1120,7 +1120,7 @@ class DatasetApiBaseUrlApi(Resource):
     @account_initialization_required
     def get(self):
         base = dify_config.SERVICE_API_URL or request.host_url.rstrip("/")
-        return {"api_base_url": normalize_api_base_url(base)}
+        return ApiBaseUrlResponse(api_base_url=normalize_api_base_url(base)).model_dump(mode="json")
 
 
 @console_ns.route("/datasets/retrieval-setting")

api/controllers/console/datasets/datasets_document.py

@@ -10,16 +10,17 @@ from uuid import UUID
 import sqlalchemy as sa
 from flask import request, send_file
 from flask_restx import Resource
-from pydantic import BaseModel, Field, RootModel, field_validator
+from pydantic import BaseModel, Field, JsonValue, field_validator
 from sqlalchemy import asc, desc, func, select
 from werkzeug.exceptions import Forbidden, NotFound
 
 import services
 from controllers.common.controller_schemas import DocumentBatchDownloadZipPayload
-from controllers.common.fields import BinaryFileResponse, SimpleResultMessageResponse, SimpleResultResponse, UrlResponse
+from controllers.common.fields import SimpleResultMessageResponse, SimpleResultResponse, UrlResponse
 from controllers.common.schema import register_response_schema_models, register_schema_models
 from controllers.console import console_ns
 from controllers.console.wraps import RBACPermission, RBACResourceScope, rbac_permission_required
+from core.entities.knowledge_entities import IndexingEstimate
 from core.errors.error import (
     LLMBadRequestError,
     ModelCurrentlyNotSupportError,
@@ -29,6 +30,7 @@ from core.errors.error import (
 from core.indexing_runner import IndexingRunner
 from core.model_manager import ModelManager
 from core.plugin.impl.exc import PluginDaemonClientSideError
+from core.rag.entities import Rule
 from core.rag.extractor.entity.datasource_type import DatasourceType
 from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo, WebsiteInfo
 from core.rag.index_processor.constant.index_type import IndexTechniqueType
@@ -49,7 +51,7 @@ from libs.login import login_required
 from libs.pagination import paginate_query
 from models import Account, DatasetProcessRule, Document, DocumentSegment, UploadFile
 from models.dataset import DocumentPipelineExecutionLog
-from models.enums import IndexingStatus, SegmentStatus
+from models.enums import IndexingStatus, ProcessRuleMode, SegmentStatus
 from services.dataset_ref_service import DatasetRefService
 from services.dataset_service import DatasetService, DocumentService
 from services.entities.knowledge_entities.knowledge_entities import KnowledgeConfig, ProcessRule, RetrievalModel
@@ -148,8 +150,91 @@ class DocumentWithSegmentsListResponse(ResponseModel):
     page: int
 
 
-class OpaqueObjectResponse(RootModel[dict[str, Any]]):
+class IndexingEstimateResponse(IndexingEstimate):
-    root: dict[str, Any]
+    tokens: int
+    total_price: float | int
+    currency: str
+
+
+class DocumentDetailResponse(ResponseModel):
+    id: str
+    position: int | None = None
+    data_source_type: str | None = None
+    data_source_info: Any = None
+    data_source_detail_dict: Any = None
+    dataset_process_rule_id: str | None = None
+    dataset_process_rule: Any = None
+    document_process_rule: Any = None
+    name: str | None = None
+    created_from: str | None = None
+    created_by: str | None = None
+    created_at: int | None = None
+    tokens: int | None = None
+    indexing_status: str | None = None
+    completed_at: int | None = None
+    updated_at: int | None = None
+    indexing_latency: float | None = None
+    error: str | None = None
+    enabled: bool | None = None
+    disabled_at: int | None = None
+    disabled_by: str | None = None
+    archived: bool | None = None
+    doc_type: str | None = None
+    doc_metadata: list[DocumentMetadataResponse] | None = None
+    segment_count: int | None = None
+    average_segment_length: float | None = None
+    hit_count: int | None = None
+    display_status: str | None = None
+    doc_form: str | None = None
+    doc_language: str | None = None
+    need_summary: bool | None = None
+
+    @field_validator("data_source_type", "indexing_status", "display_status", "doc_form", mode="before")
+    @classmethod
+    def _normalize_enum_fields(cls, value: Any) -> Any:
+        return normalize_enum(value)
+
+
+class SummaryStatusResponse(ResponseModel):
+    completed: int = 0
+    generating: int = 0
+    error: int = 0
+    not_started: int = 0
+    timeout: int = 0
+
+
+class SummaryEntryResponse(ResponseModel):
+    segment_id: str
+    segment_position: int
+    status: str
+    summary_preview: str | None = None
+    error: str | None = None
+    created_at: int | None = None
+    updated_at: int | None = None
+
+    @field_validator("status", mode="before")
+    @classmethod
+    def _normalize_status(cls, value: Any) -> Any:
+        return normalize_enum(value)
+
+
+class DocumentSummaryStatusResponse(ResponseModel):
+    total_segments: int
+    summary_status: SummaryStatusResponse
+    summaries: list[SummaryEntryResponse]
+
+
+class ProcessRuleResponse(ResponseModel):
+    mode: ProcessRuleMode
+    rules: Rule | None = None
+    limits: dict[str, Any]
+
+
+class DocumentPipelineExecutionLogResponse(ResponseModel):
+    datasource_info: JsonValue | None = None
+    datasource_type: str | None = None
+    input_data: JsonValue | None = None
+    datasource_node_id: str | None = None
 
 
 register_schema_models(
@@ -165,7 +250,6 @@ register_schema_models(
 )
 register_response_schema_models(
     console_ns,
-    BinaryFileResponse,
     SimpleResultMessageResponse,
     SimpleResultResponse,
     UrlResponse,
@@ -175,7 +259,11 @@ register_response_schema_models(
     DocumentWithSegmentsResponse,
     DatasetAndDocumentResponse,
     DocumentWithSegmentsListResponse,
-    OpaqueObjectResponse,
+    IndexingEstimateResponse,
+    DocumentDetailResponse,
+    DocumentSummaryStatusResponse,
+    ProcessRuleResponse,
+    DocumentPipelineExecutionLogResponse,
 )
 
 
@@ -225,7 +313,7 @@ class GetProcessRuleApi(Resource):
     @console_ns.doc("get_process_rule")
     @console_ns.doc(description="Get dataset document processing rules")
     @console_ns.doc(params={"document_id": "Document ID (optional)"})
-    @console_ns.response(200, "Process rules retrieved successfully", console_ns.models[OpaqueObjectResponse.__name__])
+    @console_ns.response(200, "Process rules retrieved successfully", console_ns.models[ProcessRuleResponse.__name__])
     @setup_required
     @login_required
     @account_initialization_required
@@ -264,7 +352,7 @@ class GetProcessRuleApi(Resource):
                 mode = dataset_process_rule.mode
                 rules = dataset_process_rule.rules_dict
 
-        return {"mode": mode, "rules": rules, "limits": limits}
+        return dump_response(ProcessRuleResponse, {"mode": mode, "rules": rules, "limits": limits})
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/documents")
@@ -491,7 +579,7 @@ class DatasetInitApi(Resource):
     @console_ns.doc(description="Initialize dataset with documents")
     @console_ns.expect(console_ns.models[KnowledgeConfig.__name__])
     @console_ns.response(
-        201, "Dataset initialized successfully", console_ns.models[DatasetAndDocumentResponse.__name__]
+        200, "Dataset initialized successfully", console_ns.models[DatasetAndDocumentResponse.__name__]
     )
     @console_ns.response(400, "Invalid request parameters")
     @setup_required
@@ -557,7 +645,7 @@ class DocumentIndexingEstimateApi(DocumentResource):
     @console_ns.response(
         200,
         "Indexing estimate calculated successfully",
-        console_ns.models[OpaqueObjectResponse.__name__],
+        console_ns.models[IndexingEstimateResponse.__name__],
     )
     @console_ns.response(404, "Document not found")
     @console_ns.response(400, "Document already finished")
@@ -578,8 +666,6 @@ class DocumentIndexingEstimateApi(DocumentResource):
         data_process_rule = document.dataset_process_rule
         data_process_rule_dict = data_process_rule.to_dict() if data_process_rule else {}
 
-        response = {"tokens": 0, "total_price": 0, "currency": "USD", "total_segments": 0, "preview": []}
-
         if document.data_source_type == "upload_file":
             data_source_info = document.data_source_info_dict
             if data_source_info and "upload_file_id" in data_source_info:
@@ -610,7 +696,18 @@ class DocumentIndexingEstimateApi(DocumentResource):
                         "English",
                         dataset_id_str,
                     )
-                    return estimate_response.model_dump(), 200
+                    return (
+                        # TODO: why using zero here? the same for the below endpoint
+                        IndexingEstimateResponse(
+                            tokens=0,
+                            total_price=0,
+                            currency="USD",
+                            total_segments=estimate_response.total_segments,
+                            preview=estimate_response.preview,
+                            qa_preview=estimate_response.qa_preview,
+                        ).model_dump(mode="json", exclude_none=True),
+                        200,
+                    )
                 except LLMBadRequestError:
                     raise ProviderNotInitializeError(
                         "No Embedding Model available. Please configure a valid provider "
@@ -623,15 +720,24 @@ class DocumentIndexingEstimateApi(DocumentResource):
                 except Exception as e:
                     raise IndexingEstimateError(str(e))
 
-        return response, 200
+        return (
+            IndexingEstimateResponse(
+                tokens=0,
+                total_price=0,
+                currency="USD",
+                total_segments=0,
+                preview=[],
+            ).model_dump(mode="json", exclude_none=True),
+            200,
+        )
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/batch/<string:batch>/indexing-estimate")
 class DocumentBatchIndexingEstimateApi(DocumentResource):
     @console_ns.response(
         200,
-        "Batch indexing estimate calculated successfully",
+        "Indexing estimate calculated successfully",
-        console_ns.models[OpaqueObjectResponse.__name__],
+        console_ns.models[IndexingEstimateResponse.__name__],
     )
     @setup_required
     @login_required
@@ -643,7 +749,16 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
         dataset_id_str = str(dataset_id)
         documents = self.get_batch_documents(dataset_id_str, batch, current_user)
         if not documents:
-            return {"tokens": 0, "total_price": 0, "currency": "USD", "total_segments": 0, "preview": []}, 200
+            return (
+                IndexingEstimateResponse(
+                    tokens=0,
+                    total_price=0,
+                    currency="USD",
+                    total_segments=0,
+                    preview=[],
+                ).model_dump(mode="json", exclude_none=True),
+                200,
+            )
         data_process_rule = documents[0].dataset_process_rule
         data_process_rule_dict = data_process_rule.to_dict() if data_process_rule else {}
         extract_settings = []
@@ -717,7 +832,17 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                     "English",
                     dataset_id_str,
                 )
-                return response.model_dump(), 200
+                return (
+                    IndexingEstimateResponse(
+                        tokens=0,
+                        total_price=0,
+                        currency="USD",
+                        total_segments=response.total_segments,
+                        preview=response.preview,
+                        qa_preview=response.qa_preview,
+                    ).model_dump(mode="json", exclude_none=True),
+                    200,
+                )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
                     "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
@@ -854,7 +979,7 @@ class DocumentApi(DocumentResource):
             "metadata": "Metadata inclusion (all/only/without)",
         }
     )
-    @console_ns.response(200, "Document retrieved successfully", console_ns.models[OpaqueObjectResponse.__name__])
+    @console_ns.response(200, "Document retrieved successfully", console_ns.models[DocumentDetailResponse.__name__])
     @console_ns.response(404, "Document not found")
     @setup_required
     @login_required
@@ -871,46 +996,21 @@ class DocumentApi(DocumentResource):
         if metadata not in self.METADATA_CHOICES:
             raise InvalidMetadataError(f"Invalid metadata value: {metadata}")
 
+        metadata_fields = {"doc_type", "doc_metadata"}
         if metadata == "only":
-            response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
+            response = DocumentDetailResponse.model_validate(
-        elif metadata == "without":
+                {
-            dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session())
+                    "id": document.id,
-            document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
+                    "doc_type": document.doc_type,
-            response = {
+                    "doc_metadata": document.doc_metadata_details,
-                "id": document.id,
+                }
-                "position": document.position,
+            )
-                "data_source_type": document.data_source_type,
+            return response.model_dump(mode="json", include={"id", *metadata_fields}, exclude_unset=True), 200
-                "data_source_info": document.data_source_info_dict,
+
-                "data_source_detail_dict": document.data_source_detail_dict,
+        dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session())
-                "dataset_process_rule_id": document.dataset_process_rule_id,
+        document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
-                "dataset_process_rule": dataset_process_rules,
+        response = DocumentDetailResponse.model_validate(
-                "document_process_rule": document_process_rules,
+            {
-                "name": document.name,
-                "created_from": document.created_from,
-                "created_by": document.created_by,
-                "created_at": int(document.created_at.timestamp()),
-                "tokens": document.tokens,
-                "indexing_status": document.indexing_status,
-                "completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,
-                "updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,
-                "indexing_latency": document.indexing_latency,
-                "error": document.error,
-                "enabled": document.enabled,
-                "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,
-                "disabled_by": document.disabled_by,
-                "archived": document.archived,
-                "segment_count": document.segment_count,
-                "average_segment_length": document.average_segment_length,
-                "hit_count": document.hit_count,
-                "display_status": document.display_status,
-                "doc_form": document.doc_form,
-                "doc_language": document.doc_language,
-                "need_summary": document.need_summary if document.need_summary is not None else False,
-            }
-        else:
-            dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session())
-            document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
-            response = {
                 "id": document.id,
                 "position": document.position,
                 "data_source_type": document.data_source_type,
@@ -943,8 +1043,9 @@ class DocumentApi(DocumentResource):
                 "doc_language": document.doc_language,
                 "need_summary": document.need_summary if document.need_summary is not None else False,
             }
-
+        )
-        return response, 200
+        exclude = metadata_fields if metadata == "without" else None
+        return response.model_dump(mode="json", exclude=exclude, exclude_unset=True), 200
 
     @setup_required
     @login_required
@@ -990,7 +1091,9 @@ class DocumentDownloadApi(DocumentResource):
     def get(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID) -> dict[str, Any]:
         # Reuse the shared permission/tenant checks implemented in DocumentResource.
         document = self.get_document(str(dataset_id), str(document_id), current_user, current_tenant_id)
-        return {"url": DocumentService.get_document_download_url(document, db.session())}
+        return UrlResponse(url=DocumentService.get_document_download_url(document, db.session())).model_dump(
+            mode="json"
+        )
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/documents/download-zip")
@@ -999,7 +1102,7 @@ class DocumentBatchDownloadZipApi(DocumentResource):
 
     @console_ns.doc("download_dataset_documents_as_zip")
     @console_ns.doc(description="Download selected dataset documents as a single ZIP archive (upload-file only)")
-    @console_ns.response(200, "ZIP archive generated successfully", console_ns.models[BinaryFileResponse.__name__])
+    @console_ns.response(200, "ZIP archive downloaded successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -1034,6 +1137,7 @@ class DocumentBatchDownloadZipApi(DocumentResource):
             )
             cleanup = stack.pop_all()
             response.call_on_close(cleanup.close)
+        # response-contract:ignore binary ZIP download response
         return response
 
 
@@ -1093,7 +1197,7 @@ class DocumentProcessingApi(DocumentResource):
                 document.is_paused = False
                 db.session.commit()
 
-        return {"result": "success"}, 200
+        return SimpleResultResponse(result="success").model_dump(mode="json"), 200
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/metadata")
@@ -1152,7 +1256,9 @@ class DocumentMetadataApi(DocumentResource):
         document.updated_at = naive_utc_now()
         db.session.commit()
 
-        return {"result": "success", "message": "Document metadata updated."}, 200
+        return SimpleResultMessageResponse(result="success", message="Document metadata updated.").model_dump(
+            mode="json"
+        ), 200
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/documents/status/<string:action>/batch")
@@ -1194,7 +1300,7 @@ class DocumentStatusApi(DocumentResource):
         except NotFound as e:
             raise NotFound(str(e))
 
-        return {"result": "success"}, 200
+        return SimpleResultResponse(result="success").model_dump(mode="json"), 200
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/pause")
@@ -1321,7 +1427,7 @@ class DocumentRenameApi(DocumentResource):
         # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
         if not current_user.is_dataset_editor:
             raise Forbidden()
-        dataset = DatasetService.get_dataset(dataset_id, db.session())
+        dataset = DatasetService.get_dataset(str(dataset_id), db.session())
         if not dataset:
             raise NotFound("Dataset not found.")
         DatasetService.check_dataset_operator_permission(current_user, dataset, session=db.session())
@@ -1363,15 +1469,15 @@ class WebsiteDocumentSyncApi(DocumentResource):
         # sync document
         DocumentService.sync_website_document(dataset_id_str, document, db.session())
 
-        return {"result": "success"}, 200
+        return SimpleResultResponse(result="success").model_dump(mode="json"), 200
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/pipeline-execution-log")
 class DocumentPipelineExecutionLogApi(DocumentResource):
     @console_ns.response(
         200,
-        "Document pipeline execution log retrieved successfully",
+        "Pipeline execution log retrieved successfully",
-        console_ns.models[OpaqueObjectResponse.__name__],
+        console_ns.models[DocumentPipelineExecutionLogResponse.__name__],
     )
     @setup_required
     @login_required
@@ -1394,18 +1500,16 @@ class DocumentPipelineExecutionLogApi(DocumentResource):
             .limit(1)
         )
         if not log:
-            return {
+            return DocumentPipelineExecutionLogResponse().model_dump(mode="json"), 200
-                "datasource_info": None,
+        return dump_response(
-                "datasource_type": None,
+            DocumentPipelineExecutionLogResponse,
-                "input_data": None,
+            {
-                "datasource_node_id": None,
+                "datasource_info": json.loads(log.datasource_info),
-            }, 200
+                "datasource_type": log.datasource_type,
-        return {
+                "input_data": log.input_data,
-            "datasource_info": json.loads(log.datasource_info),
+                "datasource_node_id": log.datasource_node_id,
-            "datasource_type": log.datasource_type,
+            },
-            "input_data": log.input_data,
+        ), 200
-            "datasource_node_id": log.datasource_node_id,
-        }, 200
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/documents/generate-summary")
@@ -1508,7 +1612,7 @@ class DocumentGenerateSummaryApi(Resource):
                 dataset_id_str,
             )
 
-        return {"result": "success"}, 200
+        return SimpleResultResponse(result="success").model_dump(mode="json"), 200
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/summary-status")
@@ -1516,7 +1620,11 @@ class DocumentSummaryStatusApi(DocumentResource):
     @console_ns.doc("get_document_summary_status")
     @console_ns.doc(description="Get summary index generation status for a document")
     @console_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
-    @console_ns.response(200, "Summary status retrieved successfully", console_ns.models[OpaqueObjectResponse.__name__])
+    @console_ns.response(
+        200,
+        "Summary status retrieved successfully",
+        console_ns.models[DocumentSummaryStatusResponse.__name__],
+    )
     @console_ns.response(404, "Document not found")
     @setup_required
     @login_required
@@ -1534,6 +1642,7 @@ class DocumentSummaryStatusApi(DocumentResource):
           - generating: Number of summaries being generated
           - error: Number of summaries with errors
           - not_started: Number of segments without summary records
+          - timeout: Number of summaries that timed out
         - summaries: List of summary records with status and content preview
         """
         dataset_id_str = str(dataset_id)
@@ -1559,4 +1668,4 @@ class DocumentSummaryStatusApi(DocumentResource):
             session=db.session(),
         )
 
-        return result, 200
+        return dump_response(DocumentSummaryStatusResponse, result), 200

api/controllers/console/datasets/datasets_segments.py

@@ -391,7 +391,7 @@ class DatasetDocumentSegmentApi(Resource):
             SegmentService.update_segments_status(segment_ids, action, dataset, document, db.session())
         except Exception as e:
             raise InvalidActionError(str(e))
-        return dump_response(SimpleResultResponse, {"result": "success"}), 200
+        return SimpleResultResponse(result="success").model_dump(mode="json"), 200
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segment")

api/controllers/console/datasets/external.py

@@ -1,20 +1,16 @@
+from datetime import datetime
 from typing import Any
 from uuid import UUID
 
 from flask import request
-from flask_restx import Resource, fields, marshal
+from flask_restx import Resource
-from pydantic import BaseModel, Field, RootModel
+from pydantic import AliasChoices, BaseModel, Field, field_validator
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services
 from controllers.common.fields import UsageCountResponse
-from controllers.common.schema import (
+from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
-    get_or_create_model,
-    query_params_from_model,
-    register_response_schema_models,
-    register_schema_models,
-)
 from controllers.console import console_ns
 from controllers.console.app.wraps import with_session
 from controllers.console.datasets.error import DatasetNameDuplicateError
@@ -28,21 +24,9 @@ from controllers.console.wraps import (
     with_current_tenant_id,
     with_current_user,
 )
-from extensions.ext_database import db
 from fields.base import ResponseModel
-from fields.dataset_fields import (
+from fields.dataset_fields import DatasetDetailResponse
-    dataset_detail_fields,
+from libs.helper import dump_response
-    dataset_retrieval_model_fields,
-    doc_metadata_fields,
-    external_knowledge_info_fields,
-    external_retrieval_model_fields,
-    icon_info_fields,
-    keyword_setting_fields,
-    reranking_model_fields,
-    tag_fields,
-    vector_setting_fields,
-    weighted_score_fields,
-)
 from libs.login import login_required
 from models import Account
 from services.dataset_service import DatasetService
@@ -51,50 +35,10 @@ from services.external_knowledge_service import ExternalDatasetService
 from services.hit_testing_service import HitTestingService
 from services.knowledge_service import BedrockRetrievalSetting, ExternalDatasetTestService
 
-register_response_schema_models(console_ns, UsageCountResponse)
-
-
-def _build_dataset_detail_model():
-    keyword_setting_model = get_or_create_model("DatasetKeywordSetting", keyword_setting_fields)
-    vector_setting_model = get_or_create_model("DatasetVectorSetting", vector_setting_fields)
-
-    weighted_score_fields_copy = weighted_score_fields.copy()
-    weighted_score_fields_copy["keyword_setting"] = fields.Nested(keyword_setting_model)
-    weighted_score_fields_copy["vector_setting"] = fields.Nested(vector_setting_model)
-    weighted_score_model = get_or_create_model("DatasetWeightedScore", weighted_score_fields_copy)
-
-    reranking_model = get_or_create_model("DatasetRerankingModel", reranking_model_fields)
-
-    dataset_retrieval_model_fields_copy = dataset_retrieval_model_fields.copy()
-    dataset_retrieval_model_fields_copy["reranking_model"] = fields.Nested(reranking_model)
-    dataset_retrieval_model_fields_copy["weights"] = fields.Nested(weighted_score_model, allow_null=True)
-    dataset_retrieval_model = get_or_create_model("DatasetRetrievalModel", dataset_retrieval_model_fields_copy)
-
-    tag_model = get_or_create_model("Tag", tag_fields)
-    doc_metadata_model = get_or_create_model("DatasetDocMetadata", doc_metadata_fields)
-    external_knowledge_info_model = get_or_create_model("ExternalKnowledgeInfo", external_knowledge_info_fields)
-    external_retrieval_model = get_or_create_model("ExternalRetrievalModel", external_retrieval_model_fields)
-    icon_info_model = get_or_create_model("DatasetIconInfo", icon_info_fields)
-
-    dataset_detail_fields_copy = dataset_detail_fields.copy()
-    dataset_detail_fields_copy["retrieval_model_dict"] = fields.Nested(dataset_retrieval_model)
-    dataset_detail_fields_copy["tags"] = fields.List(fields.Nested(tag_model))
-    dataset_detail_fields_copy["external_knowledge_info"] = fields.Nested(external_knowledge_info_model)
-    dataset_detail_fields_copy["external_retrieval_model"] = fields.Nested(external_retrieval_model, allow_null=True)
-    dataset_detail_fields_copy["doc_metadata"] = fields.List(fields.Nested(doc_metadata_model))
-    dataset_detail_fields_copy["icon_info"] = fields.Nested(icon_info_model)
-    return get_or_create_model("DatasetDetail", dataset_detail_fields_copy)
-
-
-try:
-    dataset_detail_model = console_ns.models["DatasetDetail"]
-except KeyError:
-    dataset_detail_model = _build_dataset_detail_model()
-
 
 class ExternalKnowledgeApiPayload(BaseModel):
     name: str = Field(..., min_length=1, max_length=40)
-    settings: dict[str, object]
+    settings: dict[str, Any]
 
 
 class ExternalDatasetCreatePayload(BaseModel):
@@ -102,15 +46,13 @@ class ExternalDatasetCreatePayload(BaseModel):
     external_knowledge_id: str
     name: str = Field(..., min_length=1, max_length=100)
     description: str | None = Field(None, max_length=400)
-    external_retrieval_model: dict[str, object] | None = Field(default=None)
+    external_retrieval_model: dict[str, Any] | None = None
 
 
 class ExternalHitTestingPayload(BaseModel):
     query: str
-    external_retrieval_model: dict[str, object] | None = Field(default=None)
+    external_retrieval_model: dict[str, Any] | None = None
-    metadata_filtering_conditions: dict[str, object] | None = Field(
+    metadata_filtering_conditions: dict[str, Any] | None = None
-        default=None,
-    )
 
 
 class BedrockRetrievalPayload(BaseModel):
@@ -125,7 +67,7 @@ class ExternalApiTemplateListQuery(BaseModel):
     keyword: str | None = Field(default=None, description="Search keyword")
 
 
-class ExternalKnowledgeDatasetBindingResponse(ResponseModel):
+class ExternalKnowledgeApiBindingResponse(ResponseModel):
     id: str
     name: str
 
@@ -135,22 +77,52 @@ class ExternalKnowledgeApiResponse(ResponseModel):
     tenant_id: str
     name: str
     description: str
-    settings: dict[str, Any] | None = Field(default=None)
+    settings: dict[str, Any] | None = Field(validation_alias=AliasChoices("settings_dict", "settings"))
-    dataset_bindings: list[ExternalKnowledgeDatasetBindingResponse] = Field(default_factory=list)
+    dataset_bindings: list[ExternalKnowledgeApiBindingResponse]
     created_by: str
     created_at: str
 
+    @field_validator("created_at", mode="before")
+    @classmethod
+    def _normalize_created_at(cls, value: datetime | str) -> str:
+        if isinstance(value, datetime):
+            return value.isoformat()
+        return value
+
 
 class ExternalKnowledgeApiListResponse(ResponseModel):
     data: list[ExternalKnowledgeApiResponse]
     has_more: bool
     limit: int
-    total: int
+    total: int | None
     page: int
 
 
-class ExternalRetrievalTestResponse(RootModel[dict[str, Any] | list[dict[str, Any]]]):
+class ExternalHitTestingQueryResponse(ResponseModel):
-    root: dict[str, Any] | list[dict[str, Any]]
+    content: str
+
+
+class ExternalHitTestingRecordResponse(ResponseModel):
+    content: str | None = None
+    title: str | None = None
+    score: float | None = None
+    metadata: dict[str, Any] | None = None
+
+
+class ExternalHitTestingResponse(ResponseModel):
+    query: ExternalHitTestingQueryResponse
+    records: list[ExternalHitTestingRecordResponse]
+
+
+class BedrockRetrievalRecordResponse(ResponseModel):
+    metadata: dict[str, Any] | None = None
+    score: float
+    title: str | None = None
+    content: str | None = None
+
+
+class BedrockRetrievalResponse(ResponseModel):
+    records: list[BedrockRetrievalRecordResponse]
 
 
 register_schema_models(
@@ -163,9 +135,16 @@ register_schema_models(
 )
 register_response_schema_models(
     console_ns,
+    UsageCountResponse,
+    DatasetDetailResponse,
+    ExternalKnowledgeApiBindingResponse,
     ExternalKnowledgeApiResponse,
     ExternalKnowledgeApiListResponse,
-    ExternalRetrievalTestResponse,
+    ExternalHitTestingQueryResponse,
+    ExternalHitTestingRecordResponse,
+    ExternalHitTestingResponse,
+    BedrockRetrievalRecordResponse,
+    BedrockRetrievalResponse,
 )
 
 
@@ -189,24 +168,26 @@ class ExternalApiTemplateListApi(Resource):
         external_knowledge_apis, total = ExternalDatasetService.get_external_knowledge_apis(
             query.page, query.limit, current_tenant_id, query.keyword
         )
-        response = {
+        return ExternalKnowledgeApiListResponse(
-            "data": [item.to_dict() for item in external_knowledge_apis],
+            data=[ExternalKnowledgeApiResponse.model_validate(item) for item in external_knowledge_apis],
-            "has_more": len(external_knowledge_apis) == query.limit,
+            has_more=len(external_knowledge_apis) == query.limit,
-            "limit": query.limit,
+            limit=query.limit,
-            "total": total,
+            total=total,
-            "page": query.page,
+            page=query.page,
-        }
+        ).model_dump(mode="json"), 200
-        return response, 200
 
-    @setup_required
+    @console_ns.doc("create_external_api_template")
-    @login_required
+    @console_ns.doc(description="Create external knowledge API template")
-    @account_initialization_required
     @console_ns.expect(console_ns.models[ExternalKnowledgeApiPayload.__name__])
     @console_ns.response(
         201,
         "External API template created successfully",
         console_ns.models[ExternalKnowledgeApiResponse.__name__],
     )
+    @console_ns.response(403, "Permission denied")
+    @setup_required
+    @login_required
+    @account_initialization_required
     @with_current_user
     @with_current_tenant_id
     @with_session
@@ -229,7 +210,7 @@ class ExternalApiTemplateListApi(Resource):
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()
 
-        return external_knowledge_api.to_dict(), 201
+        return dump_response(ExternalKnowledgeApiResponse, external_knowledge_api), 201
 
 
 @console_ns.route("/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>")
@@ -256,17 +237,21 @@ class ExternalApiTemplateApi(Resource):
         if external_knowledge_api is None:
             raise NotFound("API template not found.")
 
-        return external_knowledge_api.to_dict(), 200
+        return dump_response(ExternalKnowledgeApiResponse, external_knowledge_api), 200
 
+    @console_ns.doc("update_external_api_template")
+    @console_ns.doc(description="Update external knowledge API template")
+    @console_ns.doc(params={"external_knowledge_api_id": "External knowledge API ID"})
+    @console_ns.expect(console_ns.models[ExternalKnowledgeApiPayload.__name__])
     @console_ns.response(
         200,
         "External API template updated successfully",
         console_ns.models[ExternalKnowledgeApiResponse.__name__],
     )
+    @console_ns.response(404, "Template not found")
     @setup_required
     @login_required
     @account_initialization_required
-    @console_ns.expect(console_ns.models[ExternalKnowledgeApiPayload.__name__])
     @with_current_user
     @with_current_tenant_id
     @with_session
@@ -284,7 +269,7 @@ class ExternalApiTemplateApi(Resource):
             session=session,
         )
 
-        return external_knowledge_api.to_dict(), 200
+        return dump_response(ExternalKnowledgeApiResponse, external_knowledge_api), 200
 
     @setup_required
     @login_required
@@ -322,7 +307,7 @@ class ExternalApiUseCheckApi(Resource):
         external_knowledge_api_is_using, count = ExternalDatasetService.external_knowledge_api_use_check(
             external_knowledge_api_id_str, current_tenant_id, session=session
         )
-        return {"is_using": external_knowledge_api_is_using, "count": count}, 200
+        return UsageCountResponse(is_using=external_knowledge_api_is_using, count=count).model_dump(mode="json"), 200
 
 
 @console_ns.route("/datasets/external")
@@ -330,7 +315,9 @@ class ExternalDatasetCreateApi(Resource):
     @console_ns.doc("create_external_dataset")
     @console_ns.doc(description="Create external knowledge dataset")
     @console_ns.expect(console_ns.models[ExternalDatasetCreatePayload.__name__])
-    @console_ns.response(201, "External dataset created successfully", dataset_detail_model)
+    @console_ns.response(
+        201, "External dataset created successfully", console_ns.models[DatasetDetailResponse.__name__]
+    )
     @console_ns.response(400, "Invalid parameters")
     @console_ns.response(403, "Permission denied")
     @setup_required
@@ -360,17 +347,16 @@ class ExternalDatasetCreateApi(Resource):
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()
 
-        item = marshal(dataset, dataset_detail_fields)
+        dataset_id_str = str(dataset.id)
-        dataset_id_str = item["id"]
         permission_keys_map = enterprise_rbac_service.RBACService.DatasetPermissions.batch_get(
             str(current_tenant_id),
             current_user.id,
             [dataset_id_str],
             session=session,
         )
-        item["permission_keys"] = permission_keys_map.get(dataset_id_str, [])
+        data = DatasetDetailResponse.model_validate(dataset).model_dump(mode="json")
-
+        data["permission_keys"] = permission_keys_map.get(dataset_id_str, [])
-        return item, 201
+        return data, 201
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/external-hit-testing")
@@ -382,7 +368,7 @@ class ExternalKnowledgeHitTestingApi(Resource):
     @console_ns.response(
         200,
         "External hit testing completed successfully",
-        console_ns.models[ExternalRetrievalTestResponse.__name__],
+        console_ns.models[ExternalHitTestingResponse.__name__],
     )
     @console_ns.response(404, "Dataset not found")
     @console_ns.response(400, "Invalid parameters")
@@ -394,12 +380,12 @@ class ExternalKnowledgeHitTestingApi(Resource):
     @with_session
     def post(self, session: Session, current_user: Account, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str, db.session())
+        dataset = DatasetService.get_dataset(dataset_id_str, session)
         if dataset is None:
             raise NotFound("Dataset not found.")
 
         try:
-            DatasetService.check_dataset_permission(dataset, current_user, db.session())
+            DatasetService.check_dataset_permission(dataset, current_user, session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
@@ -416,7 +402,7 @@ class ExternalKnowledgeHitTestingApi(Resource):
                 metadata_filtering_conditions=payload.metadata_filtering_conditions,
             )
 
-            return response
+            return dump_response(ExternalHitTestingResponse, response)
         except Exception as e:
             raise InternalServerError(str(e))
 
@@ -427,11 +413,7 @@ class BedrockRetrievalApi(Resource):
     @console_ns.doc("bedrock_retrieval_test")
     @console_ns.doc(description="Bedrock retrieval test (internal use only)")
     @console_ns.expect(console_ns.models[BedrockRetrievalPayload.__name__])
-    @console_ns.response(
+    @console_ns.response(200, "Bedrock retrieval test completed", console_ns.models[BedrockRetrievalResponse.__name__])
-        200,
-        "Bedrock retrieval test completed",
-        console_ns.models[ExternalRetrievalTestResponse.__name__],
-    )
     def post(self):
         payload = BedrockRetrievalPayload.model_validate(console_ns.payload or {})
 
@@ -439,4 +421,4 @@ class BedrockRetrievalApi(Resource):
         result = ExternalDatasetTestService.knowledge_retrieval(
             payload.retrieval_setting, payload.query, payload.knowledge_id
         )
-        return result, 200
+        return dump_response(BedrockRetrievalResponse, result), 200

api/controllers/service_api/dataset/dataset.py

@@ -845,7 +845,7 @@ class DocumentStatusApi(DatasetApiResource):
         except ValueError as e:
             raise InvalidActionError(str(e))
 
-        return dump_response(SimpleResultResponse, {"result": "success"}), 200
+        return SimpleResultResponse(result="success").model_dump(mode="json"), 200
 
 
 @service_api_ns.route("/datasets/tags")
@@ -911,11 +911,8 @@ class DatasetTagsApi(DatasetApiResource):
         payload = TagCreatePayload.model_validate(service_api_ns.payload or {})
         tag = TagService.save_tags(SaveTagPayload(name=payload.name, type=TagType.KNOWLEDGE), db.session())
 
-        response = dump_response(
+        response = KnowledgeTagResponse(id=tag.id, name=tag.name, type=tag.type, binding_count="0")
-            KnowledgeTagResponse,
+        return response.model_dump(mode="json"), 200
-            {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": 0},
-        )
-        return response, 200
 
     @service_api_ns.doc(
         summary="Update Knowledge Tag",
@@ -953,11 +950,8 @@ class DatasetTagsApi(DatasetApiResource):
 
         binding_count = TagService.get_tag_binding_count(tag_id, db.session(), tag_type=TagType.KNOWLEDGE)
 
-        response = dump_response(
+        response = KnowledgeTagResponse(id=tag.id, name=tag.name, type=tag.type, binding_count=str(binding_count))
-            KnowledgeTagResponse,
+        return response.model_dump(mode="json"), 200
-            {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": binding_count},
-        )
-        return response, 200
 
     @service_api_ns.doc(
         summary="Delete Knowledge Tag",
@@ -1088,5 +1082,8 @@ class DatasetTagsBindingStatusApi(DatasetApiResource):
         tags = TagService.get_tags_by_target_id(
             "knowledge", current_user.current_tenant_id, str(dataset_id), db.session()
         )
-        tags_list = [{"id": tag.id, "name": tag.name} for tag in tags]
+        response = DatasetBoundTagListResponse(
-        return dump_response(DatasetBoundTagListResponse, {"data": tags_list, "total": len(tags)}), 200
+            data=[DatasetBoundTagResponse(id=tag.id, name=tag.name) for tag in tags],
+            total=len(tags),
+        )
+        return response.model_dump(mode="json"), 200

api/controllers/service_api/dataset/document.py

@@ -6,14 +6,22 @@ deprecated in generated API docs so clients migrate toward the canonical paths.
 """
 
 import json
-from collections.abc import Mapping
 from contextlib import ExitStack
 from copy import deepcopy
 from typing import Annotated, Any, Literal, Self, override
 from uuid import UUID
 
 from flask import request, send_file
-from pydantic import BaseModel, Field, GetJsonSchemaHandler, WithJsonSchema, field_validator, model_validator
+from pydantic import (
+    BaseModel,
+    Field,
+    GetJsonSchemaHandler,
+    ValidationError,
+    WithJsonSchema,
+    field_validator,
+    model_validator,
+)
+from pydantic.json_schema import SkipJsonSchema
 from sqlalchemy import desc, func, select
 from werkzeug.exceptions import Forbidden, NotFound
 
@@ -26,9 +34,10 @@ from controllers.common.errors import (
     TooManyFilesError,
     UnsupportedFileTypeError,
 )
-from controllers.common.fields import BinaryFileResponse, UrlResponse
+from controllers.common.fields import UrlResponse
 from controllers.common.schema import (
     query_params_from_model,
+    query_params_from_request,
     register_enum_models,
     register_response_schema_models,
     register_schema_models,
@@ -56,6 +65,7 @@ from fields.document_fields import (
     DocumentMetadataResponse,
     DocumentResponse,
     DocumentStatusListResponse,
+    normalize_enum,
 )
 from libs.helper import dump_response
 from libs.login import current_user
@@ -281,38 +291,44 @@ class DocumentAndBatchResponse(ResponseModel):
     batch: str
 
 
+# Use SkipJsonSchema to support 3 metadata modes
 class DocumentDetailResponse(ResponseModel):
     id: str
-    position: int | None = None
+    position: int | SkipJsonSchema[None] = None
-    data_source_type: str | None = None
+    data_source_type: str | SkipJsonSchema[None] = None
-    data_source_info: dict[str, Any] | None = Field(default=None)
+    data_source_info: dict[str, Any] | SkipJsonSchema[None] = None
     dataset_process_rule_id: str | None = None
-    dataset_process_rule: dict[str, Any] | None = Field(default=None)
+    dataset_process_rule: dict[str, Any] | SkipJsonSchema[None] = None
-    document_process_rule: dict[str, Any] | None = Field(default=None)
+    document_process_rule: dict[str, Any] | SkipJsonSchema[None] = None
-    name: str | None = None
+    name: str | SkipJsonSchema[None] = None
-    created_from: str | None = None
+    created_from: str | SkipJsonSchema[None] = None
-    created_by: str | None = None
+    created_by: str | SkipJsonSchema[None] = None
-    created_at: int | None = None
+    created_at: int | SkipJsonSchema[None] = None
     tokens: int | None = None
-    indexing_status: str | None = None
+    indexing_status: str | SkipJsonSchema[None] = None
     completed_at: int | None = None
     updated_at: int | None = None
     indexing_latency: float | None = None
     error: str | None = None
-    enabled: bool | None = None
+    enabled: bool | SkipJsonSchema[None] = None
     disabled_at: int | None = None
     disabled_by: str | None = None
-    archived: bool | None = None
+    archived: bool | SkipJsonSchema[None] = None
     doc_type: str | None = None
-    doc_metadata: list[DocumentMetadataResponse] | None = None
+    doc_metadata: list[DocumentMetadataResponse] | dict[str, Any] | None = None
-    segment_count: int | None = None
+    segment_count: int | SkipJsonSchema[None] = None
-    average_segment_length: float | None = None
+    average_segment_length: int | float | SkipJsonSchema[None] = None
-    hit_count: int | None = None
+    hit_count: int | SkipJsonSchema[None] = None
     display_status: str | None = None
-    doc_form: str | None = None
+    doc_form: str | SkipJsonSchema[None] = None
     doc_language: str | None = None
     summary_index_status: str | None = None
-    need_summary: bool | None = None
+    need_summary: bool | SkipJsonSchema[None] = None
+
+    @field_validator("data_source_type", "indexing_status", "display_status", "doc_form", mode="before")
+    @classmethod
+    def _normalize_enum_fields(cls, value: Any) -> Any:
+        return normalize_enum(value)
 
 
 register_enum_models(service_api_ns, RetrievalMethod)
@@ -332,7 +348,6 @@ register_schema_models(
 )
 register_response_schema_models(
     service_api_ns,
-    BinaryFileResponse,
     UrlResponse,
     DocumentResponse,
     DocumentAndBatchResponse,
@@ -342,13 +357,13 @@ register_response_schema_models(
 )
 
 
-def _create_document_by_text(tenant_id: str, dataset_id: UUID) -> tuple[Mapping[str, object], int]:
+def _create_document_by_text(tenant_id: str, dataset_id: UUID) -> tuple[Document, str]:
     """Create a document from text for both canonical and legacy routes."""
     payload = DocumentTextCreatePayload.model_validate(service_api_ns.payload or {})
     args = payload.model_dump(exclude_none=True)
 
     dataset_id_str = str(dataset_id)
-    tenant_id_str = str(tenant_id)
+    tenant_id_str = tenant_id
     dataset = db.session.scalar(
         select(Dataset).where(Dataset.tenant_id == tenant_id_str, Dataset.id == dataset_id_str).limit(1)
     )
@@ -407,10 +422,10 @@ def _create_document_by_text(tenant_id: str, dataset_id: UUID) -> tuple[Mapping[
         raise ProviderNotInitializeError(ex.description)
     document = documents[0]
 
-    return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
+    return document, batch
 
 
-def _update_document_by_text(tenant_id: str, dataset_id: UUID, document_id: UUID) -> tuple[Mapping[str, object], int]:
+def _update_document_by_text(tenant_id: str, dataset_id: UUID, document_id: UUID) -> tuple[Document, str]:
     """Update a document from text for both canonical and legacy routes."""
     payload = DocumentTextUpdate.model_validate(service_api_ns.payload or {})
     dataset = db.session.scalar(
@@ -467,7 +482,7 @@ def _update_document_by_text(tenant_id: str, dataset_id: UUID, document_id: UUID
         raise ProviderNotInitializeError(ex.description)
     document = documents[0]
 
-    return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
+    return document, batch
 
 
 @service_api_ns.route("/datasets/<uuid:dataset_id>/document/create-by-text")
@@ -511,7 +526,8 @@ class DocumentAddByTextApi(DatasetApiResource):
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def post(self, tenant_id: str, dataset_id: UUID):
         """Create document by text."""
-        return _create_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id)
+        document, batch = _create_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id)
+        return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
 
 
 @service_api_ns.route("/datasets/<uuid:dataset_id>/document/create_by_text")
@@ -543,7 +559,8 @@ class DeprecatedDocumentAddByTextApi(DatasetApiResource):
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def post(self, tenant_id: str, dataset_id: UUID):
         """Create document by text through the deprecated underscore alias."""
-        return _create_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id)
+        document, batch = _create_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id)
+        return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
 
 
 @service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update-by-text")
@@ -587,7 +604,8 @@ class DocumentUpdateByTextApi(DatasetApiResource):
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def post(self, tenant_id: str, dataset_id: UUID, document_id: UUID):
         """Update document by text."""
-        return _update_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
+        document, batch = _update_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
+        return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
 
 
 @service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_text")
@@ -618,7 +636,8 @@ class DeprecatedDocumentUpdateByTextApi(DatasetApiResource):
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def post(self, tenant_id: str, dataset_id: UUID, document_id: UUID):
         """Update document by text through the deprecated underscore alias."""
-        return _update_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
+        document, batch = _update_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
+        return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
 
 
 @service_api_ns.route(
@@ -767,10 +786,10 @@ class DocumentAddByFileApi(DatasetApiResource):
         return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
 
 
-def _update_document_by_file(tenant_id: str, dataset_id: UUID, document_id: UUID) -> tuple[Mapping[str, object], int]:
+def _update_document_by_file(tenant_id: str, dataset_id: UUID, document_id: UUID) -> tuple[Document, str]:
     """Update a document from an uploaded file for canonical and deprecated routes."""
     dataset_id_str = str(dataset_id)
-    tenant_id_str = str(tenant_id)
+    tenant_id_str = tenant_id
     dataset = db.session.scalar(
         select(Dataset).where(Dataset.tenant_id == tenant_id_str, Dataset.id == dataset_id_str).limit(1)
     )
@@ -841,7 +860,7 @@ def _update_document_by_file(tenant_id: str, dataset_id: UUID, document_id: UUID
     except ProviderTokenNotInitError as ex:
         raise ProviderNotInitializeError(ex.description)
     document = documents[0]
-    return dump_response(DocumentAndBatchResponse, {"document": document, "batch": document.batch}), 200
+    return document, document.batch
 
 
 @service_api_ns.route(
@@ -895,7 +914,8 @@ class DeprecatedDocumentUpdateByFileApi(DatasetApiResource):
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def post(self, tenant_id: str, dataset_id: UUID, document_id: UUID):
         """Update document by file through the deprecated file-update aliases."""
-        return _update_document_by_file(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
+        document, batch = _update_document_by_file(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
+        return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
 
 
 @service_api_ns.route("/datasets/<uuid:dataset_id>/documents")
@@ -928,7 +948,7 @@ class DocumentListApi(DatasetApiResource):
     def get(self, tenant_id, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
         tenant_id = str(tenant_id)
-        query_params = DocumentListQuery.model_validate(request.args.to_dict())
+        query_params = query_params_from_request(DocumentListQuery)
         dataset = db.session.scalar(
             select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
         )
@@ -1021,6 +1041,7 @@ class DocumentBatchDownloadZipApi(DatasetApiResource):
             )
             cleanup = stack.pop_all()
             response.call_on_close(cleanup.close)
+        # response-contract:ignore binary send_file response
         return response
 
 
@@ -1149,7 +1170,9 @@ class DocumentDownloadApi(DatasetApiResource):
         if document.tenant_id != str(tenant_id):
             raise Forbidden("No permission.")
 
-        return {"url": DocumentService.get_document_download_url(document, db.session())}
+        return UrlResponse(url=DocumentService.get_document_download_url(document, db.session())).model_dump(
+            mode="json"
+        )
 
 
 @service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
@@ -1176,8 +1199,13 @@ class DocumentApi(DatasetApiResource):
     )
     @service_api_ns.doc("get_document")
     @service_api_ns.doc(description="Get a specific document by ID")
-    @service_api_ns.doc(params={"dataset_id": "Knowledge base ID.", "document_id": "Document ID."})
+    @service_api_ns.doc(
-    @service_api_ns.doc(params=query_params_from_model(DocumentGetQuery))
+        params={
+            "dataset_id": "Knowledge base ID.",
+            "document_id": "Document ID.",
+            **query_params_from_model(DocumentGetQuery),
+        }
+    )
     @service_api_ns.doc(
         responses={
             200: "Document retrieved successfully",
@@ -1205,9 +1233,14 @@ class DocumentApi(DatasetApiResource):
         if document.tenant_id != str(tenant_id):
             raise Forbidden("No permission.")
 
-        metadata = request.args.get("metadata", "all")
+        try:
-        if metadata not in self.METADATA_CHOICES:
+            query_params = query_params_from_request(DocumentGetQuery)
-            raise InvalidMetadataError(f"Invalid metadata value: {metadata}")
+        except ValidationError as exc:
+            metadata = request.args.get("metadata", "all")
+            raise InvalidMetadataError(f"Invalid metadata value: {metadata}") from exc
+        metadata = query_params.metadata
+        response_include: set[str] | None = None
+        response_exclude: set[str] | None = None
 
         # Calculate summary_index_status if needed
         summary_index_status = None
@@ -1221,8 +1254,10 @@ class DocumentApi(DatasetApiResource):
             )
 
         if metadata == "only":
+            response_include = {"id", "doc_type", "doc_metadata"}
             response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
         elif metadata == "without":
+            response_exclude = {"doc_type", "doc_metadata"}
             dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session())
             document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
             data_source_info = document.data_source_detail_dict
@@ -1295,8 +1330,33 @@ class DocumentApi(DatasetApiResource):
                 "need_summary": document.need_summary if document.need_summary is not None else False,
             }
 
-        return response
+        return DocumentDetailResponse.model_validate(response).model_dump(
+            mode="json",
+            include=response_include,
+            exclude=response_exclude,
+        )
 
+    @service_api_ns.doc(
+        summary="Update Document by File",
+        description=(
+            "Update an existing document by uploading a new file. Re-triggers indexing — use the returned "
+            "`batch` ID with [Get Document Indexing Status](/api-reference/documents/"
+            "get-document-indexing-status) to track progress."
+        ),
+        tags=["Documents"],
+        responses={
+            200: "Document updated successfully.",
+            400: (
+                "- `too_many_files` : Only one file is allowed.\n"
+                "- `filename_not_exists_error` : The specified filename does not exist.\n"
+                "- `provider_not_initialize` : No valid model provider credentials found. Please go to "
+                "Settings -> Model Provider to complete your provider credentials.\n"
+                "- `invalid_param` : Knowledge base does not exist, external datasets not supported, "
+                "file too large, unsupported file type, or invalid doc_form (must be `text_model`, "
+                "`hierarchical_model`, or `qa_model`)."
+            ),
+        },
+    )
     @service_api_ns.doc("update_document_by_file")
     @service_api_ns.doc(description="Update an existing document by uploading a file")
     @service_api_ns.doc(consumes=["multipart/form-data"], params=DOCUMENT_UPDATE_BY_FILE_PARAMS)
@@ -1314,7 +1374,8 @@ class DocumentApi(DatasetApiResource):
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def patch(self, tenant_id: str, dataset_id: UUID, document_id: UUID):
         """Update document by file on the canonical document resource."""
-        return _update_document_by_file(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
+        document, batch = _update_document_by_file(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
+        return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
 
     @service_api_ns.doc(
         summary="Delete Document",

api/core/rag/entities/processing_entities.py

@@ -1,7 +1,7 @@
 from enum import StrEnum
 from typing import Annotated, Literal
 
-from pydantic import BaseModel, Field, WithJsonSchema
+from pydantic import AliasChoices, BaseModel, Field, WithJsonSchema
 
 
 class ParentMode(StrEnum):
@@ -26,7 +26,14 @@ class PreProcessingRule(BaseModel):
 
 
 class Segmentation(BaseModel):
-    separator: str = Field(default="\n", description="Custom separator for splitting text.")
+    # TODO: there are internally mismatched / inconsistent naming
+    # between `separator` and `delimiter` across the codebase.
+    # Taking `separator` as the canonical.
+    separator: str = Field(
+        default="\n",
+        description="Custom separator for splitting text.",
+        validation_alias=AliasChoices("separator", "delimiter"),
+    )
     max_tokens: int = Field(description="Maximum token count per chunk.")
     chunk_overlap: int = Field(default=0, description="Token overlap between chunks.")
 

api/fields/dataset_fields.py

@@ -1,22 +1,9 @@
 from datetime import datetime
 
-from flask_restx import fields
 from pydantic import Field, field_validator
 
 from fields.base import ResponseModel
-from libs.helper import TimestampField, to_timestamp
+from libs.helper import to_timestamp
-
-dataset_fields = {
-    "id": fields.String,
-    "name": fields.String,
-    "description": fields.String,
-    "permission": fields.String,
-    "data_source_type": fields.String,
-    "indexing_technique": fields.String,
-    "created_by": fields.String,
-    "created_at": TimestampField,
-    "permission_keys": fields.List(fields.String()),
-}
 
 
 class DatasetMetadataResponse(ResponseModel):
@@ -50,104 +37,6 @@ class DatasetMetadataActionResponse(ResponseModel):
     result: str
 
 
-reranking_model_fields = {"reranking_provider_name": fields.String, "reranking_model_name": fields.String}
-
-keyword_setting_fields = {"keyword_weight": fields.Float}
-
-vector_setting_fields = {
-    "vector_weight": fields.Float,
-    "embedding_model_name": fields.String,
-    "embedding_provider_name": fields.String,
-}
-
-weighted_score_fields = {
-    "weight_type": fields.String,
-    "keyword_setting": fields.Nested(keyword_setting_fields),
-    "vector_setting": fields.Nested(vector_setting_fields),
-}
-
-dataset_retrieval_model_fields = {
-    "search_method": fields.String,
-    "reranking_enable": fields.Boolean,
-    "reranking_mode": fields.String,
-    "reranking_model": fields.Nested(reranking_model_fields),
-    "weights": fields.Nested(weighted_score_fields, allow_null=True),
-    "top_k": fields.Integer,
-    "score_threshold_enabled": fields.Boolean,
-    "score_threshold": fields.Float,
-}
-
-dataset_summary_index_fields = {
-    "enable": fields.Boolean,
-    "model_name": fields.String,
-    "model_provider_name": fields.String,
-    "summary_prompt": fields.String,
-}
-
-external_retrieval_model_fields = {
-    "top_k": fields.Integer,
-    "score_threshold": fields.Float,
-    "score_threshold_enabled": fields.Boolean,
-}
-
-tag_fields = {"id": fields.String, "name": fields.String, "type": fields.String}
-
-external_knowledge_info_fields = {
-    "external_knowledge_id": fields.String,
-    "external_knowledge_api_id": fields.String,
-    "external_knowledge_api_name": fields.String,
-    "external_knowledge_api_endpoint": fields.String,
-}
-
-doc_metadata_fields = {"id": fields.String, "name": fields.String, "type": fields.String}
-
-icon_info_fields = {
-    "icon_type": fields.String,
-    "icon": fields.String,
-    "icon_background": fields.String,
-    "icon_url": fields.String,
-}
-
-dataset_detail_fields = {
-    "id": fields.String,
-    "name": fields.String,
-    "description": fields.String,
-    "provider": fields.String,
-    "permission": fields.String,
-    "data_source_type": fields.String,
-    "indexing_technique": fields.String,
-    "app_count": fields.Integer,
-    "document_count": fields.Integer,
-    "word_count": fields.Integer,
-    "created_by": fields.String,
-    "author_name": fields.String,
-    "created_at": TimestampField,
-    "updated_by": fields.String,
-    "updated_at": TimestampField,
-    "embedding_model": fields.String,
-    "embedding_model_provider": fields.String,
-    "embedding_available": fields.Boolean,
-    "retrieval_model_dict": fields.Nested(dataset_retrieval_model_fields),
-    "summary_index_setting": fields.Nested(dataset_summary_index_fields),
-    "tags": fields.List(fields.Nested(tag_fields)),
-    "doc_form": fields.String,
-    "external_knowledge_info": fields.Nested(external_knowledge_info_fields),
-    "external_retrieval_model": fields.Nested(external_retrieval_model_fields, allow_null=True),
-    "doc_metadata": fields.List(fields.Nested(doc_metadata_fields)),
-    "built_in_field_enabled": fields.Boolean,
-    "pipeline_id": fields.String,
-    "runtime_mode": fields.String,
-    "chunk_structure": fields.String,
-    "icon_info": fields.Nested(icon_info_fields),
-    "is_published": fields.Boolean,
-    "total_documents": fields.Integer,
-    "total_available_documents": fields.Integer,
-    "enable_api": fields.Boolean,
-    "is_multimodal": fields.Boolean,
-    "permission_keys": fields.List(fields.String()),
-}
-
-
 class DatasetRerankingModelResponse(ResponseModel):
     reranking_provider_name: str | None = None
     reranking_model_name: str | None = None

api/openapi/markdown/console-openapi.md

@@ -5492,7 +5492,7 @@ Create external knowledge dataset
 
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
-| 201 | External dataset created successfully | **application/json**: [DatasetDetail](#datasetdetail)<br> |
+| 201 | External dataset created successfully | **application/json**: [DatasetDetailResponse](#datasetdetailresponse)<br> |
 | 400 | Invalid parameters |  |
 | 403 | Permission denied |  |
 
@@ -5514,6 +5514,8 @@ Get external knowledge API templates
 | 200 | External API templates retrieved successfully | **application/json**: [ExternalKnowledgeApiListResponse](#externalknowledgeapilistresponse)<br> |
 
 ### [POST] /datasets/external-knowledge-api
+Create external knowledge API template
+
 #### Request Body
 
 | Required | Schema |
@@ -5525,6 +5527,7 @@ Get external knowledge API templates
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
 | 201 | External API template created successfully | **application/json**: [ExternalKnowledgeApiResponse](#externalknowledgeapiresponse)<br> |
+| 403 | Permission denied |  |
 
 ### [DELETE] /datasets/external-knowledge-api/{external_knowledge_api_id}
 #### Parameters
@@ -5556,11 +5559,13 @@ Get external knowledge API template details
 | 404 | Template not found |  |
 
 ### [PATCH] /datasets/external-knowledge-api/{external_knowledge_api_id}
+Update external knowledge API template
+
 #### Parameters
 
 | Name | Located in | Description | Required | Schema |
 | ---- | ---------- | ----------- | -------- | ------ |
-| external_knowledge_api_id | path |  | Yes | string (uuid) |
+| external_knowledge_api_id | path | External knowledge API ID | Yes | string (uuid) |
 
 #### Request Body
 
@@ -5573,6 +5578,7 @@ Get external knowledge API template details
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
 | 200 | External API template updated successfully | **application/json**: [ExternalKnowledgeApiResponse](#externalknowledgeapiresponse)<br> |
+| 404 | Template not found |  |
 
 ### [GET] /datasets/external-knowledge-api/{external_knowledge_api_id}/use-check
 Check if external knowledge API is being used
@@ -5617,7 +5623,7 @@ Initialize dataset with documents
 
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
-| 201 | Dataset initialized successfully | **application/json**: [DatasetAndDocumentResponse](#datasetanddocumentresponse)<br> |
+| 200 | Dataset initialized successfully | **application/json**: [DatasetAndDocumentResponse](#datasetanddocumentresponse)<br> |
 | 400 | Invalid request parameters |  |
 
 ### [GET] /datasets/metadata/built-in
@@ -5653,7 +5659,7 @@ Get dataset document processing rules
 
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
-| 200 | Process rules retrieved successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)<br> |
+| 200 | Process rules retrieved successfully | **application/json**: [ProcessRuleResponse](#processruleresponse)<br> |
 
 ### [GET] /datasets/retrieval-setting
 Get dataset retrieval settings
@@ -5774,7 +5780,7 @@ Get dataset auto disable logs
 
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
-| 200 | Batch indexing estimate calculated successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)<br> |
+| 200 | Indexing estimate calculated successfully | **application/json**: [IndexingEstimateResponse](#indexingestimateresponse)<br> |
 
 ### [GET] /datasets/{dataset_id}/batch/{batch}/indexing-status
 #### Parameters
@@ -5862,9 +5868,9 @@ Download selected dataset documents as a single ZIP archive (upload-file only)
 
 #### Responses
 
-| Code | Description | Schema |
+| Code | Description |
-| ---- | ----------- | ------ |
+| ---- | ----------- |
-| 200 | ZIP archive generated successfully | **application/json**: [BinaryFileResponse](#binaryfileresponse)<br> |
+| 200 | ZIP archive downloaded successfully |
 
 ### [POST] /datasets/{dataset_id}/documents/generate-summary
 **Generate summary index for specified documents**
@@ -5957,7 +5963,7 @@ Get document details
 
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
-| 200 | Document retrieved successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)<br> |
+| 200 | Document retrieved successfully | **application/json**: [DocumentDetailResponse](#documentdetailresponse)<br> |
 | 404 | Document not found |  |
 
 ### [GET] /datasets/{dataset_id}/documents/{document_id}/download
@@ -5990,7 +5996,7 @@ Estimate document indexing cost
 
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
-| 200 | Indexing estimate calculated successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)<br> |
+| 200 | Indexing estimate calculated successfully | **application/json**: [IndexingEstimateResponse](#indexingestimateresponse)<br> |
 | 400 | Document already finished |  |
 | 404 | Document not found |  |
 
@@ -6061,7 +6067,7 @@ Update document metadata
 
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
-| 200 | Document pipeline execution log retrieved successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)<br> |
+| 200 | Pipeline execution log retrieved successfully | **application/json**: [DocumentPipelineExecutionLogResponse](#documentpipelineexecutionlogresponse)<br> |
 
 ### [PATCH] /datasets/{dataset_id}/documents/{document_id}/processing/pause
 **pause document**
@@ -6384,6 +6390,7 @@ Returns:
   - generating: Number of summaries being generated
   - error: Number of summaries with errors
   - not_started: Number of segments without summary records
+  - timeout: Number of summaries that timed out
 - summaries: List of summary records with status and content preview
 
 #### Parameters
@@ -6397,7 +6404,7 @@ Returns:
 
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
-| 200 | Summary status retrieved successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)<br> |
+| 200 | Summary status retrieved successfully | **application/json**: [DocumentSummaryStatusResponse](#documentsummarystatusresponse)<br> |
 | 404 | Document not found |  |
 
 ### [GET] /datasets/{dataset_id}/documents/{document_id}/website-sync
@@ -6451,7 +6458,7 @@ Test external knowledge retrieval for dataset
 
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
-| 200 | External hit testing completed successfully | **application/json**: [ExternalRetrievalTestResponse](#externalretrievaltestresponse)<br> |
+| 200 | External hit testing completed successfully | **application/json**: [ExternalHitTestingResponse](#externalhittestingresponse)<br> |
 | 400 | Invalid parameters |  |
 | 404 | Dataset not found |  |
 
@@ -9388,7 +9395,7 @@ Bedrock retrieval test (internal use only)
 
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
-| 200 | Bedrock retrieval test completed | **application/json**: [ExternalRetrievalTestResponse](#externalretrievaltestresponse)<br> |
+| 200 | Bedrock retrieval test completed | **application/json**: [BedrockRetrievalResponse](#bedrockretrievalresponse)<br> |
 
 ### [GET] /trial-apps/{app_id}
 **Get app detail**
@@ -15435,6 +15442,21 @@ AppMCPServer Status Enum
 | query | string |  | Yes |
 | retrieval_setting | [BedrockRetrievalSetting](#bedrockretrievalsetting) |  | Yes |
 
+#### BedrockRetrievalRecordResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| content | string |  | No |
+| metadata | object |  | No |
+| score | number |  | Yes |
+| title | string |  | No |
+
+#### BedrockRetrievalResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| records | [ [BedrockRetrievalRecordResponse](#bedrockretrievalrecordresponse) ] |  | Yes |
+
 #### BedrockRetrievalSetting
 
 Retrieval settings for Amazon Bedrock knowledge base queries.
@@ -16325,47 +16347,6 @@ Model class for provider custom model configuration.
 | permission | [PermissionEnum](#permissionenum) |  | No |
 | provider | string, <br>**Default:** vendor |  | No |
 
-#### DatasetDetail
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| app_count | integer |  | No |
-| author_name | string |  | No |
-| built_in_field_enabled | boolean |  | No |
-| chunk_structure | string |  | No |
-| created_at | long |  | No |
-| created_by | string |  | No |
-| data_source_type | string |  | No |
-| description | string |  | No |
-| doc_form | string |  | No |
-| doc_metadata | [ [DatasetDocMetadata](#datasetdocmetadata) ] |  | No |
-| document_count | integer |  | No |
-| embedding_available | boolean |  | No |
-| embedding_model | string |  | No |
-| embedding_model_provider | string |  | No |
-| enable_api | boolean |  | No |
-| external_knowledge_info | [ExternalKnowledgeInfo](#externalknowledgeinfo) |  | No |
-| external_retrieval_model | [ExternalRetrievalModel](#externalretrievalmodel) |  | No |
-| icon_info | [DatasetIconInfo](#dataseticoninfo) |  | No |
-| id | string |  | No |
-| indexing_technique | string |  | No |
-| is_multimodal | boolean |  | No |
-| is_published | boolean |  | No |
-| name | string |  | No |
-| permission | string |  | No |
-| permission_keys | [ string ] |  | No |
-| pipeline_id | string |  | No |
-| provider | string |  | No |
-| retrieval_model_dict | [DatasetRetrievalModel](#datasetretrievalmodel) |  | No |
-| runtime_mode | string |  | No |
-| summary_index_setting | [_AnonymousInlineModel_b1954337d565](#_anonymousinlinemodel_b1954337d565) |  | No |
-| tags | [ [Tag](#tag) ] |  | No |
-| total_available_documents | integer |  | No |
-| total_documents | integer |  | No |
-| updated_at | long |  | No |
-| updated_by | string |  | No |
-| word_count | integer |  | No |
-
 #### DatasetDetailResponse
 
 | Name | Type | Description | Required |
@@ -16451,14 +16432,6 @@ Model class for provider custom model configuration.
 | updated_by | string |  | Yes |
 | word_count | integer |  | Yes |
 
-#### DatasetDocMetadata
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| id | string |  | No |
-| name | string |  | No |
-| type | string |  | No |
-
 #### DatasetDocMetadataResponse
 
 | Name | Type | Description | Required |
@@ -16484,15 +16457,6 @@ Model class for provider custom model configuration.
 | score_threshold_enabled | boolean |  | No |
 | top_k | integer |  | Yes |
 
-#### DatasetIconInfo
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| icon | string |  | No |
-| icon_background | string |  | No |
-| icon_type | string |  | No |
-| icon_url | string |  | No |
-
 #### DatasetIconInfoResponse
 
 | Name | Type | Description | Required |
@@ -16502,12 +16466,6 @@ Model class for provider custom model configuration.
 | icon_type | string |  | No |
 | icon_url | string |  | No |
 
-#### DatasetKeywordSetting
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| keyword_weight | number |  | No |
-
 #### DatasetKeywordSettingResponse
 
 | Name | Type | Description | Required |
@@ -16645,13 +16603,6 @@ Model class for provider custom model configuration.
 | page | integer |  | Yes |
 | total | integer |  | Yes |
 
-#### DatasetRerankingModel
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| reranking_model_name | string |  | No |
-| reranking_provider_name | string |  | No |
-
 #### DatasetRerankingModelResponse
 
 | Name | Type | Description | Required |
@@ -16672,19 +16623,6 @@ Model class for provider custom model configuration.
 | name | string |  | Yes |
 | permission | string |  | No |
 
-#### DatasetRetrievalModel
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| reranking_enable | boolean |  | No |
-| reranking_mode | string |  | No |
-| reranking_model | [DatasetRerankingModel](#datasetrerankingmodel) |  | No |
-| score_threshold | number |  | No |
-| score_threshold_enabled | boolean |  | No |
-| search_method | string |  | No |
-| top_k | integer |  | No |
-| weights | [DatasetWeightedScore](#datasetweightedscore) |  | No |
-
 #### DatasetRetrievalModelResponse
 
 | Name | Type | Description | Required |
@@ -16734,14 +16672,6 @@ Model class for provider custom model configuration.
 | retrieval_model | object |  | No |
 | summary_index_setting | object |  | No |
 
-#### DatasetVectorSetting
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| embedding_model_name | string |  | No |
-| embedding_provider_name | string |  | No |
-| vector_weight | number |  | No |
-
 #### DatasetVectorSettingResponse
 
 | Name | Type | Description | Required |
@@ -16750,14 +16680,6 @@ Model class for provider custom model configuration.
 | embedding_provider_name | string |  | No |
 | vector_weight | number |  | No |
 
-#### DatasetWeightedScore
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| keyword_setting | [DatasetKeywordSetting](#datasetkeywordsetting) |  | No |
-| vector_setting | [DatasetVectorSetting](#datasetvectorsetting) |  | No |
-| weight_type | string |  | No |
-
 #### DatasetWeightedScoreResponse
 
 | Name | Type | Description | Required |
@@ -17061,6 +16983,42 @@ Request payload for bulk downloading documents as a zip archive.
 | ---- | ---- | ----------- | -------- |
 | document_ids | [ string (uuid) ] | List of document IDs to include in the ZIP download. | Yes |
 
+#### DocumentDetailResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| archived | boolean |  | No |
+| average_segment_length | number |  | No |
+| completed_at | integer |  | No |
+| created_at | integer |  | No |
+| created_by | string |  | No |
+| created_from | string |  | No |
+| data_source_detail_dict |  |  | No |
+| data_source_info |  |  | No |
+| data_source_type | string |  | No |
+| dataset_process_rule |  |  | No |
+| dataset_process_rule_id | string |  | No |
+| disabled_at | integer |  | No |
+| disabled_by | string |  | No |
+| display_status | string |  | No |
+| doc_form | string |  | No |
+| doc_language | string |  | No |
+| doc_metadata | [ [DocumentMetadataResponse](#documentmetadataresponse) ] |  | No |
+| doc_type | string |  | No |
+| document_process_rule |  |  | No |
+| enabled | boolean |  | No |
+| error | string |  | No |
+| hit_count | integer |  | No |
+| id | string |  | Yes |
+| indexing_latency | number |  | No |
+| indexing_status | string |  | No |
+| name | string |  | No |
+| need_summary | boolean |  | No |
+| position | integer |  | No |
+| segment_count | integer |  | No |
+| tokens | integer |  | No |
+| updated_at | integer |  | No |
+
 #### DocumentMetadataOperation
 
 | Name | Type | Description | Required |
@@ -17085,6 +17043,15 @@ Request payload for bulk downloading documents as a zip archive.
 | doc_metadata |  |  | No |
 | doc_type | string |  | No |
 
+#### DocumentPipelineExecutionLogResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| datasource_info | [JsonValue](#jsonvalue) |  | No |
+| datasource_node_id | string |  | No |
+| datasource_type | string |  | No |
+| input_data | [JsonValue](#jsonvalue) |  | No |
+
 #### DocumentRenamePayload
 
 | Name | Type | Description | Required |
@@ -17149,6 +17116,14 @@ Request payload for bulk downloading documents as a zip archive.
 | stopped_at | integer |  | Yes |
 | total_segments | integer |  | No |
 
+#### DocumentSummaryStatusResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| summaries | [ [SummaryEntryResponse](#summaryentryresponse) ] |  | Yes |
+| summary_status | [SummaryStatusResponse](#summarystatusresponse) |  | Yes |
+| total_segments | integer |  | Yes |
+
 #### DocumentWithSegmentsListResponse
 
 | Name | Type | Description | Required |
@@ -17679,6 +17654,35 @@ Built-in tool icons are URL strings; API-based tool icons are provider-defined p
 | metadata_filtering_conditions | object |  | No |
 | query | string |  | Yes |
 
+#### ExternalHitTestingQueryResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| content | string |  | Yes |
+
+#### ExternalHitTestingRecordResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| content | string |  | No |
+| metadata | object |  | No |
+| score | number |  | No |
+| title | string |  | No |
+
+#### ExternalHitTestingResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| query | [ExternalHitTestingQueryResponse](#externalhittestingqueryresponse) |  | Yes |
+| records | [ [ExternalHitTestingRecordResponse](#externalhittestingrecordresponse) ] |  | Yes |
+
+#### ExternalKnowledgeApiBindingResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| id | string |  | Yes |
+| name | string |  | Yes |
+
 #### ExternalKnowledgeApiListResponse
 
 | Name | Type | Description | Required |
@@ -17702,43 +17706,13 @@ Built-in tool icons are URL strings; API-based tool icons are provider-defined p
 | ---- | ---- | ----------- | -------- |
 | created_at | string |  | Yes |
 | created_by | string |  | Yes |
-| dataset_bindings | [ [ExternalKnowledgeDatasetBindingResponse](#externalknowledgedatasetbindingresponse) ] |  | No |
+| dataset_bindings | [ [ExternalKnowledgeApiBindingResponse](#externalknowledgeapibindingresponse) ] |  | Yes |
 | description | string |  | Yes |
 | id | string |  | Yes |
 | name | string |  | Yes |
-| settings | object |  | No |
+| settings | object |  | Yes |
 | tenant_id | string |  | Yes |
 
-#### ExternalKnowledgeDatasetBindingResponse
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| id | string |  | Yes |
-| name | string |  | Yes |
-
-#### ExternalKnowledgeInfo
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| external_knowledge_api_endpoint | string |  | No |
-| external_knowledge_api_id | string |  | No |
-| external_knowledge_api_name | string |  | No |
-| external_knowledge_id | string |  | No |
-
-#### ExternalRetrievalModel
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| score_threshold | number |  | No |
-| score_threshold_enabled | boolean |  | No |
-| top_k | integer |  | No |
-
-#### ExternalRetrievalTestResponse
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| ExternalRetrievalTestResponse | object<br>[ object ] |  |  |
-
 #### FeatureModel
 
 | Name | Type | Description | Required |
@@ -18243,27 +18217,15 @@ Query parameter for including secret variables in export.
 | info_list | object |  | Yes |
 | process_rule | object |  | Yes |
 
-#### IndexingEstimatePreviewItemResponse
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| child_chunks | [ string ] |  | No |
-| content | string |  | Yes |
-| summary | string |  | No |
-
-#### IndexingEstimateQaPreviewItemResponse
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| answer | string |  | Yes |
-| question | string |  | Yes |
-
 #### IndexingEstimateResponse
 
 | Name | Type | Description | Required |
 | ---- | ---- | ----------- | -------- |
-| preview | [ [IndexingEstimatePreviewItemResponse](#indexingestimatepreviewitemresponse) ] |  | Yes |
+| currency | string |  | Yes |
-| qa_preview | [ [IndexingEstimateQaPreviewItemResponse](#indexingestimateqapreviewitemresponse) ] |  | No |
+| preview | [ [PreviewDetail](#previewdetail) ] |  | Yes |
+| qa_preview | [ [QAPreviewDetail](#qapreviewdetail) ] |  | No |
+| tokens | integer |  | Yes |
+| total_price | number<br>integer |  | Yes |
 | total_segments | integer |  | Yes |
 
 #### InfoList
@@ -19259,12 +19221,6 @@ OAuth schema
 | redirect_uri | string |  | No |
 | refresh_token | string |  | No |
 
-#### OpaqueObjectResponse
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| OpaqueObjectResponse | object |  |  |
-
 #### Option
 
 | Name | Type | Description | Required |
@@ -20310,6 +20266,14 @@ Dataset Process Rule Mode
 | ---- | ---- | ----------- | -------- |
 | ProcessRuleMode | string | Dataset Process Rule Mode |  |
 
+#### ProcessRuleResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| limits | object |  | Yes |
+| mode | [ProcessRuleMode](#processrulemode) |  | Yes |
+| rules | [Rule](#rule) |  | No |
+
 #### ProviderConfig
 
 Model class for common provider settings like credentials
@@ -21581,6 +21545,28 @@ The subscription constructor of the trigger provider
 | ---- | ---- | ----------- | -------- |
 | data | [ string ] |  | Yes |
 
+#### SummaryEntryResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| created_at | integer |  | No |
+| error | string |  | No |
+| segment_id | string |  | Yes |
+| segment_position | integer |  | Yes |
+| status | string |  | Yes |
+| summary_preview | string |  | No |
+| updated_at | integer |  | No |
+
+#### SummaryStatusResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| completed | integer |  | No |
+| error | integer |  | No |
+| generating | integer |  | No |
+| not_started | integer |  | No |
+| timeout | integer |  | No |
+
 #### SwitchWorkspacePayload
 
 | Name | Type | Description | Required |
@@ -23702,15 +23688,6 @@ Workflow tool configuration
 | data | [ [AccessPolicy](#accesspolicy) ] |  | No |
 | pagination | [Pagination](#pagination) |  | No |
 
-#### _AnonymousInlineModel_b1954337d565
-
-| Name | Type | Description | Required |
-| ---- | ---- | ----------- | -------- |
-| enable | boolean |  | No |
-| model_name | string |  | No |
-| model_provider_name | string |  | No |
-| summary_prompt | string |  | No |
-
 #### _DeleteMemberBindingsRequest
 
 | Name | Type | Description | Required |

api/openapi/markdown/service-openapi.md

@@ -46,76 +46,6 @@ Deprecated legacy alias for creating a new document by providing text content. U
 | 401 | Unauthorized - invalid API token |  |
 | 403 | Forbidden - dataset API access or workspace access denied |  |
 
-### [DELETE] /datasets/{dataset_id}/documents/{document_id}
-**Delete Document**
-
-Permanently delete a document and all its chunks from the knowledge base.
-
-#### Parameters
-
-| Name | Located in | Description | Required | Schema |
-| ---- | ---------- | ----------- | -------- | ------ |
-| dataset_id | path | Knowledge base ID. | Yes | string (uuid) |
-| document_id | path | Document ID. | Yes | string (uuid) |
-
-#### Responses
-
-| Code | Description |
-| ---- | ----------- |
-| 204 | Success. |
-| 400 | `document_indexing` : Cannot delete document during indexing. |
-| 401 | Unauthorized - invalid API token |
-| 403 | `archived_document_immutable` : The archived document is not editable. |
-| 404 | `not_found` : Document Not Exists. |
-
-### [GET] /datasets/{dataset_id}/documents/{document_id}
-**Get Document**
-
-Retrieve detailed information about a specific document, including its indexing status, metadata, and processing statistics.
-
-#### Parameters
-
-| Name | Located in | Description | Required | Schema |
-| ---- | ---------- | ----------- | -------- | ------ |
-| dataset_id | path | Knowledge base ID. | Yes | string (uuid) |
-| document_id | path | Document ID. | Yes | string (uuid) |
-| metadata | query | `all` returns all fields including metadata. `only` returns only `id`, `doc_type`, and `doc_metadata`. `without` returns all fields except `doc_metadata`. | No | string, <br>**Available values:** "all", "only", "without", <br>**Default:** all |
-
-#### Responses
-
-| Code | Description | Schema |
-| ---- | ----------- | ------ |
-| 200 | Document details. The response shape varies based on the `metadata` query parameter. When `metadata` is `only`, only `id`, `doc_type`, and `doc_metadata` are returned. When `metadata` is `without`, `doc_type` and `doc_metadata` are omitted. | **application/json**: [DocumentDetailResponse](#documentdetailresponse)<br> |
-| 400 | `invalid_metadata` : Invalid metadata value for the specified key. |  |
-| 401 | Unauthorized - invalid API token |  |
-| 403 | `forbidden` : No permission. |  |
-| 404 | `not_found` : Document not found. |  |
-
-### [PATCH] /datasets/{dataset_id}/documents/{document_id}
-Update an existing document by uploading a file
-
-#### Parameters
-
-| Name | Located in | Description | Required | Schema |
-| ---- | ---------- | ----------- | -------- | ------ |
-| dataset_id | path | Knowledge base ID. | Yes | string (uuid) |
-| document_id | path | Document ID. | Yes | string (uuid) |
-
-#### Request Body
-
-| Required | Schema |
-| -------- | ------ |
-|  No | **multipart/form-data**: { **"data"**: string, **"file"**: binary }<br> |
-
-#### Responses
-
-| Code | Description | Schema |
-| ---- | ----------- | ------ |
-| 200 | Document updated successfully | **application/json**: [DocumentAndBatchResponse](#documentandbatchresponse)<br> |
-| 401 | Unauthorized - invalid API token |  |
-| 403 | Forbidden - dataset API access or workspace access denied |  |
-| 404 | Document not found |  |
-
 ### ~~[POST] /datasets/{dataset_id}/documents/{document_id}/update_by_text~~
 
 ***DEPRECATED***
@@ -1439,7 +1369,9 @@ Retrieve detailed information about a specific document, including its indexing
 | 404 | `not_found` : Document not found. |  |
 
 ### [PATCH] /datasets/{dataset_id}/documents/{document_id}
-Update an existing document by uploading a file
+**Update Document by File**
+
+Update an existing document by uploading a new file. Re-triggers indexing — use the returned `batch` ID with [Get Document Indexing Status](/api-reference/documents/get-document-indexing-status) to track progress.
 
 #### Parameters
 
@@ -1458,7 +1390,8 @@ Update an existing document by uploading a file
 
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
-| 200 | Document updated successfully | **application/json**: [DocumentAndBatchResponse](#documentandbatchresponse)<br> |
+| 200 | Document updated successfully. | **application/json**: [DocumentAndBatchResponse](#documentandbatchresponse)<br> |
+| 400 | - `too_many_files` : Only one file is allowed. - `filename_not_exists_error` : The specified filename does not exist. - `provider_not_initialize` : No valid model provider credentials found. Please go to Settings -> Model Provider to complete your provider credentials. - `invalid_param` : Knowledge base does not exist, external datasets not supported, file too large, unsupported file type, or invalid doc_form (must be `text_model`, `hierarchical_model`, or `qa_model`). |  |
 | 401 | Unauthorized - invalid API token |  |
 | 403 | Forbidden - dataset API access or workspace access denied |  |
 | 404 | Document not found |  |
@@ -2995,7 +2928,7 @@ Request payload for bulk downloading documents as a zip archive.
 | Name | Type | Description | Required |
 | ---- | ---- | ----------- | -------- |
 | archived | boolean |  | No |
-| average_segment_length | number |  | No |
+| average_segment_length | integer<br>number |  | No |
 | completed_at | integer |  | No |
 | created_at | integer |  | No |
 | created_by | string |  | No |
@@ -3009,7 +2942,7 @@ Request payload for bulk downloading documents as a zip archive.
 | display_status | string |  | No |
 | doc_form | string |  | No |
 | doc_language | string |  | No |
-| doc_metadata | [ [DocumentMetadataResponse](#documentmetadataresponse) ] |  | No |
+| doc_metadata | [ [DocumentMetadataResponse](#documentmetadataresponse) ]<br>object |  | No |
 | doc_type | string |  | No |
 | document_process_rule | object |  | No |
 | enabled | boolean |  | No |

api/tests/unit_tests/controllers/console/datasets/test_datasets.py

@@ -31,6 +31,7 @@ from controllers.console.datasets.datasets import (
     DatasetUseCheckApi,
 )
 from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError
+from core.entities.knowledge_entities import IndexingEstimate
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.provider_manager import ProviderManager
 from core.rag.index_processor.constant.index_type import IndexStructureType
@@ -1391,8 +1392,7 @@ class TestDatasetIndexingEstimateApi:
 
         mock_file = self._upload_file()
 
-        mock_response = MagicMock()
+        mock_response = IndexingEstimate(total_segments=100, preview=[])
-        mock_response.model_dump.return_value = {"tokens": 100}
 
         with (
             app.test_request_context("/"),
@@ -1418,7 +1418,13 @@ class TestDatasetIndexingEstimateApi:
             response, status = method(api, "tenant-1")
 
         assert status == 200
-        assert response == {"tokens": 100}
+        assert response == {
+            "tokens": 0,
+            "total_price": 0,
+            "currency": "USD",
+            "total_segments": 100,
+            "preview": [],
+        }
 
     def test_post_file_not_found(self, app: Flask):
         api = DatasetIndexingEstimateApi()

api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py

@@ -1,3 +1,4 @@
+import datetime
 import inspect
 from unittest.mock import ANY, MagicMock, patch
 
@@ -34,6 +35,7 @@ from controllers.console.datasets.error import (
     InvalidActionError,
     InvalidMetadataError,
 )
+from core.entities.knowledge_entities import IndexingEstimate
 from core.rag.index_processor.constant.index_type import IndexStructureType
 from models.dataset import Dataset
 from models.dataset import Document as DatasetDocument
@@ -72,8 +74,46 @@ def make_serializable_document(**overrides):
     }
     attrs.update(overrides)
     document = MagicMock(spec_set=list(attrs))
-    for name, value in attrs.items():
+    document.configure_mock(**attrs)
-        setattr(document, name, value)
+    return document
+
+
+def make_document_detail(**overrides):
+    attrs = {
+        "id": "doc-1",
+        "position": 1,
+        "data_source_type": "upload_file",
+        "data_source_info_dict": {"upload_file_id": "file-1"},
+        "data_source_detail_dict": {},
+        "dataset_process_rule_id": None,
+        "dataset_process_rule": None,
+        "name": "Document",
+        "created_from": "web",
+        "created_by": "u1",
+        "created_at": datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC),
+        "tokens": 10,
+        "indexing_status": "completed",
+        "completed_at": None,
+        "updated_at": None,
+        "indexing_latency": None,
+        "error": None,
+        "enabled": True,
+        "disabled_at": None,
+        "disabled_by": None,
+        "archived": False,
+        "doc_type": "others",
+        "doc_metadata_details": [],
+        "segment_count": 0,
+        "average_segment_length": 0,
+        "hit_count": 0,
+        "display_status": "available",
+        "doc_form": "text_model",
+        "doc_language": "English",
+        "need_summary": False,
+    }
+    attrs.update(overrides)
+    document = MagicMock(spec_set=list(attrs))
+    document.configure_mock(**attrs)
     return document
 
 
@@ -173,6 +213,42 @@ class TestGetProcessRuleApi:
 
         assert "rules" in response
 
+    def test_get_with_document_preserves_legacy_segmentation_delimiter(self, app: Flask, patch_tenant):
+        api = GetProcessRuleApi()
+        method = inspect.unwrap(api.get)
+        user, _ = patch_tenant
+
+        document = MagicMock(dataset_id="ds-1")
+        process_rule = MagicMock(
+            mode="custom",
+            rules_dict={"segmentation": {"delimiter": "---", "max_tokens": 123}},
+        )
+
+        with (
+            app.test_request_context("/?document_id=doc-1"),
+            patch(
+                "controllers.console.datasets.datasets_document.db.get_or_404",
+                return_value=document,
+            ),
+            patch(
+                "controllers.console.datasets.datasets_document.DatasetService.get_dataset",
+                return_value=MagicMock(),
+            ),
+            patch(
+                "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission",
+                return_value=None,
+            ),
+            patch(
+                "controllers.console.datasets.datasets_document.db.session.scalar",
+                return_value=process_rule,
+            ),
+        ):
+            response = method(api, user)
+
+        assert response["rules"]["segmentation"]["separator"] == "---"
+        assert response["rules"]["segmentation"]["max_tokens"] == 123
+        assert "delimiter" not in response["rules"]["segmentation"]
+
     def test_get_with_document_dataset_not_found(self, app: Flask, patch_tenant):
         api = GetProcessRuleApi()
         method = inspect.unwrap(api.get)
@@ -414,7 +490,7 @@ class TestDocumentApi:
         method = inspect.unwrap(api.get)
         user, tenant_id = patch_tenant
 
-        document = MagicMock(dataset_process_rule=None)
+        document = make_document_detail()
 
         with (
             app.test_request_context("/"),
@@ -926,12 +1002,24 @@ class TestDocumentSummaryStatusApi:
             ),
             patch(
                 "services.summary_index_service.SummaryIndexService.get_document_summary_status_detail",
-                return_value={"total_segments": 0},
+                return_value={
+                    "total_segments": 1,
+                    "summary_status": {"timeout": 1},
+                    "summaries": [
+                        {
+                            "segment_id": "segment-1",
+                            "segment_position": 1,
+                            "status": "timeout",
+                        }
+                    ],
+                },
             ),
         ):
             response, status = method(api, user, "ds-1", "doc-1")
 
         assert status == 200
+        assert response["summary_status"]["timeout"] == 1
+        assert response["summaries"][0]["status"] == "timeout"
 
 
 class TestDocumentIndexingEstimateApi:
@@ -1103,7 +1191,7 @@ class TestDocumentBatchIndexingEstimateApi:
             patch.object(api, "get_batch_documents", return_value=[doc]),
             patch(
                 "controllers.console.datasets.datasets_document.IndexingRunner.indexing_estimate",
-                return_value=MagicMock(model_dump=lambda: {"tokens": 2}),
+                return_value=IndexingEstimate(total_segments=2, preview=[]),
             ),
         ):
             resp, status = method(api, tenant_id, user, "ds-1", "batch-1")
@@ -1132,7 +1220,7 @@ class TestDocumentBatchIndexingEstimateApi:
             patch.object(api, "get_batch_documents", return_value=[doc]),
             patch(
                 "controllers.console.datasets.datasets_document.IndexingRunner.indexing_estimate",
-                return_value=MagicMock(model_dump=lambda: {"tokens": 1}),
+                return_value=IndexingEstimate(total_segments=1, preview=[]),
             ),
         ):
             resp, status = method(api, tenant_id, user, "ds-1", "batch-1")
@@ -1314,7 +1402,7 @@ class TestDocumentApiMetadata:
         method = inspect.unwrap(api.get)
         user, tenant_id = patch_tenant
 
-        document = MagicMock(dataset_process_rule=None, doc_metadata_details=[])
+        document = make_document_detail(doc_metadata_details=[])
 
         with (
             app.test_request_context("/?metadata=only"),
@@ -1334,7 +1422,7 @@ class TestDocumentApiMetadata:
         method = inspect.unwrap(api.get)
         user, tenant_id = patch_tenant
 
-        document = MagicMock(dataset_process_rule=None)
+        document = make_document_detail()
 
         with (
             app.test_request_context("/?metadata=without"),
@@ -1611,7 +1699,7 @@ class TestDocumentIndexingEdgeCases:
             ),
             patch(
                 "controllers.console.datasets.datasets_document.IndexingRunner.indexing_estimate",
-                return_value=MagicMock(model_dump=lambda: {"tokens": 5}),
+                return_value=IndexingEstimate(total_segments=5, preview=[]),
             ),
         ):
             response, status = method(api, tenant_id, user, "ds-1", "doc-1")

api/tests/unit_tests/controllers/console/datasets/test_external.py

@@ -1,5 +1,6 @@
 import inspect
 from types import SimpleNamespace
+from typing import Any
 from unittest.mock import ANY, MagicMock, PropertyMock, patch
 
 import pytest
@@ -8,7 +9,6 @@ from werkzeug.exceptions import Forbidden, NotFound
 
 import services
 from controllers.console import console_ns
-from controllers.console.datasets import external as external_module
 from controllers.console.datasets.error import DatasetNameDuplicateError
 from controllers.console.datasets.external import (
     BedrockRetrievalApi,
@@ -40,28 +40,185 @@ def current_user() -> Account:
     return user
 
 
+def _external_api_dict(api_id: str = "api-1") -> dict:
+    return {
+        "id": api_id,
+        "tenant_id": "tenant-1",
+        "name": f"External API {api_id}",
+        "description": f"Description for {api_id}",
+        "settings": {
+            "endpoint": f"https://external.example.com/{api_id}",
+            "api_key": "secret",
+            "headers": {"X-Source": "unit-test"},
+            "timeout": 30,
+        },
+        "dataset_bindings": [
+            {"id": f"dataset-{api_id}", "name": f"Dataset {api_id}"},
+        ],
+        "created_by": "user-1",
+        "created_at": "2024-01-01T00:00:00",
+    }
+
+
+def _external_api_object(api_id: str = "api-1") -> SimpleNamespace:
+    payload = _external_api_dict(api_id)
+    return SimpleNamespace(
+        **{
+            **payload,
+            "dataset_bindings": [SimpleNamespace(**binding) for binding in payload["dataset_bindings"]],
+        }
+    )
+
+
+def _expected_dataset_detail_payload() -> dict[str, Any]:
+    return {
+        "id": "dataset-1",
+        "name": "Support knowledge",
+        "description": "External support articles",
+        "provider": "external",
+        "permission": "only_me",
+        "data_source_type": "external",
+        "indexing_technique": "economy",
+        "app_count": 2,
+        "document_count": 7,
+        "word_count": 2048,
+        "created_by": "user-1",
+        "author_name": "Test User",
+        "created_at": 1710000000,
+        "updated_by": "user-2",
+        "updated_at": 1710003600,
+        "embedding_model": None,
+        "embedding_model_provider": None,
+        "embedding_available": False,
+        "retrieval_model_dict": {
+            "search_method": "semantic_search",
+            "reranking_enable": False,
+            "reranking_mode": None,
+            "reranking_model": {"reranking_provider_name": None, "reranking_model_name": None},
+            "weights": None,
+            "top_k": 4,
+            "score_threshold_enabled": True,
+            "score_threshold": 0.5,
+        },
+        "summary_index_setting": {
+            "enable": True,
+            "model_name": "summary-model",
+            "model_provider_name": "provider-a",
+            "summary_prompt": "Summarize this.",
+        },
+        "tags": [{"id": "tag-1", "name": "Support", "type": "knowledge"}],
+        "doc_form": "text_model",
+        "external_knowledge_info": {
+            "external_knowledge_id": "knowledge-1",
+            "external_knowledge_api_id": "api-1",
+            "external_knowledge_api_name": "External API api-1",
+            "external_knowledge_api_endpoint": "https://external.example.com/api-1",
+        },
+        "external_retrieval_model": {
+            "top_k": 4,
+            "score_threshold": 0.5,
+            "score_threshold_enabled": True,
+        },
+        "doc_metadata": [{"id": "metadata-1", "name": "source", "type": "string"}],
+        "built_in_field_enabled": True,
+        "pipeline_id": None,
+        "runtime_mode": "external",
+        "chunk_structure": "general",
+        "icon_info": {
+            "icon_type": "emoji",
+            "icon": "book",
+            "icon_background": "#FFF4ED",
+            "icon_url": None,
+        },
+        "is_published": True,
+        "total_documents": 7,
+        "total_available_documents": 6,
+        "enable_api": True,
+        "is_multimodal": False,
+        "maintainer": None,
+        "permission_keys": [],
+    }
+
+
+def _dataset_detail_object() -> SimpleNamespace:
+    payload = _expected_dataset_detail_payload()
+    return SimpleNamespace(
+        **{
+            **payload,
+            "summary_index_setting": SimpleNamespace(**payload["summary_index_setting"]),
+            "tags": [SimpleNamespace(**tag) for tag in payload["tags"]],
+            "external_knowledge_info": SimpleNamespace(**payload["external_knowledge_info"]),
+            "external_retrieval_model": SimpleNamespace(**payload["external_retrieval_model"]),
+            "doc_metadata": [SimpleNamespace(**item) for item in payload["doc_metadata"]],
+            "icon_info": SimpleNamespace(**payload["icon_info"]),
+        }
+    )
+
+
 class TestExternalApiTemplateListApi:
     def test_get_success(self, app: Flask):
         api = ExternalApiTemplateListApi()
         method = inspect.unwrap(api.get)
 
-        api_item = MagicMock()
+        api_item = _external_api_object("api-1")
-        api_item.to_dict.return_value = {"id": "1"}
 
         with (
-            app.test_request_context("/?page=1&limit=20"),
+            app.test_request_context("/?page=2&limit=1&keyword=vector"),
             patch.object(
                 ExternalDatasetService,
                 "get_external_knowledge_apis",
-                return_value=([api_item], 1),
+                return_value=([api_item], 3),
             ) as get_external_knowledge_apis,
         ):
             resp, status = method(api, "tenant-1")
 
         assert status == 200
-        assert resp["total"] == 1
+        assert resp == {
-        assert resp["data"][0]["id"] == "1"
+            "data": [_external_api_dict("api-1")],
-        get_external_knowledge_apis.assert_called_once_with(1, 20, "tenant-1", None)
+            "has_more": True,
+            "limit": 1,
+            "total": 3,
+            "page": 2,
+        }
+        get_external_knowledge_apis.assert_called_once_with(2, 1, "tenant-1", "vector")
+
+    def test_post_success_uses_validated_payload_and_returns_template(self, app: Flask, current_user: Account):
+        api = ExternalApiTemplateListApi()
+        method = inspect.unwrap(api.post)
+
+        payload = {
+            "name": "Vendor Search",
+            "settings": {
+                "endpoint": "https://external.example.com/search",
+                "api_key": "secret",
+                "headers": {"X-Source": "unit-test"},
+                "timeout": 30,
+            },
+        }
+        created = _external_api_object("api-created")
+        session = MagicMock()
+
+        with (
+            app.test_request_context("/", json=payload),
+            patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
+            patch.object(ExternalDatasetService, "validate_api_list") as validate_api_list,
+            patch.object(
+                ExternalDatasetService,
+                "create_external_knowledge_api",
+                return_value=created,
+            ) as create_external_knowledge_api,
+        ):
+            resp, status = method(api, session, "tenant-1", current_user)
+
+        assert status == 201
+        assert resp == _external_api_dict("api-created")
+        validate_api_list.assert_called_once_with(payload["settings"])
+        create_external_knowledge_api.assert_called_once_with(
+            tenant_id="tenant-1",
+            user_id="user-1",
+            args=payload,
+            session=session,
+        )
 
     def test_post_forbidden(self, app: Flask, current_user: Account):
         current_user.role = TenantAccountRole.NORMAL
@@ -99,6 +256,28 @@ class TestExternalApiTemplateListApi:
 
 
 class TestExternalApiTemplateApi:
+    def test_get_success_returns_template_contract(self, app: Flask):
+        api = ExternalApiTemplateApi()
+        method = inspect.unwrap(api.get)
+        template = _external_api_object("api-detail")
+        session = MagicMock()
+
+        with (
+            app.test_request_context("/"),
+            patch.object(
+                ExternalDatasetService,
+                "get_external_knowledge_api",
+                return_value=template,
+            ) as get_external_knowledge_api,
+        ):
+            resp, status = method(api, session, "tenant-1", "api-detail")
+
+        assert status == 200
+        assert resp == _external_api_dict("api-detail")
+        get_external_knowledge_api.assert_called_once_with(
+            external_knowledge_api_id="api-detail", tenant_id="tenant-1", session=session
+        )
+
     def test_get_not_found(self, app: Flask):
         api = ExternalApiTemplateApi()
         method = inspect.unwrap(api.get)
@@ -114,6 +293,44 @@ class TestExternalApiTemplateApi:
             with pytest.raises(NotFound):
                 method(api, MagicMock(), "tenant-1", "api-id")
 
+    def test_patch_success_uses_validated_payload_and_returns_template(self, app: Flask, current_user: Account):
+        api = ExternalApiTemplateApi()
+        method = inspect.unwrap(api.patch)
+
+        payload = {
+            "name": "Updated API",
+            "settings": {
+                "endpoint": "https://external.example.com/updated",
+                "api_key": "new-secret",
+                "headers": {"X-Version": "2"},
+            },
+        }
+        updated = _external_api_object("api-updated")
+        session = MagicMock()
+
+        with (
+            app.test_request_context("/", json=payload),
+            patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
+            patch.object(ExternalDatasetService, "validate_api_list") as validate_api_list,
+            patch.object(
+                ExternalDatasetService,
+                "update_external_knowledge_api",
+                return_value=updated,
+            ) as update_external_knowledge_api,
+        ):
+            resp, status = method(api, session, "tenant-1", current_user, "api-updated")
+
+        assert status == 200
+        assert resp == _external_api_dict("api-updated")
+        validate_api_list.assert_called_once_with(payload["settings"])
+        update_external_knowledge_api.assert_called_once_with(
+            tenant_id="tenant-1",
+            user_id="user-1",
+            external_knowledge_api_id="api-updated",
+            args=payload,
+            session=session,
+        )
+
     def test_delete_forbidden(self, app: Flask, current_user: Account):
         current_user.role = TenantAccountRole.NORMAL
 
@@ -153,46 +370,39 @@ class TestExternalDatasetCreateApi:
         method = inspect.unwrap(api.post)
 
         payload = {
-            "external_knowledge_api_id": "api",
+            "external_knowledge_api_id": "api-1",
-            "external_knowledge_id": "kid",
+            "external_knowledge_id": "knowledge-1",
-            "name": "dataset",
+            "name": "Support knowledge",
+            "description": "External support articles",
+            "external_retrieval_model": {
+                "top_k": 4,
+                "score_threshold": 0.5,
+                "score_threshold_enabled": True,
+            },
         }
 
-        dataset = MagicMock()
+        dataset = _dataset_detail_object()
-
-        dataset.embedding_available = False
-        dataset.built_in_field_enabled = False
-        dataset.is_published = False
-        dataset.enable_api = False
-        dataset.enable_qa = False
-        dataset.enable_vector_store = False
-        dataset.vector_store_setting = None
-        dataset.is_multimodal = False
-
-        dataset.retrieval_model_dict = {}
-        dataset.tags = []
-        dataset.external_knowledge_info = None
-        dataset.external_retrieval_model = None
-        dataset.doc_metadata = []
-        dataset.icon_info = None
-        dataset.permission_keys = []
-
-        dataset.summary_index_setting = MagicMock()
-        dataset.summary_index_setting.enable = False
 
         with (
-            app.test_request_context("/"),
+            app.test_request_context("/", json=payload),
             patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
             patch.object(
                 ExternalDatasetService,
                 "create_external_dataset",
                 return_value=dataset,
-            ),
+            ) as create_external_dataset,
-            patch.object(external_module, "db", SimpleNamespace(session=lambda: MagicMock())),
         ):
-            _, status = method(api, MagicMock(), "tenant-1", current_user)
+            session = MagicMock()
+            resp, status = method(api, session, "tenant-1", current_user)
 
         assert status == 201
+        assert resp == _expected_dataset_detail_payload()
+        create_external_dataset.assert_called_once_with(
+            tenant_id="tenant-1",
+            user_id="user-1",
+            args=payload,
+            session=session,
+        )
 
     def test_create_forbidden(self, app: Flask, current_user: Account):
         current_user.role = TenantAccountRole.NORMAL
@@ -233,24 +443,59 @@ class TestExternalKnowledgeHitTestingApi:
         api = ExternalKnowledgeHitTestingApi()
         method = inspect.unwrap(api.post)
 
-        payload = {"query": "hello"}
+        payload = {
+            "query": "hello",
+            "external_retrieval_model": {
+                "top_k": 3,
+                "score_threshold": 0.25,
+                "score_threshold_enabled": True,
+            },
+            "metadata_filtering_conditions": {
+                "logical_operator": "and",
+                "conditions": [{"name": "source", "comparison_operator": "contains", "value": "external"}],
+            },
+        }
 
         dataset = MagicMock()
+        retrieve_response = {
+            "query": {"content": "hello"},
+            "records": [
+                {
+                    "content": "answer",
+                    "title": "doc",
+                    "score": 0.9,
+                    "metadata": {"source": "external", "page": 2},
+                }
+            ],
+        }
+        session = MagicMock()
 
         with (
-            app.test_request_context("/"),
+            app.test_request_context("/", json=payload),
             patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
             patch.object(DatasetService, "get_dataset", return_value=dataset),
-            patch.object(DatasetService, "check_dataset_permission"),
+            patch.object(DatasetService, "check_dataset_permission") as check_dataset_permission,
+            patch.object(HitTestingService, "hit_testing_args_check") as hit_testing_args_check,
             patch.object(
                 HitTestingService,
                 "external_retrieve",
-                return_value={"ok": True},
+                return_value=retrieve_response,
-            ),
+            ) as external_retrieve,
+            patch("controllers.console.datasets.external.dump_response", side_effect=lambda _model, value: value),
         ):
-            resp = method(api, MagicMock(), current_user, "dataset-id")
+            resp = method(api, session, current_user, "dataset-id")
 
-        assert resp["ok"] is True
+        assert resp == retrieve_response
+        check_dataset_permission.assert_called_once_with(dataset, current_user, session)
+        hit_testing_args_check.assert_called_once_with(payload)
+        external_retrieve.assert_called_once_with(
+            session=session,
+            dataset=dataset,
+            query="hello",
+            account=current_user,
+            external_retrieval_model=payload["external_retrieval_model"],
+            metadata_filtering_conditions=payload["metadata_filtering_conditions"],
+        )
 
 
 class TestBedrockRetrievalApi:
@@ -259,24 +504,46 @@ class TestBedrockRetrievalApi:
         method = inspect.unwrap(api.post)
 
         payload = {
-            "retrieval_setting": {},
+            "retrieval_setting": {"top_k": 5, "score_threshold": 0.72},
-            "query": "hello",
+            "query": "hello bedrock",
-            "knowledge_id": "kid",
+            "knowledge_id": "knowledge-base-1",
+        }
+        retrieval_response = {
+            "records": [
+                {
+                    "metadata": {"source": "bedrock", "uri": "s3://bucket/doc.txt"},
+                    "score": 0.8,
+                    "title": "doc",
+                    "content": "answer",
+                },
+                {
+                    "metadata": {"source": "bedrock", "uri": "s3://bucket/other.txt"},
+                    "score": 0.65,
+                    "title": None,
+                    "content": None,
+                },
+            ]
         }
 
+        session = MagicMock()
+
         with (
-            app.test_request_context("/"),
+            app.test_request_context("/", json=payload),
             patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
             patch.object(
                 ExternalDatasetTestService,
                 "knowledge_retrieval",
-                return_value={"ok": True},
+                return_value=retrieval_response,
-            ),
+            ) as knowledge_retrieval,
         ):
             resp, status = method()
 
         assert status == 200
-        assert resp["ok"] is True
+        assert resp == retrieval_response
+        retrieval_setting, query, knowledge_id = knowledge_retrieval.call_args.args
+        assert retrieval_setting.model_dump() == payload["retrieval_setting"]
+        assert query == "hello bedrock"
+        assert knowledge_id == "knowledge-base-1"
 
 
 class TestExternalApiTemplateListApiAdvanced:
@@ -302,10 +569,10 @@ class TestExternalApiTemplateListApiAdvanced:
         api = ExternalApiTemplateListApi()
         method = inspect.unwrap(api.get)
 
-        templates = [MagicMock(id=f"api-{i}") for i in range(3)]
+        templates = [_external_api_object(f"api-{i}") for i in range(3)]
 
         with (
-            app.test_request_context("/?page=1&limit=20"),
+            app.test_request_context("/?page=2&limit=3"),
             patch(
                 "controllers.console.datasets.external.ExternalDatasetService.get_external_knowledge_apis",
                 return_value=(templates, 25),
@@ -314,9 +581,14 @@ class TestExternalApiTemplateListApiAdvanced:
             resp, status = method(api, "tenant-1")
 
         assert status == 200
-        assert resp["total"] == 25
+        assert resp == {
-        assert len(resp["data"]) == 3
+            "data": [_external_api_dict(f"api-{i}") for i in range(3)],
-        get_external_knowledge_apis.assert_called_once_with(1, 20, "tenant-1", None)
+            "has_more": True,
+            "limit": 3,
+            "total": 25,
+            "page": 2,
+        }
+        get_external_knowledge_apis.assert_called_once_with(2, 3, "tenant-1", None)
 
 
 class TestExternalDatasetCreateApiAdvanced:
@@ -371,6 +643,7 @@ class TestExternalKnowledgeHitTestingApiAdvanced:
             "external_retrieval_model": {"type": "bm25"},
             "metadata_filtering_conditions": {"status": "active"},
         }
+        session = MagicMock()
 
         with (
             app.test_request_context("/", json=payload),
@@ -379,15 +652,46 @@ class TestExternalKnowledgeHitTestingApiAdvanced:
                 "controllers.console.datasets.external.DatasetService.get_dataset",
                 return_value=dataset,
             ),
-            patch("controllers.console.datasets.external.DatasetService.check_dataset_permission"),
+            patch("controllers.console.datasets.external.DatasetService.check_dataset_permission") as check_permission,
+            patch("controllers.console.datasets.external.HitTestingService.hit_testing_args_check") as args_check,
             patch(
                 "controllers.console.datasets.external.HitTestingService.external_retrieve",
-                return_value={"results": []},
+                return_value={
-            ),
+                    "query": {"content": "test query"},
+                    "records": [
+                        {
+                            "content": None,
+                            "title": "metadata-only",
+                            "score": None,
+                            "metadata": {"status": "active"},
+                        }
+                    ],
+                },
+            ) as external_retrieve,
         ):
-            resp = method(api, MagicMock(), current_user, "ds-1")
+            resp = method(api, session, current_user, "ds-1")
 
-        assert resp["results"] == []
+        assert resp == {
+            "query": {"content": "test query"},
+            "records": [
+                {
+                    "content": None,
+                    "title": "metadata-only",
+                    "score": None,
+                    "metadata": {"status": "active"},
+                }
+            ],
+        }
+        check_permission.assert_called_once_with(dataset, current_user, session)
+        args_check.assert_called_once_with(payload)
+        external_retrieve.assert_called_once_with(
+            session=session,
+            dataset=dataset,
+            query="test query",
+            account=current_user,
+            external_retrieval_model={"type": "bm25"},
+            metadata_filtering_conditions={"status": "active"},
+        )
 
 
 class TestBedrockRetrievalApiAdvanced:

packages/contracts/generated/api/console/datasets/orpc.gen.ts

@@ -359,8 +359,12 @@ export const get5 = oc
   .input(z.object({ params: zGetDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdPath }))
   .output(zGetDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdResponse)
 
+/**
+ * Update external knowledge API template
+ */
 export const patch = oc
   .route({
+    description: 'Update external knowledge API template',
     inputStructure: 'detailed',
     method: 'PATCH',
     operationId: 'patchDatasetsExternalKnowledgeApiByExternalKnowledgeApiId',
@@ -397,8 +401,12 @@ export const get6 = oc
   .input(z.object({ query: zGetDatasetsExternalKnowledgeApiQuery.optional() }))
   .output(zGetDatasetsExternalKnowledgeApiResponse)
 
+/**
+ * Create external knowledge API template
+ */
 export const post4 = oc
   .route({
+    description: 'Create external knowledge API template',
     inputStructure: 'detailed',
     method: 'POST',
     operationId: 'postDatasetsExternalKnowledgeApi',
@@ -444,7 +452,6 @@ export const post6 = oc
     method: 'POST',
     operationId: 'postDatasetsInit',
     path: '/datasets/init',
-    successStatus: 201,
     tags: ['console'],
   })
   .input(z.object({ body: zPostDatasetsInitBody }))
@@ -1184,12 +1191,13 @@ export const segments = {
  * - generating: Number of summaries being generated
  * - error: Number of summaries with errors
  * - not_started: Number of segments without summary records
+ * - timeout: Number of summaries that timed out
  * - summaries: List of summary records with status and content preview
  */
 export const get22 = oc
   .route({
     description:
-      'Get summary index generation status for a document\nReturns:\n- total_segments: Total number of segments in the document\n- summary_status: Dictionary with status counts\n  - completed: Number of summaries completed\n  - generating: Number of summaries being generated\n  - error: Number of summaries with errors\n  - not_started: Number of segments without summary records\n- summaries: List of summary records with status and content preview',
+      'Get summary index generation status for a document\nReturns:\n- total_segments: Total number of segments in the document\n- summary_status: Dictionary with status counts\n  - completed: Number of summaries completed\n  - generating: Number of summaries being generated\n  - error: Number of summaries with errors\n  - not_started: Number of segments without summary records\n  - timeout: Number of summaries that timed out\n- summaries: List of summary records with status and content preview',
     inputStructure: 'detailed',
     method: 'GET',
     operationId: 'getDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatus',

packages/contracts/generated/api/console/datasets/types.gen.ts

@@ -97,51 +97,12 @@ export type ExternalDatasetCreatePayload = {
   name: string
 }
 
-export type DatasetDetail = {
-  app_count?: number
-  author_name?: string
-  built_in_field_enabled?: boolean
-  chunk_structure?: string
-  created_at?: number
-  created_by?: string
-  data_source_type?: string
-  description?: string
-  doc_form?: string
-  doc_metadata?: Array<DatasetDocMetadata>
-  document_count?: number
-  embedding_available?: boolean
-  embedding_model?: string
-  embedding_model_provider?: string
-  enable_api?: boolean
-  external_knowledge_info?: ExternalKnowledgeInfo
-  external_retrieval_model?: ExternalRetrievalModel
-  icon_info?: DatasetIconInfo
-  id?: string
-  indexing_technique?: string
-  is_multimodal?: boolean
-  is_published?: boolean
-  name?: string
-  permission?: string
-  permission_keys?: Array<string>
-  pipeline_id?: string
-  provider?: string
-  retrieval_model_dict?: DatasetRetrievalModel
-  runtime_mode?: string
-  summary_index_setting?: AnonymousInlineModelB1954337D565
-  tags?: Array<Tag>
-  total_available_documents?: number
-  total_documents?: number
-  updated_at?: number
-  updated_by?: string
-  word_count?: number
-}
-
 export type ExternalKnowledgeApiListResponse = {
   data: Array<ExternalKnowledgeApiResponse>
   has_more: boolean
   limit: number
   page: number
-  total: number
+  total: number | null
 }
 
 export type ExternalKnowledgeApiPayload = {
@@ -154,11 +115,11 @@ export type ExternalKnowledgeApiPayload = {
 export type ExternalKnowledgeApiResponse = {
   created_at: string
   created_by: string
-  dataset_bindings?: Array<ExternalKnowledgeDatasetBindingResponse>
+  dataset_bindings: Array<ExternalKnowledgeApiBindingResponse>
   description: string
   id: string
   name: string
-  settings?: {
+  settings: {
     [key: string]: unknown
   } | null
   tenant_id: string
@@ -183,8 +144,11 @@ export type IndexingEstimatePayload = {
 }
 
 export type IndexingEstimateResponse = {
-  preview: Array<IndexingEstimatePreviewItemResponse>
+  currency: string
-  qa_preview?: Array<IndexingEstimateQaPreviewItemResponse> | null
+  preview: Array<PreviewDetail>
+  qa_preview?: Array<QaPreviewDetail> | null
+  tokens: number
+  total_price: number | number
   total_segments: number
 }
 
@@ -236,8 +200,12 @@ export type IndexingEstimate = {
   total_segments: number
 }
 
-export type OpaqueObjectResponse = {
+export type ProcessRuleResponse = {
-  [key: string]: unknown
+  limits: {
+    [key: string]: unknown
+  }
+  mode: ProcessRuleMode
+  rules?: Rule | null
 }
 
 export type RetrievalSettingResponse = {
@@ -337,8 +305,6 @@ export type DocumentBatchDownloadZipPayload = {
   document_ids: Array<string>
 }
 
-export type BinaryFileResponse = Blob | File
-
 export type GenerateSummaryPayload = {
   document_list: Array<string>
 }
@@ -347,6 +313,40 @@ export type MetadataOperationData = {
   operation_data: Array<DocumentMetadataOperation>
 }
 
+export type DocumentDetailResponse = {
+  archived?: boolean | null
+  average_segment_length?: number | null
+  completed_at?: number | null
+  created_at?: number | null
+  created_by?: string | null
+  created_from?: string | null
+  data_source_detail_dict?: unknown
+  data_source_info?: unknown
+  data_source_type?: string | null
+  dataset_process_rule?: unknown
+  dataset_process_rule_id?: string | null
+  disabled_at?: number | null
+  disabled_by?: string | null
+  display_status?: string | null
+  doc_form?: string | null
+  doc_language?: string | null
+  doc_metadata?: Array<DocumentMetadataResponse> | null
+  doc_type?: string | null
+  document_process_rule?: unknown
+  enabled?: boolean | null
+  error?: string | null
+  hit_count?: number | null
+  id: string
+  indexing_latency?: number | null
+  indexing_status?: string | null
+  name?: string | null
+  need_summary?: boolean | null
+  position?: number | null
+  segment_count?: number | null
+  tokens?: number | null
+  updated_at?: number | null
+}
+
 export type UrlResponse = {
   url: string
 }
@@ -376,6 +376,13 @@ export type SimpleResultMessageResponse = {
   result: string
 }
 
+export type DocumentPipelineExecutionLogResponse = {
+  datasource_info?: JsonValue | null
+  datasource_node_id?: string | null
+  datasource_type?: string | null
+  input_data?: JsonValue | null
+}
+
 export type DocumentRenamePayload = {
   name: string
 }
@@ -464,6 +471,12 @@ export type ChildChunkUpdatePayload = {
   content: string
 }
 
+export type DocumentSummaryStatusResponse = {
+  summaries: Array<SummaryEntryResponse>
+  summary_status: SummaryStatusResponse
+  total_segments: number
+}
+
 export type ErrorDocsResponse = {
   data: Array<DocumentStatusResponse>
   total: number
@@ -479,13 +492,10 @@ export type ExternalHitTestingPayload = {
   query: string
 }
 
-export type ExternalRetrievalTestResponse
+export type ExternalHitTestingResponse = {
-  = | {
+  query: ExternalHitTestingQueryResponse
-    [key: string]: unknown
+  records: Array<ExternalHitTestingRecordResponse>
-  }
+}
-  | Array<{
-    [key: string]: unknown
-  }>
 
 export type HitTestingPayload = {
   attachment_ids?: Array<string> | null
@@ -641,68 +651,18 @@ export type DatasetTagResponse = {
   type: string
 }
 
-export type DatasetDocMetadata = {
+export type ExternalKnowledgeApiBindingResponse = {
-  id?: string
-  name?: string
-  type?: string
-}
-
-export type ExternalKnowledgeInfo = {
-  external_knowledge_api_endpoint?: string
-  external_knowledge_api_id?: string
-  external_knowledge_api_name?: string
-  external_knowledge_id?: string
-}
-
-export type ExternalRetrievalModel = {
-  score_threshold?: number
-  score_threshold_enabled?: boolean
-  top_k?: number
-}
-
-export type DatasetIconInfo = {
-  icon?: string
-  icon_background?: string
-  icon_type?: string
-  icon_url?: string
-}
-
-export type DatasetRetrievalModel = {
-  reranking_enable?: boolean
-  reranking_mode?: string
-  reranking_model?: DatasetRerankingModel
-  score_threshold?: number
-  score_threshold_enabled?: boolean
-  search_method?: string
-  top_k?: number
-  weights?: DatasetWeightedScore
-}
-
-export type AnonymousInlineModelB1954337D565 = {
-  enable?: boolean
-  model_name?: string
-  model_provider_name?: string
-  summary_prompt?: string
-}
-
-export type Tag = {
-  id: string
-  name: string
-  type: string
-}
-
-export type ExternalKnowledgeDatasetBindingResponse = {
   id: string
   name: string
 }
 
-export type IndexingEstimatePreviewItemResponse = {
+export type PreviewDetail = {
   child_chunks?: Array<string> | null
   content: string
   summary?: string | null
 }
 
-export type IndexingEstimateQaPreviewItemResponse = {
+export type QaPreviewDetail = {
   answer: string
   question: string
 }
@@ -744,15 +704,13 @@ export type DatasetMetadataBuiltInFieldResponse = {
   type: string
 }
 
-export type PreviewDetail = {
+export type ProcessRuleMode = 'automatic' | 'custom' | 'hierarchical'
-  child_chunks?: Array<string> | null
-  content: string
-  summary?: string | null
-}
 
-export type QaPreviewDetail = {
+export type Rule = {
-  answer: string
+  parent_mode?: 'full-doc' | 'paragraph' | null
-  question: string
+  pre_processing_rules?: Array<PreProcessingRule> | null
+  segmentation?: Segmentation | null
+  subchunk_segmentation?: Segmentation | null
 }
 
 export type DocumentWithSegmentsResponse = {
@@ -798,6 +756,8 @@ export type DocumentMetadataResponse = {
   value?: string | number | number | boolean | null
 }
 
+export type JsonValue = unknown
+
 export type SegmentResponse = {
   answer: string | null
   attachments: Array<SegmentAttachmentResponse>
@@ -844,6 +804,37 @@ export type ChildChunkUpdateArgs = {
   id?: string | null
 }
 
+export type SummaryEntryResponse = {
+  created_at?: number | null
+  error?: string | null
+  segment_id: string
+  segment_position: number
+  status: string
+  summary_preview?: string | null
+  updated_at?: number | null
+}
+
+export type SummaryStatusResponse = {
+  completed?: number
+  error?: number
+  generating?: number
+  not_started?: number
+  timeout?: number
+}
+
+export type ExternalHitTestingQueryResponse = {
+  content: string
+}
+
+export type ExternalHitTestingRecordResponse = {
+  content?: string | null
+  metadata?: {
+    [key: string]: unknown
+  } | null
+  score?: number | null
+  title?: string | null
+}
+
 export type HitTestingQuery = {
   content: string
 }
@@ -896,17 +887,6 @@ export type DatasetWeightedScoreResponse = {
   weight_type?: string | null
 }
 
-export type DatasetRerankingModel = {
-  reranking_model_name?: string
-  reranking_provider_name?: string
-}
-
-export type DatasetWeightedScore = {
-  keyword_setting?: DatasetKeywordSetting
-  vector_setting?: DatasetVectorSetting
-  weight_type?: string
-}
-
 export type InfoList = {
   data_source_type: 'notion_import' | 'upload_file' | 'website_crawl'
   file_info_list?: FileInfo | null
@@ -914,15 +894,6 @@ export type InfoList = {
   website_info_list?: WebsiteInfo | null
 }
 
-export type ProcessRuleMode = 'automatic' | 'custom' | 'hierarchical'
-
-export type Rule = {
-  parent_mode?: 'full-doc' | 'paragraph' | null
-  pre_processing_rules?: Array<PreProcessingRule> | null
-  segmentation?: Segmentation | null
-  subchunk_segmentation?: Segmentation | null
-}
-
 export type MetadataFilteringCondition = {
   conditions?: Array<Condition> | null
   logical_operator?: 'and' | 'or' | null
@@ -945,6 +916,17 @@ export type WeightModel = {
   weight_type?: 'customized' | 'keyword_first' | 'semantic_first' | null
 }
 
+export type PreProcessingRule = {
+  enabled: boolean
+  id: 'remove_extra_spaces' | 'remove_stopwords' | 'remove_urls_emails'
+}
+
+export type Segmentation = {
+  chunk_overlap?: number
+  max_tokens: number
+  separator?: string
+}
+
 export type MetadataDetail = {
   id: string
   name: string
@@ -1018,16 +1000,6 @@ export type DatasetVectorSettingResponse = {
   vector_weight?: number | null
 }
 
-export type DatasetKeywordSetting = {
-  keyword_weight?: number
-}
-
-export type DatasetVectorSetting = {
-  embedding_model_name?: string
-  embedding_provider_name?: string
-  vector_weight?: number
-}
-
 export type FileInfo = {
   file_ids: Array<string>
 }
@@ -1045,17 +1017,6 @@ export type WebsiteInfo = {
   urls: Array<string>
 }
 
-export type PreProcessingRule = {
-  enabled: boolean
-  id: 'remove_extra_spaces' | 'remove_stopwords' | 'remove_urls_emails'
-}
-
-export type Segmentation = {
-  chunk_overlap?: number
-  max_tokens: number
-  separator?: string
-}
-
 export type Condition = {
   comparison_operator:
     | '<'
@@ -1264,7 +1225,7 @@ export type PostDatasetsExternalErrors = {
 }
 
 export type PostDatasetsExternalResponses = {
-  201: DatasetDetail
+  201: DatasetDetailResponse
 }
 
 export type PostDatasetsExternalResponse
@@ -1295,6 +1256,10 @@ export type PostDatasetsExternalKnowledgeApiData = {
   url: '/datasets/external-knowledge-api'
 }
 
+export type PostDatasetsExternalKnowledgeApiErrors = {
+  403: unknown
+}
+
 export type PostDatasetsExternalKnowledgeApiResponses = {
   201: ExternalKnowledgeApiResponse
 }
@@ -1347,6 +1312,10 @@ export type PatchDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdData = {
   url: '/datasets/external-knowledge-api/{external_knowledge_api_id}'
 }
 
+export type PatchDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdErrors = {
+  404: unknown
+}
+
 export type PatchDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdResponses = {
   200: ExternalKnowledgeApiResponse
 }
@@ -1396,7 +1365,7 @@ export type PostDatasetsInitErrors = {
 }
 
 export type PostDatasetsInitResponses = {
-  201: DatasetAndDocumentResponse
+  200: DatasetAndDocumentResponse
 }
 
 export type PostDatasetsInitResponse = PostDatasetsInitResponses[keyof PostDatasetsInitResponses]
@@ -1439,7 +1408,7 @@ export type GetDatasetsProcessRuleData = {
 }
 
 export type GetDatasetsProcessRuleResponses = {
-  200: OpaqueObjectResponse
+  200: ProcessRuleResponse
 }
 
 export type GetDatasetsProcessRuleResponse
@@ -1581,7 +1550,7 @@ export type GetDatasetsByDatasetIdBatchByBatchIndexingEstimateData = {
 }
 
 export type GetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponses = {
-  200: OpaqueObjectResponse
+  200: IndexingEstimateResponse
 }
 
 export type GetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponse
@@ -1669,7 +1638,9 @@ export type PostDatasetsByDatasetIdDocumentsDownloadZipData = {
 }
 
 export type PostDatasetsByDatasetIdDocumentsDownloadZipResponses = {
-  200: BinaryFileResponse
+  200: {
+    [key: string]: unknown
+  }
 }
 
 export type PostDatasetsByDatasetIdDocumentsDownloadZipResponse
@@ -1764,7 +1735,7 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdErrors = {
 }
 
 export type GetDatasetsByDatasetIdDocumentsByDocumentIdResponses = {
-  200: OpaqueObjectResponse
+  200: DocumentDetailResponse
 }
 
 export type GetDatasetsByDatasetIdDocumentsByDocumentIdResponse
@@ -1803,7 +1774,7 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateErrors =
 }
 
 export type GetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateResponses = {
-  200: OpaqueObjectResponse
+  200: IndexingEstimateResponse
 }
 
 export type GetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateResponse
@@ -1880,7 +1851,7 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogData
 }
 
 export type GetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogResponses = {
-  200: OpaqueObjectResponse
+  200: DocumentPipelineExecutionLogResponse
 }
 
 export type GetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogResponse
@@ -2225,7 +2196,7 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusErrors = {
 }
 
 export type GetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusResponses = {
-  200: OpaqueObjectResponse
+  200: DocumentSummaryStatusResponse
 }
 
 export type GetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusResponse
@@ -2283,7 +2254,7 @@ export type PostDatasetsByDatasetIdExternalHitTestingErrors = {
 }
 
 export type PostDatasetsByDatasetIdExternalHitTestingResponses = {
-  200: ExternalRetrievalTestResponse
+  200: ExternalHitTestingResponse
 }
 
 export type PostDatasetsByDatasetIdExternalHitTestingResponse

packages/contracts/generated/api/console/datasets/zod.gen.ts

@@ -91,11 +91,6 @@ export const zNotionEstimatePayload = z.object({
   process_rule: z.record(z.string(), z.unknown()),
 })
 
-/**
- * OpaqueObjectResponse
- */
-export const zOpaqueObjectResponse = z.record(z.string(), z.unknown())
-
 /**
  * RetrievalSettingResponse
  */
@@ -127,11 +122,6 @@ export const zDocumentBatchDownloadZipPayload = z.object({
   document_ids: z.array(z.uuid()).min(1).max(100),
 })
 
-/**
- * BinaryFileResponse
- */
-export const zBinaryFileResponse = z.custom<Blob | File>()
-
 /**
  * GenerateSummaryPayload
  */
@@ -247,14 +237,6 @@ export const zExternalHitTestingPayload = z.object({
   query: z.string(),
 })
 
-/**
- * ExternalRetrievalTestResponse
- */
-export const zExternalRetrievalTestResponse = z.union([
-  z.record(z.string(), z.unknown()),
-  z.array(z.record(z.string(), z.unknown())),
-])
-
 /**
  * MetadataArgs
  */
@@ -397,52 +379,10 @@ export const zDatasetTagResponse = z.object({
   type: z.string(),
 })
 
-export const zDatasetDocMetadata = z.object({
-  id: z.string().optional(),
-  name: z.string().optional(),
-  type: z.string().optional(),
-})
-
-export const zExternalKnowledgeInfo = z.object({
-  external_knowledge_api_endpoint: z.string().optional(),
-  external_knowledge_api_id: z.string().optional(),
-  external_knowledge_api_name: z.string().optional(),
-  external_knowledge_id: z.string().optional(),
-})
-
-export const zExternalRetrievalModel = z.object({
-  score_threshold: z.number().optional(),
-  score_threshold_enabled: z.boolean().optional(),
-  top_k: z.int().optional(),
-})
-
-export const zDatasetIconInfo = z.object({
-  icon: z.string().optional(),
-  icon_background: z.string().optional(),
-  icon_type: z.string().optional(),
-  icon_url: z.string().optional(),
-})
-
-export const zAnonymousInlineModelB1954337D565 = z.object({
-  enable: z.boolean().optional(),
-  model_name: z.string().optional(),
-  model_provider_name: z.string().optional(),
-  summary_prompt: z.string().optional(),
-})
-
 /**
- * Tag
+ * ExternalKnowledgeApiBindingResponse
  */
-export const zTag = z.object({
+export const zExternalKnowledgeApiBindingResponse = z.object({
-  id: z.string(),
-  name: z.string(),
-  type: z.string(),
-})
-
-/**
- * ExternalKnowledgeDatasetBindingResponse
- */
-export const zExternalKnowledgeDatasetBindingResponse = z.object({
   id: z.string(),
   name: z.string(),
 })
@@ -453,11 +393,11 @@ export const zExternalKnowledgeDatasetBindingResponse = z.object({
 export const zExternalKnowledgeApiResponse = z.object({
   created_at: z.string(),
   created_by: z.string(),
-  dataset_bindings: z.array(zExternalKnowledgeDatasetBindingResponse).optional(),
+  dataset_bindings: z.array(zExternalKnowledgeApiBindingResponse),
   description: z.string(),
   id: z.string(),
   name: z.string(),
-  settings: z.record(z.string(), z.unknown()).nullish(),
+  settings: z.record(z.string(), z.unknown()).nullable(),
   tenant_id: z.string(),
 })
 
@@ -469,22 +409,22 @@ export const zExternalKnowledgeApiListResponse = z.object({
   has_more: z.boolean(),
   limit: z.int(),
   page: z.int(),
-  total: z.int(),
+  total: z.int().nullable(),
 })
 
 /**
- * IndexingEstimatePreviewItemResponse
+ * PreviewDetail
  */
-export const zIndexingEstimatePreviewItemResponse = z.object({
+export const zPreviewDetail = z.object({
   child_chunks: z.array(z.string()).nullish(),
   content: z.string(),
   summary: z.string().nullish(),
 })
 
 /**
- * IndexingEstimateQaPreviewItemResponse
+ * QAPreviewDetail
  */
-export const zIndexingEstimateQaPreviewItemResponse = z.object({
+export const zQaPreviewDetail = z.object({
   answer: z.string(),
   question: z.string(),
 })
@@ -493,8 +433,20 @@ export const zIndexingEstimateQaPreviewItemResponse = z.object({
  * IndexingEstimateResponse
  */
 export const zIndexingEstimateResponse = z.object({
-  preview: z.array(zIndexingEstimatePreviewItemResponse),
+  currency: z.string(),
-  qa_preview: z.array(zIndexingEstimateQaPreviewItemResponse).nullish(),
+  preview: z.array(zPreviewDetail),
+  qa_preview: z.array(zQaPreviewDetail).nullish(),
+  tokens: z.int(),
+  total_price: z.union([z.number(), z.int()]),
+  total_segments: z.int(),
+})
+
+/**
+ * IndexingEstimate
+ */
+export const zIndexingEstimate = z.object({
+  preview: z.array(zPreviewDetail),
+  qa_preview: z.array(zQaPreviewDetail).nullish(),
   total_segments: z.int(),
 })
 
@@ -528,30 +480,11 @@ export const zDatasetMetadataBuiltInFieldsResponse = z.object({
 })
 
 /**
- * PreviewDetail
+ * ProcessRuleMode
+ *
+ * Dataset Process Rule Mode
  */
-export const zPreviewDetail = z.object({
+export const zProcessRuleMode = z.enum(['automatic', 'custom', 'hierarchical'])
-  child_chunks: z.array(z.string()).nullish(),
-  content: z.string(),
-  summary: z.string().nullish(),
-})
-
-/**
- * QAPreviewDetail
- */
-export const zQaPreviewDetail = z.object({
-  answer: z.string(),
-  question: z.string(),
-})
-
-/**
- * IndexingEstimate
- */
-export const zIndexingEstimate = z.object({
-  preview: z.array(zPreviewDetail),
-  qa_preview: z.array(zQaPreviewDetail).nullish(),
-  total_segments: z.int(),
-})
 
 /**
  * DocumentMetadataResponse
@@ -563,6 +496,43 @@ export const zDocumentMetadataResponse = z.object({
   value: z.union([z.string(), z.int(), z.number(), z.boolean()]).nullish(),
 })
 
+/**
+ * DocumentDetailResponse
+ */
+export const zDocumentDetailResponse = z.object({
+  archived: z.boolean().nullish(),
+  average_segment_length: z.number().nullish(),
+  completed_at: z.int().nullish(),
+  created_at: z.int().nullish(),
+  created_by: z.string().nullish(),
+  created_from: z.string().nullish(),
+  data_source_detail_dict: z.unknown().optional(),
+  data_source_info: z.unknown().optional(),
+  data_source_type: z.string().nullish(),
+  dataset_process_rule: z.unknown().optional(),
+  dataset_process_rule_id: z.string().nullish(),
+  disabled_at: z.int().nullish(),
+  disabled_by: z.string().nullish(),
+  display_status: z.string().nullish(),
+  doc_form: z.string().nullish(),
+  doc_language: z.string().nullish(),
+  doc_metadata: z.array(zDocumentMetadataResponse).nullish(),
+  doc_type: z.string().nullish(),
+  document_process_rule: z.unknown().optional(),
+  enabled: z.boolean().nullish(),
+  error: z.string().nullish(),
+  hit_count: z.int().nullish(),
+  id: z.string(),
+  indexing_latency: z.number().nullish(),
+  indexing_status: z.string().nullish(),
+  name: z.string().nullish(),
+  need_summary: z.boolean().nullish(),
+  position: z.int().nullish(),
+  segment_count: z.int().nullish(),
+  tokens: z.int().nullish(),
+  updated_at: z.int().nullish(),
+})
+
 /**
  * DocumentResponse
  */
@@ -646,6 +616,18 @@ export const zDocumentWithSegmentsListResponse = z.object({
   total: z.int(),
 })
 
+export const zJsonValue = z.unknown()
+
+/**
+ * DocumentPipelineExecutionLogResponse
+ */
+export const zDocumentPipelineExecutionLogResponse = z.object({
+  datasource_info: zJsonValue.nullish(),
+  datasource_node_id: z.string().nullish(),
+  datasource_type: z.string().nullish(),
+  input_data: zJsonValue.nullish(),
+})
+
 /**
  * ChildChunkResponse
  */
@@ -700,6 +682,64 @@ export const zChildChunkBatchUpdatePayload = z.object({
   chunks: z.array(zChildChunkUpdateArgs),
 })
 
+/**
+ * SummaryEntryResponse
+ */
+export const zSummaryEntryResponse = z.object({
+  created_at: z.int().nullish(),
+  error: z.string().nullish(),
+  segment_id: z.string(),
+  segment_position: z.int(),
+  status: z.string(),
+  summary_preview: z.string().nullish(),
+  updated_at: z.int().nullish(),
+})
+
+/**
+ * SummaryStatusResponse
+ */
+export const zSummaryStatusResponse = z.object({
+  completed: z.int().optional().default(0),
+  error: z.int().optional().default(0),
+  generating: z.int().optional().default(0),
+  not_started: z.int().optional().default(0),
+  timeout: z.int().optional().default(0),
+})
+
+/**
+ * DocumentSummaryStatusResponse
+ */
+export const zDocumentSummaryStatusResponse = z.object({
+  summaries: z.array(zSummaryEntryResponse),
+  summary_status: zSummaryStatusResponse,
+  total_segments: z.int(),
+})
+
+/**
+ * ExternalHitTestingQueryResponse
+ */
+export const zExternalHitTestingQueryResponse = z.object({
+  content: z.string(),
+})
+
+/**
+ * ExternalHitTestingRecordResponse
+ */
+export const zExternalHitTestingRecordResponse = z.object({
+  content: z.string().nullish(),
+  metadata: z.record(z.string(), z.unknown()).nullish(),
+  score: z.number().nullish(),
+  title: z.string().nullish(),
+})
+
+/**
+ * ExternalHitTestingResponse
+ */
+export const zExternalHitTestingResponse = z.object({
+  query: zExternalHitTestingQueryResponse,
+  records: z.array(zExternalHitTestingRecordResponse),
+})
+
 /**
  * HitTestingQuery
  */
@@ -755,18 +795,6 @@ export const zDatasetRerankingModelResponse = z.object({
   reranking_provider_name: z.string().nullish(),
 })
 
-export const zDatasetRerankingModel = z.object({
-  reranking_model_name: z.string().optional(),
-  reranking_provider_name: z.string().optional(),
-})
-
-/**
- * ProcessRuleMode
- *
- * Dataset Process Rule Mode
- */
-export const zProcessRuleMode = z.enum(['automatic', 'custom', 'hierarchical'])
-
 /**
  * RerankingModel
  */
@@ -785,6 +813,50 @@ export const zRetrievalMethod = z.enum([
   'semantic_search',
 ])
 
+/**
+ * PreProcessingRule
+ */
+export const zPreProcessingRule = z.object({
+  enabled: z.boolean(),
+  id: z.enum(['remove_extra_spaces', 'remove_stopwords', 'remove_urls_emails']),
+})
+
+/**
+ * Segmentation
+ */
+export const zSegmentation = z.object({
+  chunk_overlap: z.int().optional().default(0),
+  max_tokens: z.int(),
+  separator: z.string().optional().default('\n'),
+})
+
+/**
+ * Rule
+ */
+export const zRule = z.object({
+  parent_mode: z.enum(['full-doc', 'paragraph']).nullish(),
+  pre_processing_rules: z.array(zPreProcessingRule).nullish(),
+  segmentation: zSegmentation.nullish(),
+  subchunk_segmentation: zSegmentation.nullish(),
+})
+
+/**
+ * ProcessRuleResponse
+ */
+export const zProcessRuleResponse = z.object({
+  limits: z.record(z.string(), z.unknown()),
+  mode: zProcessRuleMode,
+  rules: zRule.nullish(),
+})
+
+/**
+ * ProcessRule
+ */
+export const zProcessRule = z.object({
+  mode: zProcessRuleMode,
+  rules: zRule.nullish(),
+})
+
 /**
  * MetadataDetail
  */
@@ -1079,88 +1151,6 @@ export const zDatasetListResponse = z.object({
   total: z.int(),
 })
 
-export const zDatasetKeywordSetting = z.object({
-  keyword_weight: z.number().optional(),
-})
-
-export const zDatasetVectorSetting = z.object({
-  embedding_model_name: z.string().optional(),
-  embedding_provider_name: z.string().optional(),
-  vector_weight: z.number().optional(),
-})
-
-export const zDatasetWeightedScore = z.object({
-  keyword_setting: zDatasetKeywordSetting.optional(),
-  vector_setting: zDatasetVectorSetting.optional(),
-  weight_type: z.string().optional(),
-})
-
-export const zDatasetRetrievalModel = z.object({
-  reranking_enable: z.boolean().optional(),
-  reranking_mode: z.string().optional(),
-  reranking_model: zDatasetRerankingModel.optional(),
-  score_threshold: z.number().optional(),
-  score_threshold_enabled: z.boolean().optional(),
-  search_method: z.string().optional(),
-  top_k: z.int().optional(),
-  weights: zDatasetWeightedScore.optional(),
-})
-
-export const zDatasetDetail = z.object({
-  app_count: z.int().optional(),
-  author_name: z.string().optional(),
-  built_in_field_enabled: z.boolean().optional(),
-  chunk_structure: z.string().optional(),
-  created_at: z.coerce
-    .bigint()
-    .min(BigInt('-9223372036854775808'), {
-      error: 'Invalid value: Expected int64 to be >= -9223372036854775808',
-    })
-    .max(BigInt('9223372036854775807'), {
-      error: 'Invalid value: Expected int64 to be <= 9223372036854775807',
-    })
-    .optional(),
-  created_by: z.string().optional(),
-  data_source_type: z.string().optional(),
-  description: z.string().optional(),
-  doc_form: z.string().optional(),
-  doc_metadata: z.array(zDatasetDocMetadata).optional(),
-  document_count: z.int().optional(),
-  embedding_available: z.boolean().optional(),
-  embedding_model: z.string().optional(),
-  embedding_model_provider: z.string().optional(),
-  enable_api: z.boolean().optional(),
-  external_knowledge_info: zExternalKnowledgeInfo.optional(),
-  external_retrieval_model: zExternalRetrievalModel.optional(),
-  icon_info: zDatasetIconInfo.optional(),
-  id: z.string().optional(),
-  indexing_technique: z.string().optional(),
-  is_multimodal: z.boolean().optional(),
-  is_published: z.boolean().optional(),
-  name: z.string().optional(),
-  permission: z.string().optional(),
-  permission_keys: z.array(z.string()).optional(),
-  pipeline_id: z.string().optional(),
-  provider: z.string().optional(),
-  retrieval_model_dict: zDatasetRetrievalModel.optional(),
-  runtime_mode: z.string().optional(),
-  summary_index_setting: zAnonymousInlineModelB1954337D565.optional(),
-  tags: z.array(zTag).optional(),
-  total_available_documents: z.int().optional(),
-  total_documents: z.int().optional(),
-  updated_at: z.coerce
-    .bigint()
-    .min(BigInt('-9223372036854775808'), {
-      error: 'Invalid value: Expected int64 to be >= -9223372036854775808',
-    })
-    .max(BigInt('9223372036854775807'), {
-      error: 'Invalid value: Expected int64 to be <= 9223372036854775807',
-    })
-    .optional(),
-  updated_by: z.string().optional(),
-  word_count: z.int().optional(),
-})
-
 /**
  * FileInfo
  */
@@ -1178,41 +1168,6 @@ export const zWebsiteInfo = z.object({
   urls: z.array(z.string()),
 })
 
-/**
- * PreProcessingRule
- */
-export const zPreProcessingRule = z.object({
-  enabled: z.boolean(),
-  id: z.enum(['remove_extra_spaces', 'remove_stopwords', 'remove_urls_emails']),
-})
-
-/**
- * Segmentation
- */
-export const zSegmentation = z.object({
-  chunk_overlap: z.int().optional().default(0),
-  max_tokens: z.int(),
-  separator: z.string().optional().default('\n'),
-})
-
-/**
- * Rule
- */
-export const zRule = z.object({
-  parent_mode: z.enum(['full-doc', 'paragraph']).nullish(),
-  pre_processing_rules: z.array(zPreProcessingRule).nullish(),
-  segmentation: zSegmentation.nullish(),
-  subchunk_segmentation: zSegmentation.nullish(),
-})
-
-/**
- * ProcessRule
- */
-export const zProcessRule = z.object({
-  mode: zProcessRuleMode,
-  rules: zRule.nullish(),
-})
-
 /**
  * Condition
  *
@@ -1558,7 +1513,7 @@ export const zPostDatasetsExternalBody = zExternalDatasetCreatePayload
 /**
  * External dataset created successfully
  */
-export const zPostDatasetsExternalResponse = zDatasetDetail
+export const zPostDatasetsExternalResponse = zDatasetDetailResponse
 
 export const zGetDatasetsExternalKnowledgeApiQuery = z.object({
   keyword: z.string().optional(),
@@ -1653,7 +1608,7 @@ export const zGetDatasetsProcessRuleQuery = z.object({
 /**
  * Process rules retrieved successfully
  */
-export const zGetDatasetsProcessRuleResponse = zOpaqueObjectResponse
+export const zGetDatasetsProcessRuleResponse = zProcessRuleResponse
 
 /**
  * Retrieval settings retrieved successfully
@@ -1723,9 +1678,9 @@ export const zGetDatasetsByDatasetIdBatchByBatchIndexingEstimatePath = z.object(
 })
 
 /**
- * Batch indexing estimate calculated successfully
+ * Indexing estimate calculated successfully
  */
-export const zGetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponse = zOpaqueObjectResponse
+export const zGetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponse = zIndexingEstimateResponse
 
 export const zGetDatasetsByDatasetIdBatchByBatchIndexingStatusPath = z.object({
   batch: z.string(),
@@ -1782,9 +1737,12 @@ export const zPostDatasetsByDatasetIdDocumentsDownloadZipPath = z.object({
 })
 
 /**
- * ZIP archive generated successfully
+ * ZIP archive downloaded successfully
  */
-export const zPostDatasetsByDatasetIdDocumentsDownloadZipResponse = zBinaryFileResponse
+export const zPostDatasetsByDatasetIdDocumentsDownloadZipResponse = z.record(
+  z.string(),
+  z.unknown(),
+)
 
 export const zPostDatasetsByDatasetIdDocumentsGenerateSummaryBody = zGenerateSummaryPayload
 
@@ -1840,7 +1798,7 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdQuery = z.object({
 /**
  * Document retrieved successfully
  */
-export const zGetDatasetsByDatasetIdDocumentsByDocumentIdResponse = zOpaqueObjectResponse
+export const zGetDatasetsByDatasetIdDocumentsByDocumentIdResponse = zDocumentDetailResponse
 
 export const zGetDatasetsByDatasetIdDocumentsByDocumentIdDownloadPath = z.object({
   dataset_id: z.uuid(),
@@ -1861,7 +1819,7 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimatePath =
  * Indexing estimate calculated successfully
  */
 export const zGetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateResponse
-  = zOpaqueObjectResponse
+  = zIndexingEstimateResponse
 
 export const zGetDatasetsByDatasetIdDocumentsByDocumentIdIndexingStatusPath = z.object({
   dataset_id: z.uuid(),
@@ -1904,10 +1862,10 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogPat
 })
 
 /**
- * Document pipeline execution log retrieved successfully
+ * Pipeline execution log retrieved successfully
  */
 export const zGetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogResponse
-  = zOpaqueObjectResponse
+  = zDocumentPipelineExecutionLogResponse
 
 export const zPatchDatasetsByDatasetIdDocumentsByDocumentIdProcessingPausePath = z.object({
   dataset_id: z.uuid(),
@@ -2158,7 +2116,7 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusPath = z.o
  * Summary status retrieved successfully
  */
 export const zGetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusResponse
-  = zOpaqueObjectResponse
+  = zDocumentSummaryStatusResponse
 
 export const zGetDatasetsByDatasetIdDocumentsByDocumentIdWebsiteSyncPath = z.object({
   dataset_id: z.uuid(),
@@ -2188,7 +2146,7 @@ export const zPostDatasetsByDatasetIdExternalHitTestingPath = z.object({
 /**
  * External hit testing completed successfully
  */
-export const zPostDatasetsByDatasetIdExternalHitTestingResponse = zExternalRetrievalTestResponse
+export const zPostDatasetsByDatasetIdExternalHitTestingResponse = zExternalHitTestingResponse
 
 export const zPostDatasetsByDatasetIdHitTestingBody = zHitTestingPayload
 

packages/contracts/generated/api/console/test/types.gen.ts

@@ -10,19 +10,24 @@ export type BedrockRetrievalPayload = {
   retrieval_setting: BedrockRetrievalSetting
 }
 
-export type ExternalRetrievalTestResponse
+export type BedrockRetrievalResponse = {
-  = | {
+  records: Array<BedrockRetrievalRecordResponse>
-    [key: string]: unknown
+}
-  }
-  | Array<{
-    [key: string]: unknown
-  }>
 
 export type BedrockRetrievalSetting = {
   score_threshold?: number
   top_k?: number | null
 }
 
+export type BedrockRetrievalRecordResponse = {
+  content?: string | null
+  metadata?: {
+    [key: string]: unknown
+  } | null
+  score: number
+  title?: string | null
+}
+
 export type PostTestRetrievalData = {
   body: BedrockRetrievalPayload
   path?: never
@@ -31,7 +36,7 @@ export type PostTestRetrievalData = {
 }
 
 export type PostTestRetrievalResponses = {
-  200: ExternalRetrievalTestResponse
+  200: BedrockRetrievalResponse
 }
 
 export type PostTestRetrievalResponse = PostTestRetrievalResponses[keyof PostTestRetrievalResponses]

packages/contracts/generated/api/console/test/zod.gen.ts

@@ -2,14 +2,6 @@
 
 import * as z from 'zod'
 
-/**
- * ExternalRetrievalTestResponse
- */
-export const zExternalRetrievalTestResponse = z.union([
-  z.record(z.string(), z.unknown()),
-  z.array(z.record(z.string(), z.unknown())),
-])
-
 /**
  * BedrockRetrievalSetting
  *
@@ -29,9 +21,26 @@ export const zBedrockRetrievalPayload = z.object({
   retrieval_setting: zBedrockRetrievalSetting,
 })
 
+/**
+ * BedrockRetrievalRecordResponse
+ */
+export const zBedrockRetrievalRecordResponse = z.object({
+  content: z.string().nullish(),
+  metadata: z.record(z.string(), z.unknown()).nullish(),
+  score: z.number(),
+  title: z.string().nullish(),
+})
+
+/**
+ * BedrockRetrievalResponse
+ */
+export const zBedrockRetrievalResponse = z.object({
+  records: z.array(zBedrockRetrievalRecordResponse),
+})
+
 export const zPostTestRetrievalBody = zBedrockRetrievalPayload
 
 /**
  * Bedrock retrieval test completed
  */
-export const zPostTestRetrievalResponse = zExternalRetrievalTestResponse
+export const zPostTestRetrievalResponse = zBedrockRetrievalResponse

packages/contracts/generated/api/service/orpc.gen.ts

@@ -1474,16 +1474,20 @@ export const get13 = oc
   .output(zGetDatasetsByDatasetIdDocumentsByDocumentIdResponse)
 
 /**
- * Update an existing document by uploading a file
+ * Update Document by File
+ *
+ * Update an existing document by uploading a new file. Re-triggers indexing — use the returned `batch` ID with [Get Document Indexing Status](/api-reference/documents/get-document-indexing-status) to track progress.
  */
 export const patch4 = oc
   .route({
-    description: 'Update an existing document by uploading a file',
+    description:
+      'Update an existing document by uploading a new file. Re-triggers indexing — use the returned `batch` ID with [Get Document Indexing Status](/api-reference/documents/get-document-indexing-status) to track progress.',
     inputStructure: 'detailed',
     method: 'PATCH',
     operationId: 'patchDatasetsByDatasetIdDocumentsByDocumentId',
     path: '/datasets/{dataset_id}/documents/{document_id}',
-    tags: ['service_api'],
+    summary: 'Update Document by File',
+    tags: ['Documents'],
   })
   .input(
     z.object({

packages/contracts/generated/api/service/types.gen.ts

@@ -593,40 +593,45 @@ export type DocumentBatchDownloadZipPayload = {
 }
 
 export type DocumentDetailResponse = {
-  archived?: boolean | null
+  archived?: boolean
-  average_segment_length?: number | null
+  average_segment_length?: number | number
   completed_at?: number | null
-  created_at?: number | null
+  created_at?: number
-  created_by?: string | null
+  created_by?: string
-  created_from?: string | null
+  created_from?: string
   data_source_info?: {
     [key: string]: unknown
-  } | null
+  }
-  data_source_type?: string | null
+  data_source_type?: string
   dataset_process_rule?: {
     [key: string]: unknown
-  } | null
+  }
   dataset_process_rule_id?: string | null
   disabled_at?: number | null
   disabled_by?: string | null
   display_status?: string | null
-  doc_form?: string | null
+  doc_form?: string
   doc_language?: string | null
-  doc_metadata?: Array<DocumentMetadataResponse> | null
+  doc_metadata?:
+    | Array<DocumentMetadataResponse>
+    | {
+      [key: string]: unknown
+    }
+    | null
   doc_type?: string | null
   document_process_rule?: {
     [key: string]: unknown
-  } | null
+  }
-  enabled?: boolean | null
+  enabled?: boolean
   error?: string | null
-  hit_count?: number | null
+  hit_count?: number
   id: string
   indexing_latency?: number | null
-  indexing_status?: string | null
+  indexing_status?: string
-  name?: string | null
+  name?: string
-  need_summary?: boolean | null
+  need_summary?: boolean
-  position?: number | null
+  position?: number
-  segment_count?: number | null
+  segment_count?: number
   summary_index_status?: string | null
   tokens?: number | null
   updated_at?: number | null
@@ -2627,6 +2632,7 @@ export type PatchDatasetsByDatasetIdDocumentsByDocumentIdData = {
 }
 
 export type PatchDatasetsByDatasetIdDocumentsByDocumentIdErrors = {
+  400: unknown
   401: unknown
   403: unknown
   404: unknown

packages/contracts/generated/api/service/zod.gen.ts

@@ -761,34 +761,36 @@ export const zDocumentMetadataResponse = z.object({
  * DocumentDetailResponse
  */
 export const zDocumentDetailResponse = z.object({
-  archived: z.boolean().nullish(),
+  archived: z.boolean().optional(),
-  average_segment_length: z.number().nullish(),
+  average_segment_length: z.union([z.int(), z.number()]).optional(),
   completed_at: z.int().nullish(),
-  created_at: z.int().nullish(),
+  created_at: z.int().optional(),
-  created_by: z.string().nullish(),
+  created_by: z.string().optional(),
-  created_from: z.string().nullish(),
+  created_from: z.string().optional(),
-  data_source_info: z.record(z.string(), z.unknown()).nullish(),
+  data_source_info: z.record(z.string(), z.unknown()).optional(),
-  data_source_type: z.string().nullish(),
+  data_source_type: z.string().optional(),
-  dataset_process_rule: z.record(z.string(), z.unknown()).nullish(),
+  dataset_process_rule: z.record(z.string(), z.unknown()).optional(),
   dataset_process_rule_id: z.string().nullish(),
   disabled_at: z.int().nullish(),
   disabled_by: z.string().nullish(),
   display_status: z.string().nullish(),
-  doc_form: z.string().nullish(),
+  doc_form: z.string().optional(),
   doc_language: z.string().nullish(),
-  doc_metadata: z.array(zDocumentMetadataResponse).nullish(),
+  doc_metadata: z
+    .union([z.array(zDocumentMetadataResponse), z.record(z.string(), z.unknown())])
+    .nullish(),
   doc_type: z.string().nullish(),
-  document_process_rule: z.record(z.string(), z.unknown()).nullish(),
+  document_process_rule: z.record(z.string(), z.unknown()).optional(),
-  enabled: z.boolean().nullish(),
+  enabled: z.boolean().optional(),
   error: z.string().nullish(),
-  hit_count: z.int().nullish(),
+  hit_count: z.int().optional(),
   id: z.string(),
   indexing_latency: z.number().nullish(),
-  indexing_status: z.string().nullish(),
+  indexing_status: z.string().optional(),
-  name: z.string().nullish(),
+  name: z.string().optional(),
-  need_summary: z.boolean().nullish(),
+  need_summary: z.boolean().optional(),
-  position: z.int().nullish(),
+  position: z.int().optional(),
-  segment_count: z.int().nullish(),
+  segment_count: z.int().optional(),
   summary_index_status: z.string().nullish(),
   tokens: z.int().nullish(),
   updated_at: z.int().nullish(),
@@ -2759,7 +2761,7 @@ export const zPatchDatasetsByDatasetIdDocumentsByDocumentIdPath = z.object({
 })
 
 /**
- * Document updated successfully
+ * Document updated successfully.
  */
 export const zPatchDatasetsByDatasetIdDocumentsByDocumentIdResponse = zDocumentAndBatchResponse