From 89d5f74a407da03ec05eb021c94d92051e12d56a Mon Sep 17 00:00:00 2001 From: chariri Date: Thu, 9 Jul 2026 13:22:30 +0900 Subject: [PATCH] refactor(api): migrate dataset endpoints to BaseModel (#37957) Co-authored-by: Asuka Minato Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- api/controllers/console/datasets/datasets.py | 46 +- .../console/datasets/datasets_document.py | 279 +++++--- .../console/datasets/datasets_segments.py | 2 +- api/controllers/console/datasets/external.py | 196 +++--- .../service_api/dataset/dataset.py | 23 +- .../service_api/dataset/document.py | 149 +++-- api/core/rag/entities/processing_entities.py | 11 +- api/fields/dataset_fields.py | 113 +--- api/openapi/markdown/console-openapi.md | 333 +++++----- api/openapi/markdown/service-openapi.md | 81 +-- .../console/datasets/test_datasets.py | 12 +- .../datasets/test_datasets_document.py | 106 ++- .../console/datasets/test_external.py | 424 ++++++++++-- .../service_api/dataset/test_document.py | 614 +++++++++++------- .../api/console/datasets/orpc.gen.ts | 12 +- .../api/console/datasets/types.gen.ts | 305 ++++----- .../generated/api/console/datasets/zod.gen.ts | 428 ++++++------ .../generated/api/console/test/types.gen.ts | 21 +- .../generated/api/console/test/zod.gen.ts | 27 +- .../generated/api/service/orpc.gen.ts | 10 +- .../generated/api/service/types.gen.ts | 42 +- .../generated/api/service/zod.gen.ts | 40 +- 22 files changed, 1871 insertions(+), 1403 deletions(-) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index c8ca1d621c4..15cc4f4cc06 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -30,6 +30,7 @@ from controllers.console.wraps import ( with_current_tenant_id, with_current_user, ) +from core.entities.knowledge_entities import IndexingEstimate from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError from core.indexing_runner import IndexingRunner from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager @@ -268,21 +269,10 @@ class ErrorDocsResponse(DocumentStatusListResponse): total: int -class IndexingEstimatePreviewItemResponse(ResponseModel): - content: str - child_chunks: list[str] | None = None - summary: str | None = None - - -class IndexingEstimateQaPreviewItemResponse(ResponseModel): - question: str - answer: str - - -class IndexingEstimateResponse(ResponseModel): - total_segments: int - preview: list[IndexingEstimatePreviewItemResponse] - qa_preview: list[IndexingEstimateQaPreviewItemResponse] | None = None +class IndexingEstimateResponse(IndexingEstimate): + tokens: int + total_price: float | int + currency: str class RetrievalSettingResponse(ResponseModel): @@ -647,7 +637,7 @@ class DatasetApi(Resource): else: data["embedding_available"] = True - return data, 200 + return dump_response(DatasetDetailWithPartialMembersResponse, data), 200 @console_ns.doc("update_dataset") @console_ns.doc(description="Update dataset details") @@ -717,7 +707,7 @@ class DatasetApi(Resource): partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session()) result_data.update({"partial_member_list": partial_member_list}) - return result_data, 200 + return dump_response(DatasetDetailWithPartialMembersResponse, result_data), 200 @setup_required @login_required @@ -760,7 +750,7 @@ class DatasetUseCheckApi(Resource): dataset_id_str = str(dataset_id) dataset_is_using = DatasetService.dataset_use_check(dataset_id_str, db.session()) - return {"is_using": dataset_is_using}, 200 + return UsageCheckResponse(is_using=dataset_is_using).model_dump(mode="json"), 200 @console_ns.route("/datasets//queries") @@ -901,7 +891,17 @@ class DatasetIndexingEstimateApi(Resource): except Exception as e: raise IndexingEstimateError(str(e)) - return response.model_dump(), 200 + return ( + IndexingEstimateResponse( + tokens=0, + total_price=0, + currency="USD", + total_segments=response.total_segments, + preview=response.preview, + qa_preview=response.qa_preview, + ).model_dump(mode="json", exclude_none=True), + 200, + ) @console_ns.route("/datasets//related-apps") @@ -1018,7 +1018,7 @@ class DatasetApiKeyApi(Resource): keys = db.session.scalars( select(ApiToken).where(ApiToken.type == self.resource_type, ApiToken.tenant_id == current_tenant_id) ).all() - return ApiKeyList.model_validate({"data": keys}, from_attributes=True).model_dump(mode="json") + return dump_response(ApiKeyList, {"data": keys}) @console_ns.response(200, "API key created successfully", console_ns.models[ApiKeyItem.__name__]) @console_ns.response(400, "Maximum keys exceeded") @@ -1052,7 +1052,7 @@ class DatasetApiKeyApi(Resource): api_token.type = self.resource_type db.session.add(api_token) db.session.commit() - return ApiKeyItem.model_validate(api_token, from_attributes=True).model_dump(mode="json"), 200 + return dump_response(ApiKeyItem, api_token), 200 @console_ns.route("/datasets/api-keys/") @@ -1107,7 +1107,7 @@ class DatasetEnableApiApi(Resource): DatasetService.update_dataset_api_status(dataset_id_str, status == "enable", db.session()) - return {"result": "success"}, 200 + return SimpleResultResponse(result="success").model_dump(mode="json"), 200 @console_ns.route("/datasets/api-base-info") @@ -1120,7 +1120,7 @@ class DatasetApiBaseUrlApi(Resource): @account_initialization_required def get(self): base = dify_config.SERVICE_API_URL or request.host_url.rstrip("/") - return {"api_base_url": normalize_api_base_url(base)} + return ApiBaseUrlResponse(api_base_url=normalize_api_base_url(base)).model_dump(mode="json") @console_ns.route("/datasets/retrieval-setting") diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index ee441704b20..df4bb1bb66a 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -10,16 +10,17 @@ from uuid import UUID import sqlalchemy as sa from flask import request, send_file from flask_restx import Resource -from pydantic import BaseModel, Field, RootModel, field_validator +from pydantic import BaseModel, Field, JsonValue, field_validator from sqlalchemy import asc, desc, func, select from werkzeug.exceptions import Forbidden, NotFound import services from controllers.common.controller_schemas import DocumentBatchDownloadZipPayload -from controllers.common.fields import BinaryFileResponse, SimpleResultMessageResponse, SimpleResultResponse, UrlResponse +from controllers.common.fields import SimpleResultMessageResponse, SimpleResultResponse, UrlResponse from controllers.common.schema import register_response_schema_models, register_schema_models from controllers.console import console_ns from controllers.console.wraps import RBACPermission, RBACResourceScope, rbac_permission_required +from core.entities.knowledge_entities import IndexingEstimate from core.errors.error import ( LLMBadRequestError, ModelCurrentlyNotSupportError, @@ -29,6 +30,7 @@ from core.errors.error import ( from core.indexing_runner import IndexingRunner from core.model_manager import ModelManager from core.plugin.impl.exc import PluginDaemonClientSideError +from core.rag.entities import Rule from core.rag.extractor.entity.datasource_type import DatasourceType from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo, WebsiteInfo from core.rag.index_processor.constant.index_type import IndexTechniqueType @@ -49,7 +51,7 @@ from libs.login import login_required from libs.pagination import paginate_query from models import Account, DatasetProcessRule, Document, DocumentSegment, UploadFile from models.dataset import DocumentPipelineExecutionLog -from models.enums import IndexingStatus, SegmentStatus +from models.enums import IndexingStatus, ProcessRuleMode, SegmentStatus from services.dataset_ref_service import DatasetRefService from services.dataset_service import DatasetService, DocumentService from services.entities.knowledge_entities.knowledge_entities import KnowledgeConfig, ProcessRule, RetrievalModel @@ -148,8 +150,91 @@ class DocumentWithSegmentsListResponse(ResponseModel): page: int -class OpaqueObjectResponse(RootModel[dict[str, Any]]): - root: dict[str, Any] +class IndexingEstimateResponse(IndexingEstimate): + tokens: int + total_price: float | int + currency: str + + +class DocumentDetailResponse(ResponseModel): + id: str + position: int | None = None + data_source_type: str | None = None + data_source_info: Any = None + data_source_detail_dict: Any = None + dataset_process_rule_id: str | None = None + dataset_process_rule: Any = None + document_process_rule: Any = None + name: str | None = None + created_from: str | None = None + created_by: str | None = None + created_at: int | None = None + tokens: int | None = None + indexing_status: str | None = None + completed_at: int | None = None + updated_at: int | None = None + indexing_latency: float | None = None + error: str | None = None + enabled: bool | None = None + disabled_at: int | None = None + disabled_by: str | None = None + archived: bool | None = None + doc_type: str | None = None + doc_metadata: list[DocumentMetadataResponse] | None = None + segment_count: int | None = None + average_segment_length: float | None = None + hit_count: int | None = None + display_status: str | None = None + doc_form: str | None = None + doc_language: str | None = None + need_summary: bool | None = None + + @field_validator("data_source_type", "indexing_status", "display_status", "doc_form", mode="before") + @classmethod + def _normalize_enum_fields(cls, value: Any) -> Any: + return normalize_enum(value) + + +class SummaryStatusResponse(ResponseModel): + completed: int = 0 + generating: int = 0 + error: int = 0 + not_started: int = 0 + timeout: int = 0 + + +class SummaryEntryResponse(ResponseModel): + segment_id: str + segment_position: int + status: str + summary_preview: str | None = None + error: str | None = None + created_at: int | None = None + updated_at: int | None = None + + @field_validator("status", mode="before") + @classmethod + def _normalize_status(cls, value: Any) -> Any: + return normalize_enum(value) + + +class DocumentSummaryStatusResponse(ResponseModel): + total_segments: int + summary_status: SummaryStatusResponse + summaries: list[SummaryEntryResponse] + + +class ProcessRuleResponse(ResponseModel): + mode: ProcessRuleMode + rules: Rule | None = None + limits: dict[str, Any] + + +class DocumentPipelineExecutionLogResponse(ResponseModel): + datasource_info: JsonValue | None = None + datasource_type: str | None = None + input_data: JsonValue | None = None + datasource_node_id: str | None = None register_schema_models( @@ -165,7 +250,6 @@ register_schema_models( ) register_response_schema_models( console_ns, - BinaryFileResponse, SimpleResultMessageResponse, SimpleResultResponse, UrlResponse, @@ -175,7 +259,11 @@ register_response_schema_models( DocumentWithSegmentsResponse, DatasetAndDocumentResponse, DocumentWithSegmentsListResponse, - OpaqueObjectResponse, + IndexingEstimateResponse, + DocumentDetailResponse, + DocumentSummaryStatusResponse, + ProcessRuleResponse, + DocumentPipelineExecutionLogResponse, ) @@ -225,7 +313,7 @@ class GetProcessRuleApi(Resource): @console_ns.doc("get_process_rule") @console_ns.doc(description="Get dataset document processing rules") @console_ns.doc(params={"document_id": "Document ID (optional)"}) - @console_ns.response(200, "Process rules retrieved successfully", console_ns.models[OpaqueObjectResponse.__name__]) + @console_ns.response(200, "Process rules retrieved successfully", console_ns.models[ProcessRuleResponse.__name__]) @setup_required @login_required @account_initialization_required @@ -264,7 +352,7 @@ class GetProcessRuleApi(Resource): mode = dataset_process_rule.mode rules = dataset_process_rule.rules_dict - return {"mode": mode, "rules": rules, "limits": limits} + return dump_response(ProcessRuleResponse, {"mode": mode, "rules": rules, "limits": limits}) @console_ns.route("/datasets//documents") @@ -491,7 +579,7 @@ class DatasetInitApi(Resource): @console_ns.doc(description="Initialize dataset with documents") @console_ns.expect(console_ns.models[KnowledgeConfig.__name__]) @console_ns.response( - 201, "Dataset initialized successfully", console_ns.models[DatasetAndDocumentResponse.__name__] + 200, "Dataset initialized successfully", console_ns.models[DatasetAndDocumentResponse.__name__] ) @console_ns.response(400, "Invalid request parameters") @setup_required @@ -557,7 +645,7 @@ class DocumentIndexingEstimateApi(DocumentResource): @console_ns.response( 200, "Indexing estimate calculated successfully", - console_ns.models[OpaqueObjectResponse.__name__], + console_ns.models[IndexingEstimateResponse.__name__], ) @console_ns.response(404, "Document not found") @console_ns.response(400, "Document already finished") @@ -578,8 +666,6 @@ class DocumentIndexingEstimateApi(DocumentResource): data_process_rule = document.dataset_process_rule data_process_rule_dict = data_process_rule.to_dict() if data_process_rule else {} - response = {"tokens": 0, "total_price": 0, "currency": "USD", "total_segments": 0, "preview": []} - if document.data_source_type == "upload_file": data_source_info = document.data_source_info_dict if data_source_info and "upload_file_id" in data_source_info: @@ -610,7 +696,18 @@ class DocumentIndexingEstimateApi(DocumentResource): "English", dataset_id_str, ) - return estimate_response.model_dump(), 200 + return ( + # TODO: why using zero here? the same for the below endpoint + IndexingEstimateResponse( + tokens=0, + total_price=0, + currency="USD", + total_segments=estimate_response.total_segments, + preview=estimate_response.preview, + qa_preview=estimate_response.qa_preview, + ).model_dump(mode="json", exclude_none=True), + 200, + ) except LLMBadRequestError: raise ProviderNotInitializeError( "No Embedding Model available. Please configure a valid provider " @@ -623,15 +720,24 @@ class DocumentIndexingEstimateApi(DocumentResource): except Exception as e: raise IndexingEstimateError(str(e)) - return response, 200 + return ( + IndexingEstimateResponse( + tokens=0, + total_price=0, + currency="USD", + total_segments=0, + preview=[], + ).model_dump(mode="json", exclude_none=True), + 200, + ) @console_ns.route("/datasets//batch//indexing-estimate") class DocumentBatchIndexingEstimateApi(DocumentResource): @console_ns.response( 200, - "Batch indexing estimate calculated successfully", - console_ns.models[OpaqueObjectResponse.__name__], + "Indexing estimate calculated successfully", + console_ns.models[IndexingEstimateResponse.__name__], ) @setup_required @login_required @@ -643,7 +749,16 @@ class DocumentBatchIndexingEstimateApi(DocumentResource): dataset_id_str = str(dataset_id) documents = self.get_batch_documents(dataset_id_str, batch, current_user) if not documents: - return {"tokens": 0, "total_price": 0, "currency": "USD", "total_segments": 0, "preview": []}, 200 + return ( + IndexingEstimateResponse( + tokens=0, + total_price=0, + currency="USD", + total_segments=0, + preview=[], + ).model_dump(mode="json", exclude_none=True), + 200, + ) data_process_rule = documents[0].dataset_process_rule data_process_rule_dict = data_process_rule.to_dict() if data_process_rule else {} extract_settings = [] @@ -717,7 +832,17 @@ class DocumentBatchIndexingEstimateApi(DocumentResource): "English", dataset_id_str, ) - return response.model_dump(), 200 + return ( + IndexingEstimateResponse( + tokens=0, + total_price=0, + currency="USD", + total_segments=response.total_segments, + preview=response.preview, + qa_preview=response.qa_preview, + ).model_dump(mode="json", exclude_none=True), + 200, + ) except LLMBadRequestError: raise ProviderNotInitializeError( "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider." @@ -854,7 +979,7 @@ class DocumentApi(DocumentResource): "metadata": "Metadata inclusion (all/only/without)", } ) - @console_ns.response(200, "Document retrieved successfully", console_ns.models[OpaqueObjectResponse.__name__]) + @console_ns.response(200, "Document retrieved successfully", console_ns.models[DocumentDetailResponse.__name__]) @console_ns.response(404, "Document not found") @setup_required @login_required @@ -871,46 +996,21 @@ class DocumentApi(DocumentResource): if metadata not in self.METADATA_CHOICES: raise InvalidMetadataError(f"Invalid metadata value: {metadata}") + metadata_fields = {"doc_type", "doc_metadata"} if metadata == "only": - response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details} - elif metadata == "without": - dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session()) - document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {} - response = { - "id": document.id, - "position": document.position, - "data_source_type": document.data_source_type, - "data_source_info": document.data_source_info_dict, - "data_source_detail_dict": document.data_source_detail_dict, - "dataset_process_rule_id": document.dataset_process_rule_id, - "dataset_process_rule": dataset_process_rules, - "document_process_rule": document_process_rules, - "name": document.name, - "created_from": document.created_from, - "created_by": document.created_by, - "created_at": int(document.created_at.timestamp()), - "tokens": document.tokens, - "indexing_status": document.indexing_status, - "completed_at": int(document.completed_at.timestamp()) if document.completed_at else None, - "updated_at": int(document.updated_at.timestamp()) if document.updated_at else None, - "indexing_latency": document.indexing_latency, - "error": document.error, - "enabled": document.enabled, - "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None, - "disabled_by": document.disabled_by, - "archived": document.archived, - "segment_count": document.segment_count, - "average_segment_length": document.average_segment_length, - "hit_count": document.hit_count, - "display_status": document.display_status, - "doc_form": document.doc_form, - "doc_language": document.doc_language, - "need_summary": document.need_summary if document.need_summary is not None else False, - } - else: - dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session()) - document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {} - response = { + response = DocumentDetailResponse.model_validate( + { + "id": document.id, + "doc_type": document.doc_type, + "doc_metadata": document.doc_metadata_details, + } + ) + return response.model_dump(mode="json", include={"id", *metadata_fields}, exclude_unset=True), 200 + + dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session()) + document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {} + response = DocumentDetailResponse.model_validate( + { "id": document.id, "position": document.position, "data_source_type": document.data_source_type, @@ -943,8 +1043,9 @@ class DocumentApi(DocumentResource): "doc_language": document.doc_language, "need_summary": document.need_summary if document.need_summary is not None else False, } - - return response, 200 + ) + exclude = metadata_fields if metadata == "without" else None + return response.model_dump(mode="json", exclude=exclude, exclude_unset=True), 200 @setup_required @login_required @@ -990,7 +1091,9 @@ class DocumentDownloadApi(DocumentResource): def get(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID) -> dict[str, Any]: # Reuse the shared permission/tenant checks implemented in DocumentResource. document = self.get_document(str(dataset_id), str(document_id), current_user, current_tenant_id) - return {"url": DocumentService.get_document_download_url(document, db.session())} + return UrlResponse(url=DocumentService.get_document_download_url(document, db.session())).model_dump( + mode="json" + ) @console_ns.route("/datasets//documents/download-zip") @@ -999,7 +1102,7 @@ class DocumentBatchDownloadZipApi(DocumentResource): @console_ns.doc("download_dataset_documents_as_zip") @console_ns.doc(description="Download selected dataset documents as a single ZIP archive (upload-file only)") - @console_ns.response(200, "ZIP archive generated successfully", console_ns.models[BinaryFileResponse.__name__]) + @console_ns.response(200, "ZIP archive downloaded successfully") @setup_required @login_required @account_initialization_required @@ -1034,6 +1137,7 @@ class DocumentBatchDownloadZipApi(DocumentResource): ) cleanup = stack.pop_all() response.call_on_close(cleanup.close) + # response-contract:ignore binary ZIP download response return response @@ -1093,7 +1197,7 @@ class DocumentProcessingApi(DocumentResource): document.is_paused = False db.session.commit() - return {"result": "success"}, 200 + return SimpleResultResponse(result="success").model_dump(mode="json"), 200 @console_ns.route("/datasets//documents//metadata") @@ -1152,7 +1256,9 @@ class DocumentMetadataApi(DocumentResource): document.updated_at = naive_utc_now() db.session.commit() - return {"result": "success", "message": "Document metadata updated."}, 200 + return SimpleResultMessageResponse(result="success", message="Document metadata updated.").model_dump( + mode="json" + ), 200 @console_ns.route("/datasets//documents/status//batch") @@ -1194,7 +1300,7 @@ class DocumentStatusApi(DocumentResource): except NotFound as e: raise NotFound(str(e)) - return {"result": "success"}, 200 + return SimpleResultResponse(result="success").model_dump(mode="json"), 200 @console_ns.route("/datasets//documents//processing/pause") @@ -1321,7 +1427,7 @@ class DocumentRenameApi(DocumentResource): # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator if not current_user.is_dataset_editor: raise Forbidden() - dataset = DatasetService.get_dataset(dataset_id, db.session()) + dataset = DatasetService.get_dataset(str(dataset_id), db.session()) if not dataset: raise NotFound("Dataset not found.") DatasetService.check_dataset_operator_permission(current_user, dataset, session=db.session()) @@ -1363,15 +1469,15 @@ class WebsiteDocumentSyncApi(DocumentResource): # sync document DocumentService.sync_website_document(dataset_id_str, document, db.session()) - return {"result": "success"}, 200 + return SimpleResultResponse(result="success").model_dump(mode="json"), 200 @console_ns.route("/datasets//documents//pipeline-execution-log") class DocumentPipelineExecutionLogApi(DocumentResource): @console_ns.response( 200, - "Document pipeline execution log retrieved successfully", - console_ns.models[OpaqueObjectResponse.__name__], + "Pipeline execution log retrieved successfully", + console_ns.models[DocumentPipelineExecutionLogResponse.__name__], ) @setup_required @login_required @@ -1394,18 +1500,16 @@ class DocumentPipelineExecutionLogApi(DocumentResource): .limit(1) ) if not log: - return { - "datasource_info": None, - "datasource_type": None, - "input_data": None, - "datasource_node_id": None, - }, 200 - return { - "datasource_info": json.loads(log.datasource_info), - "datasource_type": log.datasource_type, - "input_data": log.input_data, - "datasource_node_id": log.datasource_node_id, - }, 200 + return DocumentPipelineExecutionLogResponse().model_dump(mode="json"), 200 + return dump_response( + DocumentPipelineExecutionLogResponse, + { + "datasource_info": json.loads(log.datasource_info), + "datasource_type": log.datasource_type, + "input_data": log.input_data, + "datasource_node_id": log.datasource_node_id, + }, + ), 200 @console_ns.route("/datasets//documents/generate-summary") @@ -1508,7 +1612,7 @@ class DocumentGenerateSummaryApi(Resource): dataset_id_str, ) - return {"result": "success"}, 200 + return SimpleResultResponse(result="success").model_dump(mode="json"), 200 @console_ns.route("/datasets//documents//summary-status") @@ -1516,7 +1620,11 @@ class DocumentSummaryStatusApi(DocumentResource): @console_ns.doc("get_document_summary_status") @console_ns.doc(description="Get summary index generation status for a document") @console_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"}) - @console_ns.response(200, "Summary status retrieved successfully", console_ns.models[OpaqueObjectResponse.__name__]) + @console_ns.response( + 200, + "Summary status retrieved successfully", + console_ns.models[DocumentSummaryStatusResponse.__name__], + ) @console_ns.response(404, "Document not found") @setup_required @login_required @@ -1534,6 +1642,7 @@ class DocumentSummaryStatusApi(DocumentResource): - generating: Number of summaries being generated - error: Number of summaries with errors - not_started: Number of segments without summary records + - timeout: Number of summaries that timed out - summaries: List of summary records with status and content preview """ dataset_id_str = str(dataset_id) @@ -1559,4 +1668,4 @@ class DocumentSummaryStatusApi(DocumentResource): session=db.session(), ) - return result, 200 + return dump_response(DocumentSummaryStatusResponse, result), 200 diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index e4f2abeb844..19e5670f239 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -391,7 +391,7 @@ class DatasetDocumentSegmentApi(Resource): SegmentService.update_segments_status(segment_ids, action, dataset, document, db.session()) except Exception as e: raise InvalidActionError(str(e)) - return dump_response(SimpleResultResponse, {"result": "success"}), 200 + return SimpleResultResponse(result="success").model_dump(mode="json"), 200 @console_ns.route("/datasets//documents//segment") diff --git a/api/controllers/console/datasets/external.py b/api/controllers/console/datasets/external.py index 9cdca96f69a..cff0589f9eb 100644 --- a/api/controllers/console/datasets/external.py +++ b/api/controllers/console/datasets/external.py @@ -1,20 +1,16 @@ +from datetime import datetime from typing import Any from uuid import UUID from flask import request -from flask_restx import Resource, fields, marshal -from pydantic import BaseModel, Field, RootModel +from flask_restx import Resource +from pydantic import AliasChoices, BaseModel, Field, field_validator from sqlalchemy.orm import Session from werkzeug.exceptions import Forbidden, InternalServerError, NotFound import services from controllers.common.fields import UsageCountResponse -from controllers.common.schema import ( - get_or_create_model, - query_params_from_model, - register_response_schema_models, - register_schema_models, -) +from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models from controllers.console import console_ns from controllers.console.app.wraps import with_session from controllers.console.datasets.error import DatasetNameDuplicateError @@ -28,21 +24,9 @@ from controllers.console.wraps import ( with_current_tenant_id, with_current_user, ) -from extensions.ext_database import db from fields.base import ResponseModel -from fields.dataset_fields import ( - dataset_detail_fields, - dataset_retrieval_model_fields, - doc_metadata_fields, - external_knowledge_info_fields, - external_retrieval_model_fields, - icon_info_fields, - keyword_setting_fields, - reranking_model_fields, - tag_fields, - vector_setting_fields, - weighted_score_fields, -) +from fields.dataset_fields import DatasetDetailResponse +from libs.helper import dump_response from libs.login import login_required from models import Account from services.dataset_service import DatasetService @@ -51,50 +35,10 @@ from services.external_knowledge_service import ExternalDatasetService from services.hit_testing_service import HitTestingService from services.knowledge_service import BedrockRetrievalSetting, ExternalDatasetTestService -register_response_schema_models(console_ns, UsageCountResponse) - - -def _build_dataset_detail_model(): - keyword_setting_model = get_or_create_model("DatasetKeywordSetting", keyword_setting_fields) - vector_setting_model = get_or_create_model("DatasetVectorSetting", vector_setting_fields) - - weighted_score_fields_copy = weighted_score_fields.copy() - weighted_score_fields_copy["keyword_setting"] = fields.Nested(keyword_setting_model) - weighted_score_fields_copy["vector_setting"] = fields.Nested(vector_setting_model) - weighted_score_model = get_or_create_model("DatasetWeightedScore", weighted_score_fields_copy) - - reranking_model = get_or_create_model("DatasetRerankingModel", reranking_model_fields) - - dataset_retrieval_model_fields_copy = dataset_retrieval_model_fields.copy() - dataset_retrieval_model_fields_copy["reranking_model"] = fields.Nested(reranking_model) - dataset_retrieval_model_fields_copy["weights"] = fields.Nested(weighted_score_model, allow_null=True) - dataset_retrieval_model = get_or_create_model("DatasetRetrievalModel", dataset_retrieval_model_fields_copy) - - tag_model = get_or_create_model("Tag", tag_fields) - doc_metadata_model = get_or_create_model("DatasetDocMetadata", doc_metadata_fields) - external_knowledge_info_model = get_or_create_model("ExternalKnowledgeInfo", external_knowledge_info_fields) - external_retrieval_model = get_or_create_model("ExternalRetrievalModel", external_retrieval_model_fields) - icon_info_model = get_or_create_model("DatasetIconInfo", icon_info_fields) - - dataset_detail_fields_copy = dataset_detail_fields.copy() - dataset_detail_fields_copy["retrieval_model_dict"] = fields.Nested(dataset_retrieval_model) - dataset_detail_fields_copy["tags"] = fields.List(fields.Nested(tag_model)) - dataset_detail_fields_copy["external_knowledge_info"] = fields.Nested(external_knowledge_info_model) - dataset_detail_fields_copy["external_retrieval_model"] = fields.Nested(external_retrieval_model, allow_null=True) - dataset_detail_fields_copy["doc_metadata"] = fields.List(fields.Nested(doc_metadata_model)) - dataset_detail_fields_copy["icon_info"] = fields.Nested(icon_info_model) - return get_or_create_model("DatasetDetail", dataset_detail_fields_copy) - - -try: - dataset_detail_model = console_ns.models["DatasetDetail"] -except KeyError: - dataset_detail_model = _build_dataset_detail_model() - class ExternalKnowledgeApiPayload(BaseModel): name: str = Field(..., min_length=1, max_length=40) - settings: dict[str, object] + settings: dict[str, Any] class ExternalDatasetCreatePayload(BaseModel): @@ -102,15 +46,13 @@ class ExternalDatasetCreatePayload(BaseModel): external_knowledge_id: str name: str = Field(..., min_length=1, max_length=100) description: str | None = Field(None, max_length=400) - external_retrieval_model: dict[str, object] | None = Field(default=None) + external_retrieval_model: dict[str, Any] | None = None class ExternalHitTestingPayload(BaseModel): query: str - external_retrieval_model: dict[str, object] | None = Field(default=None) - metadata_filtering_conditions: dict[str, object] | None = Field( - default=None, - ) + external_retrieval_model: dict[str, Any] | None = None + metadata_filtering_conditions: dict[str, Any] | None = None class BedrockRetrievalPayload(BaseModel): @@ -125,7 +67,7 @@ class ExternalApiTemplateListQuery(BaseModel): keyword: str | None = Field(default=None, description="Search keyword") -class ExternalKnowledgeDatasetBindingResponse(ResponseModel): +class ExternalKnowledgeApiBindingResponse(ResponseModel): id: str name: str @@ -135,22 +77,52 @@ class ExternalKnowledgeApiResponse(ResponseModel): tenant_id: str name: str description: str - settings: dict[str, Any] | None = Field(default=None) - dataset_bindings: list[ExternalKnowledgeDatasetBindingResponse] = Field(default_factory=list) + settings: dict[str, Any] | None = Field(validation_alias=AliasChoices("settings_dict", "settings")) + dataset_bindings: list[ExternalKnowledgeApiBindingResponse] created_by: str created_at: str + @field_validator("created_at", mode="before") + @classmethod + def _normalize_created_at(cls, value: datetime | str) -> str: + if isinstance(value, datetime): + return value.isoformat() + return value + class ExternalKnowledgeApiListResponse(ResponseModel): data: list[ExternalKnowledgeApiResponse] has_more: bool limit: int - total: int + total: int | None page: int -class ExternalRetrievalTestResponse(RootModel[dict[str, Any] | list[dict[str, Any]]]): - root: dict[str, Any] | list[dict[str, Any]] +class ExternalHitTestingQueryResponse(ResponseModel): + content: str + + +class ExternalHitTestingRecordResponse(ResponseModel): + content: str | None = None + title: str | None = None + score: float | None = None + metadata: dict[str, Any] | None = None + + +class ExternalHitTestingResponse(ResponseModel): + query: ExternalHitTestingQueryResponse + records: list[ExternalHitTestingRecordResponse] + + +class BedrockRetrievalRecordResponse(ResponseModel): + metadata: dict[str, Any] | None = None + score: float + title: str | None = None + content: str | None = None + + +class BedrockRetrievalResponse(ResponseModel): + records: list[BedrockRetrievalRecordResponse] register_schema_models( @@ -163,9 +135,16 @@ register_schema_models( ) register_response_schema_models( console_ns, + UsageCountResponse, + DatasetDetailResponse, + ExternalKnowledgeApiBindingResponse, ExternalKnowledgeApiResponse, ExternalKnowledgeApiListResponse, - ExternalRetrievalTestResponse, + ExternalHitTestingQueryResponse, + ExternalHitTestingRecordResponse, + ExternalHitTestingResponse, + BedrockRetrievalRecordResponse, + BedrockRetrievalResponse, ) @@ -189,24 +168,26 @@ class ExternalApiTemplateListApi(Resource): external_knowledge_apis, total = ExternalDatasetService.get_external_knowledge_apis( query.page, query.limit, current_tenant_id, query.keyword ) - response = { - "data": [item.to_dict() for item in external_knowledge_apis], - "has_more": len(external_knowledge_apis) == query.limit, - "limit": query.limit, - "total": total, - "page": query.page, - } - return response, 200 + return ExternalKnowledgeApiListResponse( + data=[ExternalKnowledgeApiResponse.model_validate(item) for item in external_knowledge_apis], + has_more=len(external_knowledge_apis) == query.limit, + limit=query.limit, + total=total, + page=query.page, + ).model_dump(mode="json"), 200 - @setup_required - @login_required - @account_initialization_required + @console_ns.doc("create_external_api_template") + @console_ns.doc(description="Create external knowledge API template") @console_ns.expect(console_ns.models[ExternalKnowledgeApiPayload.__name__]) @console_ns.response( 201, "External API template created successfully", console_ns.models[ExternalKnowledgeApiResponse.__name__], ) + @console_ns.response(403, "Permission denied") + @setup_required + @login_required + @account_initialization_required @with_current_user @with_current_tenant_id @with_session @@ -229,7 +210,7 @@ class ExternalApiTemplateListApi(Resource): except services.errors.dataset.DatasetNameDuplicateError: raise DatasetNameDuplicateError() - return external_knowledge_api.to_dict(), 201 + return dump_response(ExternalKnowledgeApiResponse, external_knowledge_api), 201 @console_ns.route("/datasets/external-knowledge-api/") @@ -256,17 +237,21 @@ class ExternalApiTemplateApi(Resource): if external_knowledge_api is None: raise NotFound("API template not found.") - return external_knowledge_api.to_dict(), 200 + return dump_response(ExternalKnowledgeApiResponse, external_knowledge_api), 200 + @console_ns.doc("update_external_api_template") + @console_ns.doc(description="Update external knowledge API template") + @console_ns.doc(params={"external_knowledge_api_id": "External knowledge API ID"}) + @console_ns.expect(console_ns.models[ExternalKnowledgeApiPayload.__name__]) @console_ns.response( 200, "External API template updated successfully", console_ns.models[ExternalKnowledgeApiResponse.__name__], ) + @console_ns.response(404, "Template not found") @setup_required @login_required @account_initialization_required - @console_ns.expect(console_ns.models[ExternalKnowledgeApiPayload.__name__]) @with_current_user @with_current_tenant_id @with_session @@ -284,7 +269,7 @@ class ExternalApiTemplateApi(Resource): session=session, ) - return external_knowledge_api.to_dict(), 200 + return dump_response(ExternalKnowledgeApiResponse, external_knowledge_api), 200 @setup_required @login_required @@ -322,7 +307,7 @@ class ExternalApiUseCheckApi(Resource): external_knowledge_api_is_using, count = ExternalDatasetService.external_knowledge_api_use_check( external_knowledge_api_id_str, current_tenant_id, session=session ) - return {"is_using": external_knowledge_api_is_using, "count": count}, 200 + return UsageCountResponse(is_using=external_knowledge_api_is_using, count=count).model_dump(mode="json"), 200 @console_ns.route("/datasets/external") @@ -330,7 +315,9 @@ class ExternalDatasetCreateApi(Resource): @console_ns.doc("create_external_dataset") @console_ns.doc(description="Create external knowledge dataset") @console_ns.expect(console_ns.models[ExternalDatasetCreatePayload.__name__]) - @console_ns.response(201, "External dataset created successfully", dataset_detail_model) + @console_ns.response( + 201, "External dataset created successfully", console_ns.models[DatasetDetailResponse.__name__] + ) @console_ns.response(400, "Invalid parameters") @console_ns.response(403, "Permission denied") @setup_required @@ -360,17 +347,16 @@ class ExternalDatasetCreateApi(Resource): except services.errors.dataset.DatasetNameDuplicateError: raise DatasetNameDuplicateError() - item = marshal(dataset, dataset_detail_fields) - dataset_id_str = item["id"] + dataset_id_str = str(dataset.id) permission_keys_map = enterprise_rbac_service.RBACService.DatasetPermissions.batch_get( str(current_tenant_id), current_user.id, [dataset_id_str], session=session, ) - item["permission_keys"] = permission_keys_map.get(dataset_id_str, []) - - return item, 201 + data = DatasetDetailResponse.model_validate(dataset).model_dump(mode="json") + data["permission_keys"] = permission_keys_map.get(dataset_id_str, []) + return data, 201 @console_ns.route("/datasets//external-hit-testing") @@ -382,7 +368,7 @@ class ExternalKnowledgeHitTestingApi(Resource): @console_ns.response( 200, "External hit testing completed successfully", - console_ns.models[ExternalRetrievalTestResponse.__name__], + console_ns.models[ExternalHitTestingResponse.__name__], ) @console_ns.response(404, "Dataset not found") @console_ns.response(400, "Invalid parameters") @@ -394,12 +380,12 @@ class ExternalKnowledgeHitTestingApi(Resource): @with_session def post(self, session: Session, current_user: Account, dataset_id: UUID): dataset_id_str = str(dataset_id) - dataset = DatasetService.get_dataset(dataset_id_str, db.session()) + dataset = DatasetService.get_dataset(dataset_id_str, session) if dataset is None: raise NotFound("Dataset not found.") try: - DatasetService.check_dataset_permission(dataset, current_user, db.session()) + DatasetService.check_dataset_permission(dataset, current_user, session) except services.errors.account.NoPermissionError as e: raise Forbidden(str(e)) @@ -416,7 +402,7 @@ class ExternalKnowledgeHitTestingApi(Resource): metadata_filtering_conditions=payload.metadata_filtering_conditions, ) - return response + return dump_response(ExternalHitTestingResponse, response) except Exception as e: raise InternalServerError(str(e)) @@ -427,11 +413,7 @@ class BedrockRetrievalApi(Resource): @console_ns.doc("bedrock_retrieval_test") @console_ns.doc(description="Bedrock retrieval test (internal use only)") @console_ns.expect(console_ns.models[BedrockRetrievalPayload.__name__]) - @console_ns.response( - 200, - "Bedrock retrieval test completed", - console_ns.models[ExternalRetrievalTestResponse.__name__], - ) + @console_ns.response(200, "Bedrock retrieval test completed", console_ns.models[BedrockRetrievalResponse.__name__]) def post(self): payload = BedrockRetrievalPayload.model_validate(console_ns.payload or {}) @@ -439,4 +421,4 @@ class BedrockRetrievalApi(Resource): result = ExternalDatasetTestService.knowledge_retrieval( payload.retrieval_setting, payload.query, payload.knowledge_id ) - return result, 200 + return dump_response(BedrockRetrievalResponse, result), 200 diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py index 66085ca0642..b9f6a6424af 100644 --- a/api/controllers/service_api/dataset/dataset.py +++ b/api/controllers/service_api/dataset/dataset.py @@ -845,7 +845,7 @@ class DocumentStatusApi(DatasetApiResource): except ValueError as e: raise InvalidActionError(str(e)) - return dump_response(SimpleResultResponse, {"result": "success"}), 200 + return SimpleResultResponse(result="success").model_dump(mode="json"), 200 @service_api_ns.route("/datasets/tags") @@ -911,11 +911,8 @@ class DatasetTagsApi(DatasetApiResource): payload = TagCreatePayload.model_validate(service_api_ns.payload or {}) tag = TagService.save_tags(SaveTagPayload(name=payload.name, type=TagType.KNOWLEDGE), db.session()) - response = dump_response( - KnowledgeTagResponse, - {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": 0}, - ) - return response, 200 + response = KnowledgeTagResponse(id=tag.id, name=tag.name, type=tag.type, binding_count="0") + return response.model_dump(mode="json"), 200 @service_api_ns.doc( summary="Update Knowledge Tag", @@ -953,11 +950,8 @@ class DatasetTagsApi(DatasetApiResource): binding_count = TagService.get_tag_binding_count(tag_id, db.session(), tag_type=TagType.KNOWLEDGE) - response = dump_response( - KnowledgeTagResponse, - {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": binding_count}, - ) - return response, 200 + response = KnowledgeTagResponse(id=tag.id, name=tag.name, type=tag.type, binding_count=str(binding_count)) + return response.model_dump(mode="json"), 200 @service_api_ns.doc( summary="Delete Knowledge Tag", @@ -1088,5 +1082,8 @@ class DatasetTagsBindingStatusApi(DatasetApiResource): tags = TagService.get_tags_by_target_id( "knowledge", current_user.current_tenant_id, str(dataset_id), db.session() ) - tags_list = [{"id": tag.id, "name": tag.name} for tag in tags] - return dump_response(DatasetBoundTagListResponse, {"data": tags_list, "total": len(tags)}), 200 + response = DatasetBoundTagListResponse( + data=[DatasetBoundTagResponse(id=tag.id, name=tag.name) for tag in tags], + total=len(tags), + ) + return response.model_dump(mode="json"), 200 diff --git a/api/controllers/service_api/dataset/document.py b/api/controllers/service_api/dataset/document.py index 5e5919a7048..47e77fd5a34 100644 --- a/api/controllers/service_api/dataset/document.py +++ b/api/controllers/service_api/dataset/document.py @@ -6,14 +6,22 @@ deprecated in generated API docs so clients migrate toward the canonical paths. """ import json -from collections.abc import Mapping from contextlib import ExitStack from copy import deepcopy from typing import Annotated, Any, Literal, Self, override from uuid import UUID from flask import request, send_file -from pydantic import BaseModel, Field, GetJsonSchemaHandler, WithJsonSchema, field_validator, model_validator +from pydantic import ( + BaseModel, + Field, + GetJsonSchemaHandler, + ValidationError, + WithJsonSchema, + field_validator, + model_validator, +) +from pydantic.json_schema import SkipJsonSchema from sqlalchemy import desc, func, select from werkzeug.exceptions import Forbidden, NotFound @@ -26,9 +34,10 @@ from controllers.common.errors import ( TooManyFilesError, UnsupportedFileTypeError, ) -from controllers.common.fields import BinaryFileResponse, UrlResponse +from controllers.common.fields import UrlResponse from controllers.common.schema import ( query_params_from_model, + query_params_from_request, register_enum_models, register_response_schema_models, register_schema_models, @@ -56,6 +65,7 @@ from fields.document_fields import ( DocumentMetadataResponse, DocumentResponse, DocumentStatusListResponse, + normalize_enum, ) from libs.helper import dump_response from libs.login import current_user @@ -281,38 +291,44 @@ class DocumentAndBatchResponse(ResponseModel): batch: str +# Use SkipJsonSchema to support 3 metadata modes class DocumentDetailResponse(ResponseModel): id: str - position: int | None = None - data_source_type: str | None = None - data_source_info: dict[str, Any] | None = Field(default=None) + position: int | SkipJsonSchema[None] = None + data_source_type: str | SkipJsonSchema[None] = None + data_source_info: dict[str, Any] | SkipJsonSchema[None] = None dataset_process_rule_id: str | None = None - dataset_process_rule: dict[str, Any] | None = Field(default=None) - document_process_rule: dict[str, Any] | None = Field(default=None) - name: str | None = None - created_from: str | None = None - created_by: str | None = None - created_at: int | None = None + dataset_process_rule: dict[str, Any] | SkipJsonSchema[None] = None + document_process_rule: dict[str, Any] | SkipJsonSchema[None] = None + name: str | SkipJsonSchema[None] = None + created_from: str | SkipJsonSchema[None] = None + created_by: str | SkipJsonSchema[None] = None + created_at: int | SkipJsonSchema[None] = None tokens: int | None = None - indexing_status: str | None = None + indexing_status: str | SkipJsonSchema[None] = None completed_at: int | None = None updated_at: int | None = None indexing_latency: float | None = None error: str | None = None - enabled: bool | None = None + enabled: bool | SkipJsonSchema[None] = None disabled_at: int | None = None disabled_by: str | None = None - archived: bool | None = None + archived: bool | SkipJsonSchema[None] = None doc_type: str | None = None - doc_metadata: list[DocumentMetadataResponse] | None = None - segment_count: int | None = None - average_segment_length: float | None = None - hit_count: int | None = None + doc_metadata: list[DocumentMetadataResponse] | dict[str, Any] | None = None + segment_count: int | SkipJsonSchema[None] = None + average_segment_length: int | float | SkipJsonSchema[None] = None + hit_count: int | SkipJsonSchema[None] = None display_status: str | None = None - doc_form: str | None = None + doc_form: str | SkipJsonSchema[None] = None doc_language: str | None = None summary_index_status: str | None = None - need_summary: bool | None = None + need_summary: bool | SkipJsonSchema[None] = None + + @field_validator("data_source_type", "indexing_status", "display_status", "doc_form", mode="before") + @classmethod + def _normalize_enum_fields(cls, value: Any) -> Any: + return normalize_enum(value) register_enum_models(service_api_ns, RetrievalMethod) @@ -332,7 +348,6 @@ register_schema_models( ) register_response_schema_models( service_api_ns, - BinaryFileResponse, UrlResponse, DocumentResponse, DocumentAndBatchResponse, @@ -342,13 +357,13 @@ register_response_schema_models( ) -def _create_document_by_text(tenant_id: str, dataset_id: UUID) -> tuple[Mapping[str, object], int]: +def _create_document_by_text(tenant_id: str, dataset_id: UUID) -> tuple[Document, str]: """Create a document from text for both canonical and legacy routes.""" payload = DocumentTextCreatePayload.model_validate(service_api_ns.payload or {}) args = payload.model_dump(exclude_none=True) dataset_id_str = str(dataset_id) - tenant_id_str = str(tenant_id) + tenant_id_str = tenant_id dataset = db.session.scalar( select(Dataset).where(Dataset.tenant_id == tenant_id_str, Dataset.id == dataset_id_str).limit(1) ) @@ -407,10 +422,10 @@ def _create_document_by_text(tenant_id: str, dataset_id: UUID) -> tuple[Mapping[ raise ProviderNotInitializeError(ex.description) document = documents[0] - return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200 + return document, batch -def _update_document_by_text(tenant_id: str, dataset_id: UUID, document_id: UUID) -> tuple[Mapping[str, object], int]: +def _update_document_by_text(tenant_id: str, dataset_id: UUID, document_id: UUID) -> tuple[Document, str]: """Update a document from text for both canonical and legacy routes.""" payload = DocumentTextUpdate.model_validate(service_api_ns.payload or {}) dataset = db.session.scalar( @@ -467,7 +482,7 @@ def _update_document_by_text(tenant_id: str, dataset_id: UUID, document_id: UUID raise ProviderNotInitializeError(ex.description) document = documents[0] - return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200 + return document, batch @service_api_ns.route("/datasets//document/create-by-text") @@ -511,7 +526,8 @@ class DocumentAddByTextApi(DatasetApiResource): @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id: str, dataset_id: UUID): """Create document by text.""" - return _create_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id) + document, batch = _create_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id) + return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200 @service_api_ns.route("/datasets//document/create_by_text") @@ -543,7 +559,8 @@ class DeprecatedDocumentAddByTextApi(DatasetApiResource): @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id: str, dataset_id: UUID): """Create document by text through the deprecated underscore alias.""" - return _create_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id) + document, batch = _create_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id) + return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200 @service_api_ns.route("/datasets//documents//update-by-text") @@ -587,7 +604,8 @@ class DocumentUpdateByTextApi(DatasetApiResource): @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id: str, dataset_id: UUID, document_id: UUID): """Update document by text.""" - return _update_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id) + document, batch = _update_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id) + return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200 @service_api_ns.route("/datasets//documents//update_by_text") @@ -618,7 +636,8 @@ class DeprecatedDocumentUpdateByTextApi(DatasetApiResource): @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id: str, dataset_id: UUID, document_id: UUID): """Update document by text through the deprecated underscore alias.""" - return _update_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id) + document, batch = _update_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id) + return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200 @service_api_ns.route( @@ -767,10 +786,10 @@ class DocumentAddByFileApi(DatasetApiResource): return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200 -def _update_document_by_file(tenant_id: str, dataset_id: UUID, document_id: UUID) -> tuple[Mapping[str, object], int]: +def _update_document_by_file(tenant_id: str, dataset_id: UUID, document_id: UUID) -> tuple[Document, str]: """Update a document from an uploaded file for canonical and deprecated routes.""" dataset_id_str = str(dataset_id) - tenant_id_str = str(tenant_id) + tenant_id_str = tenant_id dataset = db.session.scalar( select(Dataset).where(Dataset.tenant_id == tenant_id_str, Dataset.id == dataset_id_str).limit(1) ) @@ -841,7 +860,7 @@ def _update_document_by_file(tenant_id: str, dataset_id: UUID, document_id: UUID except ProviderTokenNotInitError as ex: raise ProviderNotInitializeError(ex.description) document = documents[0] - return dump_response(DocumentAndBatchResponse, {"document": document, "batch": document.batch}), 200 + return document, document.batch @service_api_ns.route( @@ -895,7 +914,8 @@ class DeprecatedDocumentUpdateByFileApi(DatasetApiResource): @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id: str, dataset_id: UUID, document_id: UUID): """Update document by file through the deprecated file-update aliases.""" - return _update_document_by_file(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id) + document, batch = _update_document_by_file(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id) + return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200 @service_api_ns.route("/datasets//documents") @@ -928,7 +948,7 @@ class DocumentListApi(DatasetApiResource): def get(self, tenant_id, dataset_id: UUID): dataset_id_str = str(dataset_id) tenant_id = str(tenant_id) - query_params = DocumentListQuery.model_validate(request.args.to_dict()) + query_params = query_params_from_request(DocumentListQuery) dataset = db.session.scalar( select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1) ) @@ -1021,6 +1041,7 @@ class DocumentBatchDownloadZipApi(DatasetApiResource): ) cleanup = stack.pop_all() response.call_on_close(cleanup.close) + # response-contract:ignore binary send_file response return response @@ -1149,7 +1170,9 @@ class DocumentDownloadApi(DatasetApiResource): if document.tenant_id != str(tenant_id): raise Forbidden("No permission.") - return {"url": DocumentService.get_document_download_url(document, db.session())} + return UrlResponse(url=DocumentService.get_document_download_url(document, db.session())).model_dump( + mode="json" + ) @service_api_ns.route("/datasets//documents/") @@ -1176,8 +1199,13 @@ class DocumentApi(DatasetApiResource): ) @service_api_ns.doc("get_document") @service_api_ns.doc(description="Get a specific document by ID") - @service_api_ns.doc(params={"dataset_id": "Knowledge base ID.", "document_id": "Document ID."}) - @service_api_ns.doc(params=query_params_from_model(DocumentGetQuery)) + @service_api_ns.doc( + params={ + "dataset_id": "Knowledge base ID.", + "document_id": "Document ID.", + **query_params_from_model(DocumentGetQuery), + } + ) @service_api_ns.doc( responses={ 200: "Document retrieved successfully", @@ -1205,9 +1233,14 @@ class DocumentApi(DatasetApiResource): if document.tenant_id != str(tenant_id): raise Forbidden("No permission.") - metadata = request.args.get("metadata", "all") - if metadata not in self.METADATA_CHOICES: - raise InvalidMetadataError(f"Invalid metadata value: {metadata}") + try: + query_params = query_params_from_request(DocumentGetQuery) + except ValidationError as exc: + metadata = request.args.get("metadata", "all") + raise InvalidMetadataError(f"Invalid metadata value: {metadata}") from exc + metadata = query_params.metadata + response_include: set[str] | None = None + response_exclude: set[str] | None = None # Calculate summary_index_status if needed summary_index_status = None @@ -1221,8 +1254,10 @@ class DocumentApi(DatasetApiResource): ) if metadata == "only": + response_include = {"id", "doc_type", "doc_metadata"} response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details} elif metadata == "without": + response_exclude = {"doc_type", "doc_metadata"} dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session()) document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {} data_source_info = document.data_source_detail_dict @@ -1295,8 +1330,33 @@ class DocumentApi(DatasetApiResource): "need_summary": document.need_summary if document.need_summary is not None else False, } - return response + return DocumentDetailResponse.model_validate(response).model_dump( + mode="json", + include=response_include, + exclude=response_exclude, + ) + @service_api_ns.doc( + summary="Update Document by File", + description=( + "Update an existing document by uploading a new file. Re-triggers indexing — use the returned " + "`batch` ID with [Get Document Indexing Status](/api-reference/documents/" + "get-document-indexing-status) to track progress." + ), + tags=["Documents"], + responses={ + 200: "Document updated successfully.", + 400: ( + "- `too_many_files` : Only one file is allowed.\n" + "- `filename_not_exists_error` : The specified filename does not exist.\n" + "- `provider_not_initialize` : No valid model provider credentials found. Please go to " + "Settings -> Model Provider to complete your provider credentials.\n" + "- `invalid_param` : Knowledge base does not exist, external datasets not supported, " + "file too large, unsupported file type, or invalid doc_form (must be `text_model`, " + "`hierarchical_model`, or `qa_model`)." + ), + }, + ) @service_api_ns.doc("update_document_by_file") @service_api_ns.doc(description="Update an existing document by uploading a file") @service_api_ns.doc(consumes=["multipart/form-data"], params=DOCUMENT_UPDATE_BY_FILE_PARAMS) @@ -1314,7 +1374,8 @@ class DocumentApi(DatasetApiResource): @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def patch(self, tenant_id: str, dataset_id: UUID, document_id: UUID): """Update document by file on the canonical document resource.""" - return _update_document_by_file(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id) + document, batch = _update_document_by_file(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id) + return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200 @service_api_ns.doc( summary="Delete Document", diff --git a/api/core/rag/entities/processing_entities.py b/api/core/rag/entities/processing_entities.py index 46360ec086f..6b08bb858c8 100644 --- a/api/core/rag/entities/processing_entities.py +++ b/api/core/rag/entities/processing_entities.py @@ -1,7 +1,7 @@ from enum import StrEnum from typing import Annotated, Literal -from pydantic import BaseModel, Field, WithJsonSchema +from pydantic import AliasChoices, BaseModel, Field, WithJsonSchema class ParentMode(StrEnum): @@ -26,7 +26,14 @@ class PreProcessingRule(BaseModel): class Segmentation(BaseModel): - separator: str = Field(default="\n", description="Custom separator for splitting text.") + # TODO: there are internally mismatched / inconsistent naming + # between `separator` and `delimiter` across the codebase. + # Taking `separator` as the canonical. + separator: str = Field( + default="\n", + description="Custom separator for splitting text.", + validation_alias=AliasChoices("separator", "delimiter"), + ) max_tokens: int = Field(description="Maximum token count per chunk.") chunk_overlap: int = Field(default=0, description="Token overlap between chunks.") diff --git a/api/fields/dataset_fields.py b/api/fields/dataset_fields.py index ea506d2a7e4..f97f5b79460 100644 --- a/api/fields/dataset_fields.py +++ b/api/fields/dataset_fields.py @@ -1,22 +1,9 @@ from datetime import datetime -from flask_restx import fields from pydantic import Field, field_validator from fields.base import ResponseModel -from libs.helper import TimestampField, to_timestamp - -dataset_fields = { - "id": fields.String, - "name": fields.String, - "description": fields.String, - "permission": fields.String, - "data_source_type": fields.String, - "indexing_technique": fields.String, - "created_by": fields.String, - "created_at": TimestampField, - "permission_keys": fields.List(fields.String()), -} +from libs.helper import to_timestamp class DatasetMetadataResponse(ResponseModel): @@ -50,104 +37,6 @@ class DatasetMetadataActionResponse(ResponseModel): result: str -reranking_model_fields = {"reranking_provider_name": fields.String, "reranking_model_name": fields.String} - -keyword_setting_fields = {"keyword_weight": fields.Float} - -vector_setting_fields = { - "vector_weight": fields.Float, - "embedding_model_name": fields.String, - "embedding_provider_name": fields.String, -} - -weighted_score_fields = { - "weight_type": fields.String, - "keyword_setting": fields.Nested(keyword_setting_fields), - "vector_setting": fields.Nested(vector_setting_fields), -} - -dataset_retrieval_model_fields = { - "search_method": fields.String, - "reranking_enable": fields.Boolean, - "reranking_mode": fields.String, - "reranking_model": fields.Nested(reranking_model_fields), - "weights": fields.Nested(weighted_score_fields, allow_null=True), - "top_k": fields.Integer, - "score_threshold_enabled": fields.Boolean, - "score_threshold": fields.Float, -} - -dataset_summary_index_fields = { - "enable": fields.Boolean, - "model_name": fields.String, - "model_provider_name": fields.String, - "summary_prompt": fields.String, -} - -external_retrieval_model_fields = { - "top_k": fields.Integer, - "score_threshold": fields.Float, - "score_threshold_enabled": fields.Boolean, -} - -tag_fields = {"id": fields.String, "name": fields.String, "type": fields.String} - -external_knowledge_info_fields = { - "external_knowledge_id": fields.String, - "external_knowledge_api_id": fields.String, - "external_knowledge_api_name": fields.String, - "external_knowledge_api_endpoint": fields.String, -} - -doc_metadata_fields = {"id": fields.String, "name": fields.String, "type": fields.String} - -icon_info_fields = { - "icon_type": fields.String, - "icon": fields.String, - "icon_background": fields.String, - "icon_url": fields.String, -} - -dataset_detail_fields = { - "id": fields.String, - "name": fields.String, - "description": fields.String, - "provider": fields.String, - "permission": fields.String, - "data_source_type": fields.String, - "indexing_technique": fields.String, - "app_count": fields.Integer, - "document_count": fields.Integer, - "word_count": fields.Integer, - "created_by": fields.String, - "author_name": fields.String, - "created_at": TimestampField, - "updated_by": fields.String, - "updated_at": TimestampField, - "embedding_model": fields.String, - "embedding_model_provider": fields.String, - "embedding_available": fields.Boolean, - "retrieval_model_dict": fields.Nested(dataset_retrieval_model_fields), - "summary_index_setting": fields.Nested(dataset_summary_index_fields), - "tags": fields.List(fields.Nested(tag_fields)), - "doc_form": fields.String, - "external_knowledge_info": fields.Nested(external_knowledge_info_fields), - "external_retrieval_model": fields.Nested(external_retrieval_model_fields, allow_null=True), - "doc_metadata": fields.List(fields.Nested(doc_metadata_fields)), - "built_in_field_enabled": fields.Boolean, - "pipeline_id": fields.String, - "runtime_mode": fields.String, - "chunk_structure": fields.String, - "icon_info": fields.Nested(icon_info_fields), - "is_published": fields.Boolean, - "total_documents": fields.Integer, - "total_available_documents": fields.Integer, - "enable_api": fields.Boolean, - "is_multimodal": fields.Boolean, - "permission_keys": fields.List(fields.String()), -} - - class DatasetRerankingModelResponse(ResponseModel): reranking_provider_name: str | None = None reranking_model_name: str | None = None diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md index 76912376340..74cbe9f20f2 100644 --- a/api/openapi/markdown/console-openapi.md +++ b/api/openapi/markdown/console-openapi.md @@ -5492,7 +5492,7 @@ Create external knowledge dataset | Code | Description | Schema | | ---- | ----------- | ------ | -| 201 | External dataset created successfully | **application/json**: [DatasetDetail](#datasetdetail)
| +| 201 | External dataset created successfully | **application/json**: [DatasetDetailResponse](#datasetdetailresponse)
| | 400 | Invalid parameters | | | 403 | Permission denied | | @@ -5514,6 +5514,8 @@ Get external knowledge API templates | 200 | External API templates retrieved successfully | **application/json**: [ExternalKnowledgeApiListResponse](#externalknowledgeapilistresponse)
| ### [POST] /datasets/external-knowledge-api +Create external knowledge API template + #### Request Body | Required | Schema | @@ -5525,6 +5527,7 @@ Get external knowledge API templates | Code | Description | Schema | | ---- | ----------- | ------ | | 201 | External API template created successfully | **application/json**: [ExternalKnowledgeApiResponse](#externalknowledgeapiresponse)
| +| 403 | Permission denied | | ### [DELETE] /datasets/external-knowledge-api/{external_knowledge_api_id} #### Parameters @@ -5556,11 +5559,13 @@ Get external knowledge API template details | 404 | Template not found | | ### [PATCH] /datasets/external-knowledge-api/{external_knowledge_api_id} +Update external knowledge API template + #### Parameters | Name | Located in | Description | Required | Schema | | ---- | ---------- | ----------- | -------- | ------ | -| external_knowledge_api_id | path | | Yes | string (uuid) | +| external_knowledge_api_id | path | External knowledge API ID | Yes | string (uuid) | #### Request Body @@ -5573,6 +5578,7 @@ Get external knowledge API template details | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | External API template updated successfully | **application/json**: [ExternalKnowledgeApiResponse](#externalknowledgeapiresponse)
| +| 404 | Template not found | | ### [GET] /datasets/external-knowledge-api/{external_knowledge_api_id}/use-check Check if external knowledge API is being used @@ -5617,7 +5623,7 @@ Initialize dataset with documents | Code | Description | Schema | | ---- | ----------- | ------ | -| 201 | Dataset initialized successfully | **application/json**: [DatasetAndDocumentResponse](#datasetanddocumentresponse)
| +| 200 | Dataset initialized successfully | **application/json**: [DatasetAndDocumentResponse](#datasetanddocumentresponse)
| | 400 | Invalid request parameters | | ### [GET] /datasets/metadata/built-in @@ -5653,7 +5659,7 @@ Get dataset document processing rules | Code | Description | Schema | | ---- | ----------- | ------ | -| 200 | Process rules retrieved successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)
| +| 200 | Process rules retrieved successfully | **application/json**: [ProcessRuleResponse](#processruleresponse)
| ### [GET] /datasets/retrieval-setting Get dataset retrieval settings @@ -5774,7 +5780,7 @@ Get dataset auto disable logs | Code | Description | Schema | | ---- | ----------- | ------ | -| 200 | Batch indexing estimate calculated successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)
| +| 200 | Indexing estimate calculated successfully | **application/json**: [IndexingEstimateResponse](#indexingestimateresponse)
| ### [GET] /datasets/{dataset_id}/batch/{batch}/indexing-status #### Parameters @@ -5862,9 +5868,9 @@ Download selected dataset documents as a single ZIP archive (upload-file only) #### Responses -| Code | Description | Schema | -| ---- | ----------- | ------ | -| 200 | ZIP archive generated successfully | **application/json**: [BinaryFileResponse](#binaryfileresponse)
| +| Code | Description | +| ---- | ----------- | +| 200 | ZIP archive downloaded successfully | ### [POST] /datasets/{dataset_id}/documents/generate-summary **Generate summary index for specified documents** @@ -5957,7 +5963,7 @@ Get document details | Code | Description | Schema | | ---- | ----------- | ------ | -| 200 | Document retrieved successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)
| +| 200 | Document retrieved successfully | **application/json**: [DocumentDetailResponse](#documentdetailresponse)
| | 404 | Document not found | | ### [GET] /datasets/{dataset_id}/documents/{document_id}/download @@ -5990,7 +5996,7 @@ Estimate document indexing cost | Code | Description | Schema | | ---- | ----------- | ------ | -| 200 | Indexing estimate calculated successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)
| +| 200 | Indexing estimate calculated successfully | **application/json**: [IndexingEstimateResponse](#indexingestimateresponse)
| | 400 | Document already finished | | | 404 | Document not found | | @@ -6061,7 +6067,7 @@ Update document metadata | Code | Description | Schema | | ---- | ----------- | ------ | -| 200 | Document pipeline execution log retrieved successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)
| +| 200 | Pipeline execution log retrieved successfully | **application/json**: [DocumentPipelineExecutionLogResponse](#documentpipelineexecutionlogresponse)
| ### [PATCH] /datasets/{dataset_id}/documents/{document_id}/processing/pause **pause document** @@ -6384,6 +6390,7 @@ Returns: - generating: Number of summaries being generated - error: Number of summaries with errors - not_started: Number of segments without summary records + - timeout: Number of summaries that timed out - summaries: List of summary records with status and content preview #### Parameters @@ -6397,7 +6404,7 @@ Returns: | Code | Description | Schema | | ---- | ----------- | ------ | -| 200 | Summary status retrieved successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)
| +| 200 | Summary status retrieved successfully | **application/json**: [DocumentSummaryStatusResponse](#documentsummarystatusresponse)
| | 404 | Document not found | | ### [GET] /datasets/{dataset_id}/documents/{document_id}/website-sync @@ -6451,7 +6458,7 @@ Test external knowledge retrieval for dataset | Code | Description | Schema | | ---- | ----------- | ------ | -| 200 | External hit testing completed successfully | **application/json**: [ExternalRetrievalTestResponse](#externalretrievaltestresponse)
| +| 200 | External hit testing completed successfully | **application/json**: [ExternalHitTestingResponse](#externalhittestingresponse)
| | 400 | Invalid parameters | | | 404 | Dataset not found | | @@ -9388,7 +9395,7 @@ Bedrock retrieval test (internal use only) | Code | Description | Schema | | ---- | ----------- | ------ | -| 200 | Bedrock retrieval test completed | **application/json**: [ExternalRetrievalTestResponse](#externalretrievaltestresponse)
| +| 200 | Bedrock retrieval test completed | **application/json**: [BedrockRetrievalResponse](#bedrockretrievalresponse)
| ### [GET] /trial-apps/{app_id} **Get app detail** @@ -15435,6 +15442,21 @@ AppMCPServer Status Enum | query | string | | Yes | | retrieval_setting | [BedrockRetrievalSetting](#bedrockretrievalsetting) | | Yes | +#### BedrockRetrievalRecordResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| content | string | | No | +| metadata | object | | No | +| score | number | | Yes | +| title | string | | No | + +#### BedrockRetrievalResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| records | [ [BedrockRetrievalRecordResponse](#bedrockretrievalrecordresponse) ] | | Yes | + #### BedrockRetrievalSetting Retrieval settings for Amazon Bedrock knowledge base queries. @@ -16325,47 +16347,6 @@ Model class for provider custom model configuration. | permission | [PermissionEnum](#permissionenum) | | No | | provider | string,
**Default:** vendor | | No | -#### DatasetDetail - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| app_count | integer | | No | -| author_name | string | | No | -| built_in_field_enabled | boolean | | No | -| chunk_structure | string | | No | -| created_at | long | | No | -| created_by | string | | No | -| data_source_type | string | | No | -| description | string | | No | -| doc_form | string | | No | -| doc_metadata | [ [DatasetDocMetadata](#datasetdocmetadata) ] | | No | -| document_count | integer | | No | -| embedding_available | boolean | | No | -| embedding_model | string | | No | -| embedding_model_provider | string | | No | -| enable_api | boolean | | No | -| external_knowledge_info | [ExternalKnowledgeInfo](#externalknowledgeinfo) | | No | -| external_retrieval_model | [ExternalRetrievalModel](#externalretrievalmodel) | | No | -| icon_info | [DatasetIconInfo](#dataseticoninfo) | | No | -| id | string | | No | -| indexing_technique | string | | No | -| is_multimodal | boolean | | No | -| is_published | boolean | | No | -| name | string | | No | -| permission | string | | No | -| permission_keys | [ string ] | | No | -| pipeline_id | string | | No | -| provider | string | | No | -| retrieval_model_dict | [DatasetRetrievalModel](#datasetretrievalmodel) | | No | -| runtime_mode | string | | No | -| summary_index_setting | [_AnonymousInlineModel_b1954337d565](#_anonymousinlinemodel_b1954337d565) | | No | -| tags | [ [Tag](#tag) ] | | No | -| total_available_documents | integer | | No | -| total_documents | integer | | No | -| updated_at | long | | No | -| updated_by | string | | No | -| word_count | integer | | No | - #### DatasetDetailResponse | Name | Type | Description | Required | @@ -16451,14 +16432,6 @@ Model class for provider custom model configuration. | updated_by | string | | Yes | | word_count | integer | | Yes | -#### DatasetDocMetadata - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| id | string | | No | -| name | string | | No | -| type | string | | No | - #### DatasetDocMetadataResponse | Name | Type | Description | Required | @@ -16484,15 +16457,6 @@ Model class for provider custom model configuration. | score_threshold_enabled | boolean | | No | | top_k | integer | | Yes | -#### DatasetIconInfo - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| icon | string | | No | -| icon_background | string | | No | -| icon_type | string | | No | -| icon_url | string | | No | - #### DatasetIconInfoResponse | Name | Type | Description | Required | @@ -16502,12 +16466,6 @@ Model class for provider custom model configuration. | icon_type | string | | No | | icon_url | string | | No | -#### DatasetKeywordSetting - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| keyword_weight | number | | No | - #### DatasetKeywordSettingResponse | Name | Type | Description | Required | @@ -16645,13 +16603,6 @@ Model class for provider custom model configuration. | page | integer | | Yes | | total | integer | | Yes | -#### DatasetRerankingModel - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| reranking_model_name | string | | No | -| reranking_provider_name | string | | No | - #### DatasetRerankingModelResponse | Name | Type | Description | Required | @@ -16672,19 +16623,6 @@ Model class for provider custom model configuration. | name | string | | Yes | | permission | string | | No | -#### DatasetRetrievalModel - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| reranking_enable | boolean | | No | -| reranking_mode | string | | No | -| reranking_model | [DatasetRerankingModel](#datasetrerankingmodel) | | No | -| score_threshold | number | | No | -| score_threshold_enabled | boolean | | No | -| search_method | string | | No | -| top_k | integer | | No | -| weights | [DatasetWeightedScore](#datasetweightedscore) | | No | - #### DatasetRetrievalModelResponse | Name | Type | Description | Required | @@ -16734,14 +16672,6 @@ Model class for provider custom model configuration. | retrieval_model | object | | No | | summary_index_setting | object | | No | -#### DatasetVectorSetting - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| embedding_model_name | string | | No | -| embedding_provider_name | string | | No | -| vector_weight | number | | No | - #### DatasetVectorSettingResponse | Name | Type | Description | Required | @@ -16750,14 +16680,6 @@ Model class for provider custom model configuration. | embedding_provider_name | string | | No | | vector_weight | number | | No | -#### DatasetWeightedScore - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| keyword_setting | [DatasetKeywordSetting](#datasetkeywordsetting) | | No | -| vector_setting | [DatasetVectorSetting](#datasetvectorsetting) | | No | -| weight_type | string | | No | - #### DatasetWeightedScoreResponse | Name | Type | Description | Required | @@ -17061,6 +16983,42 @@ Request payload for bulk downloading documents as a zip archive. | ---- | ---- | ----------- | -------- | | document_ids | [ string (uuid) ] | List of document IDs to include in the ZIP download. | Yes | +#### DocumentDetailResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| archived | boolean | | No | +| average_segment_length | number | | No | +| completed_at | integer | | No | +| created_at | integer | | No | +| created_by | string | | No | +| created_from | string | | No | +| data_source_detail_dict | | | No | +| data_source_info | | | No | +| data_source_type | string | | No | +| dataset_process_rule | | | No | +| dataset_process_rule_id | string | | No | +| disabled_at | integer | | No | +| disabled_by | string | | No | +| display_status | string | | No | +| doc_form | string | | No | +| doc_language | string | | No | +| doc_metadata | [ [DocumentMetadataResponse](#documentmetadataresponse) ] | | No | +| doc_type | string | | No | +| document_process_rule | | | No | +| enabled | boolean | | No | +| error | string | | No | +| hit_count | integer | | No | +| id | string | | Yes | +| indexing_latency | number | | No | +| indexing_status | string | | No | +| name | string | | No | +| need_summary | boolean | | No | +| position | integer | | No | +| segment_count | integer | | No | +| tokens | integer | | No | +| updated_at | integer | | No | + #### DocumentMetadataOperation | Name | Type | Description | Required | @@ -17085,6 +17043,15 @@ Request payload for bulk downloading documents as a zip archive. | doc_metadata | | | No | | doc_type | string | | No | +#### DocumentPipelineExecutionLogResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| datasource_info | [JsonValue](#jsonvalue) | | No | +| datasource_node_id | string | | No | +| datasource_type | string | | No | +| input_data | [JsonValue](#jsonvalue) | | No | + #### DocumentRenamePayload | Name | Type | Description | Required | @@ -17149,6 +17116,14 @@ Request payload for bulk downloading documents as a zip archive. | stopped_at | integer | | Yes | | total_segments | integer | | No | +#### DocumentSummaryStatusResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| summaries | [ [SummaryEntryResponse](#summaryentryresponse) ] | | Yes | +| summary_status | [SummaryStatusResponse](#summarystatusresponse) | | Yes | +| total_segments | integer | | Yes | + #### DocumentWithSegmentsListResponse | Name | Type | Description | Required | @@ -17679,6 +17654,35 @@ Built-in tool icons are URL strings; API-based tool icons are provider-defined p | metadata_filtering_conditions | object | | No | | query | string | | Yes | +#### ExternalHitTestingQueryResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| content | string | | Yes | + +#### ExternalHitTestingRecordResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| content | string | | No | +| metadata | object | | No | +| score | number | | No | +| title | string | | No | + +#### ExternalHitTestingResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| query | [ExternalHitTestingQueryResponse](#externalhittestingqueryresponse) | | Yes | +| records | [ [ExternalHitTestingRecordResponse](#externalhittestingrecordresponse) ] | | Yes | + +#### ExternalKnowledgeApiBindingResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| id | string | | Yes | +| name | string | | Yes | + #### ExternalKnowledgeApiListResponse | Name | Type | Description | Required | @@ -17702,43 +17706,13 @@ Built-in tool icons are URL strings; API-based tool icons are provider-defined p | ---- | ---- | ----------- | -------- | | created_at | string | | Yes | | created_by | string | | Yes | -| dataset_bindings | [ [ExternalKnowledgeDatasetBindingResponse](#externalknowledgedatasetbindingresponse) ] | | No | +| dataset_bindings | [ [ExternalKnowledgeApiBindingResponse](#externalknowledgeapibindingresponse) ] | | Yes | | description | string | | Yes | | id | string | | Yes | | name | string | | Yes | -| settings | object | | No | +| settings | object | | Yes | | tenant_id | string | | Yes | -#### ExternalKnowledgeDatasetBindingResponse - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| id | string | | Yes | -| name | string | | Yes | - -#### ExternalKnowledgeInfo - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| external_knowledge_api_endpoint | string | | No | -| external_knowledge_api_id | string | | No | -| external_knowledge_api_name | string | | No | -| external_knowledge_id | string | | No | - -#### ExternalRetrievalModel - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| score_threshold | number | | No | -| score_threshold_enabled | boolean | | No | -| top_k | integer | | No | - -#### ExternalRetrievalTestResponse - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| ExternalRetrievalTestResponse | object
[ object ] | | | - #### FeatureModel | Name | Type | Description | Required | @@ -18243,27 +18217,15 @@ Query parameter for including secret variables in export. | info_list | object | | Yes | | process_rule | object | | Yes | -#### IndexingEstimatePreviewItemResponse - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| child_chunks | [ string ] | | No | -| content | string | | Yes | -| summary | string | | No | - -#### IndexingEstimateQaPreviewItemResponse - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| answer | string | | Yes | -| question | string | | Yes | - #### IndexingEstimateResponse | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | -| preview | [ [IndexingEstimatePreviewItemResponse](#indexingestimatepreviewitemresponse) ] | | Yes | -| qa_preview | [ [IndexingEstimateQaPreviewItemResponse](#indexingestimateqapreviewitemresponse) ] | | No | +| currency | string | | Yes | +| preview | [ [PreviewDetail](#previewdetail) ] | | Yes | +| qa_preview | [ [QAPreviewDetail](#qapreviewdetail) ] | | No | +| tokens | integer | | Yes | +| total_price | number
integer | | Yes | | total_segments | integer | | Yes | #### InfoList @@ -19259,12 +19221,6 @@ OAuth schema | redirect_uri | string | | No | | refresh_token | string | | No | -#### OpaqueObjectResponse - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| OpaqueObjectResponse | object | | | - #### Option | Name | Type | Description | Required | @@ -20310,6 +20266,14 @@ Dataset Process Rule Mode | ---- | ---- | ----------- | -------- | | ProcessRuleMode | string | Dataset Process Rule Mode | | +#### ProcessRuleResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| limits | object | | Yes | +| mode | [ProcessRuleMode](#processrulemode) | | Yes | +| rules | [Rule](#rule) | | No | + #### ProviderConfig Model class for common provider settings like credentials @@ -21581,6 +21545,28 @@ The subscription constructor of the trigger provider | ---- | ---- | ----------- | -------- | | data | [ string ] | | Yes | +#### SummaryEntryResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| created_at | integer | | No | +| error | string | | No | +| segment_id | string | | Yes | +| segment_position | integer | | Yes | +| status | string | | Yes | +| summary_preview | string | | No | +| updated_at | integer | | No | + +#### SummaryStatusResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| completed | integer | | No | +| error | integer | | No | +| generating | integer | | No | +| not_started | integer | | No | +| timeout | integer | | No | + #### SwitchWorkspacePayload | Name | Type | Description | Required | @@ -23702,15 +23688,6 @@ Workflow tool configuration | data | [ [AccessPolicy](#accesspolicy) ] | | No | | pagination | [Pagination](#pagination) | | No | -#### _AnonymousInlineModel_b1954337d565 - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| enable | boolean | | No | -| model_name | string | | No | -| model_provider_name | string | | No | -| summary_prompt | string | | No | - #### _DeleteMemberBindingsRequest | Name | Type | Description | Required | diff --git a/api/openapi/markdown/service-openapi.md b/api/openapi/markdown/service-openapi.md index 7680ac77111..b7030f544a0 100644 --- a/api/openapi/markdown/service-openapi.md +++ b/api/openapi/markdown/service-openapi.md @@ -46,76 +46,6 @@ Deprecated legacy alias for creating a new document by providing text content. U | 401 | Unauthorized - invalid API token | | | 403 | Forbidden - dataset API access or workspace access denied | | -### [DELETE] /datasets/{dataset_id}/documents/{document_id} -**Delete Document** - -Permanently delete a document and all its chunks from the knowledge base. - -#### Parameters - -| Name | Located in | Description | Required | Schema | -| ---- | ---------- | ----------- | -------- | ------ | -| dataset_id | path | Knowledge base ID. | Yes | string (uuid) | -| document_id | path | Document ID. | Yes | string (uuid) | - -#### Responses - -| Code | Description | -| ---- | ----------- | -| 204 | Success. | -| 400 | `document_indexing` : Cannot delete document during indexing. | -| 401 | Unauthorized - invalid API token | -| 403 | `archived_document_immutable` : The archived document is not editable. | -| 404 | `not_found` : Document Not Exists. | - -### [GET] /datasets/{dataset_id}/documents/{document_id} -**Get Document** - -Retrieve detailed information about a specific document, including its indexing status, metadata, and processing statistics. - -#### Parameters - -| Name | Located in | Description | Required | Schema | -| ---- | ---------- | ----------- | -------- | ------ | -| dataset_id | path | Knowledge base ID. | Yes | string (uuid) | -| document_id | path | Document ID. | Yes | string (uuid) | -| metadata | query | `all` returns all fields including metadata. `only` returns only `id`, `doc_type`, and `doc_metadata`. `without` returns all fields except `doc_metadata`. | No | string,
**Available values:** "all", "only", "without",
**Default:** all | - -#### Responses - -| Code | Description | Schema | -| ---- | ----------- | ------ | -| 200 | Document details. The response shape varies based on the `metadata` query parameter. When `metadata` is `only`, only `id`, `doc_type`, and `doc_metadata` are returned. When `metadata` is `without`, `doc_type` and `doc_metadata` are omitted. | **application/json**: [DocumentDetailResponse](#documentdetailresponse)
| -| 400 | `invalid_metadata` : Invalid metadata value for the specified key. | | -| 401 | Unauthorized - invalid API token | | -| 403 | `forbidden` : No permission. | | -| 404 | `not_found` : Document not found. | | - -### [PATCH] /datasets/{dataset_id}/documents/{document_id} -Update an existing document by uploading a file - -#### Parameters - -| Name | Located in | Description | Required | Schema | -| ---- | ---------- | ----------- | -------- | ------ | -| dataset_id | path | Knowledge base ID. | Yes | string (uuid) | -| document_id | path | Document ID. | Yes | string (uuid) | - -#### Request Body - -| Required | Schema | -| -------- | ------ | -| No | **multipart/form-data**: { **"data"**: string, **"file"**: binary }
| - -#### Responses - -| Code | Description | Schema | -| ---- | ----------- | ------ | -| 200 | Document updated successfully | **application/json**: [DocumentAndBatchResponse](#documentandbatchresponse)
| -| 401 | Unauthorized - invalid API token | | -| 403 | Forbidden - dataset API access or workspace access denied | | -| 404 | Document not found | | - ### ~~[POST] /datasets/{dataset_id}/documents/{document_id}/update_by_text~~ ***DEPRECATED*** @@ -1439,7 +1369,9 @@ Retrieve detailed information about a specific document, including its indexing | 404 | `not_found` : Document not found. | | ### [PATCH] /datasets/{dataset_id}/documents/{document_id} -Update an existing document by uploading a file +**Update Document by File** + +Update an existing document by uploading a new file. Re-triggers indexing — use the returned `batch` ID with [Get Document Indexing Status](/api-reference/documents/get-document-indexing-status) to track progress. #### Parameters @@ -1458,7 +1390,8 @@ Update an existing document by uploading a file | Code | Description | Schema | | ---- | ----------- | ------ | -| 200 | Document updated successfully | **application/json**: [DocumentAndBatchResponse](#documentandbatchresponse)
| +| 200 | Document updated successfully. | **application/json**: [DocumentAndBatchResponse](#documentandbatchresponse)
| +| 400 | - `too_many_files` : Only one file is allowed. - `filename_not_exists_error` : The specified filename does not exist. - `provider_not_initialize` : No valid model provider credentials found. Please go to Settings -> Model Provider to complete your provider credentials. - `invalid_param` : Knowledge base does not exist, external datasets not supported, file too large, unsupported file type, or invalid doc_form (must be `text_model`, `hierarchical_model`, or `qa_model`). | | | 401 | Unauthorized - invalid API token | | | 403 | Forbidden - dataset API access or workspace access denied | | | 404 | Document not found | | @@ -2995,7 +2928,7 @@ Request payload for bulk downloading documents as a zip archive. | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | | archived | boolean | | No | -| average_segment_length | number | | No | +| average_segment_length | integer
number | | No | | completed_at | integer | | No | | created_at | integer | | No | | created_by | string | | No | @@ -3009,7 +2942,7 @@ Request payload for bulk downloading documents as a zip archive. | display_status | string | | No | | doc_form | string | | No | | doc_language | string | | No | -| doc_metadata | [ [DocumentMetadataResponse](#documentmetadataresponse) ] | | No | +| doc_metadata | [ [DocumentMetadataResponse](#documentmetadataresponse) ]
object | | No | | doc_type | string | | No | | document_process_rule | object | | No | | enabled | boolean | | No | diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py index 6913825d599..dad78d3d4ef 100644 --- a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py +++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py @@ -31,6 +31,7 @@ from controllers.console.datasets.datasets import ( DatasetUseCheckApi, ) from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError +from core.entities.knowledge_entities import IndexingEstimate from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError from core.provider_manager import ProviderManager from core.rag.index_processor.constant.index_type import IndexStructureType @@ -1391,8 +1392,7 @@ class TestDatasetIndexingEstimateApi: mock_file = self._upload_file() - mock_response = MagicMock() - mock_response.model_dump.return_value = {"tokens": 100} + mock_response = IndexingEstimate(total_segments=100, preview=[]) with ( app.test_request_context("/"), @@ -1418,7 +1418,13 @@ class TestDatasetIndexingEstimateApi: response, status = method(api, "tenant-1") assert status == 200 - assert response == {"tokens": 100} + assert response == { + "tokens": 0, + "total_price": 0, + "currency": "USD", + "total_segments": 100, + "preview": [], + } def test_post_file_not_found(self, app: Flask): api = DatasetIndexingEstimateApi() diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py index be0a4eea40a..7c415c01cdc 100644 --- a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py +++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py @@ -1,3 +1,4 @@ +import datetime import inspect from unittest.mock import ANY, MagicMock, patch @@ -34,6 +35,7 @@ from controllers.console.datasets.error import ( InvalidActionError, InvalidMetadataError, ) +from core.entities.knowledge_entities import IndexingEstimate from core.rag.index_processor.constant.index_type import IndexStructureType from models.dataset import Dataset from models.dataset import Document as DatasetDocument @@ -72,8 +74,46 @@ def make_serializable_document(**overrides): } attrs.update(overrides) document = MagicMock(spec_set=list(attrs)) - for name, value in attrs.items(): - setattr(document, name, value) + document.configure_mock(**attrs) + return document + + +def make_document_detail(**overrides): + attrs = { + "id": "doc-1", + "position": 1, + "data_source_type": "upload_file", + "data_source_info_dict": {"upload_file_id": "file-1"}, + "data_source_detail_dict": {}, + "dataset_process_rule_id": None, + "dataset_process_rule": None, + "name": "Document", + "created_from": "web", + "created_by": "u1", + "created_at": datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC), + "tokens": 10, + "indexing_status": "completed", + "completed_at": None, + "updated_at": None, + "indexing_latency": None, + "error": None, + "enabled": True, + "disabled_at": None, + "disabled_by": None, + "archived": False, + "doc_type": "others", + "doc_metadata_details": [], + "segment_count": 0, + "average_segment_length": 0, + "hit_count": 0, + "display_status": "available", + "doc_form": "text_model", + "doc_language": "English", + "need_summary": False, + } + attrs.update(overrides) + document = MagicMock(spec_set=list(attrs)) + document.configure_mock(**attrs) return document @@ -173,6 +213,42 @@ class TestGetProcessRuleApi: assert "rules" in response + def test_get_with_document_preserves_legacy_segmentation_delimiter(self, app: Flask, patch_tenant): + api = GetProcessRuleApi() + method = inspect.unwrap(api.get) + user, _ = patch_tenant + + document = MagicMock(dataset_id="ds-1") + process_rule = MagicMock( + mode="custom", + rules_dict={"segmentation": {"delimiter": "---", "max_tokens": 123}}, + ) + + with ( + app.test_request_context("/?document_id=doc-1"), + patch( + "controllers.console.datasets.datasets_document.db.get_or_404", + return_value=document, + ), + patch( + "controllers.console.datasets.datasets_document.DatasetService.get_dataset", + return_value=MagicMock(), + ), + patch( + "controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission", + return_value=None, + ), + patch( + "controllers.console.datasets.datasets_document.db.session.scalar", + return_value=process_rule, + ), + ): + response = method(api, user) + + assert response["rules"]["segmentation"]["separator"] == "---" + assert response["rules"]["segmentation"]["max_tokens"] == 123 + assert "delimiter" not in response["rules"]["segmentation"] + def test_get_with_document_dataset_not_found(self, app: Flask, patch_tenant): api = GetProcessRuleApi() method = inspect.unwrap(api.get) @@ -414,7 +490,7 @@ class TestDocumentApi: method = inspect.unwrap(api.get) user, tenant_id = patch_tenant - document = MagicMock(dataset_process_rule=None) + document = make_document_detail() with ( app.test_request_context("/"), @@ -926,12 +1002,24 @@ class TestDocumentSummaryStatusApi: ), patch( "services.summary_index_service.SummaryIndexService.get_document_summary_status_detail", - return_value={"total_segments": 0}, + return_value={ + "total_segments": 1, + "summary_status": {"timeout": 1}, + "summaries": [ + { + "segment_id": "segment-1", + "segment_position": 1, + "status": "timeout", + } + ], + }, ), ): response, status = method(api, user, "ds-1", "doc-1") assert status == 200 + assert response["summary_status"]["timeout"] == 1 + assert response["summaries"][0]["status"] == "timeout" class TestDocumentIndexingEstimateApi: @@ -1103,7 +1191,7 @@ class TestDocumentBatchIndexingEstimateApi: patch.object(api, "get_batch_documents", return_value=[doc]), patch( "controllers.console.datasets.datasets_document.IndexingRunner.indexing_estimate", - return_value=MagicMock(model_dump=lambda: {"tokens": 2}), + return_value=IndexingEstimate(total_segments=2, preview=[]), ), ): resp, status = method(api, tenant_id, user, "ds-1", "batch-1") @@ -1132,7 +1220,7 @@ class TestDocumentBatchIndexingEstimateApi: patch.object(api, "get_batch_documents", return_value=[doc]), patch( "controllers.console.datasets.datasets_document.IndexingRunner.indexing_estimate", - return_value=MagicMock(model_dump=lambda: {"tokens": 1}), + return_value=IndexingEstimate(total_segments=1, preview=[]), ), ): resp, status = method(api, tenant_id, user, "ds-1", "batch-1") @@ -1314,7 +1402,7 @@ class TestDocumentApiMetadata: method = inspect.unwrap(api.get) user, tenant_id = patch_tenant - document = MagicMock(dataset_process_rule=None, doc_metadata_details=[]) + document = make_document_detail(doc_metadata_details=[]) with ( app.test_request_context("/?metadata=only"), @@ -1334,7 +1422,7 @@ class TestDocumentApiMetadata: method = inspect.unwrap(api.get) user, tenant_id = patch_tenant - document = MagicMock(dataset_process_rule=None) + document = make_document_detail() with ( app.test_request_context("/?metadata=without"), @@ -1611,7 +1699,7 @@ class TestDocumentIndexingEdgeCases: ), patch( "controllers.console.datasets.datasets_document.IndexingRunner.indexing_estimate", - return_value=MagicMock(model_dump=lambda: {"tokens": 5}), + return_value=IndexingEstimate(total_segments=5, preview=[]), ), ): response, status = method(api, tenant_id, user, "ds-1", "doc-1") diff --git a/api/tests/unit_tests/controllers/console/datasets/test_external.py b/api/tests/unit_tests/controllers/console/datasets/test_external.py index 1cffc90ae23..6cdd8c84ddd 100644 --- a/api/tests/unit_tests/controllers/console/datasets/test_external.py +++ b/api/tests/unit_tests/controllers/console/datasets/test_external.py @@ -1,5 +1,6 @@ import inspect from types import SimpleNamespace +from typing import Any from unittest.mock import ANY, MagicMock, PropertyMock, patch import pytest @@ -8,7 +9,6 @@ from werkzeug.exceptions import Forbidden, NotFound import services from controllers.console import console_ns -from controllers.console.datasets import external as external_module from controllers.console.datasets.error import DatasetNameDuplicateError from controllers.console.datasets.external import ( BedrockRetrievalApi, @@ -40,28 +40,185 @@ def current_user() -> Account: return user +def _external_api_dict(api_id: str = "api-1") -> dict: + return { + "id": api_id, + "tenant_id": "tenant-1", + "name": f"External API {api_id}", + "description": f"Description for {api_id}", + "settings": { + "endpoint": f"https://external.example.com/{api_id}", + "api_key": "secret", + "headers": {"X-Source": "unit-test"}, + "timeout": 30, + }, + "dataset_bindings": [ + {"id": f"dataset-{api_id}", "name": f"Dataset {api_id}"}, + ], + "created_by": "user-1", + "created_at": "2024-01-01T00:00:00", + } + + +def _external_api_object(api_id: str = "api-1") -> SimpleNamespace: + payload = _external_api_dict(api_id) + return SimpleNamespace( + **{ + **payload, + "dataset_bindings": [SimpleNamespace(**binding) for binding in payload["dataset_bindings"]], + } + ) + + +def _expected_dataset_detail_payload() -> dict[str, Any]: + return { + "id": "dataset-1", + "name": "Support knowledge", + "description": "External support articles", + "provider": "external", + "permission": "only_me", + "data_source_type": "external", + "indexing_technique": "economy", + "app_count": 2, + "document_count": 7, + "word_count": 2048, + "created_by": "user-1", + "author_name": "Test User", + "created_at": 1710000000, + "updated_by": "user-2", + "updated_at": 1710003600, + "embedding_model": None, + "embedding_model_provider": None, + "embedding_available": False, + "retrieval_model_dict": { + "search_method": "semantic_search", + "reranking_enable": False, + "reranking_mode": None, + "reranking_model": {"reranking_provider_name": None, "reranking_model_name": None}, + "weights": None, + "top_k": 4, + "score_threshold_enabled": True, + "score_threshold": 0.5, + }, + "summary_index_setting": { + "enable": True, + "model_name": "summary-model", + "model_provider_name": "provider-a", + "summary_prompt": "Summarize this.", + }, + "tags": [{"id": "tag-1", "name": "Support", "type": "knowledge"}], + "doc_form": "text_model", + "external_knowledge_info": { + "external_knowledge_id": "knowledge-1", + "external_knowledge_api_id": "api-1", + "external_knowledge_api_name": "External API api-1", + "external_knowledge_api_endpoint": "https://external.example.com/api-1", + }, + "external_retrieval_model": { + "top_k": 4, + "score_threshold": 0.5, + "score_threshold_enabled": True, + }, + "doc_metadata": [{"id": "metadata-1", "name": "source", "type": "string"}], + "built_in_field_enabled": True, + "pipeline_id": None, + "runtime_mode": "external", + "chunk_structure": "general", + "icon_info": { + "icon_type": "emoji", + "icon": "book", + "icon_background": "#FFF4ED", + "icon_url": None, + }, + "is_published": True, + "total_documents": 7, + "total_available_documents": 6, + "enable_api": True, + "is_multimodal": False, + "maintainer": None, + "permission_keys": [], + } + + +def _dataset_detail_object() -> SimpleNamespace: + payload = _expected_dataset_detail_payload() + return SimpleNamespace( + **{ + **payload, + "summary_index_setting": SimpleNamespace(**payload["summary_index_setting"]), + "tags": [SimpleNamespace(**tag) for tag in payload["tags"]], + "external_knowledge_info": SimpleNamespace(**payload["external_knowledge_info"]), + "external_retrieval_model": SimpleNamespace(**payload["external_retrieval_model"]), + "doc_metadata": [SimpleNamespace(**item) for item in payload["doc_metadata"]], + "icon_info": SimpleNamespace(**payload["icon_info"]), + } + ) + + class TestExternalApiTemplateListApi: def test_get_success(self, app: Flask): api = ExternalApiTemplateListApi() method = inspect.unwrap(api.get) - api_item = MagicMock() - api_item.to_dict.return_value = {"id": "1"} + api_item = _external_api_object("api-1") with ( - app.test_request_context("/?page=1&limit=20"), + app.test_request_context("/?page=2&limit=1&keyword=vector"), patch.object( ExternalDatasetService, "get_external_knowledge_apis", - return_value=([api_item], 1), + return_value=([api_item], 3), ) as get_external_knowledge_apis, ): resp, status = method(api, "tenant-1") assert status == 200 - assert resp["total"] == 1 - assert resp["data"][0]["id"] == "1" - get_external_knowledge_apis.assert_called_once_with(1, 20, "tenant-1", None) + assert resp == { + "data": [_external_api_dict("api-1")], + "has_more": True, + "limit": 1, + "total": 3, + "page": 2, + } + get_external_knowledge_apis.assert_called_once_with(2, 1, "tenant-1", "vector") + + def test_post_success_uses_validated_payload_and_returns_template(self, app: Flask, current_user: Account): + api = ExternalApiTemplateListApi() + method = inspect.unwrap(api.post) + + payload = { + "name": "Vendor Search", + "settings": { + "endpoint": "https://external.example.com/search", + "api_key": "secret", + "headers": {"X-Source": "unit-test"}, + "timeout": 30, + }, + } + created = _external_api_object("api-created") + session = MagicMock() + + with ( + app.test_request_context("/", json=payload), + patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload), + patch.object(ExternalDatasetService, "validate_api_list") as validate_api_list, + patch.object( + ExternalDatasetService, + "create_external_knowledge_api", + return_value=created, + ) as create_external_knowledge_api, + ): + resp, status = method(api, session, "tenant-1", current_user) + + assert status == 201 + assert resp == _external_api_dict("api-created") + validate_api_list.assert_called_once_with(payload["settings"]) + create_external_knowledge_api.assert_called_once_with( + tenant_id="tenant-1", + user_id="user-1", + args=payload, + session=session, + ) def test_post_forbidden(self, app: Flask, current_user: Account): current_user.role = TenantAccountRole.NORMAL @@ -99,6 +256,28 @@ class TestExternalApiTemplateListApi: class TestExternalApiTemplateApi: + def test_get_success_returns_template_contract(self, app: Flask): + api = ExternalApiTemplateApi() + method = inspect.unwrap(api.get) + template = _external_api_object("api-detail") + session = MagicMock() + + with ( + app.test_request_context("/"), + patch.object( + ExternalDatasetService, + "get_external_knowledge_api", + return_value=template, + ) as get_external_knowledge_api, + ): + resp, status = method(api, session, "tenant-1", "api-detail") + + assert status == 200 + assert resp == _external_api_dict("api-detail") + get_external_knowledge_api.assert_called_once_with( + external_knowledge_api_id="api-detail", tenant_id="tenant-1", session=session + ) + def test_get_not_found(self, app: Flask): api = ExternalApiTemplateApi() method = inspect.unwrap(api.get) @@ -114,6 +293,44 @@ class TestExternalApiTemplateApi: with pytest.raises(NotFound): method(api, MagicMock(), "tenant-1", "api-id") + def test_patch_success_uses_validated_payload_and_returns_template(self, app: Flask, current_user: Account): + api = ExternalApiTemplateApi() + method = inspect.unwrap(api.patch) + + payload = { + "name": "Updated API", + "settings": { + "endpoint": "https://external.example.com/updated", + "api_key": "new-secret", + "headers": {"X-Version": "2"}, + }, + } + updated = _external_api_object("api-updated") + session = MagicMock() + + with ( + app.test_request_context("/", json=payload), + patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload), + patch.object(ExternalDatasetService, "validate_api_list") as validate_api_list, + patch.object( + ExternalDatasetService, + "update_external_knowledge_api", + return_value=updated, + ) as update_external_knowledge_api, + ): + resp, status = method(api, session, "tenant-1", current_user, "api-updated") + + assert status == 200 + assert resp == _external_api_dict("api-updated") + validate_api_list.assert_called_once_with(payload["settings"]) + update_external_knowledge_api.assert_called_once_with( + tenant_id="tenant-1", + user_id="user-1", + external_knowledge_api_id="api-updated", + args=payload, + session=session, + ) + def test_delete_forbidden(self, app: Flask, current_user: Account): current_user.role = TenantAccountRole.NORMAL @@ -153,46 +370,39 @@ class TestExternalDatasetCreateApi: method = inspect.unwrap(api.post) payload = { - "external_knowledge_api_id": "api", - "external_knowledge_id": "kid", - "name": "dataset", + "external_knowledge_api_id": "api-1", + "external_knowledge_id": "knowledge-1", + "name": "Support knowledge", + "description": "External support articles", + "external_retrieval_model": { + "top_k": 4, + "score_threshold": 0.5, + "score_threshold_enabled": True, + }, } - dataset = MagicMock() - - dataset.embedding_available = False - dataset.built_in_field_enabled = False - dataset.is_published = False - dataset.enable_api = False - dataset.enable_qa = False - dataset.enable_vector_store = False - dataset.vector_store_setting = None - dataset.is_multimodal = False - - dataset.retrieval_model_dict = {} - dataset.tags = [] - dataset.external_knowledge_info = None - dataset.external_retrieval_model = None - dataset.doc_metadata = [] - dataset.icon_info = None - dataset.permission_keys = [] - - dataset.summary_index_setting = MagicMock() - dataset.summary_index_setting.enable = False + dataset = _dataset_detail_object() with ( - app.test_request_context("/"), + app.test_request_context("/", json=payload), patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload), patch.object( ExternalDatasetService, "create_external_dataset", return_value=dataset, - ), - patch.object(external_module, "db", SimpleNamespace(session=lambda: MagicMock())), + ) as create_external_dataset, ): - _, status = method(api, MagicMock(), "tenant-1", current_user) + session = MagicMock() + resp, status = method(api, session, "tenant-1", current_user) assert status == 201 + assert resp == _expected_dataset_detail_payload() + create_external_dataset.assert_called_once_with( + tenant_id="tenant-1", + user_id="user-1", + args=payload, + session=session, + ) def test_create_forbidden(self, app: Flask, current_user: Account): current_user.role = TenantAccountRole.NORMAL @@ -233,24 +443,59 @@ class TestExternalKnowledgeHitTestingApi: api = ExternalKnowledgeHitTestingApi() method = inspect.unwrap(api.post) - payload = {"query": "hello"} + payload = { + "query": "hello", + "external_retrieval_model": { + "top_k": 3, + "score_threshold": 0.25, + "score_threshold_enabled": True, + }, + "metadata_filtering_conditions": { + "logical_operator": "and", + "conditions": [{"name": "source", "comparison_operator": "contains", "value": "external"}], + }, + } dataset = MagicMock() + retrieve_response = { + "query": {"content": "hello"}, + "records": [ + { + "content": "answer", + "title": "doc", + "score": 0.9, + "metadata": {"source": "external", "page": 2}, + } + ], + } + session = MagicMock() with ( - app.test_request_context("/"), + app.test_request_context("/", json=payload), patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload), patch.object(DatasetService, "get_dataset", return_value=dataset), - patch.object(DatasetService, "check_dataset_permission"), + patch.object(DatasetService, "check_dataset_permission") as check_dataset_permission, + patch.object(HitTestingService, "hit_testing_args_check") as hit_testing_args_check, patch.object( HitTestingService, "external_retrieve", - return_value={"ok": True}, - ), + return_value=retrieve_response, + ) as external_retrieve, + patch("controllers.console.datasets.external.dump_response", side_effect=lambda _model, value: value), ): - resp = method(api, MagicMock(), current_user, "dataset-id") + resp = method(api, session, current_user, "dataset-id") - assert resp["ok"] is True + assert resp == retrieve_response + check_dataset_permission.assert_called_once_with(dataset, current_user, session) + hit_testing_args_check.assert_called_once_with(payload) + external_retrieve.assert_called_once_with( + session=session, + dataset=dataset, + query="hello", + account=current_user, + external_retrieval_model=payload["external_retrieval_model"], + metadata_filtering_conditions=payload["metadata_filtering_conditions"], + ) class TestBedrockRetrievalApi: @@ -259,24 +504,46 @@ class TestBedrockRetrievalApi: method = inspect.unwrap(api.post) payload = { - "retrieval_setting": {}, - "query": "hello", - "knowledge_id": "kid", + "retrieval_setting": {"top_k": 5, "score_threshold": 0.72}, + "query": "hello bedrock", + "knowledge_id": "knowledge-base-1", + } + retrieval_response = { + "records": [ + { + "metadata": {"source": "bedrock", "uri": "s3://bucket/doc.txt"}, + "score": 0.8, + "title": "doc", + "content": "answer", + }, + { + "metadata": {"source": "bedrock", "uri": "s3://bucket/other.txt"}, + "score": 0.65, + "title": None, + "content": None, + }, + ] } + session = MagicMock() + with ( - app.test_request_context("/"), + app.test_request_context("/", json=payload), patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload), patch.object( ExternalDatasetTestService, "knowledge_retrieval", - return_value={"ok": True}, - ), + return_value=retrieval_response, + ) as knowledge_retrieval, ): resp, status = method() assert status == 200 - assert resp["ok"] is True + assert resp == retrieval_response + retrieval_setting, query, knowledge_id = knowledge_retrieval.call_args.args + assert retrieval_setting.model_dump() == payload["retrieval_setting"] + assert query == "hello bedrock" + assert knowledge_id == "knowledge-base-1" class TestExternalApiTemplateListApiAdvanced: @@ -302,10 +569,10 @@ class TestExternalApiTemplateListApiAdvanced: api = ExternalApiTemplateListApi() method = inspect.unwrap(api.get) - templates = [MagicMock(id=f"api-{i}") for i in range(3)] + templates = [_external_api_object(f"api-{i}") for i in range(3)] with ( - app.test_request_context("/?page=1&limit=20"), + app.test_request_context("/?page=2&limit=3"), patch( "controllers.console.datasets.external.ExternalDatasetService.get_external_knowledge_apis", return_value=(templates, 25), @@ -314,9 +581,14 @@ class TestExternalApiTemplateListApiAdvanced: resp, status = method(api, "tenant-1") assert status == 200 - assert resp["total"] == 25 - assert len(resp["data"]) == 3 - get_external_knowledge_apis.assert_called_once_with(1, 20, "tenant-1", None) + assert resp == { + "data": [_external_api_dict(f"api-{i}") for i in range(3)], + "has_more": True, + "limit": 3, + "total": 25, + "page": 2, + } + get_external_knowledge_apis.assert_called_once_with(2, 3, "tenant-1", None) class TestExternalDatasetCreateApiAdvanced: @@ -371,6 +643,7 @@ class TestExternalKnowledgeHitTestingApiAdvanced: "external_retrieval_model": {"type": "bm25"}, "metadata_filtering_conditions": {"status": "active"}, } + session = MagicMock() with ( app.test_request_context("/", json=payload), @@ -379,15 +652,46 @@ class TestExternalKnowledgeHitTestingApiAdvanced: "controllers.console.datasets.external.DatasetService.get_dataset", return_value=dataset, ), - patch("controllers.console.datasets.external.DatasetService.check_dataset_permission"), + patch("controllers.console.datasets.external.DatasetService.check_dataset_permission") as check_permission, + patch("controllers.console.datasets.external.HitTestingService.hit_testing_args_check") as args_check, patch( "controllers.console.datasets.external.HitTestingService.external_retrieve", - return_value={"results": []}, - ), + return_value={ + "query": {"content": "test query"}, + "records": [ + { + "content": None, + "title": "metadata-only", + "score": None, + "metadata": {"status": "active"}, + } + ], + }, + ) as external_retrieve, ): - resp = method(api, MagicMock(), current_user, "ds-1") + resp = method(api, session, current_user, "ds-1") - assert resp["results"] == [] + assert resp == { + "query": {"content": "test query"}, + "records": [ + { + "content": None, + "title": "metadata-only", + "score": None, + "metadata": {"status": "active"}, + } + ], + } + check_permission.assert_called_once_with(dataset, current_user, session) + args_check.assert_called_once_with(payload) + external_retrieve.assert_called_once_with( + session=session, + dataset=dataset, + query="test query", + account=current_user, + external_retrieval_model={"type": "bm25"}, + metadata_filtering_conditions={"status": "active"}, + ) class TestBedrockRetrievalApiAdvanced: diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py index e83724f955f..ef379afe5c6 100644 --- a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py +++ b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py @@ -15,7 +15,10 @@ Focus on: - API endpoint business logic and error handling """ +import json import uuid +from dataclasses import dataclass, field +from datetime import UTC, datetime from unittest.mock import Mock, patch import pytest @@ -39,46 +42,151 @@ from controllers.service_api.dataset.document import ( ) from controllers.service_api.dataset.error import ArchivedDocumentImmutableError from core.rag.index_processor.constant.index_type import IndexStructureType -from models.enums import IndexingStatus +from models.dataset import Dataset, Document +from models.enums import DataSourceType, DocumentCreatedFrom, DocumentDocType, IndexingStatus from services.dataset_service import DocumentService from services.entities.knowledge_entities.knowledge_entities import ProcessRule, RetrievalModel -def make_serializable_document(**overrides: object) -> Mock: - attrs: dict[str, object] = { +def _document_data_source_info() -> dict[str, str]: + return {"type": "website_crawl", "url": "https://example.com/docs", "title": "Docs"} + + +@dataclass +class _DocumentModelSessionStub: + scalar_values: list[object] = field(default_factory=list) + + def scalar(self, *args: object, **kwargs: object) -> object: + if self.scalar_values: + return self.scalar_values.pop(0) + return None + + def scalars(self, *args: object, **kwargs: object) -> object: + result = Mock() + result.all.return_value = [] + return result + + def get(self, *args: object, **kwargs: object) -> None: + return None + + +@dataclass +class _DocumentModelDbStub: + session: _DocumentModelSessionStub = field(default_factory=_DocumentModelSessionStub) + + +@dataclass +class _PaginationRecord: + items: list[Document] + total: int + + +@pytest.fixture +def mock_tenant() -> str: + return str(uuid.uuid4()) + + +@pytest.fixture +def mock_dataset(mock_tenant: str) -> Dataset: + return make_dataset(tenant_id=mock_tenant) + + +@pytest.fixture +def mock_document(mock_dataset: Dataset) -> Document: + return make_serializable_document() + + +def make_dataset(**overrides: object) -> Dataset: + attrs = { "id": str(uuid.uuid4()), + "tenant_id": str(uuid.uuid4()), + "name": "Test Dataset", + "data_source_type": DataSourceType.WEBSITE_CRAWL, + "indexing_technique": "economy", + "created_by": "user-1", + "maintainer": "user-1", + "provider": "vendor", + "summary_index_setting": None, + } + attrs.update(overrides) + return Dataset(**attrs) + + +def make_serializable_document(**overrides: object) -> Document: + data_source_info = overrides.pop("data_source_info", _document_data_source_info()) + summary_index_status = overrides.pop("summary_index_status", None) + attrs = { + "id": str(uuid.uuid4()), + "tenant_id": str(uuid.uuid4()), + "dataset_id": str(uuid.uuid4()), "position": 1, - "data_source_type": "upload_file", - "data_source_info_dict": {"upload_file_id": "file-1"}, - "data_source_detail_dict": {}, + "data_source_type": DataSourceType.WEBSITE_CRAWL, + "data_source_info": json.dumps(data_source_info), "dataset_process_rule_id": None, "batch": "batch-1", "name": "Test Document", - "created_from": "api", + "created_from": DocumentCreatedFrom.API, "created_by": "user-1", - "created_at": None, - "tokens": None, - "indexing_status": "completed", + "created_at": datetime(2021, 1, 1, tzinfo=UTC), + "tokens": 100, + "indexing_status": IndexingStatus.COMPLETED, + "completed_at": datetime(2021, 1, 1, 0, 0, 1, tzinfo=UTC), + "updated_at": datetime(2021, 1, 1, 0, 0, 2, tzinfo=UTC), + "indexing_latency": 0.5, "error": None, "enabled": True, "disabled_at": None, "disabled_by": None, "archived": False, - "display_status": "available", - "word_count": None, - "hit_count": 0, - "doc_form": "text_model", - "doc_metadata_details": None, - "summary_index_status": None, + "doc_type": DocumentDocType.BOOK, + "doc_metadata": None, + "doc_form": IndexStructureType.PARAGRAPH_INDEX, + "doc_language": "English", "need_summary": False, + "is_paused": False, + "processing_started_at": datetime(2021, 1, 1, tzinfo=UTC), + "parsing_completed_at": datetime(2021, 1, 1, 0, 0, 1, tzinfo=UTC), + "cleaning_completed_at": datetime(2021, 1, 1, 0, 0, 2, tzinfo=UTC), + "splitting_completed_at": datetime(2021, 1, 1, 0, 0, 3, tzinfo=UTC), + "paused_at": None, + "stopped_at": None, + "word_count": None, } attrs.update(overrides) - document = Mock(spec_set=list(attrs)) - for name, value in attrs.items(): - setattr(document, name, value) + document = Document(**attrs) + document.summary_index_status = summary_index_status # type: ignore[attr-defined] return document +def _expected_document_response(document: Document) -> dict[str, object]: + return { + "id": document.id, + "position": document.position, + "data_source_type": document.data_source_type, + "data_source_info": document.data_source_info_dict, + "data_source_detail_dict": document.data_source_detail_dict, + "dataset_process_rule_id": document.dataset_process_rule_id, + "name": document.name, + "created_from": document.created_from, + "created_by": document.created_by, + "created_at": int(document.created_at.timestamp()) if document.created_at else None, + "tokens": document.tokens, + "indexing_status": document.indexing_status, + "error": document.error, + "enabled": document.enabled, + "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None, + "disabled_by": document.disabled_by, + "archived": document.archived, + "display_status": document.display_status, + "word_count": document.word_count, + "hit_count": 0, + "doc_form": document.doc_form, + "doc_metadata": [], + "summary_index_status": document.summary_index_status, + "need_summary": document.need_summary, + } + + class TestDocumentTextCreatePayload: """Test suite for DocumentTextCreatePayload Pydantic model.""" @@ -263,24 +371,22 @@ class TestDocumentService: @patch.object(DocumentService, "get_document") def test_get_document_returns_document(self, mock_get: Mock) -> None: """Test get_document returns document object.""" - mock_doc = Mock() - mock_doc.id = str(uuid.uuid4()) - mock_doc.name = "Test Document" - mock_doc.indexing_status = "completed" - mock_get.return_value = mock_doc + document = make_serializable_document(name="Test Document", indexing_status="completed") + mock_get.return_value = document - result = DocumentService.get_document(dataset_id="dataset_id", document_id="doc_id", session=Mock()) + result = DocumentService.get_document(dataset_id="dataset_id", document_id="doc_id") assert result is not None + assert result == document assert result.name == "Test Document" assert result.indexing_status == "completed" @patch.object(DocumentService, "delete_document") def test_delete_document_called(self, mock_delete): """Test delete_document is called with document.""" - mock_doc = Mock() + document = make_serializable_document() session = Mock() - DocumentService.delete_document(document=mock_doc, session=session) - mock_delete.assert_called_once_with(document=mock_doc, session=session) + DocumentService.delete_document(document=document, session=session) + mock_delete.assert_called_once_with(document=document, session=session) class TestDocumentIndexingStatus: @@ -460,9 +566,9 @@ class TestDocumentServiceBatchMethods: dataset_id = str(uuid.uuid4()) doc_ids = [str(uuid.uuid4()), str(uuid.uuid4())] + session = Mock() mock_result = Mock() mock_result.all.return_value = [Mock(id=doc_ids[0]), Mock(id=doc_ids[1])] - session = Mock() session.scalars.return_value = mock_result documents = DocumentService.get_documents_by_ids(dataset_id, doc_ids, session) @@ -488,9 +594,11 @@ class TestDocumentServiceFileOperations: mock_get_file.return_value = mock_file mock_signed_url.return_value = "https://example.com/download" - url = DocumentService.get_document_download_url(mock_doc, Mock()) + session = Mock() + url = DocumentService.get_document_download_url(mock_doc, session) assert url == "https://example.com/download" + mock_get_file.assert_called_once_with(mock_doc, session) mock_signed_url.assert_called_with(upload_file_id="file_id", as_attachment=True) @@ -532,9 +640,9 @@ class TestDocumentServiceSaveValidation: # These tests call controller methods directly, bypassing the # ``DatasetApiResource.method_decorators`` (``validate_dataset_token``) by # invoking the *undecorated* method on the class instance. Every external -# dependency (``db``, service classes, ``marshal``, ``current_user``, …) is -# patched at the module where it is looked up so the real SQLAlchemy / Flask -# extensions are never touched. +# dependency (``db``, service classes, ``current_user``, …) is patched at the +# module where it is looked up so the real SQLAlchemy / Flask extensions are +# never touched. # ============================================================================= @@ -547,58 +655,33 @@ class TestDocumentApiGet: """ @pytest.fixture - def mock_doc_detail(self, mock_tenant: Mock) -> Mock: - """A document mock with every attribute ``DocumentApi.get`` reads.""" - doc = Mock() - doc.id = str(uuid.uuid4()) - doc.tenant_id = mock_tenant.id - doc.name = "test_document.txt" - doc.indexing_status = "completed" - doc.enabled = True - doc.doc_form = IndexStructureType.PARAGRAPH_INDEX - doc.doc_language = "English" - doc.doc_type = "book" - doc.doc_metadata_details = {"source": "upload"} - doc.position = 1 - doc.data_source_type = "upload_file" - doc.data_source_detail_dict = {"type": "upload_file"} - doc.dataset_process_rule_id = str(uuid.uuid4()) - doc.dataset_process_rule = None - doc.created_from = "api" - doc.created_by = str(uuid.uuid4()) - doc.created_at = Mock() - doc.created_at.timestamp.return_value = 1609459200 - doc.tokens = 100 - doc.completed_at = Mock() - doc.completed_at.timestamp.return_value = 1609459200 - doc.updated_at = Mock() - doc.updated_at.timestamp.return_value = 1609459200 - doc.indexing_latency = 0.5 - doc.error = None - doc.disabled_at = None - doc.disabled_by = None - doc.archived = False - doc.segment_count = 5 - doc.average_segment_length = 20 - doc.hit_count = 0 - doc.display_status = "available" - doc.need_summary = False - return doc + def mock_doc_detail(self, mock_tenant: str) -> Document: + """A concrete document record with every attribute ``DocumentApi.get`` reads.""" + return make_serializable_document( + id=str(uuid.uuid4()), + tenant_id=mock_tenant, + name="test_document.txt", + dataset_process_rule_id=str(uuid.uuid4()), + word_count=100, + ) @patch("controllers.service_api.dataset.document.DatasetService") @patch("controllers.service_api.dataset.document.DocumentService") def test_get_document_success_with_all_metadata( - self, mock_doc_svc: Mock, mock_dataset_svc: Mock, app: Flask, mock_tenant: Mock, mock_doc_detail: Mock + self, + mock_doc_svc: Mock, + mock_dataset_svc: Mock, + app: Flask, + mock_tenant: str, + mock_doc_detail: Document, ) -> None: """Test successful document retrieval with metadata='all'.""" # Arrange dataset_id = str(uuid.uuid4()) - mock_dataset = Mock() - mock_dataset.id = dataset_id - mock_dataset.summary_index_setting = None + mock_dataset = make_dataset(id=dataset_id, tenant_id=mock_tenant, summary_index_setting=None) mock_doc_svc.get_document.return_value = mock_doc_detail - mock_dataset_svc.get_process_rules.return_value = [] + mock_dataset_svc.get_process_rules.return_value = {"mode": "automatic", "rules": {}} # Act with app.test_request_context( @@ -606,23 +689,54 @@ class TestDocumentApiGet: method="GET", ): api = DocumentApi() - with patch.object(api, "get_dataset", return_value=mock_dataset): - response = api.get(tenant_id=mock_tenant.id, dataset_id=dataset_id, document_id=mock_doc_detail.id) + with ( + patch.object(api, "get_dataset", return_value=mock_dataset), + patch("models.dataset.db", _DocumentModelDbStub(_DocumentModelSessionStub([5, 5, 5, 5, 0]))), + ): + response = api.get(tenant_id=mock_tenant, dataset_id=dataset_id, document_id=mock_doc_detail.id) # Assert - assert response["id"] == mock_doc_detail.id - assert response["name"] == mock_doc_detail.name - assert response["indexing_status"] == mock_doc_detail.indexing_status - assert "doc_type" in response - assert "doc_metadata" in response + assert response == { + "id": mock_doc_detail.id, + "position": 1, + "data_source_type": "website_crawl", + "data_source_info": {"type": "website_crawl", "url": "https://example.com/docs", "title": "Docs"}, + "dataset_process_rule_id": mock_doc_detail.dataset_process_rule_id, + "dataset_process_rule": {"mode": "automatic", "rules": {}}, + "document_process_rule": {}, + "name": "test_document.txt", + "created_from": "api", + "created_by": "user-1", + "created_at": 1609459200, + "tokens": 100, + "indexing_status": "completed", + "completed_at": 1609459201, + "updated_at": 1609459202, + "indexing_latency": 0.5, + "error": None, + "enabled": True, + "disabled_at": None, + "disabled_by": None, + "archived": False, + "doc_type": "book", + "doc_metadata": None, + "segment_count": 5, + "average_segment_length": 20, + "hit_count": 0, + "display_status": "available", + "doc_form": "text_model", + "doc_language": "English", + "summary_index_status": None, + "need_summary": False, + } + assert response["summary_index_status"] is None @patch("controllers.service_api.dataset.document.DocumentService") - def test_get_document_not_found(self, mock_doc_svc: Mock, app: Flask, mock_tenant: Mock) -> None: + def test_get_document_not_found(self, mock_doc_svc: Mock, app: Flask, mock_tenant: str) -> None: """Test 404 when document is not found.""" # Arrange dataset_id = str(uuid.uuid4()) - mock_dataset = Mock() - mock_dataset.id = dataset_id + mock_dataset = make_dataset(id=dataset_id, tenant_id=mock_tenant) mock_doc_svc.get_document.return_value = None @@ -634,17 +748,16 @@ class TestDocumentApiGet: api = DocumentApi() with patch.object(api, "get_dataset", return_value=mock_dataset): with pytest.raises(NotFound): - api.get(tenant_id=mock_tenant.id, dataset_id=dataset_id, document_id="nonexistent") + api.get(tenant_id=mock_tenant, dataset_id=dataset_id, document_id="nonexistent") @patch("controllers.service_api.dataset.document.DocumentService") def test_get_document_forbidden_wrong_tenant( - self, mock_doc_svc: Mock, app: Flask, mock_tenant: Mock, mock_doc_detail: Mock + self, mock_doc_svc: Mock, app: Flask, mock_tenant: str, mock_doc_detail: Document ) -> None: """Test 403 when document tenant doesn't match request tenant.""" # Arrange dataset_id = str(uuid.uuid4()) - mock_dataset = Mock() - mock_dataset.id = dataset_id + mock_dataset = make_dataset(id=dataset_id, tenant_id=mock_tenant) mock_doc_detail.tenant_id = "different-tenant-id" mock_doc_svc.get_document.return_value = mock_doc_detail @@ -657,18 +770,16 @@ class TestDocumentApiGet: api = DocumentApi() with patch.object(api, "get_dataset", return_value=mock_dataset): with pytest.raises(Forbidden): - api.get(tenant_id=mock_tenant.id, dataset_id=dataset_id, document_id=mock_doc_detail.id) + api.get(tenant_id=mock_tenant, dataset_id=dataset_id, document_id=mock_doc_detail.id) @patch("controllers.service_api.dataset.document.DocumentService") def test_get_document_metadata_only( - self, mock_doc_svc: Mock, app: Flask, mock_tenant: Mock, mock_doc_detail: Mock + self, mock_doc_svc: Mock, app: Flask, mock_tenant: str, mock_doc_detail: Document ) -> None: """Test document retrieval with metadata='only'.""" # Arrange dataset_id = str(uuid.uuid4()) - mock_dataset = Mock() - mock_dataset.id = dataset_id - mock_dataset.summary_index_setting = None + mock_dataset = make_dataset(id=dataset_id, tenant_id=mock_tenant, summary_index_setting=None) mock_doc_svc.get_document.return_value = mock_doc_detail @@ -679,28 +790,33 @@ class TestDocumentApiGet: ): api = DocumentApi() with patch.object(api, "get_dataset", return_value=mock_dataset): - response = api.get(tenant_id=mock_tenant.id, dataset_id=dataset_id, document_id=mock_doc_detail.id) + response = api.get(tenant_id=mock_tenant, dataset_id=dataset_id, document_id=mock_doc_detail.id) # Assert — metadata='only' returns only id, doc_type, doc_metadata assert response["id"] == mock_doc_detail.id - assert "doc_type" in response - assert "doc_metadata" in response - assert "name" not in response + assert response == { + "id": mock_doc_detail.id, + "doc_type": mock_doc_detail.doc_type, + "doc_metadata": None, + } @patch("controllers.service_api.dataset.document.DatasetService") @patch("controllers.service_api.dataset.document.DocumentService") def test_get_document_metadata_without( - self, mock_doc_svc: Mock, mock_dataset_svc: Mock, app: Flask, mock_tenant: Mock, mock_doc_detail: Mock + self, + mock_doc_svc: Mock, + mock_dataset_svc: Mock, + app: Flask, + mock_tenant: str, + mock_doc_detail: Document, ) -> None: """Test document retrieval with metadata='without'.""" # Arrange dataset_id = str(uuid.uuid4()) - mock_dataset = Mock() - mock_dataset.id = dataset_id - mock_dataset.summary_index_setting = None + mock_dataset = make_dataset(id=dataset_id, tenant_id=mock_tenant, summary_index_setting=None) mock_doc_svc.get_document.return_value = mock_doc_detail - mock_dataset_svc.get_process_rules.return_value = [] + mock_dataset_svc.get_process_rules.return_value = {"mode": "automatic", "rules": {}} # Act with app.test_request_context( @@ -708,25 +824,60 @@ class TestDocumentApiGet: method="GET", ): api = DocumentApi() - with patch.object(api, "get_dataset", return_value=mock_dataset): - response = api.get(tenant_id=mock_tenant.id, dataset_id=dataset_id, document_id=mock_doc_detail.id) + with ( + patch.object(api, "get_dataset", return_value=mock_dataset), + patch("models.dataset.db", _DocumentModelDbStub(_DocumentModelSessionStub([5, 5, 5, 5, 0]))), + ): + response = api.get(tenant_id=mock_tenant, dataset_id=dataset_id, document_id=mock_doc_detail.id) # Assert — metadata='without' omits doc_type / doc_metadata assert response["id"] == mock_doc_detail.id assert "doc_type" not in response assert "doc_metadata" not in response assert "name" in response + assert set(response) == { + "id", + "position", + "data_source_type", + "data_source_info", + "dataset_process_rule_id", + "dataset_process_rule", + "document_process_rule", + "name", + "created_from", + "created_by", + "created_at", + "tokens", + "indexing_status", + "completed_at", + "updated_at", + "indexing_latency", + "error", + "enabled", + "disabled_at", + "disabled_by", + "archived", + "segment_count", + "average_segment_length", + "hit_count", + "display_status", + "doc_form", + "doc_language", + "summary_index_status", + "need_summary", + } + assert response["error"] is None + assert response["disabled_at"] is None + assert response["disabled_by"] is None @patch("controllers.service_api.dataset.document.DocumentService") def test_get_document_invalid_metadata_value( - self, mock_doc_svc: Mock, app: Flask, mock_tenant: Mock, mock_doc_detail: Mock + self, mock_doc_svc: Mock, app: Flask, mock_tenant: str, mock_doc_detail: Document ) -> None: """Test error when metadata parameter has invalid value.""" # Arrange dataset_id = str(uuid.uuid4()) - mock_dataset = Mock() - mock_dataset.id = dataset_id - mock_dataset.summary_index_setting = None + mock_dataset = make_dataset(id=dataset_id, tenant_id=mock_tenant, summary_index_setting=None) mock_doc_svc.get_document.return_value = mock_doc_detail @@ -738,7 +889,7 @@ class TestDocumentApiGet: api = DocumentApi() with patch.object(api, "get_dataset", return_value=mock_dataset): with pytest.raises(InvalidMetadataError): - api.get(tenant_id=mock_tenant.id, dataset_id=dataset_id, document_id=mock_doc_detail.id) + api.get(tenant_id=mock_tenant, dataset_id=dataset_id, document_id=mock_doc_detail.id) class TestDocumentApiDelete: @@ -763,8 +914,7 @@ class TestDocumentApiDelete: """Test successful document deletion.""" # Arrange dataset_id = str(uuid.uuid4()) - mock_dataset = Mock() - mock_dataset.id = dataset_id + mock_dataset = make_dataset(id=dataset_id, tenant_id=mock_tenant) mock_db.session.scalar.return_value = mock_dataset mock_doc_svc.get_document.return_value = mock_document @@ -778,7 +928,7 @@ class TestDocumentApiDelete: ): api = DocumentApi() response = self._call_delete( - api, tenant_id=mock_tenant.id, dataset_id=dataset_id, document_id=mock_document.id + api, tenant_id=mock_tenant, dataset_id=dataset_id, document_id=mock_document.id ) # Assert @@ -792,8 +942,7 @@ class TestDocumentApiDelete: # Arrange dataset_id = str(uuid.uuid4()) document_id = str(uuid.uuid4()) - mock_dataset = Mock() - mock_dataset.id = dataset_id + mock_dataset = make_dataset(id=dataset_id, tenant_id=mock_tenant) mock_db.session.scalar.return_value = mock_dataset mock_doc_svc.get_document.return_value = None @@ -805,7 +954,7 @@ class TestDocumentApiDelete: ): api = DocumentApi() with pytest.raises(NotFound): - self._call_delete(api, tenant_id=mock_tenant.id, dataset_id=dataset_id, document_id=document_id) + self._call_delete(api, tenant_id=mock_tenant, dataset_id=dataset_id, document_id=document_id) @patch("controllers.service_api.dataset.document.DocumentService") @patch("controllers.service_api.dataset.document.db") @@ -813,8 +962,7 @@ class TestDocumentApiDelete: """Test ArchivedDocumentImmutableError when deleting archived document.""" # Arrange dataset_id = str(uuid.uuid4()) - mock_dataset = Mock() - mock_dataset.id = dataset_id + mock_dataset = make_dataset(id=dataset_id, tenant_id=mock_tenant) mock_db.session.scalar.return_value = mock_dataset mock_doc_svc.get_document.return_value = mock_document @@ -827,7 +975,7 @@ class TestDocumentApiDelete: ): api = DocumentApi() with pytest.raises(ArchivedDocumentImmutableError): - self._call_delete(api, tenant_id=mock_tenant.id, dataset_id=dataset_id, document_id=mock_document.id) + self._call_delete(api, tenant_id=mock_tenant, dataset_id=dataset_id, document_id=mock_document.id) @patch("controllers.service_api.dataset.document.DocumentService") @patch("controllers.service_api.dataset.document.db") @@ -845,7 +993,7 @@ class TestDocumentApiDelete: ): api = DocumentApi() with pytest.raises(ValueError, match="Dataset does not exist."): - self._call_delete(api, tenant_id=mock_tenant.id, dataset_id=dataset_id, document_id=document_id) + self._call_delete(api, tenant_id=mock_tenant, dataset_id=dataset_id, document_id=document_id) class TestDocumentListApi: @@ -859,17 +1007,18 @@ class TestDocumentListApi: # Arrange mock_db.session.scalar.return_value = mock_dataset - mock_pagination = Mock() - mock_pagination.items = [ + documents = [ make_serializable_document( id="doc-1", name="Document 1", - doc_metadata_details=[{"id": "meta-1", "name": "amount", "type": "number", "value": 42}], + tenant_id=mock_tenant, + dataset_id=mock_dataset.id, + ), + make_serializable_document( + id="doc-2", name="Document 2", tenant_id=mock_tenant, dataset_id=mock_dataset.id ), - make_serializable_document(id="doc-2", name="Document 2"), ] - mock_pagination.total = 2 - mock_paginate.return_value = mock_pagination + mock_paginate.return_value = _PaginationRecord(items=documents, total=2) mock_doc_svc.enrich_documents_with_summary_index_status.return_value = None @@ -879,17 +1028,17 @@ class TestDocumentListApi: method="GET", ): api = DocumentListApi() - response = api.get(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id) + with patch("models.dataset.db", _DocumentModelDbStub(_DocumentModelSessionStub([0, 0]))): + response = api.get(tenant_id=mock_tenant, dataset_id=mock_dataset.id) # Assert - assert "data" in response - assert "total" in response - assert response["page"] == 1 - assert response["limit"] == 20 - assert response["total"] == 2 - assert response["data"][0]["id"] == "doc-1" - assert response["data"][0]["data_source_info"] == {"upload_file_id": "file-1"} - assert response["data"][0]["doc_metadata"][0]["value"] == 42 + assert response == { + "data": [_expected_document_response(documents[0]), _expected_document_response(documents[1])], + "has_more": False, + "limit": 20, + "total": 2, + "page": 1, + } assert "data_source_info_dict" not in response["data"][0] assert "doc_metadata_details" not in response["data"][0] @@ -906,7 +1055,7 @@ class TestDocumentListApi: ): api = DocumentListApi() with pytest.raises(NotFound): - api.get(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id) + api.get(tenant_id=mock_tenant, dataset_id=mock_dataset.id) class TestDocumentIndexingStatusApi: @@ -918,20 +1067,14 @@ class TestDocumentIndexingStatusApi: """Test successful indexing status retrieval.""" # Arrange batch_id = "batch_123" - mock_doc = Mock() - mock_doc.id = str(uuid.uuid4()) - mock_doc.is_paused = False - mock_doc.indexing_status = "completed" - mock_doc.processing_started_at = None - mock_doc.parsing_completed_at = None - mock_doc.cleaning_completed_at = None - mock_doc.splitting_completed_at = None - mock_doc.completed_at = None - mock_doc.paused_at = None - mock_doc.error = None - mock_doc.stopped_at = None + document = make_serializable_document( + id=str(uuid.uuid4()), + tenant_id=mock_tenant, + dataset_id=mock_dataset.id, + completed_at=datetime(2021, 1, 1, 0, 0, 4, tzinfo=UTC), + ) - mock_doc_svc.get_batch_documents.return_value = [mock_doc] + mock_doc_svc.get_batch_documents.return_value = [document] # scalar() called 3 times: dataset lookup, completed_segments count, total_segments count mock_db.session.scalar.side_effect = [mock_dataset, 5, 5] @@ -942,17 +1085,27 @@ class TestDocumentIndexingStatusApi: method="GET", ): api = DocumentIndexingStatusApi() - response = api.get(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id, batch=batch_id) + response = api.get(tenant_id=mock_tenant, dataset_id=mock_dataset.id, batch=batch_id) # Assert - assert "data" in response - assert len(response["data"]) == 1 - item = response["data"][0] - assert item["id"] == mock_doc.id - assert item["indexing_status"] == "completed" - assert item["completed_segments"] == 5 - assert item["total_segments"] == 5 - assert item["processing_started_at"] is None + assert response == { + "data": [ + { + "id": document.id, + "indexing_status": "completed", + "processing_started_at": 1609459200, + "parsing_completed_at": 1609459201, + "cleaning_completed_at": 1609459202, + "splitting_completed_at": 1609459203, + "completed_at": 1609459204, + "paused_at": None, + "error": None, + "stopped_at": None, + "completed_segments": 5, + "total_segments": 5, + } + ] + } @patch("controllers.service_api.dataset.document.db") def test_get_indexing_status_dataset_not_found(self, mock_db, app: Flask, mock_tenant, mock_dataset): @@ -968,7 +1121,7 @@ class TestDocumentIndexingStatusApi: ): api = DocumentIndexingStatusApi() with pytest.raises(NotFound): - api.get(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id, batch=batch_id) + api.get(tenant_id=mock_tenant, dataset_id=mock_dataset.id, batch=batch_id) @patch("controllers.service_api.dataset.document.DocumentService") @patch("controllers.service_api.dataset.document.db") @@ -988,7 +1141,7 @@ class TestDocumentIndexingStatusApi: ): api = DocumentIndexingStatusApi() with pytest.raises(NotFound): - api.get(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id, batch=batch_id) + api.get(tenant_id=mock_tenant, dataset_id=mock_dataset.id, batch=batch_id) class TestDocumentAddByTextApi: @@ -1051,7 +1204,7 @@ class TestDocumentAddByTextApi: ): """Test successful document creation by text.""" # Arrange — neutralise billing decorators - self._setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + self._setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_db.session.scalar.return_value = mock_dataset mock_dataset.indexing_technique = "economy" @@ -1082,16 +1235,14 @@ class TestDocumentAddByTextApi: headers={"Authorization": "Bearer test_token"}, ): api = DocumentAddByTextApi() - response, status = api.post(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id) + with patch("models.dataset.db", _DocumentModelDbStub(_DocumentModelSessionStub([object(), 0]))): + response, status = api.post(tenant_id=mock_tenant, dataset_id=mock_dataset.id) # Assert - assert status == 200 - assert "document" in response - assert "batch" in response - assert response["batch"] == "batch_123" - assert response["document"]["id"] == "doc-create-text" - assert response["document"]["data_source_info"] == {"upload_file_id": "file-1"} - assert response["document"]["doc_metadata"] == [] + assert (response, status) == ( + {"document": _expected_document_response(mock_doc), "batch": "batch_123"}, + 200, + ) assert "data_source_info_dict" not in response["document"] @patch("controllers.service_api.wraps.FeatureService") @@ -1102,7 +1253,7 @@ class TestDocumentAddByTextApi: ): """Test ValueError when dataset not found.""" # Arrange — neutralise billing decorators - self._setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + self._setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_db.session.scalar.return_value = None @@ -1115,7 +1266,7 @@ class TestDocumentAddByTextApi: ): api = DocumentAddByTextApi() with pytest.raises(ValueError, match="Dataset does not exist."): - api.post(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id) + api.post(tenant_id=mock_tenant, dataset_id=mock_dataset.id) @patch("controllers.service_api.wraps.FeatureService") @patch("controllers.service_api.wraps.validate_and_get_api_token") @@ -1130,7 +1281,7 @@ class TestDocumentAddByTextApi: document creation paths instead of leaking a ``KeyError`` from the dumped payload dict. """ # Arrange — neutralise billing decorators - self._setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + self._setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_dataset.indexing_technique = None mock_db.session.scalar.return_value = mock_dataset @@ -1144,7 +1295,7 @@ class TestDocumentAddByTextApi: ): api = DocumentAddByTextApi() with pytest.raises(ValueError, match="indexing_technique is required."): - api.post(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id) + api.post(tenant_id=mock_tenant, dataset_id=mock_dataset.id) class TestArchivedDocumentImmutableError: @@ -1237,9 +1388,8 @@ class TestDocumentUpdateByTextApiPost: mock_dataset, ): """Test successful document update by text.""" - _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_dataset.indexing_technique = "economy" - mock_dataset.latest_process_rule = Mock() mock_db.session.scalar.return_value = mock_dataset mock_current_user.id = "user-1" @@ -1259,17 +1409,17 @@ class TestDocumentUpdateByTextApiPost: headers={"Authorization": "Bearer test_token"}, ): api = DocumentUpdateByTextApi() - response, status = api.post( - tenant_id=mock_tenant.id, - dataset_id=mock_dataset.id, - document_id=doc_id, - ) + with patch("models.dataset.db", _DocumentModelDbStub(_DocumentModelSessionStub([object(), 0]))): + response, status = api.post( + tenant_id=mock_tenant, + dataset_id=mock_dataset.id, + document_id=doc_id, + ) - assert status == 200 - assert "document" in response - assert response["batch"] == "batch-1" - assert response["document"]["id"] == "doc-update-text" - assert response["document"]["doc_metadata"] == [] + assert (response, status) == ( + {"document": _expected_document_response(mock_document), "batch": "batch-1"}, + 200, + ) @patch("controllers.service_api.dataset.document.db") @patch("controllers.service_api.wraps.FeatureService") @@ -1284,7 +1434,7 @@ class TestDocumentUpdateByTextApiPost: mock_dataset, ): """Test ValueError when dataset not found.""" - _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_db.session.scalar.return_value = None doc_id = str(uuid.uuid4()) @@ -1297,7 +1447,7 @@ class TestDocumentUpdateByTextApiPost: api = DocumentUpdateByTextApi() with pytest.raises(ValueError, match="Dataset does not exist"): api.post( - tenant_id=mock_tenant.id, + tenant_id=mock_tenant, dataset_id=mock_dataset.id, document_id=doc_id, ) @@ -1329,12 +1479,10 @@ class TestDocumentAddByFileApiPost: mock_dataset, ): """Test successful document creation by file.""" - _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_dataset.provider = "vendor" mock_dataset.indexing_technique = "economy" mock_dataset.chunk_structure = None - mock_dataset.latest_process_rule = Mock() - mock_dataset.created_by_account = Mock() mock_db.session.scalar.return_value = mock_dataset mock_current_user.id = "user-1" @@ -1357,13 +1505,13 @@ class TestDocumentAddByFileApiPost: headers={"Authorization": "Bearer test_token"}, ): api = DocumentAddByFileApi() - response, status = api.post(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id) + with patch("models.dataset.db", _DocumentModelDbStub(_DocumentModelSessionStub([object(), 0]))): + response, status = api.post(tenant_id=mock_tenant, dataset_id=mock_dataset.id) - assert status == 200 - assert response["batch"] == "batch-file" - assert response["document"]["id"] == "doc-create-file" - assert response["document"]["data_source_info"] == {"upload_file_id": "file-1"} - assert response["document"]["doc_metadata"] == [] + assert (response, status) == ( + {"document": _expected_document_response(mock_document), "batch": "batch-file"}, + 200, + ) @patch("controllers.service_api.dataset.document.db") @patch("controllers.service_api.wraps.FeatureService") @@ -1378,7 +1526,7 @@ class TestDocumentAddByFileApiPost: mock_dataset, ): """Test ValueError when dataset not found.""" - _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_db.session.scalar.return_value = None from io import BytesIO @@ -1393,7 +1541,7 @@ class TestDocumentAddByFileApiPost: ): api = DocumentAddByFileApi() with pytest.raises(ValueError, match="Dataset does not exist"): - api.post(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id) + api.post(tenant_id=mock_tenant, dataset_id=mock_dataset.id) @patch("controllers.service_api.dataset.document.db") @patch("controllers.service_api.wraps.FeatureService") @@ -1408,7 +1556,7 @@ class TestDocumentAddByFileApiPost: mock_dataset, ): """Test ValueError when dataset is external.""" - _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_dataset.provider = "external" mock_db.session.scalar.return_value = mock_dataset @@ -1424,7 +1572,7 @@ class TestDocumentAddByFileApiPost: ): api = DocumentAddByFileApi() with pytest.raises(ValueError, match="External datasets"): - api.post(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id) + api.post(tenant_id=mock_tenant, dataset_id=mock_dataset.id) @patch("controllers.service_api.dataset.document.db") @patch("controllers.service_api.wraps.FeatureService") @@ -1441,7 +1589,7 @@ class TestDocumentAddByFileApiPost: """Test NoFileUploadedError when no file in request.""" from controllers.common.errors import NoFileUploadedError - _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_dataset.provider = "vendor" mock_dataset.indexing_technique = "economy" mock_dataset.chunk_structure = None @@ -1456,7 +1604,7 @@ class TestDocumentAddByFileApiPost: ): api = DocumentAddByFileApi() with pytest.raises(NoFileUploadedError): - api.post(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id) + api.post(tenant_id=mock_tenant, dataset_id=mock_dataset.id) @patch("controllers.service_api.dataset.document.db") @patch("controllers.service_api.wraps.FeatureService") @@ -1471,7 +1619,7 @@ class TestDocumentAddByFileApiPost: mock_dataset, ): """Test ValueError when indexing_technique is missing.""" - _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_dataset.provider = "vendor" mock_dataset.indexing_technique = None mock_dataset.chunk_structure = None @@ -1489,7 +1637,7 @@ class TestDocumentAddByFileApiPost: ): api = DocumentAddByFileApi() with pytest.raises(ValueError, match="indexing_technique is required"): - api.post(tenant_id=mock_tenant.id, dataset_id=mock_dataset.id) + api.post(tenant_id=mock_tenant, dataset_id=mock_dataset.id) class TestDocumentUpdateByFileApiPatch: @@ -1514,8 +1662,11 @@ class TestDocumentUpdateByFileApiPatch: mock_dataset, ): """Test legacy POST aliases still dispatch while marked deprecated.""" - _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) - mock_update_document_by_file.return_value = ({"document": {"id": "doc-1"}, "batch": "batch-1"}, 200) + _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) + mock_update_document_by_file.return_value = ( + make_serializable_document(id="doc-1", batch="batch-1"), + "batch-1", + ) doc_id = str(uuid.uuid4()) with app.test_request_context( @@ -1524,16 +1675,19 @@ class TestDocumentUpdateByFileApiPatch: headers={"Authorization": "Bearer test_token"}, ): api = DeprecatedDocumentUpdateByFileApi() - response, status = api.post( - tenant_id=mock_tenant.id, - dataset_id=mock_dataset.id, - document_id=doc_id, - ) + with patch("models.dataset.db", _DocumentModelDbStub(_DocumentModelSessionStub([0]))): + response, status = api.post( + tenant_id=mock_tenant, + dataset_id=mock_dataset.id, + document_id=doc_id, + ) - assert status == 200 - assert response["batch"] == "batch-1" + assert (response, status) == ( + {"document": _expected_document_response(mock_update_document_by_file.return_value[0]), "batch": "batch-1"}, + 200, + ) mock_update_document_by_file.assert_called_once_with( - tenant_id=mock_tenant.id, + tenant_id=mock_tenant, dataset_id=mock_dataset.id, document_id=doc_id, ) @@ -1551,7 +1705,7 @@ class TestDocumentUpdateByFileApiPatch: mock_dataset, ): """Test ValueError when dataset not found.""" - _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_db.session.scalar.return_value = None from io import BytesIO @@ -1568,7 +1722,7 @@ class TestDocumentUpdateByFileApiPatch: api = DocumentApi() with pytest.raises(ValueError, match="Dataset does not exist"): api.patch( - tenant_id=mock_tenant.id, + tenant_id=mock_tenant, dataset_id=mock_dataset.id, document_id=doc_id, ) @@ -1586,7 +1740,7 @@ class TestDocumentUpdateByFileApiPatch: mock_dataset, ): """Test ValueError when dataset is external.""" - _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_dataset.provider = "external" mock_db.session.scalar.return_value = mock_dataset @@ -1604,7 +1758,7 @@ class TestDocumentUpdateByFileApiPatch: api = DocumentApi() with pytest.raises(ValueError, match="External datasets"): api.patch( - tenant_id=mock_tenant.id, + tenant_id=mock_tenant, dataset_id=mock_dataset.id, document_id=doc_id, ) @@ -1628,12 +1782,10 @@ class TestDocumentUpdateByFileApiPatch: mock_dataset, ): """Test successful document update by file.""" - _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant.id) + _setup_billing_mocks(mock_validate_token, mock_feature_svc, mock_tenant) mock_dataset.indexing_technique = "economy" mock_dataset.provider = "vendor" mock_dataset.chunk_structure = None - mock_dataset.latest_process_rule = Mock() - mock_dataset.created_by_account = Mock() mock_db.session.scalar.return_value = mock_dataset mock_current_user.id = "user-1" @@ -1657,14 +1809,14 @@ class TestDocumentUpdateByFileApiPatch: headers={"Authorization": "Bearer test_token"}, ): api = DocumentApi() - response, status = api.patch( - tenant_id=mock_tenant.id, - dataset_id=mock_dataset.id, - document_id=doc_id, - ) + with patch("models.dataset.db", _DocumentModelDbStub(_DocumentModelSessionStub([object(), 0]))): + response, status = api.patch( + tenant_id=mock_tenant, + dataset_id=mock_dataset.id, + document_id=doc_id, + ) - assert status == 200 - assert "document" in response - assert response["batch"] == "batch-1" - assert response["document"]["id"] == "doc-update-file" - assert response["document"]["data_source_info"] == {"upload_file_id": "file-1"} + assert (response, status) == ( + {"document": _expected_document_response(mock_document), "batch": "batch-1"}, + 200, + ) diff --git a/packages/contracts/generated/api/console/datasets/orpc.gen.ts b/packages/contracts/generated/api/console/datasets/orpc.gen.ts index a8b096168bb..ba3b00c6358 100644 --- a/packages/contracts/generated/api/console/datasets/orpc.gen.ts +++ b/packages/contracts/generated/api/console/datasets/orpc.gen.ts @@ -359,8 +359,12 @@ export const get5 = oc .input(z.object({ params: zGetDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdPath })) .output(zGetDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdResponse) +/** + * Update external knowledge API template + */ export const patch = oc .route({ + description: 'Update external knowledge API template', inputStructure: 'detailed', method: 'PATCH', operationId: 'patchDatasetsExternalKnowledgeApiByExternalKnowledgeApiId', @@ -397,8 +401,12 @@ export const get6 = oc .input(z.object({ query: zGetDatasetsExternalKnowledgeApiQuery.optional() })) .output(zGetDatasetsExternalKnowledgeApiResponse) +/** + * Create external knowledge API template + */ export const post4 = oc .route({ + description: 'Create external knowledge API template', inputStructure: 'detailed', method: 'POST', operationId: 'postDatasetsExternalKnowledgeApi', @@ -444,7 +452,6 @@ export const post6 = oc method: 'POST', operationId: 'postDatasetsInit', path: '/datasets/init', - successStatus: 201, tags: ['console'], }) .input(z.object({ body: zPostDatasetsInitBody })) @@ -1184,12 +1191,13 @@ export const segments = { * - generating: Number of summaries being generated * - error: Number of summaries with errors * - not_started: Number of segments without summary records + * - timeout: Number of summaries that timed out * - summaries: List of summary records with status and content preview */ export const get22 = oc .route({ description: - 'Get summary index generation status for a document\nReturns:\n- total_segments: Total number of segments in the document\n- summary_status: Dictionary with status counts\n - completed: Number of summaries completed\n - generating: Number of summaries being generated\n - error: Number of summaries with errors\n - not_started: Number of segments without summary records\n- summaries: List of summary records with status and content preview', + 'Get summary index generation status for a document\nReturns:\n- total_segments: Total number of segments in the document\n- summary_status: Dictionary with status counts\n - completed: Number of summaries completed\n - generating: Number of summaries being generated\n - error: Number of summaries with errors\n - not_started: Number of segments without summary records\n - timeout: Number of summaries that timed out\n- summaries: List of summary records with status and content preview', inputStructure: 'detailed', method: 'GET', operationId: 'getDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatus', diff --git a/packages/contracts/generated/api/console/datasets/types.gen.ts b/packages/contracts/generated/api/console/datasets/types.gen.ts index f904ea7d717..1e2f253d3af 100644 --- a/packages/contracts/generated/api/console/datasets/types.gen.ts +++ b/packages/contracts/generated/api/console/datasets/types.gen.ts @@ -97,51 +97,12 @@ export type ExternalDatasetCreatePayload = { name: string } -export type DatasetDetail = { - app_count?: number - author_name?: string - built_in_field_enabled?: boolean - chunk_structure?: string - created_at?: number - created_by?: string - data_source_type?: string - description?: string - doc_form?: string - doc_metadata?: Array - document_count?: number - embedding_available?: boolean - embedding_model?: string - embedding_model_provider?: string - enable_api?: boolean - external_knowledge_info?: ExternalKnowledgeInfo - external_retrieval_model?: ExternalRetrievalModel - icon_info?: DatasetIconInfo - id?: string - indexing_technique?: string - is_multimodal?: boolean - is_published?: boolean - name?: string - permission?: string - permission_keys?: Array - pipeline_id?: string - provider?: string - retrieval_model_dict?: DatasetRetrievalModel - runtime_mode?: string - summary_index_setting?: AnonymousInlineModelB1954337D565 - tags?: Array - total_available_documents?: number - total_documents?: number - updated_at?: number - updated_by?: string - word_count?: number -} - export type ExternalKnowledgeApiListResponse = { data: Array has_more: boolean limit: number page: number - total: number + total: number | null } export type ExternalKnowledgeApiPayload = { @@ -154,11 +115,11 @@ export type ExternalKnowledgeApiPayload = { export type ExternalKnowledgeApiResponse = { created_at: string created_by: string - dataset_bindings?: Array + dataset_bindings: Array description: string id: string name: string - settings?: { + settings: { [key: string]: unknown } | null tenant_id: string @@ -183,8 +144,11 @@ export type IndexingEstimatePayload = { } export type IndexingEstimateResponse = { - preview: Array - qa_preview?: Array | null + currency: string + preview: Array + qa_preview?: Array | null + tokens: number + total_price: number | number total_segments: number } @@ -236,8 +200,12 @@ export type IndexingEstimate = { total_segments: number } -export type OpaqueObjectResponse = { - [key: string]: unknown +export type ProcessRuleResponse = { + limits: { + [key: string]: unknown + } + mode: ProcessRuleMode + rules?: Rule | null } export type RetrievalSettingResponse = { @@ -337,8 +305,6 @@ export type DocumentBatchDownloadZipPayload = { document_ids: Array } -export type BinaryFileResponse = Blob | File - export type GenerateSummaryPayload = { document_list: Array } @@ -347,6 +313,40 @@ export type MetadataOperationData = { operation_data: Array } +export type DocumentDetailResponse = { + archived?: boolean | null + average_segment_length?: number | null + completed_at?: number | null + created_at?: number | null + created_by?: string | null + created_from?: string | null + data_source_detail_dict?: unknown + data_source_info?: unknown + data_source_type?: string | null + dataset_process_rule?: unknown + dataset_process_rule_id?: string | null + disabled_at?: number | null + disabled_by?: string | null + display_status?: string | null + doc_form?: string | null + doc_language?: string | null + doc_metadata?: Array | null + doc_type?: string | null + document_process_rule?: unknown + enabled?: boolean | null + error?: string | null + hit_count?: number | null + id: string + indexing_latency?: number | null + indexing_status?: string | null + name?: string | null + need_summary?: boolean | null + position?: number | null + segment_count?: number | null + tokens?: number | null + updated_at?: number | null +} + export type UrlResponse = { url: string } @@ -376,6 +376,13 @@ export type SimpleResultMessageResponse = { result: string } +export type DocumentPipelineExecutionLogResponse = { + datasource_info?: JsonValue | null + datasource_node_id?: string | null + datasource_type?: string | null + input_data?: JsonValue | null +} + export type DocumentRenamePayload = { name: string } @@ -464,6 +471,12 @@ export type ChildChunkUpdatePayload = { content: string } +export type DocumentSummaryStatusResponse = { + summaries: Array + summary_status: SummaryStatusResponse + total_segments: number +} + export type ErrorDocsResponse = { data: Array total: number @@ -479,13 +492,10 @@ export type ExternalHitTestingPayload = { query: string } -export type ExternalRetrievalTestResponse - = | { - [key: string]: unknown - } - | Array<{ - [key: string]: unknown - }> +export type ExternalHitTestingResponse = { + query: ExternalHitTestingQueryResponse + records: Array +} export type HitTestingPayload = { attachment_ids?: Array | null @@ -641,68 +651,18 @@ export type DatasetTagResponse = { type: string } -export type DatasetDocMetadata = { - id?: string - name?: string - type?: string -} - -export type ExternalKnowledgeInfo = { - external_knowledge_api_endpoint?: string - external_knowledge_api_id?: string - external_knowledge_api_name?: string - external_knowledge_id?: string -} - -export type ExternalRetrievalModel = { - score_threshold?: number - score_threshold_enabled?: boolean - top_k?: number -} - -export type DatasetIconInfo = { - icon?: string - icon_background?: string - icon_type?: string - icon_url?: string -} - -export type DatasetRetrievalModel = { - reranking_enable?: boolean - reranking_mode?: string - reranking_model?: DatasetRerankingModel - score_threshold?: number - score_threshold_enabled?: boolean - search_method?: string - top_k?: number - weights?: DatasetWeightedScore -} - -export type AnonymousInlineModelB1954337D565 = { - enable?: boolean - model_name?: string - model_provider_name?: string - summary_prompt?: string -} - -export type Tag = { - id: string - name: string - type: string -} - -export type ExternalKnowledgeDatasetBindingResponse = { +export type ExternalKnowledgeApiBindingResponse = { id: string name: string } -export type IndexingEstimatePreviewItemResponse = { +export type PreviewDetail = { child_chunks?: Array | null content: string summary?: string | null } -export type IndexingEstimateQaPreviewItemResponse = { +export type QaPreviewDetail = { answer: string question: string } @@ -744,15 +704,13 @@ export type DatasetMetadataBuiltInFieldResponse = { type: string } -export type PreviewDetail = { - child_chunks?: Array | null - content: string - summary?: string | null -} +export type ProcessRuleMode = 'automatic' | 'custom' | 'hierarchical' -export type QaPreviewDetail = { - answer: string - question: string +export type Rule = { + parent_mode?: 'full-doc' | 'paragraph' | null + pre_processing_rules?: Array | null + segmentation?: Segmentation | null + subchunk_segmentation?: Segmentation | null } export type DocumentWithSegmentsResponse = { @@ -798,6 +756,8 @@ export type DocumentMetadataResponse = { value?: string | number | number | boolean | null } +export type JsonValue = unknown + export type SegmentResponse = { answer: string | null attachments: Array @@ -844,6 +804,37 @@ export type ChildChunkUpdateArgs = { id?: string | null } +export type SummaryEntryResponse = { + created_at?: number | null + error?: string | null + segment_id: string + segment_position: number + status: string + summary_preview?: string | null + updated_at?: number | null +} + +export type SummaryStatusResponse = { + completed?: number + error?: number + generating?: number + not_started?: number + timeout?: number +} + +export type ExternalHitTestingQueryResponse = { + content: string +} + +export type ExternalHitTestingRecordResponse = { + content?: string | null + metadata?: { + [key: string]: unknown + } | null + score?: number | null + title?: string | null +} + export type HitTestingQuery = { content: string } @@ -896,17 +887,6 @@ export type DatasetWeightedScoreResponse = { weight_type?: string | null } -export type DatasetRerankingModel = { - reranking_model_name?: string - reranking_provider_name?: string -} - -export type DatasetWeightedScore = { - keyword_setting?: DatasetKeywordSetting - vector_setting?: DatasetVectorSetting - weight_type?: string -} - export type InfoList = { data_source_type: 'notion_import' | 'upload_file' | 'website_crawl' file_info_list?: FileInfo | null @@ -914,15 +894,6 @@ export type InfoList = { website_info_list?: WebsiteInfo | null } -export type ProcessRuleMode = 'automatic' | 'custom' | 'hierarchical' - -export type Rule = { - parent_mode?: 'full-doc' | 'paragraph' | null - pre_processing_rules?: Array | null - segmentation?: Segmentation | null - subchunk_segmentation?: Segmentation | null -} - export type MetadataFilteringCondition = { conditions?: Array | null logical_operator?: 'and' | 'or' | null @@ -945,6 +916,17 @@ export type WeightModel = { weight_type?: 'customized' | 'keyword_first' | 'semantic_first' | null } +export type PreProcessingRule = { + enabled: boolean + id: 'remove_extra_spaces' | 'remove_stopwords' | 'remove_urls_emails' +} + +export type Segmentation = { + chunk_overlap?: number + max_tokens: number + separator?: string +} + export type MetadataDetail = { id: string name: string @@ -1018,16 +1000,6 @@ export type DatasetVectorSettingResponse = { vector_weight?: number | null } -export type DatasetKeywordSetting = { - keyword_weight?: number -} - -export type DatasetVectorSetting = { - embedding_model_name?: string - embedding_provider_name?: string - vector_weight?: number -} - export type FileInfo = { file_ids: Array } @@ -1045,17 +1017,6 @@ export type WebsiteInfo = { urls: Array } -export type PreProcessingRule = { - enabled: boolean - id: 'remove_extra_spaces' | 'remove_stopwords' | 'remove_urls_emails' -} - -export type Segmentation = { - chunk_overlap?: number - max_tokens: number - separator?: string -} - export type Condition = { comparison_operator: | '<' @@ -1264,7 +1225,7 @@ export type PostDatasetsExternalErrors = { } export type PostDatasetsExternalResponses = { - 201: DatasetDetail + 201: DatasetDetailResponse } export type PostDatasetsExternalResponse @@ -1295,6 +1256,10 @@ export type PostDatasetsExternalKnowledgeApiData = { url: '/datasets/external-knowledge-api' } +export type PostDatasetsExternalKnowledgeApiErrors = { + 403: unknown +} + export type PostDatasetsExternalKnowledgeApiResponses = { 201: ExternalKnowledgeApiResponse } @@ -1347,6 +1312,10 @@ export type PatchDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdData = { url: '/datasets/external-knowledge-api/{external_knowledge_api_id}' } +export type PatchDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdErrors = { + 404: unknown +} + export type PatchDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdResponses = { 200: ExternalKnowledgeApiResponse } @@ -1396,7 +1365,7 @@ export type PostDatasetsInitErrors = { } export type PostDatasetsInitResponses = { - 201: DatasetAndDocumentResponse + 200: DatasetAndDocumentResponse } export type PostDatasetsInitResponse = PostDatasetsInitResponses[keyof PostDatasetsInitResponses] @@ -1439,7 +1408,7 @@ export type GetDatasetsProcessRuleData = { } export type GetDatasetsProcessRuleResponses = { - 200: OpaqueObjectResponse + 200: ProcessRuleResponse } export type GetDatasetsProcessRuleResponse @@ -1581,7 +1550,7 @@ export type GetDatasetsByDatasetIdBatchByBatchIndexingEstimateData = { } export type GetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponses = { - 200: OpaqueObjectResponse + 200: IndexingEstimateResponse } export type GetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponse @@ -1669,7 +1638,9 @@ export type PostDatasetsByDatasetIdDocumentsDownloadZipData = { } export type PostDatasetsByDatasetIdDocumentsDownloadZipResponses = { - 200: BinaryFileResponse + 200: { + [key: string]: unknown + } } export type PostDatasetsByDatasetIdDocumentsDownloadZipResponse @@ -1764,7 +1735,7 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdErrors = { } export type GetDatasetsByDatasetIdDocumentsByDocumentIdResponses = { - 200: OpaqueObjectResponse + 200: DocumentDetailResponse } export type GetDatasetsByDatasetIdDocumentsByDocumentIdResponse @@ -1803,7 +1774,7 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateErrors = } export type GetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateResponses = { - 200: OpaqueObjectResponse + 200: IndexingEstimateResponse } export type GetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateResponse @@ -1880,7 +1851,7 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogData } export type GetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogResponses = { - 200: OpaqueObjectResponse + 200: DocumentPipelineExecutionLogResponse } export type GetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogResponse @@ -2225,7 +2196,7 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusErrors = { } export type GetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusResponses = { - 200: OpaqueObjectResponse + 200: DocumentSummaryStatusResponse } export type GetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusResponse @@ -2283,7 +2254,7 @@ export type PostDatasetsByDatasetIdExternalHitTestingErrors = { } export type PostDatasetsByDatasetIdExternalHitTestingResponses = { - 200: ExternalRetrievalTestResponse + 200: ExternalHitTestingResponse } export type PostDatasetsByDatasetIdExternalHitTestingResponse diff --git a/packages/contracts/generated/api/console/datasets/zod.gen.ts b/packages/contracts/generated/api/console/datasets/zod.gen.ts index 10ceb11cb5d..2ae01a8cfe3 100644 --- a/packages/contracts/generated/api/console/datasets/zod.gen.ts +++ b/packages/contracts/generated/api/console/datasets/zod.gen.ts @@ -91,11 +91,6 @@ export const zNotionEstimatePayload = z.object({ process_rule: z.record(z.string(), z.unknown()), }) -/** - * OpaqueObjectResponse - */ -export const zOpaqueObjectResponse = z.record(z.string(), z.unknown()) - /** * RetrievalSettingResponse */ @@ -127,11 +122,6 @@ export const zDocumentBatchDownloadZipPayload = z.object({ document_ids: z.array(z.uuid()).min(1).max(100), }) -/** - * BinaryFileResponse - */ -export const zBinaryFileResponse = z.custom() - /** * GenerateSummaryPayload */ @@ -247,14 +237,6 @@ export const zExternalHitTestingPayload = z.object({ query: z.string(), }) -/** - * ExternalRetrievalTestResponse - */ -export const zExternalRetrievalTestResponse = z.union([ - z.record(z.string(), z.unknown()), - z.array(z.record(z.string(), z.unknown())), -]) - /** * MetadataArgs */ @@ -397,52 +379,10 @@ export const zDatasetTagResponse = z.object({ type: z.string(), }) -export const zDatasetDocMetadata = z.object({ - id: z.string().optional(), - name: z.string().optional(), - type: z.string().optional(), -}) - -export const zExternalKnowledgeInfo = z.object({ - external_knowledge_api_endpoint: z.string().optional(), - external_knowledge_api_id: z.string().optional(), - external_knowledge_api_name: z.string().optional(), - external_knowledge_id: z.string().optional(), -}) - -export const zExternalRetrievalModel = z.object({ - score_threshold: z.number().optional(), - score_threshold_enabled: z.boolean().optional(), - top_k: z.int().optional(), -}) - -export const zDatasetIconInfo = z.object({ - icon: z.string().optional(), - icon_background: z.string().optional(), - icon_type: z.string().optional(), - icon_url: z.string().optional(), -}) - -export const zAnonymousInlineModelB1954337D565 = z.object({ - enable: z.boolean().optional(), - model_name: z.string().optional(), - model_provider_name: z.string().optional(), - summary_prompt: z.string().optional(), -}) - /** - * Tag + * ExternalKnowledgeApiBindingResponse */ -export const zTag = z.object({ - id: z.string(), - name: z.string(), - type: z.string(), -}) - -/** - * ExternalKnowledgeDatasetBindingResponse - */ -export const zExternalKnowledgeDatasetBindingResponse = z.object({ +export const zExternalKnowledgeApiBindingResponse = z.object({ id: z.string(), name: z.string(), }) @@ -453,11 +393,11 @@ export const zExternalKnowledgeDatasetBindingResponse = z.object({ export const zExternalKnowledgeApiResponse = z.object({ created_at: z.string(), created_by: z.string(), - dataset_bindings: z.array(zExternalKnowledgeDatasetBindingResponse).optional(), + dataset_bindings: z.array(zExternalKnowledgeApiBindingResponse), description: z.string(), id: z.string(), name: z.string(), - settings: z.record(z.string(), z.unknown()).nullish(), + settings: z.record(z.string(), z.unknown()).nullable(), tenant_id: z.string(), }) @@ -469,22 +409,22 @@ export const zExternalKnowledgeApiListResponse = z.object({ has_more: z.boolean(), limit: z.int(), page: z.int(), - total: z.int(), + total: z.int().nullable(), }) /** - * IndexingEstimatePreviewItemResponse + * PreviewDetail */ -export const zIndexingEstimatePreviewItemResponse = z.object({ +export const zPreviewDetail = z.object({ child_chunks: z.array(z.string()).nullish(), content: z.string(), summary: z.string().nullish(), }) /** - * IndexingEstimateQaPreviewItemResponse + * QAPreviewDetail */ -export const zIndexingEstimateQaPreviewItemResponse = z.object({ +export const zQaPreviewDetail = z.object({ answer: z.string(), question: z.string(), }) @@ -493,8 +433,20 @@ export const zIndexingEstimateQaPreviewItemResponse = z.object({ * IndexingEstimateResponse */ export const zIndexingEstimateResponse = z.object({ - preview: z.array(zIndexingEstimatePreviewItemResponse), - qa_preview: z.array(zIndexingEstimateQaPreviewItemResponse).nullish(), + currency: z.string(), + preview: z.array(zPreviewDetail), + qa_preview: z.array(zQaPreviewDetail).nullish(), + tokens: z.int(), + total_price: z.union([z.number(), z.int()]), + total_segments: z.int(), +}) + +/** + * IndexingEstimate + */ +export const zIndexingEstimate = z.object({ + preview: z.array(zPreviewDetail), + qa_preview: z.array(zQaPreviewDetail).nullish(), total_segments: z.int(), }) @@ -528,30 +480,11 @@ export const zDatasetMetadataBuiltInFieldsResponse = z.object({ }) /** - * PreviewDetail + * ProcessRuleMode + * + * Dataset Process Rule Mode */ -export const zPreviewDetail = z.object({ - child_chunks: z.array(z.string()).nullish(), - content: z.string(), - summary: z.string().nullish(), -}) - -/** - * QAPreviewDetail - */ -export const zQaPreviewDetail = z.object({ - answer: z.string(), - question: z.string(), -}) - -/** - * IndexingEstimate - */ -export const zIndexingEstimate = z.object({ - preview: z.array(zPreviewDetail), - qa_preview: z.array(zQaPreviewDetail).nullish(), - total_segments: z.int(), -}) +export const zProcessRuleMode = z.enum(['automatic', 'custom', 'hierarchical']) /** * DocumentMetadataResponse @@ -563,6 +496,43 @@ export const zDocumentMetadataResponse = z.object({ value: z.union([z.string(), z.int(), z.number(), z.boolean()]).nullish(), }) +/** + * DocumentDetailResponse + */ +export const zDocumentDetailResponse = z.object({ + archived: z.boolean().nullish(), + average_segment_length: z.number().nullish(), + completed_at: z.int().nullish(), + created_at: z.int().nullish(), + created_by: z.string().nullish(), + created_from: z.string().nullish(), + data_source_detail_dict: z.unknown().optional(), + data_source_info: z.unknown().optional(), + data_source_type: z.string().nullish(), + dataset_process_rule: z.unknown().optional(), + dataset_process_rule_id: z.string().nullish(), + disabled_at: z.int().nullish(), + disabled_by: z.string().nullish(), + display_status: z.string().nullish(), + doc_form: z.string().nullish(), + doc_language: z.string().nullish(), + doc_metadata: z.array(zDocumentMetadataResponse).nullish(), + doc_type: z.string().nullish(), + document_process_rule: z.unknown().optional(), + enabled: z.boolean().nullish(), + error: z.string().nullish(), + hit_count: z.int().nullish(), + id: z.string(), + indexing_latency: z.number().nullish(), + indexing_status: z.string().nullish(), + name: z.string().nullish(), + need_summary: z.boolean().nullish(), + position: z.int().nullish(), + segment_count: z.int().nullish(), + tokens: z.int().nullish(), + updated_at: z.int().nullish(), +}) + /** * DocumentResponse */ @@ -646,6 +616,18 @@ export const zDocumentWithSegmentsListResponse = z.object({ total: z.int(), }) +export const zJsonValue = z.unknown() + +/** + * DocumentPipelineExecutionLogResponse + */ +export const zDocumentPipelineExecutionLogResponse = z.object({ + datasource_info: zJsonValue.nullish(), + datasource_node_id: z.string().nullish(), + datasource_type: z.string().nullish(), + input_data: zJsonValue.nullish(), +}) + /** * ChildChunkResponse */ @@ -700,6 +682,64 @@ export const zChildChunkBatchUpdatePayload = z.object({ chunks: z.array(zChildChunkUpdateArgs), }) +/** + * SummaryEntryResponse + */ +export const zSummaryEntryResponse = z.object({ + created_at: z.int().nullish(), + error: z.string().nullish(), + segment_id: z.string(), + segment_position: z.int(), + status: z.string(), + summary_preview: z.string().nullish(), + updated_at: z.int().nullish(), +}) + +/** + * SummaryStatusResponse + */ +export const zSummaryStatusResponse = z.object({ + completed: z.int().optional().default(0), + error: z.int().optional().default(0), + generating: z.int().optional().default(0), + not_started: z.int().optional().default(0), + timeout: z.int().optional().default(0), +}) + +/** + * DocumentSummaryStatusResponse + */ +export const zDocumentSummaryStatusResponse = z.object({ + summaries: z.array(zSummaryEntryResponse), + summary_status: zSummaryStatusResponse, + total_segments: z.int(), +}) + +/** + * ExternalHitTestingQueryResponse + */ +export const zExternalHitTestingQueryResponse = z.object({ + content: z.string(), +}) + +/** + * ExternalHitTestingRecordResponse + */ +export const zExternalHitTestingRecordResponse = z.object({ + content: z.string().nullish(), + metadata: z.record(z.string(), z.unknown()).nullish(), + score: z.number().nullish(), + title: z.string().nullish(), +}) + +/** + * ExternalHitTestingResponse + */ +export const zExternalHitTestingResponse = z.object({ + query: zExternalHitTestingQueryResponse, + records: z.array(zExternalHitTestingRecordResponse), +}) + /** * HitTestingQuery */ @@ -755,18 +795,6 @@ export const zDatasetRerankingModelResponse = z.object({ reranking_provider_name: z.string().nullish(), }) -export const zDatasetRerankingModel = z.object({ - reranking_model_name: z.string().optional(), - reranking_provider_name: z.string().optional(), -}) - -/** - * ProcessRuleMode - * - * Dataset Process Rule Mode - */ -export const zProcessRuleMode = z.enum(['automatic', 'custom', 'hierarchical']) - /** * RerankingModel */ @@ -785,6 +813,50 @@ export const zRetrievalMethod = z.enum([ 'semantic_search', ]) +/** + * PreProcessingRule + */ +export const zPreProcessingRule = z.object({ + enabled: z.boolean(), + id: z.enum(['remove_extra_spaces', 'remove_stopwords', 'remove_urls_emails']), +}) + +/** + * Segmentation + */ +export const zSegmentation = z.object({ + chunk_overlap: z.int().optional().default(0), + max_tokens: z.int(), + separator: z.string().optional().default('\n'), +}) + +/** + * Rule + */ +export const zRule = z.object({ + parent_mode: z.enum(['full-doc', 'paragraph']).nullish(), + pre_processing_rules: z.array(zPreProcessingRule).nullish(), + segmentation: zSegmentation.nullish(), + subchunk_segmentation: zSegmentation.nullish(), +}) + +/** + * ProcessRuleResponse + */ +export const zProcessRuleResponse = z.object({ + limits: z.record(z.string(), z.unknown()), + mode: zProcessRuleMode, + rules: zRule.nullish(), +}) + +/** + * ProcessRule + */ +export const zProcessRule = z.object({ + mode: zProcessRuleMode, + rules: zRule.nullish(), +}) + /** * MetadataDetail */ @@ -1079,88 +1151,6 @@ export const zDatasetListResponse = z.object({ total: z.int(), }) -export const zDatasetKeywordSetting = z.object({ - keyword_weight: z.number().optional(), -}) - -export const zDatasetVectorSetting = z.object({ - embedding_model_name: z.string().optional(), - embedding_provider_name: z.string().optional(), - vector_weight: z.number().optional(), -}) - -export const zDatasetWeightedScore = z.object({ - keyword_setting: zDatasetKeywordSetting.optional(), - vector_setting: zDatasetVectorSetting.optional(), - weight_type: z.string().optional(), -}) - -export const zDatasetRetrievalModel = z.object({ - reranking_enable: z.boolean().optional(), - reranking_mode: z.string().optional(), - reranking_model: zDatasetRerankingModel.optional(), - score_threshold: z.number().optional(), - score_threshold_enabled: z.boolean().optional(), - search_method: z.string().optional(), - top_k: z.int().optional(), - weights: zDatasetWeightedScore.optional(), -}) - -export const zDatasetDetail = z.object({ - app_count: z.int().optional(), - author_name: z.string().optional(), - built_in_field_enabled: z.boolean().optional(), - chunk_structure: z.string().optional(), - created_at: z.coerce - .bigint() - .min(BigInt('-9223372036854775808'), { - error: 'Invalid value: Expected int64 to be >= -9223372036854775808', - }) - .max(BigInt('9223372036854775807'), { - error: 'Invalid value: Expected int64 to be <= 9223372036854775807', - }) - .optional(), - created_by: z.string().optional(), - data_source_type: z.string().optional(), - description: z.string().optional(), - doc_form: z.string().optional(), - doc_metadata: z.array(zDatasetDocMetadata).optional(), - document_count: z.int().optional(), - embedding_available: z.boolean().optional(), - embedding_model: z.string().optional(), - embedding_model_provider: z.string().optional(), - enable_api: z.boolean().optional(), - external_knowledge_info: zExternalKnowledgeInfo.optional(), - external_retrieval_model: zExternalRetrievalModel.optional(), - icon_info: zDatasetIconInfo.optional(), - id: z.string().optional(), - indexing_technique: z.string().optional(), - is_multimodal: z.boolean().optional(), - is_published: z.boolean().optional(), - name: z.string().optional(), - permission: z.string().optional(), - permission_keys: z.array(z.string()).optional(), - pipeline_id: z.string().optional(), - provider: z.string().optional(), - retrieval_model_dict: zDatasetRetrievalModel.optional(), - runtime_mode: z.string().optional(), - summary_index_setting: zAnonymousInlineModelB1954337D565.optional(), - tags: z.array(zTag).optional(), - total_available_documents: z.int().optional(), - total_documents: z.int().optional(), - updated_at: z.coerce - .bigint() - .min(BigInt('-9223372036854775808'), { - error: 'Invalid value: Expected int64 to be >= -9223372036854775808', - }) - .max(BigInt('9223372036854775807'), { - error: 'Invalid value: Expected int64 to be <= 9223372036854775807', - }) - .optional(), - updated_by: z.string().optional(), - word_count: z.int().optional(), -}) - /** * FileInfo */ @@ -1178,41 +1168,6 @@ export const zWebsiteInfo = z.object({ urls: z.array(z.string()), }) -/** - * PreProcessingRule - */ -export const zPreProcessingRule = z.object({ - enabled: z.boolean(), - id: z.enum(['remove_extra_spaces', 'remove_stopwords', 'remove_urls_emails']), -}) - -/** - * Segmentation - */ -export const zSegmentation = z.object({ - chunk_overlap: z.int().optional().default(0), - max_tokens: z.int(), - separator: z.string().optional().default('\n'), -}) - -/** - * Rule - */ -export const zRule = z.object({ - parent_mode: z.enum(['full-doc', 'paragraph']).nullish(), - pre_processing_rules: z.array(zPreProcessingRule).nullish(), - segmentation: zSegmentation.nullish(), - subchunk_segmentation: zSegmentation.nullish(), -}) - -/** - * ProcessRule - */ -export const zProcessRule = z.object({ - mode: zProcessRuleMode, - rules: zRule.nullish(), -}) - /** * Condition * @@ -1558,7 +1513,7 @@ export const zPostDatasetsExternalBody = zExternalDatasetCreatePayload /** * External dataset created successfully */ -export const zPostDatasetsExternalResponse = zDatasetDetail +export const zPostDatasetsExternalResponse = zDatasetDetailResponse export const zGetDatasetsExternalKnowledgeApiQuery = z.object({ keyword: z.string().optional(), @@ -1653,7 +1608,7 @@ export const zGetDatasetsProcessRuleQuery = z.object({ /** * Process rules retrieved successfully */ -export const zGetDatasetsProcessRuleResponse = zOpaqueObjectResponse +export const zGetDatasetsProcessRuleResponse = zProcessRuleResponse /** * Retrieval settings retrieved successfully @@ -1723,9 +1678,9 @@ export const zGetDatasetsByDatasetIdBatchByBatchIndexingEstimatePath = z.object( }) /** - * Batch indexing estimate calculated successfully + * Indexing estimate calculated successfully */ -export const zGetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponse = zOpaqueObjectResponse +export const zGetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponse = zIndexingEstimateResponse export const zGetDatasetsByDatasetIdBatchByBatchIndexingStatusPath = z.object({ batch: z.string(), @@ -1782,9 +1737,12 @@ export const zPostDatasetsByDatasetIdDocumentsDownloadZipPath = z.object({ }) /** - * ZIP archive generated successfully + * ZIP archive downloaded successfully */ -export const zPostDatasetsByDatasetIdDocumentsDownloadZipResponse = zBinaryFileResponse +export const zPostDatasetsByDatasetIdDocumentsDownloadZipResponse = z.record( + z.string(), + z.unknown(), +) export const zPostDatasetsByDatasetIdDocumentsGenerateSummaryBody = zGenerateSummaryPayload @@ -1840,7 +1798,7 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdQuery = z.object({ /** * Document retrieved successfully */ -export const zGetDatasetsByDatasetIdDocumentsByDocumentIdResponse = zOpaqueObjectResponse +export const zGetDatasetsByDatasetIdDocumentsByDocumentIdResponse = zDocumentDetailResponse export const zGetDatasetsByDatasetIdDocumentsByDocumentIdDownloadPath = z.object({ dataset_id: z.uuid(), @@ -1861,7 +1819,7 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimatePath = * Indexing estimate calculated successfully */ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateResponse - = zOpaqueObjectResponse + = zIndexingEstimateResponse export const zGetDatasetsByDatasetIdDocumentsByDocumentIdIndexingStatusPath = z.object({ dataset_id: z.uuid(), @@ -1904,10 +1862,10 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogPat }) /** - * Document pipeline execution log retrieved successfully + * Pipeline execution log retrieved successfully */ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogResponse - = zOpaqueObjectResponse + = zDocumentPipelineExecutionLogResponse export const zPatchDatasetsByDatasetIdDocumentsByDocumentIdProcessingPausePath = z.object({ dataset_id: z.uuid(), @@ -2158,7 +2116,7 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusPath = z.o * Summary status retrieved successfully */ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusResponse - = zOpaqueObjectResponse + = zDocumentSummaryStatusResponse export const zGetDatasetsByDatasetIdDocumentsByDocumentIdWebsiteSyncPath = z.object({ dataset_id: z.uuid(), @@ -2188,7 +2146,7 @@ export const zPostDatasetsByDatasetIdExternalHitTestingPath = z.object({ /** * External hit testing completed successfully */ -export const zPostDatasetsByDatasetIdExternalHitTestingResponse = zExternalRetrievalTestResponse +export const zPostDatasetsByDatasetIdExternalHitTestingResponse = zExternalHitTestingResponse export const zPostDatasetsByDatasetIdHitTestingBody = zHitTestingPayload diff --git a/packages/contracts/generated/api/console/test/types.gen.ts b/packages/contracts/generated/api/console/test/types.gen.ts index 421460c015d..efbf875b0ef 100644 --- a/packages/contracts/generated/api/console/test/types.gen.ts +++ b/packages/contracts/generated/api/console/test/types.gen.ts @@ -10,19 +10,24 @@ export type BedrockRetrievalPayload = { retrieval_setting: BedrockRetrievalSetting } -export type ExternalRetrievalTestResponse - = | { - [key: string]: unknown - } - | Array<{ - [key: string]: unknown - }> +export type BedrockRetrievalResponse = { + records: Array +} export type BedrockRetrievalSetting = { score_threshold?: number top_k?: number | null } +export type BedrockRetrievalRecordResponse = { + content?: string | null + metadata?: { + [key: string]: unknown + } | null + score: number + title?: string | null +} + export type PostTestRetrievalData = { body: BedrockRetrievalPayload path?: never @@ -31,7 +36,7 @@ export type PostTestRetrievalData = { } export type PostTestRetrievalResponses = { - 200: ExternalRetrievalTestResponse + 200: BedrockRetrievalResponse } export type PostTestRetrievalResponse = PostTestRetrievalResponses[keyof PostTestRetrievalResponses] diff --git a/packages/contracts/generated/api/console/test/zod.gen.ts b/packages/contracts/generated/api/console/test/zod.gen.ts index 35ec1f4b034..84cb75e6196 100644 --- a/packages/contracts/generated/api/console/test/zod.gen.ts +++ b/packages/contracts/generated/api/console/test/zod.gen.ts @@ -2,14 +2,6 @@ import * as z from 'zod' -/** - * ExternalRetrievalTestResponse - */ -export const zExternalRetrievalTestResponse = z.union([ - z.record(z.string(), z.unknown()), - z.array(z.record(z.string(), z.unknown())), -]) - /** * BedrockRetrievalSetting * @@ -29,9 +21,26 @@ export const zBedrockRetrievalPayload = z.object({ retrieval_setting: zBedrockRetrievalSetting, }) +/** + * BedrockRetrievalRecordResponse + */ +export const zBedrockRetrievalRecordResponse = z.object({ + content: z.string().nullish(), + metadata: z.record(z.string(), z.unknown()).nullish(), + score: z.number(), + title: z.string().nullish(), +}) + +/** + * BedrockRetrievalResponse + */ +export const zBedrockRetrievalResponse = z.object({ + records: z.array(zBedrockRetrievalRecordResponse), +}) + export const zPostTestRetrievalBody = zBedrockRetrievalPayload /** * Bedrock retrieval test completed */ -export const zPostTestRetrievalResponse = zExternalRetrievalTestResponse +export const zPostTestRetrievalResponse = zBedrockRetrievalResponse diff --git a/packages/contracts/generated/api/service/orpc.gen.ts b/packages/contracts/generated/api/service/orpc.gen.ts index b15cc78d07d..7ed6b163dd9 100644 --- a/packages/contracts/generated/api/service/orpc.gen.ts +++ b/packages/contracts/generated/api/service/orpc.gen.ts @@ -1474,16 +1474,20 @@ export const get13 = oc .output(zGetDatasetsByDatasetIdDocumentsByDocumentIdResponse) /** - * Update an existing document by uploading a file + * Update Document by File + * + * Update an existing document by uploading a new file. Re-triggers indexing — use the returned `batch` ID with [Get Document Indexing Status](/api-reference/documents/get-document-indexing-status) to track progress. */ export const patch4 = oc .route({ - description: 'Update an existing document by uploading a file', + description: + 'Update an existing document by uploading a new file. Re-triggers indexing — use the returned `batch` ID with [Get Document Indexing Status](/api-reference/documents/get-document-indexing-status) to track progress.', inputStructure: 'detailed', method: 'PATCH', operationId: 'patchDatasetsByDatasetIdDocumentsByDocumentId', path: '/datasets/{dataset_id}/documents/{document_id}', - tags: ['service_api'], + summary: 'Update Document by File', + tags: ['Documents'], }) .input( z.object({ diff --git a/packages/contracts/generated/api/service/types.gen.ts b/packages/contracts/generated/api/service/types.gen.ts index 4a58a619741..896c1129fa3 100644 --- a/packages/contracts/generated/api/service/types.gen.ts +++ b/packages/contracts/generated/api/service/types.gen.ts @@ -593,40 +593,45 @@ export type DocumentBatchDownloadZipPayload = { } export type DocumentDetailResponse = { - archived?: boolean | null - average_segment_length?: number | null + archived?: boolean + average_segment_length?: number | number completed_at?: number | null - created_at?: number | null - created_by?: string | null - created_from?: string | null + created_at?: number + created_by?: string + created_from?: string data_source_info?: { [key: string]: unknown - } | null - data_source_type?: string | null + } + data_source_type?: string dataset_process_rule?: { [key: string]: unknown - } | null + } dataset_process_rule_id?: string | null disabled_at?: number | null disabled_by?: string | null display_status?: string | null - doc_form?: string | null + doc_form?: string doc_language?: string | null - doc_metadata?: Array | null + doc_metadata?: + | Array + | { + [key: string]: unknown + } + | null doc_type?: string | null document_process_rule?: { [key: string]: unknown - } | null - enabled?: boolean | null + } + enabled?: boolean error?: string | null - hit_count?: number | null + hit_count?: number id: string indexing_latency?: number | null - indexing_status?: string | null - name?: string | null - need_summary?: boolean | null - position?: number | null - segment_count?: number | null + indexing_status?: string + name?: string + need_summary?: boolean + position?: number + segment_count?: number summary_index_status?: string | null tokens?: number | null updated_at?: number | null @@ -2627,6 +2632,7 @@ export type PatchDatasetsByDatasetIdDocumentsByDocumentIdData = { } export type PatchDatasetsByDatasetIdDocumentsByDocumentIdErrors = { + 400: unknown 401: unknown 403: unknown 404: unknown diff --git a/packages/contracts/generated/api/service/zod.gen.ts b/packages/contracts/generated/api/service/zod.gen.ts index a5da8668e5c..885093d0dc7 100644 --- a/packages/contracts/generated/api/service/zod.gen.ts +++ b/packages/contracts/generated/api/service/zod.gen.ts @@ -761,34 +761,36 @@ export const zDocumentMetadataResponse = z.object({ * DocumentDetailResponse */ export const zDocumentDetailResponse = z.object({ - archived: z.boolean().nullish(), - average_segment_length: z.number().nullish(), + archived: z.boolean().optional(), + average_segment_length: z.union([z.int(), z.number()]).optional(), completed_at: z.int().nullish(), - created_at: z.int().nullish(), - created_by: z.string().nullish(), - created_from: z.string().nullish(), - data_source_info: z.record(z.string(), z.unknown()).nullish(), - data_source_type: z.string().nullish(), - dataset_process_rule: z.record(z.string(), z.unknown()).nullish(), + created_at: z.int().optional(), + created_by: z.string().optional(), + created_from: z.string().optional(), + data_source_info: z.record(z.string(), z.unknown()).optional(), + data_source_type: z.string().optional(), + dataset_process_rule: z.record(z.string(), z.unknown()).optional(), dataset_process_rule_id: z.string().nullish(), disabled_at: z.int().nullish(), disabled_by: z.string().nullish(), display_status: z.string().nullish(), - doc_form: z.string().nullish(), + doc_form: z.string().optional(), doc_language: z.string().nullish(), - doc_metadata: z.array(zDocumentMetadataResponse).nullish(), + doc_metadata: z + .union([z.array(zDocumentMetadataResponse), z.record(z.string(), z.unknown())]) + .nullish(), doc_type: z.string().nullish(), - document_process_rule: z.record(z.string(), z.unknown()).nullish(), - enabled: z.boolean().nullish(), + document_process_rule: z.record(z.string(), z.unknown()).optional(), + enabled: z.boolean().optional(), error: z.string().nullish(), - hit_count: z.int().nullish(), + hit_count: z.int().optional(), id: z.string(), indexing_latency: z.number().nullish(), - indexing_status: z.string().nullish(), - name: z.string().nullish(), - need_summary: z.boolean().nullish(), - position: z.int().nullish(), - segment_count: z.int().nullish(), + indexing_status: z.string().optional(), + name: z.string().optional(), + need_summary: z.boolean().optional(), + position: z.int().optional(), + segment_count: z.int().optional(), summary_index_status: z.string().nullish(), tokens: z.int().nullish(), updated_at: z.int().nullish(), @@ -2759,7 +2761,7 @@ export const zPatchDatasetsByDatasetIdDocumentsByDocumentIdPath = z.object({ }) /** - * Document updated successfully + * Document updated successfully. */ export const zPatchDatasetsByDatasetIdDocumentsByDocumentIdResponse = zDocumentAndBatchResponse