opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-07-21 02:28:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(api): migrate dataset endpoints to BaseModel (#37957)

Browse Source 
Co-authored-by: Asuka Minato <i@asukaminato.eu.org>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
chariri 2026-07-09 13:22:30 +09:00 committed by GitHub
parent 9bb3b1fa98
commit 89d5f74a40
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 1871 additions and 1403 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
api/controllers/console/datasets/datasets.py
View File

@ -30,6 +30,7 @@ from controllers.console.wraps import (
with_current_tenant_id,
with_current_user,
)
from core.entities.knowledge_entities import IndexingEstimate
from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
from core.indexing_runner import IndexingRunner
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
@ -268,21 +269,10 @@ class ErrorDocsResponse(DocumentStatusListResponse):
total: int
class IndexingEstimatePreviewItemResponse(ResponseModel):
content: str
child_chunks: list[str] | None = None
summary: str | None = None
class IndexingEstimateQaPreviewItemResponse(ResponseModel):
question: str
answer: str
class IndexingEstimateResponse(ResponseModel):
total_segments: int
preview: list[IndexingEstimatePreviewItemResponse]
qa_preview: list[IndexingEstimateQaPreviewItemResponse] | None = None
class IndexingEstimateResponse(IndexingEstimate):
tokens: int
total_price: float | int
currency: str
class RetrievalSettingResponse(ResponseModel):
@ -647,7 +637,7 @@ class DatasetApi(Resource):
else:
data["embedding_available"] = True
return data, 200
return dump_response(DatasetDetailWithPartialMembersResponse, data), 200
@console_ns.doc("update_dataset")
@console_ns.doc(description="Update dataset details")
@ -717,7 +707,7 @@ class DatasetApi(Resource):
partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session())
result_data.update({"partial_member_list": partial_member_list})
return result_data, 200
return dump_response(DatasetDetailWithPartialMembersResponse, result_data), 200
@setup_required
@login_required
@ -760,7 +750,7 @@ class DatasetUseCheckApi(Resource):
dataset_id_str = str(dataset_id)
dataset_is_using = DatasetService.dataset_use_check(dataset_id_str, db.session())
return {"is_using": dataset_is_using}, 200
return UsageCheckResponse(is_using=dataset_is_using).model_dump(mode="json"), 200
@console_ns.route("/datasets/<uuid:dataset_id>/queries")
@ -901,7 +891,17 @@ class DatasetIndexingEstimateApi(Resource):
except Exception as e:
raise IndexingEstimateError(str(e))
return response.model_dump(), 200
return (
IndexingEstimateResponse(
tokens=0,
total_price=0,
currency="USD",
total_segments=response.total_segments,
preview=response.preview,
qa_preview=response.qa_preview,
).model_dump(mode="json", exclude_none=True),
200,
)
@console_ns.route("/datasets/<uuid:dataset_id>/related-apps")
@ -1018,7 +1018,7 @@ class DatasetApiKeyApi(Resource):
keys = db.session.scalars(
select(ApiToken).where(ApiToken.type == self.resource_type, ApiToken.tenant_id == current_tenant_id)
).all()
return ApiKeyList.model_validate({"data": keys}, from_attributes=True).model_dump(mode="json")
return dump_response(ApiKeyList, {"data": keys})
@console_ns.response(200, "API key created successfully", console_ns.models[ApiKeyItem.__name__])
@console_ns.response(400, "Maximum keys exceeded")
@ -1052,7 +1052,7 @@ class DatasetApiKeyApi(Resource):
api_token.type = self.resource_type
db.session.add(api_token)
db.session.commit()
return ApiKeyItem.model_validate(api_token, from_attributes=True).model_dump(mode="json"), 200
return dump_response(ApiKeyItem, api_token), 200
@console_ns.route("/datasets/api-keys/<uuid:api_key_id>")
@ -1107,7 +1107,7 @@ class DatasetEnableApiApi(Resource):
DatasetService.update_dataset_api_status(dataset_id_str, status == "enable", db.session())
return {"result": "success"}, 200
return SimpleResultResponse(result="success").model_dump(mode="json"), 200
@console_ns.route("/datasets/api-base-info")
@ -1120,7 +1120,7 @@ class DatasetApiBaseUrlApi(Resource):
@account_initialization_required
def get(self):
base = dify_config.SERVICE_API_URL or request.host_url.rstrip("/")
return {"api_base_url": normalize_api_base_url(base)}
return ApiBaseUrlResponse(api_base_url=normalize_api_base_url(base)).model_dump(mode="json")
@console_ns.route("/datasets/retrieval-setting")

279
api/controllers/console/datasets/datasets_document.py
View File

@ -10,16 +10,17 @@ from uuid import UUID
import sqlalchemy as sa
from flask import request, send_file
from flask_restx import Resource
from pydantic import BaseModel, Field, RootModel, field_validator
from pydantic import BaseModel, Field, JsonValue, field_validator
from sqlalchemy import asc, desc, func, select
from werkzeug.exceptions import Forbidden, NotFound
import services
from controllers.common.controller_schemas import DocumentBatchDownloadZipPayload
from controllers.common.fields import BinaryFileResponse, SimpleResultMessageResponse, SimpleResultResponse, UrlResponse
from controllers.common.fields import SimpleResultMessageResponse, SimpleResultResponse, UrlResponse
from controllers.common.schema import register_response_schema_models, register_schema_models
from controllers.console import console_ns
from controllers.console.wraps import RBACPermission, RBACResourceScope, rbac_permission_required
from core.entities.knowledge_entities import IndexingEstimate
from core.errors.error import (
LLMBadRequestError,
ModelCurrentlyNotSupportError,
@ -29,6 +30,7 @@ from core.errors.error import (
from core.indexing_runner import IndexingRunner
from core.model_manager import ModelManager
from core.plugin.impl.exc import PluginDaemonClientSideError
from core.rag.entities import Rule
from core.rag.extractor.entity.datasource_type import DatasourceType
from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo, WebsiteInfo
from core.rag.index_processor.constant.index_type import IndexTechniqueType
@ -49,7 +51,7 @@ from libs.login import login_required
from libs.pagination import paginate_query
from models import Account, DatasetProcessRule, Document, DocumentSegment, UploadFile
from models.dataset import DocumentPipelineExecutionLog
from models.enums import IndexingStatus, SegmentStatus
from models.enums import IndexingStatus, ProcessRuleMode, SegmentStatus
from services.dataset_ref_service import DatasetRefService
from services.dataset_service import DatasetService, DocumentService
from services.entities.knowledge_entities.knowledge_entities import KnowledgeConfig, ProcessRule, RetrievalModel
@ -148,8 +150,91 @@ class DocumentWithSegmentsListResponse(ResponseModel):
page: int
class OpaqueObjectResponse(RootModel[dict[str, Any]]):
root: dict[str, Any]
class IndexingEstimateResponse(IndexingEstimate):
tokens: int
total_price: float | int
currency: str
class DocumentDetailResponse(ResponseModel):
id: str
position: int | None = None
data_source_type: str | None = None
data_source_info: Any = None
data_source_detail_dict: Any = None
dataset_process_rule_id: str | None = None
dataset_process_rule: Any = None
document_process_rule: Any = None
name: str | None = None
created_from: str | None = None
created_by: str | None = None
created_at: int | None = None
tokens: int | None = None
indexing_status: str | None = None
completed_at: int | None = None
updated_at: int | None = None
indexing_latency: float | None = None
error: str | None = None
enabled: bool | None = None
disabled_at: int | None = None
disabled_by: str | None = None
archived: bool | None = None
doc_type: str | None = None
doc_metadata: list[DocumentMetadataResponse] | None = None
segment_count: int | None = None
average_segment_length: float | None = None
hit_count: int | None = None
display_status: str | None = None
doc_form: str | None = None
doc_language: str | None = None
need_summary: bool | None = None
@field_validator("data_source_type", "indexing_status", "display_status", "doc_form", mode="before")
@classmethod
def _normalize_enum_fields(cls, value: Any) -> Any:
return normalize_enum(value)
class SummaryStatusResponse(ResponseModel):
completed: int = 0
generating: int = 0
error: int = 0
not_started: int = 0
timeout: int = 0
class SummaryEntryResponse(ResponseModel):
segment_id: str
segment_position: int
status: str
summary_preview: str | None = None
error: str | None = None
created_at: int | None = None
updated_at: int | None = None
@field_validator("status", mode="before")
@classmethod
def _normalize_status(cls, value: Any) -> Any:
return normalize_enum(value)
class DocumentSummaryStatusResponse(ResponseModel):
total_segments: int
summary_status: SummaryStatusResponse
summaries: list[SummaryEntryResponse]
class ProcessRuleResponse(ResponseModel):
mode: ProcessRuleMode
rules: Rule | None = None
limits: dict[str, Any]
class DocumentPipelineExecutionLogResponse(ResponseModel):
datasource_info: JsonValue | None = None
datasource_type: str | None = None
input_data: JsonValue | None = None
datasource_node_id: str | None = None
register_schema_models(
@ -165,7 +250,6 @@ register_schema_models(
)
register_response_schema_models(
console_ns,
BinaryFileResponse,
SimpleResultMessageResponse,
SimpleResultResponse,
UrlResponse,
@ -175,7 +259,11 @@ register_response_schema_models(
DocumentWithSegmentsResponse,
DatasetAndDocumentResponse,
DocumentWithSegmentsListResponse,
OpaqueObjectResponse,
IndexingEstimateResponse,
DocumentDetailResponse,
DocumentSummaryStatusResponse,
ProcessRuleResponse,
DocumentPipelineExecutionLogResponse,
)
@ -225,7 +313,7 @@ class GetProcessRuleApi(Resource):
@console_ns.doc("get_process_rule")
@console_ns.doc(description="Get dataset document processing rules")
@console_ns.doc(params={"document_id": "Document ID (optional)"})
@console_ns.response(200, "Process rules retrieved successfully", console_ns.models[OpaqueObjectResponse.__name__])
@console_ns.response(200, "Process rules retrieved successfully", console_ns.models[ProcessRuleResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@ -264,7 +352,7 @@ class GetProcessRuleApi(Resource):
mode = dataset_process_rule.mode
rules = dataset_process_rule.rules_dict
return {"mode": mode, "rules": rules, "limits": limits}
return dump_response(ProcessRuleResponse, {"mode": mode, "rules": rules, "limits": limits})
@console_ns.route("/datasets/<uuid:dataset_id>/documents")
@ -491,7 +579,7 @@ class DatasetInitApi(Resource):
@console_ns.doc(description="Initialize dataset with documents")
@console_ns.expect(console_ns.models[KnowledgeConfig.__name__])
@console_ns.response(
201, "Dataset initialized successfully", console_ns.models[DatasetAndDocumentResponse.__name__]
200, "Dataset initialized successfully", console_ns.models[DatasetAndDocumentResponse.__name__]
)
@console_ns.response(400, "Invalid request parameters")
@setup_required
@ -557,7 +645,7 @@ class DocumentIndexingEstimateApi(DocumentResource):
@console_ns.response(
200,
"Indexing estimate calculated successfully",
console_ns.models[OpaqueObjectResponse.__name__],
console_ns.models[IndexingEstimateResponse.__name__],
)
@console_ns.response(404, "Document not found")
@console_ns.response(400, "Document already finished")
@ -578,8 +666,6 @@ class DocumentIndexingEstimateApi(DocumentResource):
data_process_rule = document.dataset_process_rule
data_process_rule_dict = data_process_rule.to_dict() if data_process_rule else {}
response = {"tokens": 0, "total_price": 0, "currency": "USD", "total_segments": 0, "preview": []}
if document.data_source_type == "upload_file":
data_source_info = document.data_source_info_dict
if data_source_info and "upload_file_id" in data_source_info:
@ -610,7 +696,18 @@ class DocumentIndexingEstimateApi(DocumentResource):
"English",
dataset_id_str,
)
return estimate_response.model_dump(), 200
return (
# TODO: why using zero here? the same for the below endpoint
IndexingEstimateResponse(
tokens=0,
total_price=0,
currency="USD",
total_segments=estimate_response.total_segments,
preview=estimate_response.preview,
qa_preview=estimate_response.qa_preview,
).model_dump(mode="json", exclude_none=True),
200,
)
except LLMBadRequestError:
raise ProviderNotInitializeError(
"No Embedding Model available. Please configure a valid provider "
@ -623,15 +720,24 @@ class DocumentIndexingEstimateApi(DocumentResource):
except Exception as e:
raise IndexingEstimateError(str(e))
return response, 200
return (
IndexingEstimateResponse(
tokens=0,
total_price=0,
currency="USD",
total_segments=0,
preview=[],
).model_dump(mode="json", exclude_none=True),
200,
)
@console_ns.route("/datasets/<uuid:dataset_id>/batch/<string:batch>/indexing-estimate")
class DocumentBatchIndexingEstimateApi(DocumentResource):
@console_ns.response(
200,
"Batch indexing estimate calculated successfully",
console_ns.models[OpaqueObjectResponse.__name__],
"Indexing estimate calculated successfully",
console_ns.models[IndexingEstimateResponse.__name__],
)
@setup_required
@login_required
@ -643,7 +749,16 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
dataset_id_str = str(dataset_id)
documents = self.get_batch_documents(dataset_id_str, batch, current_user)
if not documents:
return {"tokens": 0, "total_price": 0, "currency": "USD", "total_segments": 0, "preview": []}, 200
return (
IndexingEstimateResponse(
tokens=0,
total_price=0,
currency="USD",
total_segments=0,
preview=[],
).model_dump(mode="json", exclude_none=True),
200,
)
data_process_rule = documents[0].dataset_process_rule
data_process_rule_dict = data_process_rule.to_dict() if data_process_rule else {}
extract_settings = []
@ -717,7 +832,17 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
"English",
dataset_id_str,
)
return response.model_dump(), 200
return (
IndexingEstimateResponse(
tokens=0,
total_price=0,
currency="USD",
total_segments=response.total_segments,
preview=response.preview,
qa_preview=response.qa_preview,
).model_dump(mode="json", exclude_none=True),
200,
)
except LLMBadRequestError:
raise ProviderNotInitializeError(
"No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
@ -854,7 +979,7 @@ class DocumentApi(DocumentResource):
"metadata": "Metadata inclusion (all/only/without)",
}
)
@console_ns.response(200, "Document retrieved successfully", console_ns.models[OpaqueObjectResponse.__name__])
@console_ns.response(200, "Document retrieved successfully", console_ns.models[DocumentDetailResponse.__name__])
@console_ns.response(404, "Document not found")
@setup_required
@login_required
@ -871,46 +996,21 @@ class DocumentApi(DocumentResource):
if metadata not in self.METADATA_CHOICES:
raise InvalidMetadataError(f"Invalid metadata value: {metadata}")
metadata_fields = {"doc_type", "doc_metadata"}
if metadata == "only":
response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
elif metadata == "without":
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session())
document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
response = {
"id": document.id,
"position": document.position,
"data_source_type": document.data_source_type,
"data_source_info": document.data_source_info_dict,
"data_source_detail_dict": document.data_source_detail_dict,
"dataset_process_rule_id": document.dataset_process_rule_id,
"dataset_process_rule": dataset_process_rules,
"document_process_rule": document_process_rules,
"name": document.name,
"created_from": document.created_from,
"created_by": document.created_by,
"created_at": int(document.created_at.timestamp()),
"tokens": document.tokens,
"indexing_status": document.indexing_status,
"completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,
"updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,
"indexing_latency": document.indexing_latency,
"error": document.error,
"enabled": document.enabled,
"disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,
"disabled_by": document.disabled_by,
"archived": document.archived,
"segment_count": document.segment_count,
"average_segment_length": document.average_segment_length,
"hit_count": document.hit_count,
"display_status": document.display_status,
"doc_form": document.doc_form,
"doc_language": document.doc_language,
"need_summary": document.need_summary if document.need_summary is not None else False,
}
else:
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session())
document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
response = {
response = DocumentDetailResponse.model_validate(
{
"id": document.id,
"doc_type": document.doc_type,
"doc_metadata": document.doc_metadata_details,
}
)
return response.model_dump(mode="json", include={"id", *metadata_fields}, exclude_unset=True), 200
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session())
document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
response = DocumentDetailResponse.model_validate(
{
"id": document.id,
"position": document.position,
"data_source_type": document.data_source_type,
@ -943,8 +1043,9 @@ class DocumentApi(DocumentResource):
"doc_language": document.doc_language,
"need_summary": document.need_summary if document.need_summary is not None else False,
}
return response, 200
)
exclude = metadata_fields if metadata == "without" else None
return response.model_dump(mode="json", exclude=exclude, exclude_unset=True), 200
@setup_required
@login_required
@ -990,7 +1091,9 @@ class DocumentDownloadApi(DocumentResource):
def get(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID) -> dict[str, Any]:
# Reuse the shared permission/tenant checks implemented in DocumentResource.
document = self.get_document(str(dataset_id), str(document_id), current_user, current_tenant_id)
return {"url": DocumentService.get_document_download_url(document, db.session())}
return UrlResponse(url=DocumentService.get_document_download_url(document, db.session())).model_dump(
mode="json"
)
@console_ns.route("/datasets/<uuid:dataset_id>/documents/download-zip")
@ -999,7 +1102,7 @@ class DocumentBatchDownloadZipApi(DocumentResource):
@console_ns.doc("download_dataset_documents_as_zip")
@console_ns.doc(description="Download selected dataset documents as a single ZIP archive (upload-file only)")
@console_ns.response(200, "ZIP archive generated successfully", console_ns.models[BinaryFileResponse.__name__])
@console_ns.response(200, "ZIP archive downloaded successfully")
@setup_required
@login_required
@account_initialization_required
@ -1034,6 +1137,7 @@ class DocumentBatchDownloadZipApi(DocumentResource):
)
cleanup = stack.pop_all()
response.call_on_close(cleanup.close)
# response-contract:ignore binary ZIP download response
return response
@ -1093,7 +1197,7 @@ class DocumentProcessingApi(DocumentResource):
document.is_paused = False
db.session.commit()
return {"result": "success"}, 200
return SimpleResultResponse(result="success").model_dump(mode="json"), 200
@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/metadata")
@ -1152,7 +1256,9 @@ class DocumentMetadataApi(DocumentResource):
document.updated_at = naive_utc_now()
db.session.commit()
return {"result": "success", "message": "Document metadata updated."}, 200
return SimpleResultMessageResponse(result="success", message="Document metadata updated.").model_dump(
mode="json"
), 200
@console_ns.route("/datasets/<uuid:dataset_id>/documents/status/<string:action>/batch")
@ -1194,7 +1300,7 @@ class DocumentStatusApi(DocumentResource):
except NotFound as e:
raise NotFound(str(e))
return {"result": "success"}, 200
return SimpleResultResponse(result="success").model_dump(mode="json"), 200
@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/pause")
@ -1321,7 +1427,7 @@ class DocumentRenameApi(DocumentResource):
# The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
if not current_user.is_dataset_editor:
raise Forbidden()
dataset = DatasetService.get_dataset(dataset_id, db.session())
dataset = DatasetService.get_dataset(str(dataset_id), db.session())
if not dataset:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_operator_permission(current_user, dataset, session=db.session())
@ -1363,15 +1469,15 @@ class WebsiteDocumentSyncApi(DocumentResource):
# sync document
DocumentService.sync_website_document(dataset_id_str, document, db.session())
return {"result": "success"}, 200
return SimpleResultResponse(result="success").model_dump(mode="json"), 200
@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/pipeline-execution-log")
class DocumentPipelineExecutionLogApi(DocumentResource):
@console_ns.response(
200,
"Document pipeline execution log retrieved successfully",
console_ns.models[OpaqueObjectResponse.__name__],
"Pipeline execution log retrieved successfully",
console_ns.models[DocumentPipelineExecutionLogResponse.__name__],
)
@setup_required
@login_required
@ -1394,18 +1500,16 @@ class DocumentPipelineExecutionLogApi(DocumentResource):
.limit(1)
)
if not log:
return {
"datasource_info": None,
"datasource_type": None,
"input_data": None,
"datasource_node_id": None,
}, 200
return {
"datasource_info": json.loads(log.datasource_info),
"datasource_type": log.datasource_type,
"input_data": log.input_data,
"datasource_node_id": log.datasource_node_id,
}, 200
return DocumentPipelineExecutionLogResponse().model_dump(mode="json"), 200
return dump_response(
DocumentPipelineExecutionLogResponse,
{
"datasource_info": json.loads(log.datasource_info),
"datasource_type": log.datasource_type,
"input_data": log.input_data,
"datasource_node_id": log.datasource_node_id,
},
), 200
@console_ns.route("/datasets/<uuid:dataset_id>/documents/generate-summary")
@ -1508,7 +1612,7 @@ class DocumentGenerateSummaryApi(Resource):
dataset_id_str,
)
return {"result": "success"}, 200
return SimpleResultResponse(result="success").model_dump(mode="json"), 200
@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/summary-status")
@ -1516,7 +1620,11 @@ class DocumentSummaryStatusApi(DocumentResource):
@console_ns.doc("get_document_summary_status")
@console_ns.doc(description="Get summary index generation status for a document")
@console_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
@console_ns.response(200, "Summary status retrieved successfully", console_ns.models[OpaqueObjectResponse.__name__])
@console_ns.response(
200,
"Summary status retrieved successfully",
console_ns.models[DocumentSummaryStatusResponse.__name__],
)
@console_ns.response(404, "Document not found")
@setup_required
@login_required
@ -1534,6 +1642,7 @@ class DocumentSummaryStatusApi(DocumentResource):
- generating: Number of summaries being generated
- error: Number of summaries with errors
- not_started: Number of segments without summary records
- timeout: Number of summaries that timed out
- summaries: List of summary records with status and content preview
"""
dataset_id_str = str(dataset_id)
@ -1559,4 +1668,4 @@ class DocumentSummaryStatusApi(DocumentResource):
session=db.session(),
)
return result, 200
return dump_response(DocumentSummaryStatusResponse, result), 200

2
api/controllers/console/datasets/datasets_segments.py
View File

@ -391,7 +391,7 @@ class DatasetDocumentSegmentApi(Resource):
SegmentService.update_segments_status(segment_ids, action, dataset, document, db.session())
except Exception as e:
raise InvalidActionError(str(e))
return dump_response(SimpleResultResponse, {"result": "success"}), 200
return SimpleResultResponse(result="success").model_dump(mode="json"), 200
@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segment")

196
api/controllers/console/datasets/external.py
View File

@ -1,20 +1,16 @@
from datetime import datetime
from typing import Any
from uuid import UUID
from flask import request
from flask_restx import Resource, fields, marshal
from pydantic import BaseModel, Field, RootModel
from flask_restx import Resource
from pydantic import AliasChoices, BaseModel, Field, field_validator
from sqlalchemy.orm import Session
from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
import services
from controllers.common.fields import UsageCountResponse
from controllers.common.schema import (
get_or_create_model,
query_params_from_model,
register_response_schema_models,
register_schema_models,
)
from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
from controllers.console import console_ns
from controllers.console.app.wraps import with_session
from controllers.console.datasets.error import DatasetNameDuplicateError
@ -28,21 +24,9 @@ from controllers.console.wraps import (
with_current_tenant_id,
with_current_user,
)
from extensions.ext_database import db
from fields.base import ResponseModel
from fields.dataset_fields import (
dataset_detail_fields,
dataset_retrieval_model_fields,
doc_metadata_fields,
external_knowledge_info_fields,
external_retrieval_model_fields,
icon_info_fields,
keyword_setting_fields,
reranking_model_fields,
tag_fields,
vector_setting_fields,
weighted_score_fields,
)
from fields.dataset_fields import DatasetDetailResponse
from libs.helper import dump_response
from libs.login import login_required
from models import Account
from services.dataset_service import DatasetService
@ -51,50 +35,10 @@ from services.external_knowledge_service import ExternalDatasetService
from services.hit_testing_service import HitTestingService
from services.knowledge_service import BedrockRetrievalSetting, ExternalDatasetTestService
register_response_schema_models(console_ns, UsageCountResponse)
def _build_dataset_detail_model():
keyword_setting_model = get_or_create_model("DatasetKeywordSetting", keyword_setting_fields)
vector_setting_model = get_or_create_model("DatasetVectorSetting", vector_setting_fields)
weighted_score_fields_copy = weighted_score_fields.copy()
weighted_score_fields_copy["keyword_setting"] = fields.Nested(keyword_setting_model)
weighted_score_fields_copy["vector_setting"] = fields.Nested(vector_setting_model)
weighted_score_model = get_or_create_model("DatasetWeightedScore", weighted_score_fields_copy)
reranking_model = get_or_create_model("DatasetRerankingModel", reranking_model_fields)
dataset_retrieval_model_fields_copy = dataset_retrieval_model_fields.copy()
dataset_retrieval_model_fields_copy["reranking_model"] = fields.Nested(reranking_model)
dataset_retrieval_model_fields_copy["weights"] = fields.Nested(weighted_score_model, allow_null=True)
dataset_retrieval_model = get_or_create_model("DatasetRetrievalModel", dataset_retrieval_model_fields_copy)
tag_model = get_or_create_model("Tag", tag_fields)
doc_metadata_model = get_or_create_model("DatasetDocMetadata", doc_metadata_fields)
external_knowledge_info_model = get_or_create_model("ExternalKnowledgeInfo", external_knowledge_info_fields)
external_retrieval_model = get_or_create_model("ExternalRetrievalModel", external_retrieval_model_fields)
icon_info_model = get_or_create_model("DatasetIconInfo", icon_info_fields)
dataset_detail_fields_copy = dataset_detail_fields.copy()
dataset_detail_fields_copy["retrieval_model_dict"] = fields.Nested(dataset_retrieval_model)
dataset_detail_fields_copy["tags"] = fields.List(fields.Nested(tag_model))
dataset_detail_fields_copy["external_knowledge_info"] = fields.Nested(external_knowledge_info_model)
dataset_detail_fields_copy["external_retrieval_model"] = fields.Nested(external_retrieval_model, allow_null=True)
dataset_detail_fields_copy["doc_metadata"] = fields.List(fields.Nested(doc_metadata_model))
dataset_detail_fields_copy["icon_info"] = fields.Nested(icon_info_model)
return get_or_create_model("DatasetDetail", dataset_detail_fields_copy)
try:
dataset_detail_model = console_ns.models["DatasetDetail"]
except KeyError:
dataset_detail_model = _build_dataset_detail_model()
class ExternalKnowledgeApiPayload(BaseModel):
name: str = Field(..., min_length=1, max_length=40)
settings: dict[str, object]
settings: dict[str, Any]
class ExternalDatasetCreatePayload(BaseModel):
@ -102,15 +46,13 @@ class ExternalDatasetCreatePayload(BaseModel):
external_knowledge_id: str
name: str = Field(..., min_length=1, max_length=100)
description: str | None = Field(None, max_length=400)
external_retrieval_model: dict[str, object] | None = Field(default=None)
external_retrieval_model: dict[str, Any] | None = None
class ExternalHitTestingPayload(BaseModel):
query: str
external_retrieval_model: dict[str, object] | None = Field(default=None)
metadata_filtering_conditions: dict[str, object] | None = Field(
default=None,
)
external_retrieval_model: dict[str, Any] | None = None
metadata_filtering_conditions: dict[str, Any] | None = None
class BedrockRetrievalPayload(BaseModel):
@ -125,7 +67,7 @@ class ExternalApiTemplateListQuery(BaseModel):
keyword: str | None = Field(default=None, description="Search keyword")
class ExternalKnowledgeDatasetBindingResponse(ResponseModel):
class ExternalKnowledgeApiBindingResponse(ResponseModel):
id: str
name: str
@ -135,22 +77,52 @@ class ExternalKnowledgeApiResponse(ResponseModel):
tenant_id: str
name: str
description: str
settings: dict[str, Any] | None = Field(default=None)
dataset_bindings: list[ExternalKnowledgeDatasetBindingResponse] = Field(default_factory=list)
settings: dict[str, Any] | None = Field(validation_alias=AliasChoices("settings_dict", "settings"))
dataset_bindings: list[ExternalKnowledgeApiBindingResponse]
created_by: str
created_at: str
@field_validator("created_at", mode="before")
@classmethod
def _normalize_created_at(cls, value: datetime | str) -> str:
if isinstance(value, datetime):
return value.isoformat()
return value
class ExternalKnowledgeApiListResponse(ResponseModel):
data: list[ExternalKnowledgeApiResponse]
has_more: bool
limit: int
total: int
total: int | None
page: int
class ExternalRetrievalTestResponse(RootModel[dict[str, Any] | list[dict[str, Any]]]):
root: dict[str, Any] | list[dict[str, Any]]
class ExternalHitTestingQueryResponse(ResponseModel):
content: str
class ExternalHitTestingRecordResponse(ResponseModel):
content: str | None = None
title: str | None = None
score: float | None = None
metadata: dict[str, Any] | None = None
class ExternalHitTestingResponse(ResponseModel):
query: ExternalHitTestingQueryResponse
records: list[ExternalHitTestingRecordResponse]
class BedrockRetrievalRecordResponse(ResponseModel):
metadata: dict[str, Any] | None = None
score: float
title: str | None = None
content: str | None = None
class BedrockRetrievalResponse(ResponseModel):
records: list[BedrockRetrievalRecordResponse]
register_schema_models(
@ -163,9 +135,16 @@ register_schema_models(
)
register_response_schema_models(
console_ns,
UsageCountResponse,
DatasetDetailResponse,
ExternalKnowledgeApiBindingResponse,
ExternalKnowledgeApiResponse,
ExternalKnowledgeApiListResponse,
ExternalRetrievalTestResponse,
ExternalHitTestingQueryResponse,
ExternalHitTestingRecordResponse,
ExternalHitTestingResponse,
BedrockRetrievalRecordResponse,
BedrockRetrievalResponse,
)
@ -189,24 +168,26 @@ class ExternalApiTemplateListApi(Resource):
external_knowledge_apis, total = ExternalDatasetService.get_external_knowledge_apis(
query.page, query.limit, current_tenant_id, query.keyword
)
response = {
"data": [item.to_dict() for item in external_knowledge_apis],
"has_more": len(external_knowledge_apis) == query.limit,
"limit": query.limit,
"total": total,
"page": query.page,
}
return response, 200
return ExternalKnowledgeApiListResponse(
data=[ExternalKnowledgeApiResponse.model_validate(item) for item in external_knowledge_apis],
has_more=len(external_knowledge_apis) == query.limit,
limit=query.limit,
total=total,
page=query.page,
).model_dump(mode="json"), 200
@setup_required
@login_required
@account_initialization_required
@console_ns.doc("create_external_api_template")
@console_ns.doc(description="Create external knowledge API template")
@console_ns.expect(console_ns.models[ExternalKnowledgeApiPayload.__name__])
@console_ns.response(
201,
"External API template created successfully",
console_ns.models[ExternalKnowledgeApiResponse.__name__],
)
@console_ns.response(403, "Permission denied")
@setup_required
@login_required
@account_initialization_required
@with_current_user
@with_current_tenant_id
@with_session
@ -229,7 +210,7 @@ class ExternalApiTemplateListApi(Resource):
except services.errors.dataset.DatasetNameDuplicateError:
raise DatasetNameDuplicateError()
return external_knowledge_api.to_dict(), 201
return dump_response(ExternalKnowledgeApiResponse, external_knowledge_api), 201
@console_ns.route("/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>")
@ -256,17 +237,21 @@ class ExternalApiTemplateApi(Resource):
if external_knowledge_api is None:
raise NotFound("API template not found.")
return external_knowledge_api.to_dict(), 200
return dump_response(ExternalKnowledgeApiResponse, external_knowledge_api), 200
@console_ns.doc("update_external_api_template")
@console_ns.doc(description="Update external knowledge API template")
@console_ns.doc(params={"external_knowledge_api_id": "External knowledge API ID"})
@console_ns.expect(console_ns.models[ExternalKnowledgeApiPayload.__name__])
@console_ns.response(
200,
"External API template updated successfully",
console_ns.models[ExternalKnowledgeApiResponse.__name__],
)
@console_ns.response(404, "Template not found")
@setup_required
@login_required
@account_initialization_required
@console_ns.expect(console_ns.models[ExternalKnowledgeApiPayload.__name__])
@with_current_user
@with_current_tenant_id
@with_session
@ -284,7 +269,7 @@ class ExternalApiTemplateApi(Resource):
session=session,
)
return external_knowledge_api.to_dict(), 200
return dump_response(ExternalKnowledgeApiResponse, external_knowledge_api), 200
@setup_required
@login_required
@ -322,7 +307,7 @@ class ExternalApiUseCheckApi(Resource):
external_knowledge_api_is_using, count = ExternalDatasetService.external_knowledge_api_use_check(
external_knowledge_api_id_str, current_tenant_id, session=session
)
return {"is_using": external_knowledge_api_is_using, "count": count}, 200
return UsageCountResponse(is_using=external_knowledge_api_is_using, count=count).model_dump(mode="json"), 200
@console_ns.route("/datasets/external")
@ -330,7 +315,9 @@ class ExternalDatasetCreateApi(Resource):
@console_ns.doc("create_external_dataset")
@console_ns.doc(description="Create external knowledge dataset")
@console_ns.expect(console_ns.models[ExternalDatasetCreatePayload.__name__])
@console_ns.response(201, "External dataset created successfully", dataset_detail_model)
@console_ns.response(
201, "External dataset created successfully", console_ns.models[DatasetDetailResponse.__name__]
)
@console_ns.response(400, "Invalid parameters")
@console_ns.response(403, "Permission denied")
@setup_required
@ -360,17 +347,16 @@ class ExternalDatasetCreateApi(Resource):
except services.errors.dataset.DatasetNameDuplicateError:
raise DatasetNameDuplicateError()
item = marshal(dataset, dataset_detail_fields)
dataset_id_str = item["id"]
dataset_id_str = str(dataset.id)
permission_keys_map = enterprise_rbac_service.RBACService.DatasetPermissions.batch_get(
str(current_tenant_id),
current_user.id,
[dataset_id_str],
session=session,
)
item["permission_keys"] = permission_keys_map.get(dataset_id_str, [])
return item, 201
data = DatasetDetailResponse.model_validate(dataset).model_dump(mode="json")
data["permission_keys"] = permission_keys_map.get(dataset_id_str, [])
return data, 201
@console_ns.route("/datasets/<uuid:dataset_id>/external-hit-testing")
@ -382,7 +368,7 @@ class ExternalKnowledgeHitTestingApi(Resource):
@console_ns.response(
200,
"External hit testing completed successfully",
console_ns.models[ExternalRetrievalTestResponse.__name__],
console_ns.models[ExternalHitTestingResponse.__name__],
)
@console_ns.response(404, "Dataset not found")
@console_ns.response(400, "Invalid parameters")
@ -394,12 +380,12 @@ class ExternalKnowledgeHitTestingApi(Resource):
@with_session
def post(self, session: Session, current_user: Account, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str, db.session())
dataset = DatasetService.get_dataset(dataset_id_str, session)
if dataset is None:
raise NotFound("Dataset not found.")
try:
DatasetService.check_dataset_permission(dataset, current_user, db.session())
DatasetService.check_dataset_permission(dataset, current_user, session)
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
@ -416,7 +402,7 @@ class ExternalKnowledgeHitTestingApi(Resource):
metadata_filtering_conditions=payload.metadata_filtering_conditions,
)
return response
return dump_response(ExternalHitTestingResponse, response)
except Exception as e:
raise InternalServerError(str(e))
@ -427,11 +413,7 @@ class BedrockRetrievalApi(Resource):
@console_ns.doc("bedrock_retrieval_test")
@console_ns.doc(description="Bedrock retrieval test (internal use only)")
@console_ns.expect(console_ns.models[BedrockRetrievalPayload.__name__])
@console_ns.response(
200,
"Bedrock retrieval test completed",
console_ns.models[ExternalRetrievalTestResponse.__name__],
)
@console_ns.response(200, "Bedrock retrieval test completed", console_ns.models[BedrockRetrievalResponse.__name__])
def post(self):
payload = BedrockRetrievalPayload.model_validate(console_ns.payload or {})
@ -439,4 +421,4 @@ class BedrockRetrievalApi(Resource):
result = ExternalDatasetTestService.knowledge_retrieval(
payload.retrieval_setting, payload.query, payload.knowledge_id
)
return result, 200
return dump_response(BedrockRetrievalResponse, result), 200

23
api/controllers/service_api/dataset/dataset.py
View File

@ -845,7 +845,7 @@ class DocumentStatusApi(DatasetApiResource):
except ValueError as e:
raise InvalidActionError(str(e))
return dump_response(SimpleResultResponse, {"result": "success"}), 200
return SimpleResultResponse(result="success").model_dump(mode="json"), 200
@service_api_ns.route("/datasets/tags")
@ -911,11 +911,8 @@ class DatasetTagsApi(DatasetApiResource):
payload = TagCreatePayload.model_validate(service_api_ns.payload or {})
tag = TagService.save_tags(SaveTagPayload(name=payload.name, type=TagType.KNOWLEDGE), db.session())
response = dump_response(
KnowledgeTagResponse,
{"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": 0},
)
return response, 200
response = KnowledgeTagResponse(id=tag.id, name=tag.name, type=tag.type, binding_count="0")
return response.model_dump(mode="json"), 200
@service_api_ns.doc(
summary="Update Knowledge Tag",
@ -953,11 +950,8 @@ class DatasetTagsApi(DatasetApiResource):
binding_count = TagService.get_tag_binding_count(tag_id, db.session(), tag_type=TagType.KNOWLEDGE)
response = dump_response(
KnowledgeTagResponse,
{"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": binding_count},
)
return response, 200
response = KnowledgeTagResponse(id=tag.id, name=tag.name, type=tag.type, binding_count=str(binding_count))
return response.model_dump(mode="json"), 200
@service_api_ns.doc(
summary="Delete Knowledge Tag",
@ -1088,5 +1082,8 @@ class DatasetTagsBindingStatusApi(DatasetApiResource):
tags = TagService.get_tags_by_target_id(
"knowledge", current_user.current_tenant_id, str(dataset_id), db.session()
)
tags_list = [{"id": tag.id, "name": tag.name} for tag in tags]
return dump_response(DatasetBoundTagListResponse, {"data": tags_list, "total": len(tags)}), 200
response = DatasetBoundTagListResponse(
data=[DatasetBoundTagResponse(id=tag.id, name=tag.name) for tag in tags],
total=len(tags),
)
return response.model_dump(mode="json"), 200

149
api/controllers/service_api/dataset/document.py
View File

@ -6,14 +6,22 @@ deprecated in generated API docs so clients migrate toward the canonical paths.
"""
import json
from collections.abc import Mapping
from contextlib import ExitStack
from copy import deepcopy
from typing import Annotated, Any, Literal, Self, override
from uuid import UUID
from flask import request, send_file
from pydantic import BaseModel, Field, GetJsonSchemaHandler, WithJsonSchema, field_validator, model_validator
from pydantic import (
BaseModel,
Field,
GetJsonSchemaHandler,
ValidationError,
WithJsonSchema,
field_validator,
model_validator,
)
from pydantic.json_schema import SkipJsonSchema
from sqlalchemy import desc, func, select
from werkzeug.exceptions import Forbidden, NotFound
@ -26,9 +34,10 @@ from controllers.common.errors import (
TooManyFilesError,
UnsupportedFileTypeError,
)
from controllers.common.fields import BinaryFileResponse, UrlResponse
from controllers.common.fields import UrlResponse
from controllers.common.schema import (
query_params_from_model,
query_params_from_request,
register_enum_models,
register_response_schema_models,
register_schema_models,
@ -56,6 +65,7 @@ from fields.document_fields import (
DocumentMetadataResponse,
DocumentResponse,
DocumentStatusListResponse,
normalize_enum,
)
from libs.helper import dump_response
from libs.login import current_user
@ -281,38 +291,44 @@ class DocumentAndBatchResponse(ResponseModel):
batch: str
# Use SkipJsonSchema to support 3 metadata modes
class DocumentDetailResponse(ResponseModel):
id: str
position: int | None = None
data_source_type: str | None = None
data_source_info: dict[str, Any] | None = Field(default=None)
position: int | SkipJsonSchema[None] = None
data_source_type: str | SkipJsonSchema[None] = None
data_source_info: dict[str, Any] | SkipJsonSchema[None] = None
dataset_process_rule_id: str | None = None
dataset_process_rule: dict[str, Any] | None = Field(default=None)
document_process_rule: dict[str, Any] | None = Field(default=None)
name: str | None = None
created_from: str | None = None
created_by: str | None = None
created_at: int | None = None
dataset_process_rule: dict[str, Any] | SkipJsonSchema[None] = None
document_process_rule: dict[str, Any] | SkipJsonSchema[None] = None
name: str | SkipJsonSchema[None] = None
created_from: str | SkipJsonSchema[None] = None
created_by: str | SkipJsonSchema[None] = None
created_at: int | SkipJsonSchema[None] = None
tokens: int | None = None
indexing_status: str | None = None
indexing_status: str | SkipJsonSchema[None] = None
completed_at: int | None = None
updated_at: int | None = None
indexing_latency: float | None = None
error: str | None = None
enabled: bool | None = None
enabled: bool | SkipJsonSchema[None] = None
disabled_at: int | None = None
disabled_by: str | None = None
archived: bool | None = None
archived: bool | SkipJsonSchema[None] = None
doc_type: str | None = None
doc_metadata: list[DocumentMetadataResponse] | None = None
segment_count: int | None = None
average_segment_length: float | None = None
hit_count: int | None = None
doc_metadata: list[DocumentMetadataResponse] | dict[str, Any] | None = None
segment_count: int | SkipJsonSchema[None] = None
average_segment_length: int | float | SkipJsonSchema[None] = None
hit_count: int | SkipJsonSchema[None] = None
display_status: str | None = None
doc_form: str | None = None
doc_form: str | SkipJsonSchema[None] = None
doc_language: str | None = None
summary_index_status: str | None = None
need_summary: bool | None = None
need_summary: bool | SkipJsonSchema[None] = None
@field_validator("data_source_type", "indexing_status", "display_status", "doc_form", mode="before")
@classmethod
def _normalize_enum_fields(cls, value: Any) -> Any:
return normalize_enum(value)
register_enum_models(service_api_ns, RetrievalMethod)
@ -332,7 +348,6 @@ register_schema_models(
)
register_response_schema_models(
service_api_ns,
BinaryFileResponse,
UrlResponse,
DocumentResponse,
DocumentAndBatchResponse,
@ -342,13 +357,13 @@ register_response_schema_models(
)
def _create_document_by_text(tenant_id: str, dataset_id: UUID) -> tuple[Mapping[str, object], int]:
def _create_document_by_text(tenant_id: str, dataset_id: UUID) -> tuple[Document, str]:
"""Create a document from text for both canonical and legacy routes."""
payload = DocumentTextCreatePayload.model_validate(service_api_ns.payload or {})
args = payload.model_dump(exclude_none=True)
dataset_id_str = str(dataset_id)
tenant_id_str = str(tenant_id)
tenant_id_str = tenant_id
dataset = db.session.scalar(
select(Dataset).where(Dataset.tenant_id == tenant_id_str, Dataset.id == dataset_id_str).limit(1)
)
@ -407,10 +422,10 @@ def _create_document_by_text(tenant_id: str, dataset_id: UUID) -> tuple[Mapping[
raise ProviderNotInitializeError(ex.description)
document = documents[0]
return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
return document, batch
def _update_document_by_text(tenant_id: str, dataset_id: UUID, document_id: UUID) -> tuple[Mapping[str, object], int]:
def _update_document_by_text(tenant_id: str, dataset_id: UUID, document_id: UUID) -> tuple[Document, str]:
"""Update a document from text for both canonical and legacy routes."""
payload = DocumentTextUpdate.model_validate(service_api_ns.payload or {})
dataset = db.session.scalar(
@ -467,7 +482,7 @@ def _update_document_by_text(tenant_id: str, dataset_id: UUID, document_id: UUID
raise ProviderNotInitializeError(ex.description)
document = documents[0]
return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
return document, batch
@service_api_ns.route("/datasets/<uuid:dataset_id>/document/create-by-text")
@ -511,7 +526,8 @@ class DocumentAddByTextApi(DatasetApiResource):
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
def post(self, tenant_id: str, dataset_id: UUID):
"""Create document by text."""
return _create_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id)
document, batch = _create_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id)
return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
@service_api_ns.route("/datasets/<uuid:dataset_id>/document/create_by_text")
@ -543,7 +559,8 @@ class DeprecatedDocumentAddByTextApi(DatasetApiResource):
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
def post(self, tenant_id: str, dataset_id: UUID):
"""Create document by text through the deprecated underscore alias."""
return _create_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id)
document, batch = _create_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id)
return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update-by-text")
@ -587,7 +604,8 @@ class DocumentUpdateByTextApi(DatasetApiResource):
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
def post(self, tenant_id: str, dataset_id: UUID, document_id: UUID):
"""Update document by text."""
return _update_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
document, batch = _update_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_text")
@ -618,7 +636,8 @@ class DeprecatedDocumentUpdateByTextApi(DatasetApiResource):
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
def post(self, tenant_id: str, dataset_id: UUID, document_id: UUID):
"""Update document by text through the deprecated underscore alias."""
return _update_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
document, batch = _update_document_by_text(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
@service_api_ns.route(
@ -767,10 +786,10 @@ class DocumentAddByFileApi(DatasetApiResource):
return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
def _update_document_by_file(tenant_id: str, dataset_id: UUID, document_id: UUID) -> tuple[Mapping[str, object], int]:
def _update_document_by_file(tenant_id: str, dataset_id: UUID, document_id: UUID) -> tuple[Document, str]:
"""Update a document from an uploaded file for canonical and deprecated routes."""
dataset_id_str = str(dataset_id)
tenant_id_str = str(tenant_id)
tenant_id_str = tenant_id
dataset = db.session.scalar(
select(Dataset).where(Dataset.tenant_id == tenant_id_str, Dataset.id == dataset_id_str).limit(1)
)
@ -841,7 +860,7 @@ def _update_document_by_file(tenant_id: str, dataset_id: UUID, document_id: UUID
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description)
document = documents[0]
return dump_response(DocumentAndBatchResponse, {"document": document, "batch": document.batch}), 200
return document, document.batch
@service_api_ns.route(
@ -895,7 +914,8 @@ class DeprecatedDocumentUpdateByFileApi(DatasetApiResource):
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
def post(self, tenant_id: str, dataset_id: UUID, document_id: UUID):
"""Update document by file through the deprecated file-update aliases."""
return _update_document_by_file(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
document, batch = _update_document_by_file(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
@service_api_ns.route("/datasets/<uuid:dataset_id>/documents")
@ -928,7 +948,7 @@ class DocumentListApi(DatasetApiResource):
def get(self, tenant_id, dataset_id: UUID):
dataset_id_str = str(dataset_id)
tenant_id = str(tenant_id)
query_params = DocumentListQuery.model_validate(request.args.to_dict())
query_params = query_params_from_request(DocumentListQuery)
dataset = db.session.scalar(
select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str).limit(1)
)
@ -1021,6 +1041,7 @@ class DocumentBatchDownloadZipApi(DatasetApiResource):
)
cleanup = stack.pop_all()
response.call_on_close(cleanup.close)
# response-contract:ignore binary send_file response
return response
@ -1149,7 +1170,9 @@ class DocumentDownloadApi(DatasetApiResource):
if document.tenant_id != str(tenant_id):
raise Forbidden("No permission.")
return {"url": DocumentService.get_document_download_url(document, db.session())}
return UrlResponse(url=DocumentService.get_document_download_url(document, db.session())).model_dump(
mode="json"
)
@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
@ -1176,8 +1199,13 @@ class DocumentApi(DatasetApiResource):
)
@service_api_ns.doc("get_document")
@service_api_ns.doc(description="Get a specific document by ID")
@service_api_ns.doc(params={"dataset_id": "Knowledge base ID.", "document_id": "Document ID."})
@service_api_ns.doc(params=query_params_from_model(DocumentGetQuery))
@service_api_ns.doc(
params={
"dataset_id": "Knowledge base ID.",
"document_id": "Document ID.",
**query_params_from_model(DocumentGetQuery),
}
)
@service_api_ns.doc(
responses={
200: "Document retrieved successfully",
@ -1205,9 +1233,14 @@ class DocumentApi(DatasetApiResource):
if document.tenant_id != str(tenant_id):
raise Forbidden("No permission.")
metadata = request.args.get("metadata", "all")
if metadata not in self.METADATA_CHOICES:
raise InvalidMetadataError(f"Invalid metadata value: {metadata}")
try:
query_params = query_params_from_request(DocumentGetQuery)
except ValidationError as exc:
metadata = request.args.get("metadata", "all")
raise InvalidMetadataError(f"Invalid metadata value: {metadata}") from exc
metadata = query_params.metadata
response_include: set[str] | None = None
response_exclude: set[str] | None = None
# Calculate summary_index_status if needed
summary_index_status = None
@ -1221,8 +1254,10 @@ class DocumentApi(DatasetApiResource):
)
if metadata == "only":
response_include = {"id", "doc_type", "doc_metadata"}
response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
elif metadata == "without":
response_exclude = {"doc_type", "doc_metadata"}
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session())
document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
data_source_info = document.data_source_detail_dict
@ -1295,8 +1330,33 @@ class DocumentApi(DatasetApiResource):
"need_summary": document.need_summary if document.need_summary is not None else False,
}
return response
return DocumentDetailResponse.model_validate(response).model_dump(
mode="json",
include=response_include,
exclude=response_exclude,
)
@service_api_ns.doc(
summary="Update Document by File",
description=(
"Update an existing document by uploading a new file. Re-triggers indexing — use the returned "
"`batch` ID with [Get Document Indexing Status](/api-reference/documents/"
"get-document-indexing-status) to track progress."
),
tags=["Documents"],
responses={
200: "Document updated successfully.",
400: (
"- `too_many_files` : Only one file is allowed.\n"
"- `filename_not_exists_error` : The specified filename does not exist.\n"
"- `provider_not_initialize` : No valid model provider credentials found. Please go to "
"Settings -> Model Provider to complete your provider credentials.\n"
"- `invalid_param` : Knowledge base does not exist, external datasets not supported, "
"file too large, unsupported file type, or invalid doc_form (must be `text_model`, "
"`hierarchical_model`, or `qa_model`)."
),
},
)
@service_api_ns.doc("update_document_by_file")
@service_api_ns.doc(description="Update an existing document by uploading a file")
@service_api_ns.doc(consumes=["multipart/form-data"], params=DOCUMENT_UPDATE_BY_FILE_PARAMS)
@ -1314,7 +1374,8 @@ class DocumentApi(DatasetApiResource):
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
def patch(self, tenant_id: str, dataset_id: UUID, document_id: UUID):
"""Update document by file on the canonical document resource."""
return _update_document_by_file(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
document, batch = _update_document_by_file(tenant_id=tenant_id, dataset_id=dataset_id, document_id=document_id)
return dump_response(DocumentAndBatchResponse, {"document": document, "batch": batch}), 200
@service_api_ns.doc(
summary="Delete Document",

11
api/core/rag/entities/processing_entities.py
View File

@ -1,7 +1,7 @@
from enum import StrEnum
from typing import Annotated, Literal
from pydantic import BaseModel, Field, WithJsonSchema
from pydantic import AliasChoices, BaseModel, Field, WithJsonSchema
class ParentMode(StrEnum):
@ -26,7 +26,14 @@ class PreProcessingRule(BaseModel):
class Segmentation(BaseModel):
separator: str = Field(default="\n", description="Custom separator for splitting text.")
# TODO: there are internally mismatched / inconsistent naming
# between `separator` and `delimiter` across the codebase.
# Taking `separator` as the canonical.
separator: str = Field(
default="\n",
description="Custom separator for splitting text.",
validation_alias=AliasChoices("separator", "delimiter"),
)
max_tokens: int = Field(description="Maximum token count per chunk.")
chunk_overlap: int = Field(default=0, description="Token overlap between chunks.")

113
api/fields/dataset_fields.py
View File

@ -1,22 +1,9 @@
from datetime import datetime
from flask_restx import fields
from pydantic import Field, field_validator
from fields.base import ResponseModel
from libs.helper import TimestampField, to_timestamp
dataset_fields = {
"id": fields.String,
"name": fields.String,
"description": fields.String,
"permission": fields.String,
"data_source_type": fields.String,
"indexing_technique": fields.String,
"created_by": fields.String,
"created_at": TimestampField,
"permission_keys": fields.List(fields.String()),
}
from libs.helper import to_timestamp
class DatasetMetadataResponse(ResponseModel):
@ -50,104 +37,6 @@ class DatasetMetadataActionResponse(ResponseModel):
result: str
reranking_model_fields = {"reranking_provider_name": fields.String, "reranking_model_name": fields.String}
keyword_setting_fields = {"keyword_weight": fields.Float}
vector_setting_fields = {
"vector_weight": fields.Float,
"embedding_model_name": fields.String,
"embedding_provider_name": fields.String,
}
weighted_score_fields = {
"weight_type": fields.String,
"keyword_setting": fields.Nested(keyword_setting_fields),
"vector_setting": fields.Nested(vector_setting_fields),
}
dataset_retrieval_model_fields = {
"search_method": fields.String,
"reranking_enable": fields.Boolean,
"reranking_mode": fields.String,
"reranking_model": fields.Nested(reranking_model_fields),
"weights": fields.Nested(weighted_score_fields, allow_null=True),
"top_k": fields.Integer,
"score_threshold_enabled": fields.Boolean,
"score_threshold": fields.Float,
}
dataset_summary_index_fields = {
"enable": fields.Boolean,
"model_name": fields.String,
"model_provider_name": fields.String,
"summary_prompt": fields.String,
}
external_retrieval_model_fields = {
"top_k": fields.Integer,
"score_threshold": fields.Float,
"score_threshold_enabled": fields.Boolean,
}
tag_fields = {"id": fields.String, "name": fields.String, "type": fields.String}
external_knowledge_info_fields = {
"external_knowledge_id": fields.String,
"external_knowledge_api_id": fields.String,
"external_knowledge_api_name": fields.String,
"external_knowledge_api_endpoint": fields.String,
}
doc_metadata_fields = {"id": fields.String, "name": fields.String, "type": fields.String}
icon_info_fields = {
"icon_type": fields.String,
"icon": fields.String,
"icon_background": fields.String,
"icon_url": fields.String,
}
dataset_detail_fields = {
"id": fields.String,
"name": fields.String,
"description": fields.String,
"provider": fields.String,
"permission": fields.String,
"data_source_type": fields.String,
"indexing_technique": fields.String,
"app_count": fields.Integer,
"document_count": fields.Integer,
"word_count": fields.Integer,
"created_by": fields.String,
"author_name": fields.String,
"created_at": TimestampField,
"updated_by": fields.String,
"updated_at": TimestampField,
"embedding_model": fields.String,
"embedding_model_provider": fields.String,
"embedding_available": fields.Boolean,
"retrieval_model_dict": fields.Nested(dataset_retrieval_model_fields),
"summary_index_setting": fields.Nested(dataset_summary_index_fields),
"tags": fields.List(fields.Nested(tag_fields)),
"doc_form": fields.String,
"external_knowledge_info": fields.Nested(external_knowledge_info_fields),
"external_retrieval_model": fields.Nested(external_retrieval_model_fields, allow_null=True),
"doc_metadata": fields.List(fields.Nested(doc_metadata_fields)),
"built_in_field_enabled": fields.Boolean,
"pipeline_id": fields.String,
"runtime_mode": fields.String,
"chunk_structure": fields.String,
"icon_info": fields.Nested(icon_info_fields),
"is_published": fields.Boolean,
"total_documents": fields.Integer,
"total_available_documents": fields.Integer,
"enable_api": fields.Boolean,
"is_multimodal": fields.Boolean,
"permission_keys": fields.List(fields.String()),
}
class DatasetRerankingModelResponse(ResponseModel):
reranking_provider_name: str | None = None
reranking_model_name: str | None = None

333
api/openapi/markdown/console-openapi.md
View File

@ -5492,7 +5492,7 @@ Create external knowledge dataset
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 201 | External dataset created successfully | **application/json**: [DatasetDetail](#datasetdetail)<br> |
| 201 | External dataset created successfully | **application/json**: [DatasetDetailResponse](#datasetdetailresponse)<br> |
| 400 | Invalid parameters | |
| 403 | Permission denied | |
@ -5514,6 +5514,8 @@ Get external knowledge API templates
| 200 | External API templates retrieved successfully | **application/json**: [ExternalKnowledgeApiListResponse](#externalknowledgeapilistresponse)<br> |
### [POST] /datasets/external-knowledge-api
Create external knowledge API template
#### Request Body
| Required | Schema |
@ -5525,6 +5527,7 @@ Get external knowledge API templates
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 201 | External API template created successfully | **application/json**: [ExternalKnowledgeApiResponse](#externalknowledgeapiresponse)<br> |
| 403 | Permission denied | |
### [DELETE] /datasets/external-knowledge-api/{external_knowledge_api_id}
#### Parameters
@ -5556,11 +5559,13 @@ Get external knowledge API template details
| 404 | Template not found | |
### [PATCH] /datasets/external-knowledge-api/{external_knowledge_api_id}
Update external knowledge API template
#### Parameters
| Name | Located in | Description | Required | Schema |
| ---- | ---------- | ----------- | -------- | ------ |
| external_knowledge_api_id | path | | Yes | string (uuid) |
| external_knowledge_api_id | path | External knowledge API ID | Yes | string (uuid) |
#### Request Body
@ -5573,6 +5578,7 @@ Get external knowledge API template details
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | External API template updated successfully | **application/json**: [ExternalKnowledgeApiResponse](#externalknowledgeapiresponse)<br> |
| 404 | Template not found | |
### [GET] /datasets/external-knowledge-api/{external_knowledge_api_id}/use-check
Check if external knowledge API is being used
@ -5617,7 +5623,7 @@ Initialize dataset with documents
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 201 | Dataset initialized successfully | **application/json**: [DatasetAndDocumentResponse](#datasetanddocumentresponse)<br> |
| 200 | Dataset initialized successfully | **application/json**: [DatasetAndDocumentResponse](#datasetanddocumentresponse)<br> |
| 400 | Invalid request parameters | |
### [GET] /datasets/metadata/built-in
@ -5653,7 +5659,7 @@ Get dataset document processing rules
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Process rules retrieved successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)<br> |
| 200 | Process rules retrieved successfully | **application/json**: [ProcessRuleResponse](#processruleresponse)<br> |
### [GET] /datasets/retrieval-setting
Get dataset retrieval settings
@ -5774,7 +5780,7 @@ Get dataset auto disable logs
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Batch indexing estimate calculated successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)<br> |
| 200 | Indexing estimate calculated successfully | **application/json**: [IndexingEstimateResponse](#indexingestimateresponse)<br> |
### [GET] /datasets/{dataset_id}/batch/{batch}/indexing-status
#### Parameters
@ -5862,9 +5868,9 @@ Download selected dataset documents as a single ZIP archive (upload-file only)
#### Responses
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | ZIP archive generated successfully | **application/json**: [BinaryFileResponse](#binaryfileresponse)<br> |
| Code | Description |
| ---- | ----------- |
| 200 | ZIP archive downloaded successfully |
### [POST] /datasets/{dataset_id}/documents/generate-summary
**Generate summary index for specified documents**
@ -5957,7 +5963,7 @@ Get document details
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Document retrieved successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)<br> |
| 200 | Document retrieved successfully | **application/json**: [DocumentDetailResponse](#documentdetailresponse)<br> |
| 404 | Document not found | |
### [GET] /datasets/{dataset_id}/documents/{document_id}/download
@ -5990,7 +5996,7 @@ Estimate document indexing cost
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Indexing estimate calculated successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)<br> |
| 200 | Indexing estimate calculated successfully | **application/json**: [IndexingEstimateResponse](#indexingestimateresponse)<br> |
| 400 | Document already finished | |
| 404 | Document not found | |
@ -6061,7 +6067,7 @@ Update document metadata
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Document pipeline execution log retrieved successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)<br> |
| 200 | Pipeline execution log retrieved successfully | **application/json**: [DocumentPipelineExecutionLogResponse](#documentpipelineexecutionlogresponse)<br> |
### [PATCH] /datasets/{dataset_id}/documents/{document_id}/processing/pause
**pause document**
@ -6384,6 +6390,7 @@ Returns:
- generating: Number of summaries being generated
- error: Number of summaries with errors
- not_started: Number of segments without summary records
- timeout: Number of summaries that timed out
- summaries: List of summary records with status and content preview
#### Parameters
@ -6397,7 +6404,7 @@ Returns:
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Summary status retrieved successfully | **application/json**: [OpaqueObjectResponse](#opaqueobjectresponse)<br> |
| 200 | Summary status retrieved successfully | **application/json**: [DocumentSummaryStatusResponse](#documentsummarystatusresponse)<br> |
| 404 | Document not found | |
### [GET] /datasets/{dataset_id}/documents/{document_id}/website-sync
@ -6451,7 +6458,7 @@ Test external knowledge retrieval for dataset
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | External hit testing completed successfully | **application/json**: [ExternalRetrievalTestResponse](#externalretrievaltestresponse)<br> |
| 200 | External hit testing completed successfully | **application/json**: [ExternalHitTestingResponse](#externalhittestingresponse)<br> |
| 400 | Invalid parameters | |
| 404 | Dataset not found | |
@ -9388,7 +9395,7 @@ Bedrock retrieval test (internal use only)
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Bedrock retrieval test completed | **application/json**: [ExternalRetrievalTestResponse](#externalretrievaltestresponse)<br> |
| 200 | Bedrock retrieval test completed | **application/json**: [BedrockRetrievalResponse](#bedrockretrievalresponse)<br> |
### [GET] /trial-apps/{app_id}
**Get app detail**
@ -15435,6 +15442,21 @@ AppMCPServer Status Enum
| query | string | | Yes |
| retrieval_setting | [BedrockRetrievalSetting](#bedrockretrievalsetting) | | Yes |
#### BedrockRetrievalRecordResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| content | string | | No |
| metadata | object | | No |
| score | number | | Yes |
| title | string | | No |
#### BedrockRetrievalResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| records | [ [BedrockRetrievalRecordResponse](#bedrockretrievalrecordresponse) ] | | Yes |
#### BedrockRetrievalSetting
Retrieval settings for Amazon Bedrock knowledge base queries.
@ -16325,47 +16347,6 @@ Model class for provider custom model configuration.
| permission | [PermissionEnum](#permissionenum) | | No |
| provider | string, <br>**Default:** vendor | | No |
#### DatasetDetail
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| app_count | integer | | No |
| author_name | string | | No |
| built_in_field_enabled | boolean | | No |
| chunk_structure | string | | No |
| created_at | long | | No |
| created_by | string | | No |
| data_source_type | string | | No |
| description | string | | No |
| doc_form | string | | No |
| doc_metadata | [ [DatasetDocMetadata](#datasetdocmetadata) ] | | No |
| document_count | integer | | No |
| embedding_available | boolean | | No |
| embedding_model | string | | No |
| embedding_model_provider | string | | No |
| enable_api | boolean | | No |
| external_knowledge_info | [ExternalKnowledgeInfo](#externalknowledgeinfo) | | No |
| external_retrieval_model | [ExternalRetrievalModel](#externalretrievalmodel) | | No |
| icon_info | [DatasetIconInfo](#dataseticoninfo) | | No |
| id | string | | No |
| indexing_technique | string | | No |
| is_multimodal | boolean | | No |
| is_published | boolean | | No |
| name | string | | No |
| permission | string | | No |
| permission_keys | [ string ] | | No |
| pipeline_id | string | | No |
| provider | string | | No |
| retrieval_model_dict | [DatasetRetrievalModel](#datasetretrievalmodel) | | No |
| runtime_mode | string | | No |
| summary_index_setting | [_AnonymousInlineModel_b1954337d565](#_anonymousinlinemodel_b1954337d565) | | No |
| tags | [ [Tag](#tag) ] | | No |
| total_available_documents | integer | | No |
| total_documents | integer | | No |
| updated_at | long | | No |
| updated_by | string | | No |
| word_count | integer | | No |
#### DatasetDetailResponse
| Name | Type | Description | Required |
@ -16451,14 +16432,6 @@ Model class for provider custom model configuration.
| updated_by | string | | Yes |
| word_count | integer | | Yes |
#### DatasetDocMetadata
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| id | string | | No |
| name | string | | No |
| type | string | | No |
#### DatasetDocMetadataResponse
| Name | Type | Description | Required |
@ -16484,15 +16457,6 @@ Model class for provider custom model configuration.
| score_threshold_enabled | boolean | | No |
| top_k | integer | | Yes |
#### DatasetIconInfo
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| icon | string | | No |
| icon_background | string | | No |
| icon_type | string | | No |
| icon_url | string | | No |
#### DatasetIconInfoResponse
| Name | Type | Description | Required |
@ -16502,12 +16466,6 @@ Model class for provider custom model configuration.
| icon_type | string | | No |
| icon_url | string | | No |
#### DatasetKeywordSetting
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| keyword_weight | number | | No |
#### DatasetKeywordSettingResponse
| Name | Type | Description | Required |
@ -16645,13 +16603,6 @@ Model class for provider custom model configuration.
| page | integer | | Yes |
| total | integer | | Yes |
#### DatasetRerankingModel
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| reranking_model_name | string | | No |
| reranking_provider_name | string | | No |
#### DatasetRerankingModelResponse
| Name | Type | Description | Required |
@ -16672,19 +16623,6 @@ Model class for provider custom model configuration.
| name | string | | Yes |
| permission | string | | No |
#### DatasetRetrievalModel
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| reranking_enable | boolean | | No |
| reranking_mode | string | | No |
| reranking_model | [DatasetRerankingModel](#datasetrerankingmodel) | | No |
| score_threshold | number | | No |
| score_threshold_enabled | boolean | | No |
| search_method | string | | No |
| top_k | integer | | No |
| weights | [DatasetWeightedScore](#datasetweightedscore) | | No |
#### DatasetRetrievalModelResponse
| Name | Type | Description | Required |
@ -16734,14 +16672,6 @@ Model class for provider custom model configuration.
| retrieval_model | object | | No |
| summary_index_setting | object | | No |
#### DatasetVectorSetting
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| embedding_model_name | string | | No |
| embedding_provider_name | string | | No |
| vector_weight | number | | No |
#### DatasetVectorSettingResponse
| Name | Type | Description | Required |
@ -16750,14 +16680,6 @@ Model class for provider custom model configuration.
| embedding_provider_name | string | | No |
| vector_weight | number | | No |
#### DatasetWeightedScore
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| keyword_setting | [DatasetKeywordSetting](#datasetkeywordsetting) | | No |
| vector_setting | [DatasetVectorSetting](#datasetvectorsetting) | | No |
| weight_type | string | | No |
#### DatasetWeightedScoreResponse
| Name | Type | Description | Required |
@ -17061,6 +16983,42 @@ Request payload for bulk downloading documents as a zip archive.
| ---- | ---- | ----------- | -------- |
| document_ids | [ string (uuid) ] | List of document IDs to include in the ZIP download. | Yes |
#### DocumentDetailResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| archived | boolean | | No |
| average_segment_length | number | | No |
| completed_at | integer | | No |
| created_at | integer | | No |
| created_by | string | | No |
| created_from | string | | No |
| data_source_detail_dict | | | No |
| data_source_info | | | No |
| data_source_type | string | | No |
| dataset_process_rule | | | No |
| dataset_process_rule_id | string | | No |
| disabled_at | integer | | No |
| disabled_by | string | | No |
| display_status | string | | No |
| doc_form | string | | No |
| doc_language | string | | No |
| doc_metadata | [ [DocumentMetadataResponse](#documentmetadataresponse) ] | | No |
| doc_type | string | | No |
| document_process_rule | | | No |
| enabled | boolean | | No |
| error | string | | No |
| hit_count | integer | | No |
| id | string | | Yes |
| indexing_latency | number | | No |
| indexing_status | string | | No |
| name | string | | No |
| need_summary | boolean | | No |
| position | integer | | No |
| segment_count | integer | | No |
| tokens | integer | | No |
| updated_at | integer | | No |
#### DocumentMetadataOperation
| Name | Type | Description | Required |
@ -17085,6 +17043,15 @@ Request payload for bulk downloading documents as a zip archive.
| doc_metadata | | | No |
| doc_type | string | | No |
#### DocumentPipelineExecutionLogResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| datasource_info | [JsonValue](#jsonvalue) | | No |
| datasource_node_id | string | | No |
| datasource_type | string | | No |
| input_data | [JsonValue](#jsonvalue) | | No |
#### DocumentRenamePayload
| Name | Type | Description | Required |
@ -17149,6 +17116,14 @@ Request payload for bulk downloading documents as a zip archive.
| stopped_at | integer | | Yes |
| total_segments | integer | | No |
#### DocumentSummaryStatusResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| summaries | [ [SummaryEntryResponse](#summaryentryresponse) ] | | Yes |
| summary_status | [SummaryStatusResponse](#summarystatusresponse) | | Yes |
| total_segments | integer | | Yes |
#### DocumentWithSegmentsListResponse
| Name | Type | Description | Required |
@ -17679,6 +17654,35 @@ Built-in tool icons are URL strings; API-based tool icons are provider-defined p
| metadata_filtering_conditions | object | | No |
| query | string | | Yes |
#### ExternalHitTestingQueryResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| content | string | | Yes |
#### ExternalHitTestingRecordResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| content | string | | No |
| metadata | object | | No |
| score | number | | No |
| title | string | | No |
#### ExternalHitTestingResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| query | [ExternalHitTestingQueryResponse](#externalhittestingqueryresponse) | | Yes |
| records | [ [ExternalHitTestingRecordResponse](#externalhittestingrecordresponse) ] | | Yes |
#### ExternalKnowledgeApiBindingResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| id | string | | Yes |
| name | string | | Yes |
#### ExternalKnowledgeApiListResponse
| Name | Type | Description | Required |
@ -17702,43 +17706,13 @@ Built-in tool icons are URL strings; API-based tool icons are provider-defined p
| ---- | ---- | ----------- | -------- |
| created_at | string | | Yes |
| created_by | string | | Yes |
| dataset_bindings | [ [ExternalKnowledgeDatasetBindingResponse](#externalknowledgedatasetbindingresponse) ] | | No |
| dataset_bindings | [ [ExternalKnowledgeApiBindingResponse](#externalknowledgeapibindingresponse) ] | | Yes |
| description | string | | Yes |
| id | string | | Yes |
| name | string | | Yes |
| settings | object | | No |
| settings | object | | Yes |
| tenant_id | string | | Yes |
#### ExternalKnowledgeDatasetBindingResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| id | string | | Yes |
| name | string | | Yes |
#### ExternalKnowledgeInfo
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| external_knowledge_api_endpoint | string | | No |
| external_knowledge_api_id | string | | No |
| external_knowledge_api_name | string | | No |
| external_knowledge_id | string | | No |
#### ExternalRetrievalModel
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| score_threshold | number | | No |
| score_threshold_enabled | boolean | | No |
| top_k | integer | | No |
#### ExternalRetrievalTestResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| ExternalRetrievalTestResponse | object<br>[ object ] | | |
#### FeatureModel
| Name | Type | Description | Required |
@ -18243,27 +18217,15 @@ Query parameter for including secret variables in export.
| info_list | object | | Yes |
| process_rule | object | | Yes |
#### IndexingEstimatePreviewItemResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| child_chunks | [ string ] | | No |
| content | string | | Yes |
| summary | string | | No |
#### IndexingEstimateQaPreviewItemResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| answer | string | | Yes |
| question | string | | Yes |
#### IndexingEstimateResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| preview | [ [IndexingEstimatePreviewItemResponse](#indexingestimatepreviewitemresponse) ] | | Yes |
| qa_preview | [ [IndexingEstimateQaPreviewItemResponse](#indexingestimateqapreviewitemresponse) ] | | No |
| currency | string | | Yes |
| preview | [ [PreviewDetail](#previewdetail) ] | | Yes |
| qa_preview | [ [QAPreviewDetail](#qapreviewdetail) ] | | No |
| tokens | integer | | Yes |
| total_price | number<br>integer | | Yes |
| total_segments | integer | | Yes |
#### InfoList
@ -19259,12 +19221,6 @@ OAuth schema
| redirect_uri | string | | No |
| refresh_token | string | | No |
#### OpaqueObjectResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| OpaqueObjectResponse | object | | |
#### Option
| Name | Type | Description | Required |
@ -20310,6 +20266,14 @@ Dataset Process Rule Mode
| ---- | ---- | ----------- | -------- |
| ProcessRuleMode | string | Dataset Process Rule Mode | |
#### ProcessRuleResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| limits | object | | Yes |
| mode | [ProcessRuleMode](#processrulemode) | | Yes |
| rules | [Rule](#rule) | | No |
#### ProviderConfig
Model class for common provider settings like credentials
@ -21581,6 +21545,28 @@ The subscription constructor of the trigger provider
| ---- | ---- | ----------- | -------- |
| data | [ string ] | | Yes |
#### SummaryEntryResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| created_at | integer | | No |
| error | string | | No |
| segment_id | string | | Yes |
| segment_position | integer | | Yes |
| status | string | | Yes |
| summary_preview | string | | No |
| updated_at | integer | | No |
#### SummaryStatusResponse
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| completed | integer | | No |
| error | integer | | No |
| generating | integer | | No |
| not_started | integer | | No |
| timeout | integer | | No |
#### SwitchWorkspacePayload
| Name | Type | Description | Required |
@ -23702,15 +23688,6 @@ Workflow tool configuration
| data | [ [AccessPolicy](#accesspolicy) ] | | No |
| pagination | [Pagination](#pagination) | | No |
#### _AnonymousInlineModel_b1954337d565
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| enable | boolean | | No |
| model_name | string | | No |
| model_provider_name | string | | No |
| summary_prompt | string | | No |
#### _DeleteMemberBindingsRequest
| Name | Type | Description | Required |

81
api/openapi/markdown/service-openapi.md
View File

@ -46,76 +46,6 @@ Deprecated legacy alias for creating a new document by providing text content. U
| 401 | Unauthorized - invalid API token | |
| 403 | Forbidden - dataset API access or workspace access denied | |
### [DELETE] /datasets/{dataset_id}/documents/{document_id}
**Delete Document**
Permanently delete a document and all its chunks from the knowledge base.
#### Parameters
| Name | Located in | Description | Required | Schema |
| ---- | ---------- | ----------- | -------- | ------ |
| dataset_id | path | Knowledge base ID. | Yes | string (uuid) |
| document_id | path | Document ID. | Yes | string (uuid) |
#### Responses
| Code | Description |
| ---- | ----------- |
| 204 | Success. |
| 400 | `document_indexing` : Cannot delete document during indexing. |
| 401 | Unauthorized - invalid API token |
| 403 | `archived_document_immutable` : The archived document is not editable. |
| 404 | `not_found` : Document Not Exists. |
### [GET] /datasets/{dataset_id}/documents/{document_id}
**Get Document**
Retrieve detailed information about a specific document, including its indexing status, metadata, and processing statistics.
#### Parameters
| Name | Located in | Description | Required | Schema |
| ---- | ---------- | ----------- | -------- | ------ |
| dataset_id | path | Knowledge base ID. | Yes | string (uuid) |
| document_id | path | Document ID. | Yes | string (uuid) |
| metadata | query | `all` returns all fields including metadata. `only` returns only `id`, `doc_type`, and `doc_metadata`. `without` returns all fields except `doc_metadata`. | No | string, <br>**Available values:** "all", "only", "without", <br>**Default:** all |
#### Responses
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Document details. The response shape varies based on the `metadata` query parameter. When `metadata` is `only`, only `id`, `doc_type`, and `doc_metadata` are returned. When `metadata` is `without`, `doc_type` and `doc_metadata` are omitted. | **application/json**: [DocumentDetailResponse](#documentdetailresponse)<br> |
| 400 | `invalid_metadata` : Invalid metadata value for the specified key. | |
| 401 | Unauthorized - invalid API token | |
| 403 | `forbidden` : No permission. | |
| 404 | `not_found` : Document not found. | |
### [PATCH] /datasets/{dataset_id}/documents/{document_id}
Update an existing document by uploading a file
#### Parameters
| Name | Located in | Description | Required | Schema |
| ---- | ---------- | ----------- | -------- | ------ |
| dataset_id | path | Knowledge base ID. | Yes | string (uuid) |
| document_id | path | Document ID. | Yes | string (uuid) |
#### Request Body
| Required | Schema |
| -------- | ------ |
| No | **multipart/form-data**: { **"data"**: string, **"file"**: binary }<br> |
#### Responses
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Document updated successfully | **application/json**: [DocumentAndBatchResponse](#documentandbatchresponse)<br> |
| 401 | Unauthorized - invalid API token | |
| 403 | Forbidden - dataset API access or workspace access denied | |
| 404 | Document not found | |
### ~~[POST] /datasets/{dataset_id}/documents/{document_id}/update_by_text~~
***DEPRECATED***
@ -1439,7 +1369,9 @@ Retrieve detailed information about a specific document, including its indexing
| 404 | `not_found` : Document not found. | |
### [PATCH] /datasets/{dataset_id}/documents/{document_id}
Update an existing document by uploading a file
**Update Document by File**
Update an existing document by uploading a new file. Re-triggers indexing — use the returned `batch` ID with [Get Document Indexing Status](/api-reference/documents/get-document-indexing-status) to track progress.
#### Parameters
@ -1458,7 +1390,8 @@ Update an existing document by uploading a file
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Document updated successfully | **application/json**: [DocumentAndBatchResponse](#documentandbatchresponse)<br> |
| 200 | Document updated successfully. | **application/json**: [DocumentAndBatchResponse](#documentandbatchresponse)<br> |
| 400 | - `too_many_files` : Only one file is allowed. - `filename_not_exists_error` : The specified filename does not exist. - `provider_not_initialize` : No valid model provider credentials found. Please go to Settings -> Model Provider to complete your provider credentials. - `invalid_param` : Knowledge base does not exist, external datasets not supported, file too large, unsupported file type, or invalid doc_form (must be `text_model`, `hierarchical_model`, or `qa_model`). | |
| 401 | Unauthorized - invalid API token | |
| 403 | Forbidden - dataset API access or workspace access denied | |
| 404 | Document not found | |
@ -2995,7 +2928,7 @@ Request payload for bulk downloading documents as a zip archive.
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| archived | boolean | | No |
| average_segment_length | number | | No |
| average_segment_length | integer<br>number | | No |
| completed_at | integer | | No |
| created_at | integer | | No |
| created_by | string | | No |
@ -3009,7 +2942,7 @@ Request payload for bulk downloading documents as a zip archive.
| display_status | string | | No |
| doc_form | string | | No |
| doc_language | string | | No |
| doc_metadata | [ [DocumentMetadataResponse](#documentmetadataresponse) ] | | No |
| doc_metadata | [ [DocumentMetadataResponse](#documentmetadataresponse) ]<br>object | | No |
| doc_type | string | | No |
| document_process_rule | object | | No |
| enabled | boolean | | No |

12
api/tests/unit_tests/controllers/console/datasets/test_datasets.py
View File

@ -31,6 +31,7 @@ from controllers.console.datasets.datasets import (
DatasetUseCheckApi,
)
from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError
from core.entities.knowledge_entities import IndexingEstimate
from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
from core.provider_manager import ProviderManager
from core.rag.index_processor.constant.index_type import IndexStructureType
@ -1391,8 +1392,7 @@ class TestDatasetIndexingEstimateApi:
mock_file = self._upload_file()
mock_response = MagicMock()
mock_response.model_dump.return_value = {"tokens": 100}
mock_response = IndexingEstimate(total_segments=100, preview=[])
with (
app.test_request_context("/"),
@ -1418,7 +1418,13 @@ class TestDatasetIndexingEstimateApi:
response, status = method(api, "tenant-1")
assert status == 200
assert response == {"tokens": 100}
assert response == {
"tokens": 0,
"total_price": 0,
"currency": "USD",
"total_segments": 100,
"preview": [],
}
def test_post_file_not_found(self, app: Flask):
api = DatasetIndexingEstimateApi()

106
api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py
View File

@ -1,3 +1,4 @@
import datetime
import inspect
from unittest.mock import ANY, MagicMock, patch
@ -34,6 +35,7 @@ from controllers.console.datasets.error import (
InvalidActionError,
InvalidMetadataError,
)
from core.entities.knowledge_entities import IndexingEstimate
from core.rag.index_processor.constant.index_type import IndexStructureType
from models.dataset import Dataset
from models.dataset import Document as DatasetDocument
@ -72,8 +74,46 @@ def make_serializable_document(**overrides):
}
attrs.update(overrides)
document = MagicMock(spec_set=list(attrs))
for name, value in attrs.items():
setattr(document, name, value)
document.configure_mock(**attrs)
return document
def make_document_detail(**overrides):
attrs = {
"id": "doc-1",
"position": 1,
"data_source_type": "upload_file",
"data_source_info_dict": {"upload_file_id": "file-1"},
"data_source_detail_dict": {},
"dataset_process_rule_id": None,
"dataset_process_rule": None,
"name": "Document",
"created_from": "web",
"created_by": "u1",
"created_at": datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC),
"tokens": 10,
"indexing_status": "completed",
"completed_at": None,
"updated_at": None,
"indexing_latency": None,
"error": None,
"enabled": True,
"disabled_at": None,
"disabled_by": None,
"archived": False,
"doc_type": "others",
"doc_metadata_details": [],
"segment_count": 0,
"average_segment_length": 0,
"hit_count": 0,
"display_status": "available",
"doc_form": "text_model",
"doc_language": "English",
"need_summary": False,
}
attrs.update(overrides)
document = MagicMock(spec_set=list(attrs))
document.configure_mock(**attrs)
return document
@ -173,6 +213,42 @@ class TestGetProcessRuleApi:
assert "rules" in response
def test_get_with_document_preserves_legacy_segmentation_delimiter(self, app: Flask, patch_tenant):
api = GetProcessRuleApi()
method = inspect.unwrap(api.get)
user, _ = patch_tenant
document = MagicMock(dataset_id="ds-1")
process_rule = MagicMock(
mode="custom",
rules_dict={"segmentation": {"delimiter": "---", "max_tokens": 123}},
)
with (
app.test_request_context("/?document_id=doc-1"),
patch(
"controllers.console.datasets.datasets_document.db.get_or_404",
return_value=document,
),
patch(
"controllers.console.datasets.datasets_document.DatasetService.get_dataset",
return_value=MagicMock(),
),
patch(
"controllers.console.datasets.datasets_document.DatasetService.check_dataset_permission",
return_value=None,
),
patch(
"controllers.console.datasets.datasets_document.db.session.scalar",
return_value=process_rule,
),
):
response = method(api, user)
assert response["rules"]["segmentation"]["separator"] == "---"
assert response["rules"]["segmentation"]["max_tokens"] == 123
assert "delimiter" not in response["rules"]["segmentation"]
def test_get_with_document_dataset_not_found(self, app: Flask, patch_tenant):
api = GetProcessRuleApi()
method = inspect.unwrap(api.get)
@ -414,7 +490,7 @@ class TestDocumentApi:
method = inspect.unwrap(api.get)
user, tenant_id = patch_tenant
document = MagicMock(dataset_process_rule=None)
document = make_document_detail()
with (
app.test_request_context("/"),
@ -926,12 +1002,24 @@ class TestDocumentSummaryStatusApi:
),
patch(
"services.summary_index_service.SummaryIndexService.get_document_summary_status_detail",
return_value={"total_segments": 0},
return_value={
"total_segments": 1,
"summary_status": {"timeout": 1},
"summaries": [
{
"segment_id": "segment-1",
"segment_position": 1,
"status": "timeout",
}
],
},
),
):
response, status = method(api, user, "ds-1", "doc-1")
assert status == 200
assert response["summary_status"]["timeout"] == 1
assert response["summaries"][0]["status"] == "timeout"
class TestDocumentIndexingEstimateApi:
@ -1103,7 +1191,7 @@ class TestDocumentBatchIndexingEstimateApi:
patch.object(api, "get_batch_documents", return_value=[doc]),
patch(
"controllers.console.datasets.datasets_document.IndexingRunner.indexing_estimate",
return_value=MagicMock(model_dump=lambda: {"tokens": 2}),
return_value=IndexingEstimate(total_segments=2, preview=[]),
),
):
resp, status = method(api, tenant_id, user, "ds-1", "batch-1")
@ -1132,7 +1220,7 @@ class TestDocumentBatchIndexingEstimateApi:
patch.object(api, "get_batch_documents", return_value=[doc]),
patch(
"controllers.console.datasets.datasets_document.IndexingRunner.indexing_estimate",
return_value=MagicMock(model_dump=lambda: {"tokens": 1}),
return_value=IndexingEstimate(total_segments=1, preview=[]),
),
):
resp, status = method(api, tenant_id, user, "ds-1", "batch-1")
@ -1314,7 +1402,7 @@ class TestDocumentApiMetadata:
method = inspect.unwrap(api.get)
user, tenant_id = patch_tenant
document = MagicMock(dataset_process_rule=None, doc_metadata_details=[])
document = make_document_detail(doc_metadata_details=[])
with (
app.test_request_context("/?metadata=only"),
@ -1334,7 +1422,7 @@ class TestDocumentApiMetadata:
method = inspect.unwrap(api.get)
user, tenant_id = patch_tenant
document = MagicMock(dataset_process_rule=None)
document = make_document_detail()
with (
app.test_request_context("/?metadata=without"),
@ -1611,7 +1699,7 @@ class TestDocumentIndexingEdgeCases:
),
patch(
"controllers.console.datasets.datasets_document.IndexingRunner.indexing_estimate",
return_value=MagicMock(model_dump=lambda: {"tokens": 5}),
return_value=IndexingEstimate(total_segments=5, preview=[]),
),
):
response, status = method(api, tenant_id, user, "ds-1", "doc-1")

424
api/tests/unit_tests/controllers/console/datasets/test_external.py
View File

@ -1,5 +1,6 @@
import inspect
from types import SimpleNamespace
from typing import Any
from unittest.mock import ANY, MagicMock, PropertyMock, patch
import pytest
@ -8,7 +9,6 @@ from werkzeug.exceptions import Forbidden, NotFound
import services
from controllers.console import console_ns
from controllers.console.datasets import external as external_module
from controllers.console.datasets.error import DatasetNameDuplicateError
from controllers.console.datasets.external import (
BedrockRetrievalApi,
@ -40,28 +40,185 @@ def current_user() -> Account:
return user
def _external_api_dict(api_id: str = "api-1") -> dict:
return {
"id": api_id,
"tenant_id": "tenant-1",
"name": f"External API {api_id}",
"description": f"Description for {api_id}",
"settings": {
"endpoint": f"https://external.example.com/{api_id}",
"api_key": "secret",
"headers": {"X-Source": "unit-test"},
"timeout": 30,
},
"dataset_bindings": [
{"id": f"dataset-{api_id}", "name": f"Dataset {api_id}"},
],
"created_by": "user-1",
"created_at": "2024-01-01T00:00:00",
}
def _external_api_object(api_id: str = "api-1") -> SimpleNamespace:
payload = _external_api_dict(api_id)
return SimpleNamespace(
**{
**payload,
"dataset_bindings": [SimpleNamespace(**binding) for binding in payload["dataset_bindings"]],
}
)
def _expected_dataset_detail_payload() -> dict[str, Any]:
return {
"id": "dataset-1",
"name": "Support knowledge",
"description": "External support articles",
"provider": "external",
"permission": "only_me",
"data_source_type": "external",
"indexing_technique": "economy",
"app_count": 2,
"document_count": 7,
"word_count": 2048,
"created_by": "user-1",
"author_name": "Test User",
"created_at": 1710000000,
"updated_by": "user-2",
"updated_at": 1710003600,
"embedding_model": None,
"embedding_model_provider": None,
"embedding_available": False,
"retrieval_model_dict": {
"search_method": "semantic_search",
"reranking_enable": False,
"reranking_mode": None,
"reranking_model": {"reranking_provider_name": None, "reranking_model_name": None},
"weights": None,
"top_k": 4,
"score_threshold_enabled": True,
"score_threshold": 0.5,
},
"summary_index_setting": {
"enable": True,
"model_name": "summary-model",
"model_provider_name": "provider-a",
"summary_prompt": "Summarize this.",
},
"tags": [{"id": "tag-1", "name": "Support", "type": "knowledge"}],
"doc_form": "text_model",
"external_knowledge_info": {
"external_knowledge_id": "knowledge-1",
"external_knowledge_api_id": "api-1",
"external_knowledge_api_name": "External API api-1",
"external_knowledge_api_endpoint": "https://external.example.com/api-1",
},
"external_retrieval_model": {
"top_k": 4,
"score_threshold": 0.5,
"score_threshold_enabled": True,
},
"doc_metadata": [{"id": "metadata-1", "name": "source", "type": "string"}],
"built_in_field_enabled": True,
"pipeline_id": None,
"runtime_mode": "external",
"chunk_structure": "general",
"icon_info": {
"icon_type": "emoji",
"icon": "book",
"icon_background": "#FFF4ED",
"icon_url": None,
},
"is_published": True,
"total_documents": 7,
"total_available_documents": 6,
"enable_api": True,
"is_multimodal": False,
"maintainer": None,
"permission_keys": [],
}
def _dataset_detail_object() -> SimpleNamespace:
payload = _expected_dataset_detail_payload()
return SimpleNamespace(
**{
**payload,
"summary_index_setting": SimpleNamespace(**payload["summary_index_setting"]),
"tags": [SimpleNamespace(**tag) for tag in payload["tags"]],
"external_knowledge_info": SimpleNamespace(**payload["external_knowledge_info"]),
"external_retrieval_model": SimpleNamespace(**payload["external_retrieval_model"]),
"doc_metadata": [SimpleNamespace(**item) for item in payload["doc_metadata"]],
"icon_info": SimpleNamespace(**payload["icon_info"]),
}
)
class TestExternalApiTemplateListApi:
def test_get_success(self, app: Flask):
api = ExternalApiTemplateListApi()
method = inspect.unwrap(api.get)
api_item = MagicMock()
api_item.to_dict.return_value = {"id": "1"}
api_item = _external_api_object("api-1")
with (
app.test_request_context("/?page=1&limit=20"),
app.test_request_context("/?page=2&limit=1&keyword=vector"),
patch.object(
ExternalDatasetService,
"get_external_knowledge_apis",
return_value=([api_item], 1),
return_value=([api_item], 3),
) as get_external_knowledge_apis,
):
resp, status = method(api, "tenant-1")
assert status == 200
assert resp["total"] == 1
assert resp["data"][0]["id"] == "1"
get_external_knowledge_apis.assert_called_once_with(1, 20, "tenant-1", None)
assert resp == {
"data": [_external_api_dict("api-1")],
"has_more": True,
"limit": 1,
"total": 3,
"page": 2,
}
get_external_knowledge_apis.assert_called_once_with(2, 1, "tenant-1", "vector")
def test_post_success_uses_validated_payload_and_returns_template(self, app: Flask, current_user: Account):
api = ExternalApiTemplateListApi()
method = inspect.unwrap(api.post)
payload = {
"name": "Vendor Search",
"settings": {
"endpoint": "https://external.example.com/search",
"api_key": "secret",
"headers": {"X-Source": "unit-test"},
"timeout": 30,
},
}
created = _external_api_object("api-created")
session = MagicMock()
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
patch.object(ExternalDatasetService, "validate_api_list") as validate_api_list,
patch.object(
ExternalDatasetService,
"create_external_knowledge_api",
return_value=created,
) as create_external_knowledge_api,
):
resp, status = method(api, session, "tenant-1", current_user)
assert status == 201
assert resp == _external_api_dict("api-created")
validate_api_list.assert_called_once_with(payload["settings"])
create_external_knowledge_api.assert_called_once_with(
tenant_id="tenant-1",
user_id="user-1",
args=payload,
session=session,
)
def test_post_forbidden(self, app: Flask, current_user: Account):
current_user.role = TenantAccountRole.NORMAL
@ -99,6 +256,28 @@ class TestExternalApiTemplateListApi:
class TestExternalApiTemplateApi:
def test_get_success_returns_template_contract(self, app: Flask):
api = ExternalApiTemplateApi()
method = inspect.unwrap(api.get)
template = _external_api_object("api-detail")
session = MagicMock()
with (
app.test_request_context("/"),
patch.object(
ExternalDatasetService,
"get_external_knowledge_api",
return_value=template,
) as get_external_knowledge_api,
):
resp, status = method(api, session, "tenant-1", "api-detail")
assert status == 200
assert resp == _external_api_dict("api-detail")
get_external_knowledge_api.assert_called_once_with(
external_knowledge_api_id="api-detail", tenant_id="tenant-1", session=session
)
def test_get_not_found(self, app: Flask):
api = ExternalApiTemplateApi()
method = inspect.unwrap(api.get)
@ -114,6 +293,44 @@ class TestExternalApiTemplateApi:
with pytest.raises(NotFound):
method(api, MagicMock(), "tenant-1", "api-id")
def test_patch_success_uses_validated_payload_and_returns_template(self, app: Flask, current_user: Account):
api = ExternalApiTemplateApi()
method = inspect.unwrap(api.patch)
payload = {
"name": "Updated API",
"settings": {
"endpoint": "https://external.example.com/updated",
"api_key": "new-secret",
"headers": {"X-Version": "2"},
},
}
updated = _external_api_object("api-updated")
session = MagicMock()
with (
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
patch.object(ExternalDatasetService, "validate_api_list") as validate_api_list,
patch.object(
ExternalDatasetService,
"update_external_knowledge_api",
return_value=updated,
) as update_external_knowledge_api,
):
resp, status = method(api, session, "tenant-1", current_user, "api-updated")
assert status == 200
assert resp == _external_api_dict("api-updated")
validate_api_list.assert_called_once_with(payload["settings"])
update_external_knowledge_api.assert_called_once_with(
tenant_id="tenant-1",
user_id="user-1",
external_knowledge_api_id="api-updated",
args=payload,
session=session,
)
def test_delete_forbidden(self, app: Flask, current_user: Account):
current_user.role = TenantAccountRole.NORMAL
@ -153,46 +370,39 @@ class TestExternalDatasetCreateApi:
method = inspect.unwrap(api.post)
payload = {
"external_knowledge_api_id": "api",
"external_knowledge_id": "kid",
"name": "dataset",
"external_knowledge_api_id": "api-1",
"external_knowledge_id": "knowledge-1",
"name": "Support knowledge",
"description": "External support articles",
"external_retrieval_model": {
"top_k": 4,
"score_threshold": 0.5,
"score_threshold_enabled": True,
},
}
dataset = MagicMock()
dataset.embedding_available = False
dataset.built_in_field_enabled = False
dataset.is_published = False
dataset.enable_api = False
dataset.enable_qa = False
dataset.enable_vector_store = False
dataset.vector_store_setting = None
dataset.is_multimodal = False
dataset.retrieval_model_dict = {}
dataset.tags = []
dataset.external_knowledge_info = None
dataset.external_retrieval_model = None
dataset.doc_metadata = []
dataset.icon_info = None
dataset.permission_keys = []
dataset.summary_index_setting = MagicMock()
dataset.summary_index_setting.enable = False
dataset = _dataset_detail_object()
with (
app.test_request_context("/"),
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
patch.object(
ExternalDatasetService,
"create_external_dataset",
return_value=dataset,
),
patch.object(external_module, "db", SimpleNamespace(session=lambda: MagicMock())),
) as create_external_dataset,
):
_, status = method(api, MagicMock(), "tenant-1", current_user)
session = MagicMock()
resp, status = method(api, session, "tenant-1", current_user)
assert status == 201
assert resp == _expected_dataset_detail_payload()
create_external_dataset.assert_called_once_with(
tenant_id="tenant-1",
user_id="user-1",
args=payload,
session=session,
)
def test_create_forbidden(self, app: Flask, current_user: Account):
current_user.role = TenantAccountRole.NORMAL
@ -233,24 +443,59 @@ class TestExternalKnowledgeHitTestingApi:
api = ExternalKnowledgeHitTestingApi()
method = inspect.unwrap(api.post)
payload = {"query": "hello"}
payload = {
"query": "hello",
"external_retrieval_model": {
"top_k": 3,
"score_threshold": 0.25,
"score_threshold_enabled": True,
},
"metadata_filtering_conditions": {
"logical_operator": "and",
"conditions": [{"name": "source", "comparison_operator": "contains", "value": "external"}],
},
}
dataset = MagicMock()
retrieve_response = {
"query": {"content": "hello"},
"records": [
{
"content": "answer",
"title": "doc",
"score": 0.9,
"metadata": {"source": "external", "page": 2},
}
],
}
session = MagicMock()
with (
app.test_request_context("/"),
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
patch.object(DatasetService, "get_dataset", return_value=dataset),
patch.object(DatasetService, "check_dataset_permission"),
patch.object(DatasetService, "check_dataset_permission") as check_dataset_permission,
patch.object(HitTestingService, "hit_testing_args_check") as hit_testing_args_check,
patch.object(
HitTestingService,
"external_retrieve",
return_value={"ok": True},
),
return_value=retrieve_response,
) as external_retrieve,
patch("controllers.console.datasets.external.dump_response", side_effect=lambda _model, value: value),
):
resp = method(api, MagicMock(), current_user, "dataset-id")
resp = method(api, session, current_user, "dataset-id")
assert resp["ok"] is True
assert resp == retrieve_response
check_dataset_permission.assert_called_once_with(dataset, current_user, session)
hit_testing_args_check.assert_called_once_with(payload)
external_retrieve.assert_called_once_with(
session=session,
dataset=dataset,
query="hello",
account=current_user,
external_retrieval_model=payload["external_retrieval_model"],
metadata_filtering_conditions=payload["metadata_filtering_conditions"],
)
class TestBedrockRetrievalApi:
@ -259,24 +504,46 @@ class TestBedrockRetrievalApi:
method = inspect.unwrap(api.post)
payload = {
"retrieval_setting": {},
"query": "hello",
"knowledge_id": "kid",
"retrieval_setting": {"top_k": 5, "score_threshold": 0.72},
"query": "hello bedrock",
"knowledge_id": "knowledge-base-1",
}
retrieval_response = {
"records": [
{
"metadata": {"source": "bedrock", "uri": "s3://bucket/doc.txt"},
"score": 0.8,
"title": "doc",
"content": "answer",
},
{
"metadata": {"source": "bedrock", "uri": "s3://bucket/other.txt"},
"score": 0.65,
"title": None,
"content": None,
},
]
}
session = MagicMock()
with (
app.test_request_context("/"),
app.test_request_context("/", json=payload),
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
patch.object(
ExternalDatasetTestService,
"knowledge_retrieval",
return_value={"ok": True},
),
return_value=retrieval_response,
) as knowledge_retrieval,
):
resp, status = method()
assert status == 200
assert resp["ok"] is True
assert resp == retrieval_response
retrieval_setting, query, knowledge_id = knowledge_retrieval.call_args.args
assert retrieval_setting.model_dump() == payload["retrieval_setting"]
assert query == "hello bedrock"
assert knowledge_id == "knowledge-base-1"
class TestExternalApiTemplateListApiAdvanced:
@ -302,10 +569,10 @@ class TestExternalApiTemplateListApiAdvanced:
api = ExternalApiTemplateListApi()
method = inspect.unwrap(api.get)
templates = [MagicMock(id=f"api-{i}") for i in range(3)]
templates = [_external_api_object(f"api-{i}") for i in range(3)]
with (
app.test_request_context("/?page=1&limit=20"),
app.test_request_context("/?page=2&limit=3"),
patch(
"controllers.console.datasets.external.ExternalDatasetService.get_external_knowledge_apis",
return_value=(templates, 25),
@ -314,9 +581,14 @@ class TestExternalApiTemplateListApiAdvanced:
resp, status = method(api, "tenant-1")
assert status == 200
assert resp["total"] == 25
assert len(resp["data"]) == 3
get_external_knowledge_apis.assert_called_once_with(1, 20, "tenant-1", None)
assert resp == {
"data": [_external_api_dict(f"api-{i}") for i in range(3)],
"has_more": True,
"limit": 3,
"total": 25,
"page": 2,
}
get_external_knowledge_apis.assert_called_once_with(2, 3, "tenant-1", None)
class TestExternalDatasetCreateApiAdvanced:
@ -371,6 +643,7 @@ class TestExternalKnowledgeHitTestingApiAdvanced:
"external_retrieval_model": {"type": "bm25"},
"metadata_filtering_conditions": {"status": "active"},
}
session = MagicMock()
with (
app.test_request_context("/", json=payload),
@ -379,15 +652,46 @@ class TestExternalKnowledgeHitTestingApiAdvanced:
"controllers.console.datasets.external.DatasetService.get_dataset",
return_value=dataset,
),
patch("controllers.console.datasets.external.DatasetService.check_dataset_permission"),
patch("controllers.console.datasets.external.DatasetService.check_dataset_permission") as check_permission,
patch("controllers.console.datasets.external.HitTestingService.hit_testing_args_check") as args_check,
patch(
"controllers.console.datasets.external.HitTestingService.external_retrieve",
return_value={"results": []},
),
return_value={
"query": {"content": "test query"},
"records": [
{
"content": None,
"title": "metadata-only",
"score": None,
"metadata": {"status": "active"},
}
],
},
) as external_retrieve,
):
resp = method(api, MagicMock(), current_user, "ds-1")
resp = method(api, session, current_user, "ds-1")
assert resp["results"] == []
assert resp == {
"query": {"content": "test query"},
"records": [
{
"content": None,
"title": "metadata-only",
"score": None,
"metadata": {"status": "active"},
}
],
}
check_permission.assert_called_once_with(dataset, current_user, session)
args_check.assert_called_once_with(payload)
external_retrieve.assert_called_once_with(
session=session,
dataset=dataset,
query="test query",
account=current_user,
external_retrieval_model={"type": "bm25"},
metadata_filtering_conditions={"status": "active"},
)
class TestBedrockRetrievalApiAdvanced:

614
api/tests/unit_tests/controllers/service_api/dataset/test_document.py
View File

File diff suppressed because it is too large Load Diff

12
packages/contracts/generated/api/console/datasets/orpc.gen.ts
View File

@ -359,8 +359,12 @@ export const get5 = oc
.input(z.object({ params: zGetDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdPath }))
.output(zGetDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdResponse)
/**
* Update external knowledge API template
*/
export const patch = oc
.route({
description: 'Update external knowledge API template',
inputStructure: 'detailed',
method: 'PATCH',
operationId: 'patchDatasetsExternalKnowledgeApiByExternalKnowledgeApiId',
@ -397,8 +401,12 @@ export const get6 = oc
.input(z.object({ query: zGetDatasetsExternalKnowledgeApiQuery.optional() }))
.output(zGetDatasetsExternalKnowledgeApiResponse)
/**
* Create external knowledge API template
*/
export const post4 = oc
.route({
description: 'Create external knowledge API template',
inputStructure: 'detailed',
method: 'POST',
operationId: 'postDatasetsExternalKnowledgeApi',
@ -444,7 +452,6 @@ export const post6 = oc
method: 'POST',
operationId: 'postDatasetsInit',
path: '/datasets/init',
successStatus: 201,
tags: ['console'],
})
.input(z.object({ body: zPostDatasetsInitBody }))
@ -1184,12 +1191,13 @@ export const segments = {
* - generating: Number of summaries being generated
* - error: Number of summaries with errors
* - not_started: Number of segments without summary records
* - timeout: Number of summaries that timed out
* - summaries: List of summary records with status and content preview
*/
export const get22 = oc
.route({
description:
'Get summary index generation status for a document\nReturns:\n- total_segments: Total number of segments in the document\n- summary_status: Dictionary with status counts\n - completed: Number of summaries completed\n - generating: Number of summaries being generated\n - error: Number of summaries with errors\n - not_started: Number of segments without summary records\n- summaries: List of summary records with status and content preview',
'Get summary index generation status for a document\nReturns:\n- total_segments: Total number of segments in the document\n- summary_status: Dictionary with status counts\n - completed: Number of summaries completed\n - generating: Number of summaries being generated\n - error: Number of summaries with errors\n - not_started: Number of segments without summary records\n - timeout: Number of summaries that timed out\n- summaries: List of summary records with status and content preview',
inputStructure: 'detailed',
method: 'GET',
operationId: 'getDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatus',

305
packages/contracts/generated/api/console/datasets/types.gen.ts
View File

@ -97,51 +97,12 @@ export type ExternalDatasetCreatePayload = {
name: string
}
export type DatasetDetail = {
app_count?: number
author_name?: string
built_in_field_enabled?: boolean
chunk_structure?: string
created_at?: number
created_by?: string
data_source_type?: string
description?: string
doc_form?: string
doc_metadata?: Array<DatasetDocMetadata>
document_count?: number
embedding_available?: boolean
embedding_model?: string
embedding_model_provider?: string
enable_api?: boolean
external_knowledge_info?: ExternalKnowledgeInfo
external_retrieval_model?: ExternalRetrievalModel
icon_info?: DatasetIconInfo
id?: string
indexing_technique?: string
is_multimodal?: boolean
is_published?: boolean
name?: string
permission?: string
permission_keys?: Array<string>
pipeline_id?: string
provider?: string
retrieval_model_dict?: DatasetRetrievalModel
runtime_mode?: string
summary_index_setting?: AnonymousInlineModelB1954337D565
tags?: Array<Tag>
total_available_documents?: number
total_documents?: number
updated_at?: number
updated_by?: string
word_count?: number
}
export type ExternalKnowledgeApiListResponse = {
data: Array<ExternalKnowledgeApiResponse>
has_more: boolean
limit: number
page: number
total: number
total: number | null
}
export type ExternalKnowledgeApiPayload = {
@ -154,11 +115,11 @@ export type ExternalKnowledgeApiPayload = {
export type ExternalKnowledgeApiResponse = {
created_at: string
created_by: string
dataset_bindings?: Array<ExternalKnowledgeDatasetBindingResponse>
dataset_bindings: Array<ExternalKnowledgeApiBindingResponse>
description: string
id: string
name: string
settings?: {
settings: {
[key: string]: unknown
} | null
tenant_id: string
@ -183,8 +144,11 @@ export type IndexingEstimatePayload = {
}
export type IndexingEstimateResponse = {
preview: Array<IndexingEstimatePreviewItemResponse>
qa_preview?: Array<IndexingEstimateQaPreviewItemResponse> | null
currency: string
preview: Array<PreviewDetail>
qa_preview?: Array<QaPreviewDetail> | null
tokens: number
total_price: number | number
total_segments: number
}
@ -236,8 +200,12 @@ export type IndexingEstimate = {
total_segments: number
}
export type OpaqueObjectResponse = {
[key: string]: unknown
export type ProcessRuleResponse = {
limits: {
[key: string]: unknown
}
mode: ProcessRuleMode
rules?: Rule | null
}
export type RetrievalSettingResponse = {
@ -337,8 +305,6 @@ export type DocumentBatchDownloadZipPayload = {
document_ids: Array<string>
}
export type BinaryFileResponse = Blob | File
export type GenerateSummaryPayload = {
document_list: Array<string>
}
@ -347,6 +313,40 @@ export type MetadataOperationData = {
operation_data: Array<DocumentMetadataOperation>
}
export type DocumentDetailResponse = {
archived?: boolean | null
average_segment_length?: number | null
completed_at?: number | null
created_at?: number | null
created_by?: string | null
created_from?: string | null
data_source_detail_dict?: unknown
data_source_info?: unknown
data_source_type?: string | null
dataset_process_rule?: unknown
dataset_process_rule_id?: string | null
disabled_at?: number | null
disabled_by?: string | null
display_status?: string | null
doc_form?: string | null
doc_language?: string | null
doc_metadata?: Array<DocumentMetadataResponse> | null
doc_type?: string | null
document_process_rule?: unknown
enabled?: boolean | null
error?: string | null
hit_count?: number | null
id: string
indexing_latency?: number | null
indexing_status?: string | null
name?: string | null
need_summary?: boolean | null
position?: number | null
segment_count?: number | null
tokens?: number | null
updated_at?: number | null
}
export type UrlResponse = {
url: string
}
@ -376,6 +376,13 @@ export type SimpleResultMessageResponse = {
result: string
}
export type DocumentPipelineExecutionLogResponse = {
datasource_info?: JsonValue | null
datasource_node_id?: string | null
datasource_type?: string | null
input_data?: JsonValue | null
}
export type DocumentRenamePayload = {
name: string
}
@ -464,6 +471,12 @@ export type ChildChunkUpdatePayload = {
content: string
}
export type DocumentSummaryStatusResponse = {
summaries: Array<SummaryEntryResponse>
summary_status: SummaryStatusResponse
total_segments: number
}
export type ErrorDocsResponse = {
data: Array<DocumentStatusResponse>
total: number
@ -479,13 +492,10 @@ export type ExternalHitTestingPayload = {
query: string
}
export type ExternalRetrievalTestResponse
= | {
[key: string]: unknown
}
| Array<{
[key: string]: unknown
}>
export type ExternalHitTestingResponse = {
query: ExternalHitTestingQueryResponse
records: Array<ExternalHitTestingRecordResponse>
}
export type HitTestingPayload = {
attachment_ids?: Array<string> | null
@ -641,68 +651,18 @@ export type DatasetTagResponse = {
type: string
}
export type DatasetDocMetadata = {
id?: string
name?: string
type?: string
}
export type ExternalKnowledgeInfo = {
external_knowledge_api_endpoint?: string
external_knowledge_api_id?: string
external_knowledge_api_name?: string
external_knowledge_id?: string
}
export type ExternalRetrievalModel = {
score_threshold?: number
score_threshold_enabled?: boolean
top_k?: number
}
export type DatasetIconInfo = {
icon?: string
icon_background?: string
icon_type?: string
icon_url?: string
}
export type DatasetRetrievalModel = {
reranking_enable?: boolean
reranking_mode?: string
reranking_model?: DatasetRerankingModel
score_threshold?: number
score_threshold_enabled?: boolean
search_method?: string
top_k?: number
weights?: DatasetWeightedScore
}
export type AnonymousInlineModelB1954337D565 = {
enable?: boolean
model_name?: string
model_provider_name?: string
summary_prompt?: string
}
export type Tag = {
id: string
name: string
type: string
}
export type ExternalKnowledgeDatasetBindingResponse = {
export type ExternalKnowledgeApiBindingResponse = {
id: string
name: string
}
export type IndexingEstimatePreviewItemResponse = {
export type PreviewDetail = {
child_chunks?: Array<string> | null
content: string
summary?: string | null
}
export type IndexingEstimateQaPreviewItemResponse = {
export type QaPreviewDetail = {
answer: string
question: string
}
@ -744,15 +704,13 @@ export type DatasetMetadataBuiltInFieldResponse = {
type: string
}
export type PreviewDetail = {
child_chunks?: Array<string> | null
content: string
summary?: string | null
}
export type ProcessRuleMode = 'automatic' | 'custom' | 'hierarchical'
export type QaPreviewDetail = {
answer: string
question: string
export type Rule = {
parent_mode?: 'full-doc' | 'paragraph' | null
pre_processing_rules?: Array<PreProcessingRule> | null
segmentation?: Segmentation | null
subchunk_segmentation?: Segmentation | null
}
export type DocumentWithSegmentsResponse = {
@ -798,6 +756,8 @@ export type DocumentMetadataResponse = {
value?: string | number | number | boolean | null
}
export type JsonValue = unknown
export type SegmentResponse = {
answer: string | null
attachments: Array<SegmentAttachmentResponse>
@ -844,6 +804,37 @@ export type ChildChunkUpdateArgs = {
id?: string | null
}
export type SummaryEntryResponse = {
created_at?: number | null
error?: string | null
segment_id: string
segment_position: number
status: string
summary_preview?: string | null
updated_at?: number | null
}
export type SummaryStatusResponse = {
completed?: number
error?: number
generating?: number
not_started?: number
timeout?: number
}
export type ExternalHitTestingQueryResponse = {
content: string
}
export type ExternalHitTestingRecordResponse = {
content?: string | null
metadata?: {
[key: string]: unknown
} | null
score?: number | null
title?: string | null
}
export type HitTestingQuery = {
content: string
}
@ -896,17 +887,6 @@ export type DatasetWeightedScoreResponse = {
weight_type?: string | null
}
export type DatasetRerankingModel = {
reranking_model_name?: string
reranking_provider_name?: string
}
export type DatasetWeightedScore = {
keyword_setting?: DatasetKeywordSetting
vector_setting?: DatasetVectorSetting
weight_type?: string
}
export type InfoList = {
data_source_type: 'notion_import' | 'upload_file' | 'website_crawl'
file_info_list?: FileInfo | null
@ -914,15 +894,6 @@ export type InfoList = {
website_info_list?: WebsiteInfo | null
}
export type ProcessRuleMode = 'automatic' | 'custom' | 'hierarchical'
export type Rule = {
parent_mode?: 'full-doc' | 'paragraph' | null
pre_processing_rules?: Array<PreProcessingRule> | null
segmentation?: Segmentation | null
subchunk_segmentation?: Segmentation | null
}
export type MetadataFilteringCondition = {
conditions?: Array<Condition> | null
logical_operator?: 'and' | 'or' | null
@ -945,6 +916,17 @@ export type WeightModel = {
weight_type?: 'customized' | 'keyword_first' | 'semantic_first' | null
}
export type PreProcessingRule = {
enabled: boolean
id: 'remove_extra_spaces' | 'remove_stopwords' | 'remove_urls_emails'
}
export type Segmentation = {
chunk_overlap?: number
max_tokens: number
separator?: string
}
export type MetadataDetail = {
id: string
name: string
@ -1018,16 +1000,6 @@ export type DatasetVectorSettingResponse = {
vector_weight?: number | null
}
export type DatasetKeywordSetting = {
keyword_weight?: number
}
export type DatasetVectorSetting = {
embedding_model_name?: string
embedding_provider_name?: string
vector_weight?: number
}
export type FileInfo = {
file_ids: Array<string>
}
@ -1045,17 +1017,6 @@ export type WebsiteInfo = {
urls: Array<string>
}
export type PreProcessingRule = {
enabled: boolean
id: 'remove_extra_spaces' | 'remove_stopwords' | 'remove_urls_emails'
}
export type Segmentation = {
chunk_overlap?: number
max_tokens: number
separator?: string
}
export type Condition = {
comparison_operator:
| '<'
@ -1264,7 +1225,7 @@ export type PostDatasetsExternalErrors = {
}
export type PostDatasetsExternalResponses = {
201: DatasetDetail
201: DatasetDetailResponse
}
export type PostDatasetsExternalResponse
@ -1295,6 +1256,10 @@ export type PostDatasetsExternalKnowledgeApiData = {
url: '/datasets/external-knowledge-api'
}
export type PostDatasetsExternalKnowledgeApiErrors = {
403: unknown
}
export type PostDatasetsExternalKnowledgeApiResponses = {
201: ExternalKnowledgeApiResponse
}
@ -1347,6 +1312,10 @@ export type PatchDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdData = {
url: '/datasets/external-knowledge-api/{external_knowledge_api_id}'
}
export type PatchDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdErrors = {
404: unknown
}
export type PatchDatasetsExternalKnowledgeApiByExternalKnowledgeApiIdResponses = {
200: ExternalKnowledgeApiResponse
}
@ -1396,7 +1365,7 @@ export type PostDatasetsInitErrors = {
}
export type PostDatasetsInitResponses = {
201: DatasetAndDocumentResponse
200: DatasetAndDocumentResponse
}
export type PostDatasetsInitResponse = PostDatasetsInitResponses[keyof PostDatasetsInitResponses]
@ -1439,7 +1408,7 @@ export type GetDatasetsProcessRuleData = {
}
export type GetDatasetsProcessRuleResponses = {
200: OpaqueObjectResponse
200: ProcessRuleResponse
}
export type GetDatasetsProcessRuleResponse
@ -1581,7 +1550,7 @@ export type GetDatasetsByDatasetIdBatchByBatchIndexingEstimateData = {
}
export type GetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponses = {
200: OpaqueObjectResponse
200: IndexingEstimateResponse
}
export type GetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponse
@ -1669,7 +1638,9 @@ export type PostDatasetsByDatasetIdDocumentsDownloadZipData = {
}
export type PostDatasetsByDatasetIdDocumentsDownloadZipResponses = {
200: BinaryFileResponse
200: {
[key: string]: unknown
}
}
export type PostDatasetsByDatasetIdDocumentsDownloadZipResponse
@ -1764,7 +1735,7 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdErrors = {
}
export type GetDatasetsByDatasetIdDocumentsByDocumentIdResponses = {
200: OpaqueObjectResponse
200: DocumentDetailResponse
}
export type GetDatasetsByDatasetIdDocumentsByDocumentIdResponse
@ -1803,7 +1774,7 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateErrors =
}
export type GetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateResponses = {
200: OpaqueObjectResponse
200: IndexingEstimateResponse
}
export type GetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateResponse
@ -1880,7 +1851,7 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogData
}
export type GetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogResponses = {
200: OpaqueObjectResponse
200: DocumentPipelineExecutionLogResponse
}
export type GetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogResponse
@ -2225,7 +2196,7 @@ export type GetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusErrors = {
}
export type GetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusResponses = {
200: OpaqueObjectResponse
200: DocumentSummaryStatusResponse
}
export type GetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusResponse
@ -2283,7 +2254,7 @@ export type PostDatasetsByDatasetIdExternalHitTestingErrors = {
}
export type PostDatasetsByDatasetIdExternalHitTestingResponses = {
200: ExternalRetrievalTestResponse
200: ExternalHitTestingResponse
}
export type PostDatasetsByDatasetIdExternalHitTestingResponse

428
packages/contracts/generated/api/console/datasets/zod.gen.ts
View File

@ -91,11 +91,6 @@ export const zNotionEstimatePayload = z.object({
process_rule: z.record(z.string(), z.unknown()),
})
/**
* OpaqueObjectResponse
*/
export const zOpaqueObjectResponse = z.record(z.string(), z.unknown())
/**
* RetrievalSettingResponse
*/
@ -127,11 +122,6 @@ export const zDocumentBatchDownloadZipPayload = z.object({
document_ids: z.array(z.uuid()).min(1).max(100),
})
/**
* BinaryFileResponse
*/
export const zBinaryFileResponse = z.custom<Blob | File>()
/**
* GenerateSummaryPayload
*/
@ -247,14 +237,6 @@ export const zExternalHitTestingPayload = z.object({
query: z.string(),
})
/**
* ExternalRetrievalTestResponse
*/
export const zExternalRetrievalTestResponse = z.union([
z.record(z.string(), z.unknown()),
z.array(z.record(z.string(), z.unknown())),
])
/**
* MetadataArgs
*/
@ -397,52 +379,10 @@ export const zDatasetTagResponse = z.object({
type: z.string(),
})
export const zDatasetDocMetadata = z.object({
id: z.string().optional(),
name: z.string().optional(),
type: z.string().optional(),
})
export const zExternalKnowledgeInfo = z.object({
external_knowledge_api_endpoint: z.string().optional(),
external_knowledge_api_id: z.string().optional(),
external_knowledge_api_name: z.string().optional(),
external_knowledge_id: z.string().optional(),
})
export const zExternalRetrievalModel = z.object({
score_threshold: z.number().optional(),
score_threshold_enabled: z.boolean().optional(),
top_k: z.int().optional(),
})
export const zDatasetIconInfo = z.object({
icon: z.string().optional(),
icon_background: z.string().optional(),
icon_type: z.string().optional(),
icon_url: z.string().optional(),
})
export const zAnonymousInlineModelB1954337D565 = z.object({
enable: z.boolean().optional(),
model_name: z.string().optional(),
model_provider_name: z.string().optional(),
summary_prompt: z.string().optional(),
})
/**
* Tag
* ExternalKnowledgeApiBindingResponse
*/
export const zTag = z.object({
id: z.string(),
name: z.string(),
type: z.string(),
})
/**
* ExternalKnowledgeDatasetBindingResponse
*/
export const zExternalKnowledgeDatasetBindingResponse = z.object({
export const zExternalKnowledgeApiBindingResponse = z.object({
id: z.string(),
name: z.string(),
})
@ -453,11 +393,11 @@ export const zExternalKnowledgeDatasetBindingResponse = z.object({
export const zExternalKnowledgeApiResponse = z.object({
created_at: z.string(),
created_by: z.string(),
dataset_bindings: z.array(zExternalKnowledgeDatasetBindingResponse).optional(),
dataset_bindings: z.array(zExternalKnowledgeApiBindingResponse),
description: z.string(),
id: z.string(),
name: z.string(),
settings: z.record(z.string(), z.unknown()).nullish(),
settings: z.record(z.string(), z.unknown()).nullable(),
tenant_id: z.string(),
})
@ -469,22 +409,22 @@ export const zExternalKnowledgeApiListResponse = z.object({
has_more: z.boolean(),
limit: z.int(),
page: z.int(),
total: z.int(),
total: z.int().nullable(),
})
/**
* IndexingEstimatePreviewItemResponse
* PreviewDetail
*/
export const zIndexingEstimatePreviewItemResponse = z.object({
export const zPreviewDetail = z.object({
child_chunks: z.array(z.string()).nullish(),
content: z.string(),
summary: z.string().nullish(),
})
/**
* IndexingEstimateQaPreviewItemResponse
* QAPreviewDetail
*/
export const zIndexingEstimateQaPreviewItemResponse = z.object({
export const zQaPreviewDetail = z.object({
answer: z.string(),
question: z.string(),
})
@ -493,8 +433,20 @@ export const zIndexingEstimateQaPreviewItemResponse = z.object({
* IndexingEstimateResponse
*/
export const zIndexingEstimateResponse = z.object({
preview: z.array(zIndexingEstimatePreviewItemResponse),
qa_preview: z.array(zIndexingEstimateQaPreviewItemResponse).nullish(),
currency: z.string(),
preview: z.array(zPreviewDetail),
qa_preview: z.array(zQaPreviewDetail).nullish(),
tokens: z.int(),
total_price: z.union([z.number(), z.int()]),
total_segments: z.int(),
})
/**
* IndexingEstimate
*/
export const zIndexingEstimate = z.object({
preview: z.array(zPreviewDetail),
qa_preview: z.array(zQaPreviewDetail).nullish(),
total_segments: z.int(),
})
@ -528,30 +480,11 @@ export const zDatasetMetadataBuiltInFieldsResponse = z.object({
})
/**
* PreviewDetail
* ProcessRuleMode
*
* Dataset Process Rule Mode
*/
export const zPreviewDetail = z.object({
child_chunks: z.array(z.string()).nullish(),
content: z.string(),
summary: z.string().nullish(),
})
/**
* QAPreviewDetail
*/
export const zQaPreviewDetail = z.object({
answer: z.string(),
question: z.string(),
})
/**
* IndexingEstimate
*/
export const zIndexingEstimate = z.object({
preview: z.array(zPreviewDetail),
qa_preview: z.array(zQaPreviewDetail).nullish(),
total_segments: z.int(),
})
export const zProcessRuleMode = z.enum(['automatic', 'custom', 'hierarchical'])
/**
* DocumentMetadataResponse
@ -563,6 +496,43 @@ export const zDocumentMetadataResponse = z.object({
value: z.union([z.string(), z.int(), z.number(), z.boolean()]).nullish(),
})
/**
* DocumentDetailResponse
*/
export const zDocumentDetailResponse = z.object({
archived: z.boolean().nullish(),
average_segment_length: z.number().nullish(),
completed_at: z.int().nullish(),
created_at: z.int().nullish(),
created_by: z.string().nullish(),
created_from: z.string().nullish(),
data_source_detail_dict: z.unknown().optional(),
data_source_info: z.unknown().optional(),
data_source_type: z.string().nullish(),
dataset_process_rule: z.unknown().optional(),
dataset_process_rule_id: z.string().nullish(),
disabled_at: z.int().nullish(),
disabled_by: z.string().nullish(),
display_status: z.string().nullish(),
doc_form: z.string().nullish(),
doc_language: z.string().nullish(),
doc_metadata: z.array(zDocumentMetadataResponse).nullish(),
doc_type: z.string().nullish(),
document_process_rule: z.unknown().optional(),
enabled: z.boolean().nullish(),
error: z.string().nullish(),
hit_count: z.int().nullish(),
id: z.string(),
indexing_latency: z.number().nullish(),
indexing_status: z.string().nullish(),
name: z.string().nullish(),
need_summary: z.boolean().nullish(),
position: z.int().nullish(),
segment_count: z.int().nullish(),
tokens: z.int().nullish(),
updated_at: z.int().nullish(),
})
/**
* DocumentResponse
*/
@ -646,6 +616,18 @@ export const zDocumentWithSegmentsListResponse = z.object({
total: z.int(),
})
export const zJsonValue = z.unknown()
/**
* DocumentPipelineExecutionLogResponse
*/
export const zDocumentPipelineExecutionLogResponse = z.object({
datasource_info: zJsonValue.nullish(),
datasource_node_id: z.string().nullish(),
datasource_type: z.string().nullish(),
input_data: zJsonValue.nullish(),
})
/**
* ChildChunkResponse
*/
@ -700,6 +682,64 @@ export const zChildChunkBatchUpdatePayload = z.object({
chunks: z.array(zChildChunkUpdateArgs),
})
/**
* SummaryEntryResponse
*/
export const zSummaryEntryResponse = z.object({
created_at: z.int().nullish(),
error: z.string().nullish(),
segment_id: z.string(),
segment_position: z.int(),
status: z.string(),
summary_preview: z.string().nullish(),
updated_at: z.int().nullish(),
})
/**
* SummaryStatusResponse
*/
export const zSummaryStatusResponse = z.object({
completed: z.int().optional().default(0),
error: z.int().optional().default(0),
generating: z.int().optional().default(0),
not_started: z.int().optional().default(0),
timeout: z.int().optional().default(0),
})
/**
* DocumentSummaryStatusResponse
*/
export const zDocumentSummaryStatusResponse = z.object({
summaries: z.array(zSummaryEntryResponse),
summary_status: zSummaryStatusResponse,
total_segments: z.int(),
})
/**
* ExternalHitTestingQueryResponse
*/
export const zExternalHitTestingQueryResponse = z.object({
content: z.string(),
})
/**
* ExternalHitTestingRecordResponse
*/
export const zExternalHitTestingRecordResponse = z.object({
content: z.string().nullish(),
metadata: z.record(z.string(), z.unknown()).nullish(),
score: z.number().nullish(),
title: z.string().nullish(),
})
/**
* ExternalHitTestingResponse
*/
export const zExternalHitTestingResponse = z.object({
query: zExternalHitTestingQueryResponse,
records: z.array(zExternalHitTestingRecordResponse),
})
/**
* HitTestingQuery
*/
@ -755,18 +795,6 @@ export const zDatasetRerankingModelResponse = z.object({
reranking_provider_name: z.string().nullish(),
})
export const zDatasetRerankingModel = z.object({
reranking_model_name: z.string().optional(),
reranking_provider_name: z.string().optional(),
})
/**
* ProcessRuleMode
*
* Dataset Process Rule Mode
*/
export const zProcessRuleMode = z.enum(['automatic', 'custom', 'hierarchical'])
/**
* RerankingModel
*/
@ -785,6 +813,50 @@ export const zRetrievalMethod = z.enum([
'semantic_search',
])
/**
* PreProcessingRule
*/
export const zPreProcessingRule = z.object({
enabled: z.boolean(),
id: z.enum(['remove_extra_spaces', 'remove_stopwords', 'remove_urls_emails']),
})
/**
* Segmentation
*/
export const zSegmentation = z.object({
chunk_overlap: z.int().optional().default(0),
max_tokens: z.int(),
separator: z.string().optional().default('\n'),
})
/**
* Rule
*/
export const zRule = z.object({
parent_mode: z.enum(['full-doc', 'paragraph']).nullish(),
pre_processing_rules: z.array(zPreProcessingRule).nullish(),
segmentation: zSegmentation.nullish(),
subchunk_segmentation: zSegmentation.nullish(),
})
/**
* ProcessRuleResponse
*/
export const zProcessRuleResponse = z.object({
limits: z.record(z.string(), z.unknown()),
mode: zProcessRuleMode,
rules: zRule.nullish(),
})
/**
* ProcessRule
*/
export const zProcessRule = z.object({
mode: zProcessRuleMode,
rules: zRule.nullish(),
})
/**
* MetadataDetail
*/
@ -1079,88 +1151,6 @@ export const zDatasetListResponse = z.object({
total: z.int(),
})
export const zDatasetKeywordSetting = z.object({
keyword_weight: z.number().optional(),
})
export const zDatasetVectorSetting = z.object({
embedding_model_name: z.string().optional(),
embedding_provider_name: z.string().optional(),
vector_weight: z.number().optional(),
})
export const zDatasetWeightedScore = z.object({
keyword_setting: zDatasetKeywordSetting.optional(),
vector_setting: zDatasetVectorSetting.optional(),
weight_type: z.string().optional(),
})
export const zDatasetRetrievalModel = z.object({
reranking_enable: z.boolean().optional(),
reranking_mode: z.string().optional(),
reranking_model: zDatasetRerankingModel.optional(),
score_threshold: z.number().optional(),
score_threshold_enabled: z.boolean().optional(),
search_method: z.string().optional(),
top_k: z.int().optional(),
weights: zDatasetWeightedScore.optional(),
})
export const zDatasetDetail = z.object({
app_count: z.int().optional(),
author_name: z.string().optional(),
built_in_field_enabled: z.boolean().optional(),
chunk_structure: z.string().optional(),
created_at: z.coerce
.bigint()
.min(BigInt('-9223372036854775808'), {
error: 'Invalid value: Expected int64 to be >= -9223372036854775808',
})
.max(BigInt('9223372036854775807'), {
error: 'Invalid value: Expected int64 to be <= 9223372036854775807',
})
.optional(),
created_by: z.string().optional(),
data_source_type: z.string().optional(),
description: z.string().optional(),
doc_form: z.string().optional(),
doc_metadata: z.array(zDatasetDocMetadata).optional(),
document_count: z.int().optional(),
embedding_available: z.boolean().optional(),
embedding_model: z.string().optional(),
embedding_model_provider: z.string().optional(),
enable_api: z.boolean().optional(),
external_knowledge_info: zExternalKnowledgeInfo.optional(),
external_retrieval_model: zExternalRetrievalModel.optional(),
icon_info: zDatasetIconInfo.optional(),
id: z.string().optional(),
indexing_technique: z.string().optional(),
is_multimodal: z.boolean().optional(),
is_published: z.boolean().optional(),
name: z.string().optional(),
permission: z.string().optional(),
permission_keys: z.array(z.string()).optional(),
pipeline_id: z.string().optional(),
provider: z.string().optional(),
retrieval_model_dict: zDatasetRetrievalModel.optional(),
runtime_mode: z.string().optional(),
summary_index_setting: zAnonymousInlineModelB1954337D565.optional(),
tags: z.array(zTag).optional(),
total_available_documents: z.int().optional(),
total_documents: z.int().optional(),
updated_at: z.coerce
.bigint()
.min(BigInt('-9223372036854775808'), {
error: 'Invalid value: Expected int64 to be >= -9223372036854775808',
})
.max(BigInt('9223372036854775807'), {
error: 'Invalid value: Expected int64 to be <= 9223372036854775807',
})
.optional(),
updated_by: z.string().optional(),
word_count: z.int().optional(),
})
/**
* FileInfo
*/
@ -1178,41 +1168,6 @@ export const zWebsiteInfo = z.object({
urls: z.array(z.string()),
})
/**
* PreProcessingRule
*/
export const zPreProcessingRule = z.object({
enabled: z.boolean(),
id: z.enum(['remove_extra_spaces', 'remove_stopwords', 'remove_urls_emails']),
})
/**
* Segmentation
*/
export const zSegmentation = z.object({
chunk_overlap: z.int().optional().default(0),
max_tokens: z.int(),
separator: z.string().optional().default('\n'),
})
/**
* Rule
*/
export const zRule = z.object({
parent_mode: z.enum(['full-doc', 'paragraph']).nullish(),
pre_processing_rules: z.array(zPreProcessingRule).nullish(),
segmentation: zSegmentation.nullish(),
subchunk_segmentation: zSegmentation.nullish(),
})
/**
* ProcessRule
*/
export const zProcessRule = z.object({
mode: zProcessRuleMode,
rules: zRule.nullish(),
})
/**
* Condition
*
@ -1558,7 +1513,7 @@ export const zPostDatasetsExternalBody = zExternalDatasetCreatePayload
/**
* External dataset created successfully
*/
export const zPostDatasetsExternalResponse = zDatasetDetail
export const zPostDatasetsExternalResponse = zDatasetDetailResponse
export const zGetDatasetsExternalKnowledgeApiQuery = z.object({
keyword: z.string().optional(),
@ -1653,7 +1608,7 @@ export const zGetDatasetsProcessRuleQuery = z.object({
/**
* Process rules retrieved successfully
*/
export const zGetDatasetsProcessRuleResponse = zOpaqueObjectResponse
export const zGetDatasetsProcessRuleResponse = zProcessRuleResponse
/**
* Retrieval settings retrieved successfully
@ -1723,9 +1678,9 @@ export const zGetDatasetsByDatasetIdBatchByBatchIndexingEstimatePath = z.object(
})
/**
* Batch indexing estimate calculated successfully
* Indexing estimate calculated successfully
*/
export const zGetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponse = zOpaqueObjectResponse
export const zGetDatasetsByDatasetIdBatchByBatchIndexingEstimateResponse = zIndexingEstimateResponse
export const zGetDatasetsByDatasetIdBatchByBatchIndexingStatusPath = z.object({
batch: z.string(),
@ -1782,9 +1737,12 @@ export const zPostDatasetsByDatasetIdDocumentsDownloadZipPath = z.object({
})
/**
* ZIP archive generated successfully
* ZIP archive downloaded successfully
*/
export const zPostDatasetsByDatasetIdDocumentsDownloadZipResponse = zBinaryFileResponse
export const zPostDatasetsByDatasetIdDocumentsDownloadZipResponse = z.record(
z.string(),
z.unknown(),
)
export const zPostDatasetsByDatasetIdDocumentsGenerateSummaryBody = zGenerateSummaryPayload
@ -1840,7 +1798,7 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdQuery = z.object({
/**
* Document retrieved successfully
*/
export const zGetDatasetsByDatasetIdDocumentsByDocumentIdResponse = zOpaqueObjectResponse
export const zGetDatasetsByDatasetIdDocumentsByDocumentIdResponse = zDocumentDetailResponse
export const zGetDatasetsByDatasetIdDocumentsByDocumentIdDownloadPath = z.object({
dataset_id: z.uuid(),
@ -1861,7 +1819,7 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimatePath =
* Indexing estimate calculated successfully
*/
export const zGetDatasetsByDatasetIdDocumentsByDocumentIdIndexingEstimateResponse
= zOpaqueObjectResponse
= zIndexingEstimateResponse
export const zGetDatasetsByDatasetIdDocumentsByDocumentIdIndexingStatusPath = z.object({
dataset_id: z.uuid(),
@ -1904,10 +1862,10 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogPat
})
/**
* Document pipeline execution log retrieved successfully
* Pipeline execution log retrieved successfully
*/
export const zGetDatasetsByDatasetIdDocumentsByDocumentIdPipelineExecutionLogResponse
= zOpaqueObjectResponse
= zDocumentPipelineExecutionLogResponse
export const zPatchDatasetsByDatasetIdDocumentsByDocumentIdProcessingPausePath = z.object({
dataset_id: z.uuid(),
@ -2158,7 +2116,7 @@ export const zGetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusPath = z.o
* Summary status retrieved successfully
*/
export const zGetDatasetsByDatasetIdDocumentsByDocumentIdSummaryStatusResponse
= zOpaqueObjectResponse
= zDocumentSummaryStatusResponse
export const zGetDatasetsByDatasetIdDocumentsByDocumentIdWebsiteSyncPath = z.object({
dataset_id: z.uuid(),
@ -2188,7 +2146,7 @@ export const zPostDatasetsByDatasetIdExternalHitTestingPath = z.object({
/**
* External hit testing completed successfully
*/
export const zPostDatasetsByDatasetIdExternalHitTestingResponse = zExternalRetrievalTestResponse
export const zPostDatasetsByDatasetIdExternalHitTestingResponse = zExternalHitTestingResponse
export const zPostDatasetsByDatasetIdHitTestingBody = zHitTestingPayload

21
packages/contracts/generated/api/console/test/types.gen.ts
View File

@ -10,19 +10,24 @@ export type BedrockRetrievalPayload = {
retrieval_setting: BedrockRetrievalSetting
}
export type ExternalRetrievalTestResponse
= | {
[key: string]: unknown
}
| Array<{
[key: string]: unknown
}>
export type BedrockRetrievalResponse = {
records: Array<BedrockRetrievalRecordResponse>
}
export type BedrockRetrievalSetting = {
score_threshold?: number
top_k?: number | null
}
export type BedrockRetrievalRecordResponse = {
content?: string | null
metadata?: {
[key: string]: unknown
} | null
score: number
title?: string | null
}
export type PostTestRetrievalData = {
body: BedrockRetrievalPayload
path?: never
@ -31,7 +36,7 @@ export type PostTestRetrievalData = {
}
export type PostTestRetrievalResponses = {
200: ExternalRetrievalTestResponse
200: BedrockRetrievalResponse
}
export type PostTestRetrievalResponse = PostTestRetrievalResponses[keyof PostTestRetrievalResponses]

27
packages/contracts/generated/api/console/test/zod.gen.ts
View File

@ -2,14 +2,6 @@
import * as z from 'zod'
/**
* ExternalRetrievalTestResponse
*/
export const zExternalRetrievalTestResponse = z.union([
z.record(z.string(), z.unknown()),
z.array(z.record(z.string(), z.unknown())),
])
/**
* BedrockRetrievalSetting
*
@ -29,9 +21,26 @@ export const zBedrockRetrievalPayload = z.object({
retrieval_setting: zBedrockRetrievalSetting,
})
/**
* BedrockRetrievalRecordResponse
*/
export const zBedrockRetrievalRecordResponse = z.object({
content: z.string().nullish(),
metadata: z.record(z.string(), z.unknown()).nullish(),
score: z.number(),
title: z.string().nullish(),
})
/**
* BedrockRetrievalResponse
*/
export const zBedrockRetrievalResponse = z.object({
records: z.array(zBedrockRetrievalRecordResponse),
})
export const zPostTestRetrievalBody = zBedrockRetrievalPayload
/**
* Bedrock retrieval test completed
*/
export const zPostTestRetrievalResponse = zExternalRetrievalTestResponse
export const zPostTestRetrievalResponse = zBedrockRetrievalResponse

10
packages/contracts/generated/api/service/orpc.gen.ts
View File

@ -1474,16 +1474,20 @@ export const get13 = oc
.output(zGetDatasetsByDatasetIdDocumentsByDocumentIdResponse)
/**
* Update an existing document by uploading a file
* Update Document by File
*
* Update an existing document by uploading a new file. Re-triggers indexing — use the returned `batch` ID with [Get Document Indexing Status](/api-reference/documents/get-document-indexing-status) to track progress.
*/
export const patch4 = oc
.route({
description: 'Update an existing document by uploading a file',
description:
'Update an existing document by uploading a new file. Re-triggers indexing — use the returned `batch` ID with [Get Document Indexing Status](/api-reference/documents/get-document-indexing-status) to track progress.',
inputStructure: 'detailed',
method: 'PATCH',
operationId: 'patchDatasetsByDatasetIdDocumentsByDocumentId',
path: '/datasets/{dataset_id}/documents/{document_id}',
tags: ['service_api'],
summary: 'Update Document by File',
tags: ['Documents'],
})
.input(
z.object({

42
packages/contracts/generated/api/service/types.gen.ts
View File

@ -593,40 +593,45 @@ export type DocumentBatchDownloadZipPayload = {
}
export type DocumentDetailResponse = {
archived?: boolean | null
average_segment_length?: number | null
archived?: boolean
average_segment_length?: number | number
completed_at?: number | null
created_at?: number | null
created_by?: string | null
created_from?: string | null
created_at?: number
created_by?: string
created_from?: string
data_source_info?: {
[key: string]: unknown
} | null
data_source_type?: string | null
}
data_source_type?: string
dataset_process_rule?: {
[key: string]: unknown
} | null
}
dataset_process_rule_id?: string | null
disabled_at?: number | null
disabled_by?: string | null
display_status?: string | null
doc_form?: string | null
doc_form?: string
doc_language?: string | null
doc_metadata?: Array<DocumentMetadataResponse> | null
doc_metadata?:
| Array<DocumentMetadataResponse>
| {
[key: string]: unknown
}
| null
doc_type?: string | null
document_process_rule?: {
[key: string]: unknown
} | null
enabled?: boolean | null
}
enabled?: boolean
error?: string | null
hit_count?: number | null
hit_count?: number
id: string
indexing_latency?: number | null
indexing_status?: string | null
name?: string | null
need_summary?: boolean | null
position?: number | null
segment_count?: number | null
indexing_status?: string
name?: string
need_summary?: boolean
position?: number
segment_count?: number
summary_index_status?: string | null
tokens?: number | null
updated_at?: number | null
@ -2627,6 +2632,7 @@ export type PatchDatasetsByDatasetIdDocumentsByDocumentIdData = {
}
export type PatchDatasetsByDatasetIdDocumentsByDocumentIdErrors = {
400: unknown
401: unknown
403: unknown
404: unknown

40
packages/contracts/generated/api/service/zod.gen.ts
View File

@ -761,34 +761,36 @@ export const zDocumentMetadataResponse = z.object({
* DocumentDetailResponse
*/
export const zDocumentDetailResponse = z.object({
archived: z.boolean().nullish(),
average_segment_length: z.number().nullish(),
archived: z.boolean().optional(),
average_segment_length: z.union([z.int(), z.number()]).optional(),
completed_at: z.int().nullish(),
created_at: z.int().nullish(),
created_by: z.string().nullish(),
created_from: z.string().nullish(),
data_source_info: z.record(z.string(), z.unknown()).nullish(),
data_source_type: z.string().nullish(),
dataset_process_rule: z.record(z.string(), z.unknown()).nullish(),
created_at: z.int().optional(),
created_by: z.string().optional(),
created_from: z.string().optional(),
data_source_info: z.record(z.string(), z.unknown()).optional(),
data_source_type: z.string().optional(),
dataset_process_rule: z.record(z.string(), z.unknown()).optional(),
dataset_process_rule_id: z.string().nullish(),
disabled_at: z.int().nullish(),
disabled_by: z.string().nullish(),
display_status: z.string().nullish(),
doc_form: z.string().nullish(),
doc_form: z.string().optional(),
doc_language: z.string().nullish(),
doc_metadata: z.array(zDocumentMetadataResponse).nullish(),
doc_metadata: z
.union([z.array(zDocumentMetadataResponse), z.record(z.string(), z.unknown())])
.nullish(),
doc_type: z.string().nullish(),
document_process_rule: z.record(z.string(), z.unknown()).nullish(),
enabled: z.boolean().nullish(),
document_process_rule: z.record(z.string(), z.unknown()).optional(),
enabled: z.boolean().optional(),
error: z.string().nullish(),
hit_count: z.int().nullish(),
hit_count: z.int().optional(),
id: z.string(),
indexing_latency: z.number().nullish(),
indexing_status: z.string().nullish(),
name: z.string().nullish(),
need_summary: z.boolean().nullish(),
position: z.int().nullish(),
segment_count: z.int().nullish(),
indexing_status: z.string().optional(),
name: z.string().optional(),
need_summary: z.boolean().optional(),
position: z.int().optional(),
segment_count: z.int().optional(),
summary_index_status: z.string().nullish(),
tokens: z.int().nullish(),
updated_at: z.int().nullish(),
@ -2759,7 +2761,7 @@ export const zPatchDatasetsByDatasetIdDocumentsByDocumentIdPath = z.object({
})
/**
* Document updated successfully
* Document updated successfully.
*/
export const zPatchDatasetsByDatasetIdDocumentsByDocumentIdResponse = zDocumentAndBatchResponse