From aab95d06263e01e4605ef1eb88e6892b57221840 Mon Sep 17 00:00:00 2001 From: changkeke <33918095+changkeke@users.noreply.github.com> Date: Mon, 24 Nov 2025 20:44:09 +0800 Subject: [PATCH] fix: Failed to load API definition (#28509) Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Asuka Minato --- api/controllers/console/apikey.py | 18 +- api/controllers/console/app/annotation.py | 22 +- api/controllers/console/app/app.py | 147 +++++++-- api/controllers/console/app/app_import.py | 27 +- api/controllers/console/app/conversation.py | 284 +++++++++++++++++- .../console/app/conversation_variables.py | 24 +- api/controllers/console/app/mcp_server.py | 19 +- api/controllers/console/app/message.py | 135 ++++++++- api/controllers/console/app/site.py | 11 +- api/controllers/console/app/workflow.py | 75 ++++- .../console/app/workflow_app_log.py | 9 +- .../console/app/workflow_draft_variable.py | 59 +++- api/controllers/console/app/workflow_run.py | 96 +++++- api/controllers/console/datasets/datasets.py | 93 +++++- .../console/datasets/datasets_document.py | 41 ++- api/controllers/console/datasets/external.py | 61 +++- api/controllers/console/extension.py | 20 +- 17 files changed, 996 insertions(+), 145 deletions(-) diff --git a/api/controllers/console/apikey.py b/api/controllers/console/apikey.py index d93858d3fc..9b0d4b1a78 100644 --- a/api/controllers/console/apikey.py +++ b/api/controllers/console/apikey.py @@ -24,6 +24,12 @@ api_key_fields = { api_key_list = {"data": fields.List(fields.Nested(api_key_fields), attribute="items")} +api_key_item_model = console_ns.model("ApiKeyItem", api_key_fields) + +api_key_list_model = console_ns.model( + "ApiKeyList", {"data": fields.List(fields.Nested(api_key_item_model), attribute="items")} +) + def _get_resource(resource_id, tenant_id, resource_model): if resource_model == App: @@ -52,7 +58,7 @@ class BaseApiKeyListResource(Resource): token_prefix: str | None = None max_keys = 10 - @marshal_with(api_key_list) + @marshal_with(api_key_list_model) def get(self, resource_id): assert self.resource_id_field is not None, "resource_id_field must be set" resource_id = str(resource_id) @@ -66,7 +72,7 @@ class BaseApiKeyListResource(Resource): ).all() return {"items": keys} - @marshal_with(api_key_fields) + @marshal_with(api_key_item_model) @edit_permission_required def post(self, resource_id): assert self.resource_id_field is not None, "resource_id_field must be set" @@ -136,7 +142,7 @@ class AppApiKeyListResource(BaseApiKeyListResource): @console_ns.doc("get_app_api_keys") @console_ns.doc(description="Get all API keys for an app") @console_ns.doc(params={"resource_id": "App ID"}) - @console_ns.response(200, "Success", api_key_list) + @console_ns.response(200, "Success", api_key_list_model) def get(self, resource_id): # type: ignore """Get all API keys for an app""" return super().get(resource_id) @@ -144,7 +150,7 @@ class AppApiKeyListResource(BaseApiKeyListResource): @console_ns.doc("create_app_api_key") @console_ns.doc(description="Create a new API key for an app") @console_ns.doc(params={"resource_id": "App ID"}) - @console_ns.response(201, "API key created successfully", api_key_fields) + @console_ns.response(201, "API key created successfully", api_key_item_model) @console_ns.response(400, "Maximum keys exceeded") def post(self, resource_id): # type: ignore """Create a new API key for an app""" @@ -176,7 +182,7 @@ class DatasetApiKeyListResource(BaseApiKeyListResource): @console_ns.doc("get_dataset_api_keys") @console_ns.doc(description="Get all API keys for a dataset") @console_ns.doc(params={"resource_id": "Dataset ID"}) - @console_ns.response(200, "Success", api_key_list) + @console_ns.response(200, "Success", api_key_list_model) def get(self, resource_id): # type: ignore """Get all API keys for a dataset""" return super().get(resource_id) @@ -184,7 +190,7 @@ class DatasetApiKeyListResource(BaseApiKeyListResource): @console_ns.doc("create_dataset_api_key") @console_ns.doc(description="Create a new API key for a dataset") @console_ns.doc(params={"resource_id": "Dataset ID"}) - @console_ns.response(201, "API key created successfully", api_key_fields) + @console_ns.response(201, "API key created successfully", api_key_item_model) @console_ns.response(400, "Maximum keys exceeded") def post(self, resource_id): # type: ignore """Create a new API key for a dataset""" diff --git a/api/controllers/console/app/annotation.py b/api/controllers/console/app/annotation.py index 0be39c9178..edf0cc2cec 100644 --- a/api/controllers/console/app/annotation.py +++ b/api/controllers/console/app/annotation.py @@ -15,6 +15,7 @@ from extensions.ext_redis import redis_client from fields.annotation_fields import ( annotation_fields, annotation_hit_history_fields, + build_annotation_model, ) from libs.helper import uuid_value from libs.login import login_required @@ -184,7 +185,7 @@ class AnnotationApi(Resource): }, ) ) - @console_ns.response(201, "Annotation created successfully", annotation_fields) + @console_ns.response(201, "Annotation created successfully", build_annotation_model(console_ns)) @console_ns.response(403, "Insufficient permissions") @setup_required @login_required @@ -238,7 +239,11 @@ class AnnotationExportApi(Resource): @console_ns.doc("export_annotations") @console_ns.doc(description="Export all annotations for an app") @console_ns.doc(params={"app_id": "Application ID"}) - @console_ns.response(200, "Annotations exported successfully", fields.List(fields.Nested(annotation_fields))) + @console_ns.response( + 200, + "Annotations exported successfully", + console_ns.model("AnnotationList", {"data": fields.List(fields.Nested(build_annotation_model(console_ns)))}), + ) @console_ns.response(403, "Insufficient permissions") @setup_required @login_required @@ -263,7 +268,7 @@ class AnnotationUpdateDeleteApi(Resource): @console_ns.doc("update_delete_annotation") @console_ns.doc(description="Update or delete an annotation") @console_ns.doc(params={"app_id": "Application ID", "annotation_id": "Annotation ID"}) - @console_ns.response(200, "Annotation updated successfully", annotation_fields) + @console_ns.response(200, "Annotation updated successfully", build_annotation_model(console_ns)) @console_ns.response(204, "Annotation deleted successfully") @console_ns.response(403, "Insufficient permissions") @console_ns.expect(parser) @@ -359,7 +364,16 @@ class AnnotationHitHistoryListApi(Resource): .add_argument("limit", type=int, location="args", default=20, help="Page size") ) @console_ns.response( - 200, "Hit histories retrieved successfully", fields.List(fields.Nested(annotation_hit_history_fields)) + 200, + "Hit histories retrieved successfully", + console_ns.model( + "AnnotationHitHistoryList", + { + "data": fields.List( + fields.Nested(console_ns.model("AnnotationHitHistoryItem", annotation_hit_history_fields)) + ) + }, + ), ) @console_ns.response(403, "Insufficient permissions") @setup_required diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index 85a46aa9c3..e6687de03e 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -18,7 +18,15 @@ from controllers.console.wraps import ( from core.ops.ops_trace_manager import OpsTraceManager from core.workflow.enums import NodeType from extensions.ext_database import db -from fields.app_fields import app_detail_fields, app_detail_fields_with_site, app_pagination_fields +from fields.app_fields import ( + deleted_tool_fields, + model_config_fields, + model_config_partial_fields, + site_fields, + tag_fields, +) +from fields.workflow_fields import workflow_partial_fields as _workflow_partial_fields_dict +from libs.helper import AppIconUrlField, TimestampField from libs.login import current_account_with_tenant, login_required from libs.validators import validate_description_length from models import App, Workflow @@ -29,6 +37,111 @@ from services.feature_service import FeatureService ALLOW_CREATE_APP_MODES = ["chat", "agent-chat", "advanced-chat", "workflow", "completion"] +# Register models for flask_restx to avoid dict type issues in Swagger +# Register base models first +tag_model = console_ns.model("Tag", tag_fields) + +workflow_partial_model = console_ns.model("WorkflowPartial", _workflow_partial_fields_dict) + +model_config_model = console_ns.model("ModelConfig", model_config_fields) + +model_config_partial_model = console_ns.model("ModelConfigPartial", model_config_partial_fields) + +deleted_tool_model = console_ns.model("DeletedTool", deleted_tool_fields) + +site_model = console_ns.model("Site", site_fields) + +app_partial_model = console_ns.model( + "AppPartial", + { + "id": fields.String, + "name": fields.String, + "max_active_requests": fields.Raw(), + "description": fields.String(attribute="desc_or_prompt"), + "mode": fields.String(attribute="mode_compatible_with_agent"), + "icon_type": fields.String, + "icon": fields.String, + "icon_background": fields.String, + "icon_url": AppIconUrlField, + "model_config": fields.Nested(model_config_partial_model, attribute="app_model_config", allow_null=True), + "workflow": fields.Nested(workflow_partial_model, allow_null=True), + "use_icon_as_answer_icon": fields.Boolean, + "created_by": fields.String, + "created_at": TimestampField, + "updated_by": fields.String, + "updated_at": TimestampField, + "tags": fields.List(fields.Nested(tag_model)), + "access_mode": fields.String, + "create_user_name": fields.String, + "author_name": fields.String, + "has_draft_trigger": fields.Boolean, + }, +) + +app_detail_model = console_ns.model( + "AppDetail", + { + "id": fields.String, + "name": fields.String, + "description": fields.String, + "mode": fields.String(attribute="mode_compatible_with_agent"), + "icon": fields.String, + "icon_background": fields.String, + "enable_site": fields.Boolean, + "enable_api": fields.Boolean, + "model_config": fields.Nested(model_config_model, attribute="app_model_config", allow_null=True), + "workflow": fields.Nested(workflow_partial_model, allow_null=True), + "tracing": fields.Raw, + "use_icon_as_answer_icon": fields.Boolean, + "created_by": fields.String, + "created_at": TimestampField, + "updated_by": fields.String, + "updated_at": TimestampField, + "access_mode": fields.String, + "tags": fields.List(fields.Nested(tag_model)), + }, +) + +app_detail_with_site_model = console_ns.model( + "AppDetailWithSite", + { + "id": fields.String, + "name": fields.String, + "description": fields.String, + "mode": fields.String(attribute="mode_compatible_with_agent"), + "icon_type": fields.String, + "icon": fields.String, + "icon_background": fields.String, + "icon_url": AppIconUrlField, + "enable_site": fields.Boolean, + "enable_api": fields.Boolean, + "model_config": fields.Nested(model_config_model, attribute="app_model_config", allow_null=True), + "workflow": fields.Nested(workflow_partial_model, allow_null=True), + "api_base_url": fields.String, + "use_icon_as_answer_icon": fields.Boolean, + "max_active_requests": fields.Integer, + "created_by": fields.String, + "created_at": TimestampField, + "updated_by": fields.String, + "updated_at": TimestampField, + "deleted_tools": fields.List(fields.Nested(deleted_tool_model)), + "access_mode": fields.String, + "tags": fields.List(fields.Nested(tag_model)), + "site": fields.Nested(site_model), + }, +) + +app_pagination_model = console_ns.model( + "AppPagination", + { + "page": fields.Integer, + "limit": fields.Integer(attribute="per_page"), + "total": fields.Integer, + "has_more": fields.Boolean(attribute="has_next"), + "data": fields.List(fields.Nested(app_partial_model), attribute="items"), + }, +) + @console_ns.route("/apps") class AppListApi(Resource): @@ -50,7 +163,7 @@ class AppListApi(Resource): .add_argument("tag_ids", type=str, location="args", help="Comma-separated tag IDs") .add_argument("is_created_by_me", type=bool, location="args", help="Filter by creator") ) - @console_ns.response(200, "Success", app_pagination_fields) + @console_ns.response(200, "Success", app_pagination_model) @setup_required @login_required @account_initialization_required @@ -137,7 +250,7 @@ class AppListApi(Resource): for app in app_pagination.items: app.has_draft_trigger = str(app.id) in draft_trigger_app_ids - return marshal(app_pagination, app_pagination_fields), 200 + return marshal(app_pagination, app_pagination_model), 200 @console_ns.doc("create_app") @console_ns.doc(description="Create a new application") @@ -154,13 +267,13 @@ class AppListApi(Resource): }, ) ) - @console_ns.response(201, "App created successfully", app_detail_fields) + @console_ns.response(201, "App created successfully", app_detail_model) @console_ns.response(403, "Insufficient permissions") @console_ns.response(400, "Invalid request parameters") @setup_required @login_required @account_initialization_required - @marshal_with(app_detail_fields) + @marshal_with(app_detail_model) @cloud_edition_billing_resource_check("apps") @edit_permission_required def post(self): @@ -191,13 +304,13 @@ class AppApi(Resource): @console_ns.doc("get_app_detail") @console_ns.doc(description="Get application details") @console_ns.doc(params={"app_id": "Application ID"}) - @console_ns.response(200, "Success", app_detail_fields_with_site) + @console_ns.response(200, "Success", app_detail_with_site_model) @setup_required @login_required @account_initialization_required @enterprise_license_required @get_app_model - @marshal_with(app_detail_fields_with_site) + @marshal_with(app_detail_with_site_model) def get(self, app_model): """Get app detail""" app_service = AppService() @@ -227,7 +340,7 @@ class AppApi(Resource): }, ) ) - @console_ns.response(200, "App updated successfully", app_detail_fields_with_site) + @console_ns.response(200, "App updated successfully", app_detail_with_site_model) @console_ns.response(403, "Insufficient permissions") @console_ns.response(400, "Invalid request parameters") @setup_required @@ -235,7 +348,7 @@ class AppApi(Resource): @account_initialization_required @get_app_model @edit_permission_required - @marshal_with(app_detail_fields_with_site) + @marshal_with(app_detail_with_site_model) def put(self, app_model): """Update app""" parser = ( @@ -300,14 +413,14 @@ class AppCopyApi(Resource): }, ) ) - @console_ns.response(201, "App copied successfully", app_detail_fields_with_site) + @console_ns.response(201, "App copied successfully", app_detail_with_site_model) @console_ns.response(403, "Insufficient permissions") @setup_required @login_required @account_initialization_required @get_app_model @edit_permission_required - @marshal_with(app_detail_fields_with_site) + @marshal_with(app_detail_with_site_model) def post(self, app_model): """Copy app""" # The role of the current user in the ta table must be admin, owner, or editor @@ -396,7 +509,7 @@ class AppNameApi(Resource): @login_required @account_initialization_required @get_app_model - @marshal_with(app_detail_fields) + @marshal_with(app_detail_model) @edit_permission_required def post(self, app_model): args = parser.parse_args() @@ -428,7 +541,7 @@ class AppIconApi(Resource): @login_required @account_initialization_required @get_app_model - @marshal_with(app_detail_fields) + @marshal_with(app_detail_model) @edit_permission_required def post(self, app_model): parser = ( @@ -454,13 +567,13 @@ class AppSiteStatus(Resource): "AppSiteStatusRequest", {"enable_site": fields.Boolean(required=True, description="Enable or disable site")} ) ) - @console_ns.response(200, "Site status updated successfully", app_detail_fields) + @console_ns.response(200, "Site status updated successfully", app_detail_model) @console_ns.response(403, "Insufficient permissions") @setup_required @login_required @account_initialization_required @get_app_model - @marshal_with(app_detail_fields) + @marshal_with(app_detail_model) @edit_permission_required def post(self, app_model): parser = reqparse.RequestParser().add_argument("enable_site", type=bool, required=True, location="json") @@ -482,14 +595,14 @@ class AppApiStatus(Resource): "AppApiStatusRequest", {"enable_api": fields.Boolean(required=True, description="Enable or disable API")} ) ) - @console_ns.response(200, "API status updated successfully", app_detail_fields) + @console_ns.response(200, "API status updated successfully", app_detail_model) @console_ns.response(403, "Insufficient permissions") @setup_required @login_required @is_admin_or_owner_required @account_initialization_required @get_app_model - @marshal_with(app_detail_fields) + @marshal_with(app_detail_model) def post(self, app_model): parser = reqparse.RequestParser().add_argument("enable_api", type=bool, required=True, location="json") args = parser.parse_args() diff --git a/api/controllers/console/app/app_import.py b/api/controllers/console/app/app_import.py index 35a3393742..1b02edd489 100644 --- a/api/controllers/console/app/app_import.py +++ b/api/controllers/console/app/app_import.py @@ -1,4 +1,4 @@ -from flask_restx import Resource, marshal_with, reqparse +from flask_restx import Resource, fields, marshal_with, reqparse from sqlalchemy.orm import Session from controllers.console.app.wraps import get_app_model @@ -9,7 +9,11 @@ from controllers.console.wraps import ( setup_required, ) from extensions.ext_database import db -from fields.app_fields import app_import_check_dependencies_fields, app_import_fields +from fields.app_fields import ( + app_import_check_dependencies_fields, + app_import_fields, + leaked_dependency_fields, +) from libs.login import current_account_with_tenant, login_required from models.model import App from services.app_dsl_service import AppDslService, ImportStatus @@ -18,6 +22,19 @@ from services.feature_service import FeatureService from .. import console_ns +# Register models for flask_restx to avoid dict type issues in Swagger +# Register base model first +leaked_dependency_model = console_ns.model("LeakedDependency", leaked_dependency_fields) + +app_import_model = console_ns.model("AppImport", app_import_fields) + +# For nested models, need to replace nested dict with registered model +app_import_check_dependencies_fields_copy = app_import_check_dependencies_fields.copy() +app_import_check_dependencies_fields_copy["leaked_dependencies"] = fields.List(fields.Nested(leaked_dependency_model)) +app_import_check_dependencies_model = console_ns.model( + "AppImportCheckDependencies", app_import_check_dependencies_fields_copy +) + parser = ( reqparse.RequestParser() .add_argument("mode", type=str, required=True, location="json") @@ -38,7 +55,7 @@ class AppImportApi(Resource): @setup_required @login_required @account_initialization_required - @marshal_with(app_import_fields) + @marshal_with(app_import_model) @cloud_edition_billing_resource_check("apps") @edit_permission_required def post(self): @@ -81,7 +98,7 @@ class AppImportConfirmApi(Resource): @setup_required @login_required @account_initialization_required - @marshal_with(app_import_fields) + @marshal_with(app_import_model) @edit_permission_required def post(self, import_id): # Check user role first @@ -107,7 +124,7 @@ class AppImportCheckDependenciesApi(Resource): @login_required @get_app_model @account_initialization_required - @marshal_with(app_import_check_dependencies_fields) + @marshal_with(app_import_check_dependencies_model) @edit_permission_required def get(self, app_model: App): with Session(db.engine) as session: diff --git a/api/controllers/console/app/conversation.py b/api/controllers/console/app/conversation.py index e102300438..3d92c46756 100644 --- a/api/controllers/console/app/conversation.py +++ b/api/controllers/console/app/conversation.py @@ -1,6 +1,6 @@ import sqlalchemy as sa from flask import abort -from flask_restx import Resource, marshal_with, reqparse +from flask_restx import Resource, fields, marshal_with, reqparse from flask_restx.inputs import int_range from sqlalchemy import func, or_ from sqlalchemy.orm import joinedload @@ -11,20 +11,272 @@ from controllers.console.app.wraps import get_app_model from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required from core.app.entities.app_invoke_entities import InvokeFrom from extensions.ext_database import db -from fields.conversation_fields import ( - conversation_detail_fields, - conversation_message_detail_fields, - conversation_pagination_fields, - conversation_with_summary_pagination_fields, -) +from fields.conversation_fields import MessageTextField +from fields.raws import FilesContainedField from libs.datetime_utils import naive_utc_now, parse_time_range -from libs.helper import DatetimeString +from libs.helper import DatetimeString, TimestampField from libs.login import current_account_with_tenant, login_required from models import Conversation, EndUser, Message, MessageAnnotation from models.model import AppMode from services.conversation_service import ConversationService from services.errors.conversation import ConversationNotExistsError +# Register models for flask_restx to avoid dict type issues in Swagger +# Register in dependency order: base models first, then dependent models + +# Base models +simple_account_model = console_ns.model( + "SimpleAccount", + { + "id": fields.String, + "name": fields.String, + "email": fields.String, + }, +) + +feedback_stat_model = console_ns.model( + "FeedbackStat", + { + "like": fields.Integer, + "dislike": fields.Integer, + }, +) + +status_count_model = console_ns.model( + "StatusCount", + { + "success": fields.Integer, + "failed": fields.Integer, + "partial_success": fields.Integer, + }, +) + +message_file_model = console_ns.model( + "MessageFile", + { + "id": fields.String, + "filename": fields.String, + "type": fields.String, + "url": fields.String, + "mime_type": fields.String, + "size": fields.Integer, + "transfer_method": fields.String, + "belongs_to": fields.String(default="user"), + "upload_file_id": fields.String(default=None), + }, +) + +agent_thought_model = console_ns.model( + "AgentThought", + { + "id": fields.String, + "chain_id": fields.String, + "message_id": fields.String, + "position": fields.Integer, + "thought": fields.String, + "tool": fields.String, + "tool_labels": fields.Raw, + "tool_input": fields.String, + "created_at": TimestampField, + "observation": fields.String, + "files": fields.List(fields.String), + }, +) + +simple_model_config_model = console_ns.model( + "SimpleModelConfig", + { + "model": fields.Raw(attribute="model_dict"), + "pre_prompt": fields.String, + }, +) + +model_config_model = console_ns.model( + "ModelConfig", + { + "opening_statement": fields.String, + "suggested_questions": fields.Raw, + "model": fields.Raw, + "user_input_form": fields.Raw, + "pre_prompt": fields.String, + "agent_mode": fields.Raw, + }, +) + +# Models that depend on simple_account_model +feedback_model = console_ns.model( + "Feedback", + { + "rating": fields.String, + "content": fields.String, + "from_source": fields.String, + "from_end_user_id": fields.String, + "from_account": fields.Nested(simple_account_model, allow_null=True), + }, +) + +annotation_model = console_ns.model( + "Annotation", + { + "id": fields.String, + "question": fields.String, + "content": fields.String, + "account": fields.Nested(simple_account_model, allow_null=True), + "created_at": TimestampField, + }, +) + +annotation_hit_history_model = console_ns.model( + "AnnotationHitHistory", + { + "annotation_id": fields.String(attribute="id"), + "annotation_create_account": fields.Nested(simple_account_model, allow_null=True), + "created_at": TimestampField, + }, +) + +# Simple message detail model +simple_message_detail_model = console_ns.model( + "SimpleMessageDetail", + { + "inputs": FilesContainedField, + "query": fields.String, + "message": MessageTextField, + "answer": fields.String, + }, +) + +# Message detail model that depends on multiple models +message_detail_model = console_ns.model( + "MessageDetail", + { + "id": fields.String, + "conversation_id": fields.String, + "inputs": FilesContainedField, + "query": fields.String, + "message": fields.Raw, + "message_tokens": fields.Integer, + "answer": fields.String(attribute="re_sign_file_url_answer"), + "answer_tokens": fields.Integer, + "provider_response_latency": fields.Float, + "from_source": fields.String, + "from_end_user_id": fields.String, + "from_account_id": fields.String, + "feedbacks": fields.List(fields.Nested(feedback_model)), + "workflow_run_id": fields.String, + "annotation": fields.Nested(annotation_model, allow_null=True), + "annotation_hit_history": fields.Nested(annotation_hit_history_model, allow_null=True), + "created_at": TimestampField, + "agent_thoughts": fields.List(fields.Nested(agent_thought_model)), + "message_files": fields.List(fields.Nested(message_file_model)), + "metadata": fields.Raw(attribute="message_metadata_dict"), + "status": fields.String, + "error": fields.String, + "parent_message_id": fields.String, + }, +) + +# Conversation models +conversation_fields_model = console_ns.model( + "Conversation", + { + "id": fields.String, + "status": fields.String, + "from_source": fields.String, + "from_end_user_id": fields.String, + "from_end_user_session_id": fields.String(), + "from_account_id": fields.String, + "from_account_name": fields.String, + "read_at": TimestampField, + "created_at": TimestampField, + "updated_at": TimestampField, + "annotation": fields.Nested(annotation_model, allow_null=True), + "model_config": fields.Nested(simple_model_config_model), + "user_feedback_stats": fields.Nested(feedback_stat_model), + "admin_feedback_stats": fields.Nested(feedback_stat_model), + "message": fields.Nested(simple_message_detail_model, attribute="first_message"), + }, +) + +conversation_pagination_model = console_ns.model( + "ConversationPagination", + { + "page": fields.Integer, + "limit": fields.Integer(attribute="per_page"), + "total": fields.Integer, + "has_more": fields.Boolean(attribute="has_next"), + "data": fields.List(fields.Nested(conversation_fields_model), attribute="items"), + }, +) + +conversation_message_detail_model = console_ns.model( + "ConversationMessageDetail", + { + "id": fields.String, + "status": fields.String, + "from_source": fields.String, + "from_end_user_id": fields.String, + "from_account_id": fields.String, + "created_at": TimestampField, + "model_config": fields.Nested(model_config_model), + "message": fields.Nested(message_detail_model, attribute="first_message"), + }, +) + +conversation_with_summary_model = console_ns.model( + "ConversationWithSummary", + { + "id": fields.String, + "status": fields.String, + "from_source": fields.String, + "from_end_user_id": fields.String, + "from_end_user_session_id": fields.String, + "from_account_id": fields.String, + "from_account_name": fields.String, + "name": fields.String, + "summary": fields.String(attribute="summary_or_query"), + "read_at": TimestampField, + "created_at": TimestampField, + "updated_at": TimestampField, + "annotated": fields.Boolean, + "model_config": fields.Nested(simple_model_config_model), + "message_count": fields.Integer, + "user_feedback_stats": fields.Nested(feedback_stat_model), + "admin_feedback_stats": fields.Nested(feedback_stat_model), + "status_count": fields.Nested(status_count_model), + }, +) + +conversation_with_summary_pagination_model = console_ns.model( + "ConversationWithSummaryPagination", + { + "page": fields.Integer, + "limit": fields.Integer(attribute="per_page"), + "total": fields.Integer, + "has_more": fields.Boolean(attribute="has_next"), + "data": fields.List(fields.Nested(conversation_with_summary_model), attribute="items"), + }, +) + +conversation_detail_model = console_ns.model( + "ConversationDetail", + { + "id": fields.String, + "status": fields.String, + "from_source": fields.String, + "from_end_user_id": fields.String, + "from_account_id": fields.String, + "created_at": TimestampField, + "updated_at": TimestampField, + "annotated": fields.Boolean, + "introduction": fields.String, + "model_config": fields.Nested(model_config_model), + "message_count": fields.Integer, + "user_feedback_stats": fields.Nested(feedback_stat_model), + "admin_feedback_stats": fields.Nested(feedback_stat_model), + }, +) + @console_ns.route("/apps//completion-conversations") class CompletionConversationApi(Resource): @@ -47,13 +299,13 @@ class CompletionConversationApi(Resource): .add_argument("page", type=int, location="args", default=1, help="Page number") .add_argument("limit", type=int, location="args", default=20, help="Page size (1-100)") ) - @console_ns.response(200, "Success", conversation_pagination_fields) + @console_ns.response(200, "Success", conversation_pagination_model) @console_ns.response(403, "Insufficient permissions") @setup_required @login_required @account_initialization_required @get_app_model(mode=AppMode.COMPLETION) - @marshal_with(conversation_pagination_fields) + @marshal_with(conversation_pagination_model) @edit_permission_required def get(self, app_model): current_user, _ = current_account_with_tenant() @@ -125,14 +377,14 @@ class CompletionConversationDetailApi(Resource): @console_ns.doc("get_completion_conversation") @console_ns.doc(description="Get completion conversation details with messages") @console_ns.doc(params={"app_id": "Application ID", "conversation_id": "Conversation ID"}) - @console_ns.response(200, "Success", conversation_message_detail_fields) + @console_ns.response(200, "Success", conversation_message_detail_model) @console_ns.response(403, "Insufficient permissions") @console_ns.response(404, "Conversation not found") @setup_required @login_required @account_initialization_required @get_app_model(mode=AppMode.COMPLETION) - @marshal_with(conversation_message_detail_fields) + @marshal_with(conversation_message_detail_model) @edit_permission_required def get(self, app_model, conversation_id): conversation_id = str(conversation_id) @@ -192,13 +444,13 @@ class ChatConversationApi(Resource): help="Sort field and direction", ) ) - @console_ns.response(200, "Success", conversation_with_summary_pagination_fields) + @console_ns.response(200, "Success", conversation_with_summary_pagination_model) @console_ns.response(403, "Insufficient permissions") @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]) - @marshal_with(conversation_with_summary_pagination_fields) + @marshal_with(conversation_with_summary_pagination_model) @edit_permission_required def get(self, app_model): current_user, _ = current_account_with_tenant() @@ -325,14 +577,14 @@ class ChatConversationDetailApi(Resource): @console_ns.doc("get_chat_conversation") @console_ns.doc(description="Get chat conversation details") @console_ns.doc(params={"app_id": "Application ID", "conversation_id": "Conversation ID"}) - @console_ns.response(200, "Success", conversation_detail_fields) + @console_ns.response(200, "Success", conversation_detail_model) @console_ns.response(403, "Insufficient permissions") @console_ns.response(404, "Conversation not found") @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]) - @marshal_with(conversation_detail_fields) + @marshal_with(conversation_detail_model) @edit_permission_required def get(self, app_model, conversation_id): conversation_id = str(conversation_id) diff --git a/api/controllers/console/app/conversation_variables.py b/api/controllers/console/app/conversation_variables.py index 15ea004143..c612041fab 100644 --- a/api/controllers/console/app/conversation_variables.py +++ b/api/controllers/console/app/conversation_variables.py @@ -1,4 +1,4 @@ -from flask_restx import Resource, marshal_with, reqparse +from flask_restx import Resource, fields, marshal_with, reqparse from sqlalchemy import select from sqlalchemy.orm import Session @@ -6,11 +6,27 @@ from controllers.console import console_ns from controllers.console.app.wraps import get_app_model from controllers.console.wraps import account_initialization_required, setup_required from extensions.ext_database import db -from fields.conversation_variable_fields import paginated_conversation_variable_fields +from fields.conversation_variable_fields import ( + conversation_variable_fields, + paginated_conversation_variable_fields, +) from libs.login import login_required from models import ConversationVariable from models.model import AppMode +# Register models for flask_restx to avoid dict type issues in Swagger +# Register base model first +conversation_variable_model = console_ns.model("ConversationVariable", conversation_variable_fields) + +# For nested models, need to replace nested dict with registered model +paginated_conversation_variable_fields_copy = paginated_conversation_variable_fields.copy() +paginated_conversation_variable_fields_copy["data"] = fields.List( + fields.Nested(conversation_variable_model), attribute="data" +) +paginated_conversation_variable_model = console_ns.model( + "PaginatedConversationVariable", paginated_conversation_variable_fields_copy +) + @console_ns.route("/apps//conversation-variables") class ConversationVariablesApi(Resource): @@ -22,12 +38,12 @@ class ConversationVariablesApi(Resource): "conversation_id", type=str, location="args", help="Conversation ID to filter variables" ) ) - @console_ns.response(200, "Conversation variables retrieved successfully", paginated_conversation_variable_fields) + @console_ns.response(200, "Conversation variables retrieved successfully", paginated_conversation_variable_model) @setup_required @login_required @account_initialization_required @get_app_model(mode=AppMode.ADVANCED_CHAT) - @marshal_with(paginated_conversation_variable_fields) + @marshal_with(paginated_conversation_variable_model) def get(self, app_model): parser = reqparse.RequestParser().add_argument("conversation_id", type=str, location="args") args = parser.parse_args() diff --git a/api/controllers/console/app/mcp_server.py b/api/controllers/console/app/mcp_server.py index 7454d87068..58d1fb4a2d 100644 --- a/api/controllers/console/app/mcp_server.py +++ b/api/controllers/console/app/mcp_server.py @@ -12,6 +12,9 @@ from fields.app_fields import app_server_fields from libs.login import current_account_with_tenant, login_required from models.model import AppMCPServer +# Register model for flask_restx to avoid dict type issues in Swagger +app_server_model = console_ns.model("AppServer", app_server_fields) + class AppMCPServerStatus(StrEnum): ACTIVE = "active" @@ -23,12 +26,12 @@ class AppMCPServerController(Resource): @console_ns.doc("get_app_mcp_server") @console_ns.doc(description="Get MCP server configuration for an application") @console_ns.doc(params={"app_id": "Application ID"}) - @console_ns.response(200, "MCP server configuration retrieved successfully", app_server_fields) + @console_ns.response(200, "MCP server configuration retrieved successfully", app_server_model) @login_required @account_initialization_required @setup_required @get_app_model - @marshal_with(app_server_fields) + @marshal_with(app_server_model) def get(self, app_model): server = db.session.query(AppMCPServer).where(AppMCPServer.app_id == app_model.id).first() return server @@ -45,13 +48,13 @@ class AppMCPServerController(Resource): }, ) ) - @console_ns.response(201, "MCP server configuration created successfully", app_server_fields) + @console_ns.response(201, "MCP server configuration created successfully", app_server_model) @console_ns.response(403, "Insufficient permissions") @account_initialization_required @get_app_model @login_required @setup_required - @marshal_with(app_server_fields) + @marshal_with(app_server_model) @edit_permission_required def post(self, app_model): _, current_tenant_id = current_account_with_tenant() @@ -93,14 +96,14 @@ class AppMCPServerController(Resource): }, ) ) - @console_ns.response(200, "MCP server configuration updated successfully", app_server_fields) + @console_ns.response(200, "MCP server configuration updated successfully", app_server_model) @console_ns.response(403, "Insufficient permissions") @console_ns.response(404, "Server not found") @get_app_model @login_required @setup_required @account_initialization_required - @marshal_with(app_server_fields) + @marshal_with(app_server_model) @edit_permission_required def put(self, app_model): parser = ( @@ -137,13 +140,13 @@ class AppMCPServerRefreshController(Resource): @console_ns.doc("refresh_app_mcp_server") @console_ns.doc(description="Refresh MCP server configuration and regenerate server code") @console_ns.doc(params={"server_id": "Server ID"}) - @console_ns.response(200, "MCP server refreshed successfully", app_server_fields) + @console_ns.response(200, "MCP server refreshed successfully", app_server_model) @console_ns.response(403, "Insufficient permissions") @console_ns.response(404, "Server not found") @setup_required @login_required @account_initialization_required - @marshal_with(app_server_fields) + @marshal_with(app_server_model) @edit_permission_required def get(self, server_id): _, current_tenant_id = current_account_with_tenant() diff --git a/api/controllers/console/app/message.py b/api/controllers/console/app/message.py index b6672c88e0..7fdf49c3fa 100644 --- a/api/controllers/console/app/message.py +++ b/api/controllers/console/app/message.py @@ -23,8 +23,8 @@ from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from core.model_runtime.errors.invoke import InvokeError from extensions.ext_database import db -from fields.conversation_fields import message_detail_fields -from libs.helper import uuid_value +from fields.raws import FilesContainedField +from libs.helper import TimestampField, uuid_value from libs.infinite_scroll_pagination import InfiniteScrollPagination from libs.login import current_account_with_tenant, login_required from models.model import AppMode, Conversation, Message, MessageAnnotation, MessageFeedback @@ -34,15 +34,126 @@ from services.message_service import MessageService logger = logging.getLogger(__name__) +# Register models for flask_restx to avoid dict type issues in Swagger +# Register in dependency order: base models first, then dependent models + +# Base models +simple_account_model = console_ns.model( + "SimpleAccount", + { + "id": fields.String, + "name": fields.String, + "email": fields.String, + }, +) + +message_file_model = console_ns.model( + "MessageFile", + { + "id": fields.String, + "filename": fields.String, + "type": fields.String, + "url": fields.String, + "mime_type": fields.String, + "size": fields.Integer, + "transfer_method": fields.String, + "belongs_to": fields.String(default="user"), + "upload_file_id": fields.String(default=None), + }, +) + +agent_thought_model = console_ns.model( + "AgentThought", + { + "id": fields.String, + "chain_id": fields.String, + "message_id": fields.String, + "position": fields.Integer, + "thought": fields.String, + "tool": fields.String, + "tool_labels": fields.Raw, + "tool_input": fields.String, + "created_at": TimestampField, + "observation": fields.String, + "files": fields.List(fields.String), + }, +) + +# Models that depend on simple_account_model +feedback_model = console_ns.model( + "Feedback", + { + "rating": fields.String, + "content": fields.String, + "from_source": fields.String, + "from_end_user_id": fields.String, + "from_account": fields.Nested(simple_account_model, allow_null=True), + }, +) + +annotation_model = console_ns.model( + "Annotation", + { + "id": fields.String, + "question": fields.String, + "content": fields.String, + "account": fields.Nested(simple_account_model, allow_null=True), + "created_at": TimestampField, + }, +) + +annotation_hit_history_model = console_ns.model( + "AnnotationHitHistory", + { + "annotation_id": fields.String(attribute="id"), + "annotation_create_account": fields.Nested(simple_account_model, allow_null=True), + "created_at": TimestampField, + }, +) + +# Message detail model that depends on multiple models +message_detail_model = console_ns.model( + "MessageDetail", + { + "id": fields.String, + "conversation_id": fields.String, + "inputs": FilesContainedField, + "query": fields.String, + "message": fields.Raw, + "message_tokens": fields.Integer, + "answer": fields.String(attribute="re_sign_file_url_answer"), + "answer_tokens": fields.Integer, + "provider_response_latency": fields.Float, + "from_source": fields.String, + "from_end_user_id": fields.String, + "from_account_id": fields.String, + "feedbacks": fields.List(fields.Nested(feedback_model)), + "workflow_run_id": fields.String, + "annotation": fields.Nested(annotation_model, allow_null=True), + "annotation_hit_history": fields.Nested(annotation_hit_history_model, allow_null=True), + "created_at": TimestampField, + "agent_thoughts": fields.List(fields.Nested(agent_thought_model)), + "message_files": fields.List(fields.Nested(message_file_model)), + "metadata": fields.Raw(attribute="message_metadata_dict"), + "status": fields.String, + "error": fields.String, + "parent_message_id": fields.String, + }, +) + +# Message infinite scroll pagination model +message_infinite_scroll_pagination_model = console_ns.model( + "MessageInfiniteScrollPagination", + { + "limit": fields.Integer, + "has_more": fields.Boolean, + "data": fields.List(fields.Nested(message_detail_model)), + }, +) + @console_ns.route("/apps//chat-messages") class ChatMessageListApi(Resource): - message_infinite_scroll_pagination_fields = { - "limit": fields.Integer, - "has_more": fields.Boolean, - "data": fields.List(fields.Nested(message_detail_fields)), - } - @console_ns.doc("list_chat_messages") @console_ns.doc(description="Get chat messages for a conversation with pagination") @console_ns.doc(params={"app_id": "Application ID"}) @@ -52,13 +163,13 @@ class ChatMessageListApi(Resource): .add_argument("first_id", type=str, location="args", help="First message ID for pagination") .add_argument("limit", type=int, location="args", default=20, help="Number of messages to return (1-100)") ) - @console_ns.response(200, "Success", message_infinite_scroll_pagination_fields) + @console_ns.response(200, "Success", message_infinite_scroll_pagination_model) @console_ns.response(404, "Conversation not found") @login_required @account_initialization_required @setup_required @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]) - @marshal_with(message_infinite_scroll_pagination_fields) + @marshal_with(message_infinite_scroll_pagination_model) @edit_permission_required def get(self, app_model): parser = ( @@ -263,13 +374,13 @@ class MessageApi(Resource): @console_ns.doc("get_message") @console_ns.doc(description="Get message details by ID") @console_ns.doc(params={"app_id": "Application ID", "message_id": "Message ID"}) - @console_ns.response(200, "Message retrieved successfully", message_detail_fields) + @console_ns.response(200, "Message retrieved successfully", message_detail_model) @console_ns.response(404, "Message not found") @get_app_model @setup_required @login_required @account_initialization_required - @marshal_with(message_detail_fields) + @marshal_with(message_detail_model) def get(self, app_model, message_id: str): message_id = str(message_id) diff --git a/api/controllers/console/app/site.py b/api/controllers/console/app/site.py index b2f1997620..d46b8c5c9d 100644 --- a/api/controllers/console/app/site.py +++ b/api/controllers/console/app/site.py @@ -16,6 +16,9 @@ from libs.datetime_utils import naive_utc_now from libs.login import current_account_with_tenant, login_required from models import Site +# Register model for flask_restx to avoid dict type issues in Swagger +app_site_model = console_ns.model("AppSite", app_site_fields) + def parse_app_site_args(): parser = ( @@ -76,7 +79,7 @@ class AppSite(Resource): }, ) ) - @console_ns.response(200, "Site configuration updated successfully", app_site_fields) + @console_ns.response(200, "Site configuration updated successfully", app_site_model) @console_ns.response(403, "Insufficient permissions") @console_ns.response(404, "App not found") @setup_required @@ -84,7 +87,7 @@ class AppSite(Resource): @edit_permission_required @account_initialization_required @get_app_model - @marshal_with(app_site_fields) + @marshal_with(app_site_model) def post(self, app_model): args = parse_app_site_args() current_user, _ = current_account_with_tenant() @@ -126,7 +129,7 @@ class AppSiteAccessTokenReset(Resource): @console_ns.doc("reset_app_site_access_token") @console_ns.doc(description="Reset access token for application site") @console_ns.doc(params={"app_id": "Application ID"}) - @console_ns.response(200, "Access token reset successfully", app_site_fields) + @console_ns.response(200, "Access token reset successfully", app_site_model) @console_ns.response(403, "Insufficient permissions (admin/owner required)") @console_ns.response(404, "App or site not found") @setup_required @@ -134,7 +137,7 @@ class AppSiteAccessTokenReset(Resource): @is_admin_or_owner_required @account_initialization_required @get_app_model - @marshal_with(app_site_fields) + @marshal_with(app_site_model) def post(self, app_model): current_user, _ = current_account_with_tenant() site = db.session.query(Site).where(Site.app_id == app_model.id).first() diff --git a/api/controllers/console/app/workflow.py b/api/controllers/console/app/workflow.py index 9b5a4e895c..7b7a8defa5 100644 --- a/api/controllers/console/app/workflow.py +++ b/api/controllers/console/app/workflow.py @@ -32,6 +32,7 @@ from core.workflow.enums import NodeType from core.workflow.graph_engine.manager import GraphEngineManager from extensions.ext_database import db from factories import file_factory, variable_factory +from fields.member_fields import simple_account_fields from fields.workflow_fields import workflow_fields, workflow_pagination_fields from fields.workflow_run_fields import workflow_run_node_execution_fields from libs import helper @@ -49,6 +50,56 @@ from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseE logger = logging.getLogger(__name__) LISTENING_RETRY_IN = 2000 +# Register models for flask_restx to avoid dict type issues in Swagger +# Register in dependency order: base models first, then dependent models + +# Base models +simple_account_model = console_ns.model("SimpleAccount", simple_account_fields) + +from fields.workflow_fields import pipeline_variable_fields, serialize_value_type + +conversation_variable_model = console_ns.model( + "ConversationVariable", + { + "id": fields.String, + "name": fields.String, + "value_type": fields.String(attribute=serialize_value_type), + "value": fields.Raw, + "description": fields.String, + }, +) + +pipeline_variable_model = console_ns.model("PipelineVariable", pipeline_variable_fields) + +# Workflow model with nested dependencies +workflow_fields_copy = workflow_fields.copy() +workflow_fields_copy["created_by"] = fields.Nested(simple_account_model, attribute="created_by_account") +workflow_fields_copy["updated_by"] = fields.Nested( + simple_account_model, attribute="updated_by_account", allow_null=True +) +workflow_fields_copy["conversation_variables"] = fields.List(fields.Nested(conversation_variable_model)) +workflow_fields_copy["rag_pipeline_variables"] = fields.List(fields.Nested(pipeline_variable_model)) +workflow_model = console_ns.model("Workflow", workflow_fields_copy) + +# Workflow pagination model +workflow_pagination_fields_copy = workflow_pagination_fields.copy() +workflow_pagination_fields_copy["items"] = fields.List(fields.Nested(workflow_model), attribute="items") +workflow_pagination_model = console_ns.model("WorkflowPagination", workflow_pagination_fields_copy) + +# Reuse workflow_run_node_execution_model from workflow_run.py if already registered +# Otherwise register it here +from fields.end_user_fields import simple_end_user_fields + +try: + simple_end_user_model = console_ns.models.get("SimpleEndUser") +except (KeyError, AttributeError): + simple_end_user_model = console_ns.model("SimpleEndUser", simple_end_user_fields) + +try: + workflow_run_node_execution_model = console_ns.models.get("WorkflowRunNodeExecution") +except (KeyError, AttributeError): + workflow_run_node_execution_model = console_ns.model("WorkflowRunNodeExecution", workflow_run_node_execution_fields) + # TODO(QuantumGhost): Refactor existing node run API to handle file parameter parsing # at the controller level rather than in the workflow logic. This would improve separation @@ -73,13 +124,13 @@ class DraftWorkflowApi(Resource): @console_ns.doc("get_draft_workflow") @console_ns.doc(description="Get draft workflow for an application") @console_ns.doc(params={"app_id": "Application ID"}) - @console_ns.response(200, "Draft workflow retrieved successfully", workflow_fields) + @console_ns.response(200, "Draft workflow retrieved successfully", workflow_model) @console_ns.response(404, "Draft workflow not found") @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @marshal_with(workflow_fields) + @marshal_with(workflow_model) @edit_permission_required def get(self, app_model: App): """ @@ -539,14 +590,14 @@ class DraftWorkflowNodeRunApi(Resource): }, ) ) - @console_ns.response(200, "Node run started successfully", workflow_run_node_execution_fields) + @console_ns.response(200, "Node run started successfully", workflow_run_node_execution_model) @console_ns.response(403, "Permission denied") @console_ns.response(404, "Node not found") @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @marshal_with(workflow_run_node_execution_fields) + @marshal_with(workflow_run_node_execution_model) @edit_permission_required def post(self, app_model: App, node_id: str): """ @@ -598,13 +649,13 @@ class PublishedWorkflowApi(Resource): @console_ns.doc("get_published_workflow") @console_ns.doc(description="Get published workflow for an application") @console_ns.doc(params={"app_id": "Application ID"}) - @console_ns.response(200, "Published workflow retrieved successfully", workflow_fields) + @console_ns.response(200, "Published workflow retrieved successfully", workflow_model) @console_ns.response(404, "Published workflow not found") @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @marshal_with(workflow_fields) + @marshal_with(workflow_model) @edit_permission_required def get(self, app_model: App): """ @@ -781,12 +832,12 @@ class PublishedAllWorkflowApi(Resource): @console_ns.doc("get_all_published_workflows") @console_ns.doc(description="Get all published workflows for an application") @console_ns.doc(params={"app_id": "Application ID"}) - @console_ns.response(200, "Published workflows retrieved successfully", workflow_pagination_fields) + @console_ns.response(200, "Published workflows retrieved successfully", workflow_pagination_model) @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @marshal_with(workflow_pagination_fields) + @marshal_with(workflow_pagination_model) @edit_permission_required def get(self, app_model: App): """ @@ -838,14 +889,14 @@ class WorkflowByIdApi(Resource): }, ) ) - @console_ns.response(200, "Workflow updated successfully", workflow_fields) + @console_ns.response(200, "Workflow updated successfully", workflow_model) @console_ns.response(404, "Workflow not found") @console_ns.response(403, "Permission denied") @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @marshal_with(workflow_fields) + @marshal_with(workflow_model) @edit_permission_required def patch(self, app_model: App, workflow_id: str): """ @@ -929,14 +980,14 @@ class DraftWorkflowNodeLastRunApi(Resource): @console_ns.doc("get_draft_workflow_node_last_run") @console_ns.doc(description="Get last run result for draft workflow node") @console_ns.doc(params={"app_id": "Application ID", "node_id": "Node ID"}) - @console_ns.response(200, "Node last run retrieved successfully", workflow_run_node_execution_fields) + @console_ns.response(200, "Node last run retrieved successfully", workflow_run_node_execution_model) @console_ns.response(404, "Node last run not found") @console_ns.response(403, "Permission denied") @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @marshal_with(workflow_run_node_execution_fields) + @marshal_with(workflow_run_node_execution_model) def get(self, app_model: App, node_id: str): srv = WorkflowService() workflow = srv.get_draft_workflow(app_model) diff --git a/api/controllers/console/app/workflow_app_log.py b/api/controllers/console/app/workflow_app_log.py index fc1fa9cb13..677678cb8f 100644 --- a/api/controllers/console/app/workflow_app_log.py +++ b/api/controllers/console/app/workflow_app_log.py @@ -8,12 +8,15 @@ from controllers.console.app.wraps import get_app_model from controllers.console.wraps import account_initialization_required, setup_required from core.workflow.enums import WorkflowExecutionStatus from extensions.ext_database import db -from fields.workflow_app_log_fields import workflow_app_log_pagination_fields +from fields.workflow_app_log_fields import build_workflow_app_log_pagination_model from libs.login import login_required from models import App from models.model import AppMode from services.workflow_app_service import WorkflowAppService +# Register model for flask_restx to avoid dict type issues in Swagger +workflow_app_log_pagination_model = build_workflow_app_log_pagination_model(console_ns) + @console_ns.route("/apps//workflow-app-logs") class WorkflowAppLogApi(Resource): @@ -33,12 +36,12 @@ class WorkflowAppLogApi(Resource): "limit": "Number of items per page (1-100)", } ) - @console_ns.response(200, "Workflow app logs retrieved successfully", workflow_app_log_pagination_fields) + @console_ns.response(200, "Workflow app logs retrieved successfully", workflow_app_log_pagination_model) @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.WORKFLOW]) - @marshal_with(workflow_app_log_pagination_fields) + @marshal_with(workflow_app_log_pagination_model) def get(self, app_model: App): """ Get workflow app logs diff --git a/api/controllers/console/app/workflow_draft_variable.py b/api/controllers/console/app/workflow_draft_variable.py index 007061ae7a..41ae8727de 100644 --- a/api/controllers/console/app/workflow_draft_variable.py +++ b/api/controllers/console/app/workflow_draft_variable.py @@ -141,6 +141,37 @@ _WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS = { "items": fields.List(fields.Nested(_WORKFLOW_DRAFT_VARIABLE_FIELDS), attribute=_get_items), } +# Register models for flask_restx to avoid dict type issues in Swagger +workflow_draft_variable_without_value_model = console_ns.model( + "WorkflowDraftVariableWithoutValue", _WORKFLOW_DRAFT_VARIABLE_WITHOUT_VALUE_FIELDS +) + +workflow_draft_variable_model = console_ns.model("WorkflowDraftVariable", _WORKFLOW_DRAFT_VARIABLE_FIELDS) + +workflow_draft_env_variable_model = console_ns.model("WorkflowDraftEnvVariable", _WORKFLOW_DRAFT_ENV_VARIABLE_FIELDS) + +workflow_draft_env_variable_list_fields_copy = _WORKFLOW_DRAFT_ENV_VARIABLE_LIST_FIELDS.copy() +workflow_draft_env_variable_list_fields_copy["items"] = fields.List(fields.Nested(workflow_draft_env_variable_model)) +workflow_draft_env_variable_list_model = console_ns.model( + "WorkflowDraftEnvVariableList", workflow_draft_env_variable_list_fields_copy +) + +workflow_draft_variable_list_without_value_fields_copy = _WORKFLOW_DRAFT_VARIABLE_LIST_WITHOUT_VALUE_FIELDS.copy() +workflow_draft_variable_list_without_value_fields_copy["items"] = fields.List( + fields.Nested(workflow_draft_variable_without_value_model), attribute=_get_items +) +workflow_draft_variable_list_without_value_model = console_ns.model( + "WorkflowDraftVariableListWithoutValue", workflow_draft_variable_list_without_value_fields_copy +) + +workflow_draft_variable_list_fields_copy = _WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS.copy() +workflow_draft_variable_list_fields_copy["items"] = fields.List( + fields.Nested(workflow_draft_variable_model), attribute=_get_items +) +workflow_draft_variable_list_model = console_ns.model( + "WorkflowDraftVariableList", workflow_draft_variable_list_fields_copy +) + P = ParamSpec("P") R = TypeVar("R") @@ -176,10 +207,10 @@ class WorkflowVariableCollectionApi(Resource): @console_ns.doc(params={"app_id": "Application ID"}) @console_ns.doc(params={"page": "Page number (1-100000)", "limit": "Number of items per page (1-100)"}) @console_ns.response( - 200, "Workflow variables retrieved successfully", _WORKFLOW_DRAFT_VARIABLE_LIST_WITHOUT_VALUE_FIELDS + 200, "Workflow variables retrieved successfully", workflow_draft_variable_list_without_value_model ) @_api_prerequisite - @marshal_with(_WORKFLOW_DRAFT_VARIABLE_LIST_WITHOUT_VALUE_FIELDS) + @marshal_with(workflow_draft_variable_list_without_value_model) def get(self, app_model: App): """ Get draft workflow @@ -242,9 +273,9 @@ class NodeVariableCollectionApi(Resource): @console_ns.doc("get_node_variables") @console_ns.doc(description="Get variables for a specific node") @console_ns.doc(params={"app_id": "Application ID", "node_id": "Node ID"}) - @console_ns.response(200, "Node variables retrieved successfully", _WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS) + @console_ns.response(200, "Node variables retrieved successfully", workflow_draft_variable_list_model) @_api_prerequisite - @marshal_with(_WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS) + @marshal_with(workflow_draft_variable_list_model) def get(self, app_model: App, node_id: str): validate_node_id(node_id) with Session(bind=db.engine, expire_on_commit=False) as session: @@ -275,10 +306,10 @@ class VariableApi(Resource): @console_ns.doc("get_variable") @console_ns.doc(description="Get a specific workflow variable") @console_ns.doc(params={"app_id": "Application ID", "variable_id": "Variable ID"}) - @console_ns.response(200, "Variable retrieved successfully", _WORKFLOW_DRAFT_VARIABLE_FIELDS) + @console_ns.response(200, "Variable retrieved successfully", workflow_draft_variable_model) @console_ns.response(404, "Variable not found") @_api_prerequisite - @marshal_with(_WORKFLOW_DRAFT_VARIABLE_FIELDS) + @marshal_with(workflow_draft_variable_model) def get(self, app_model: App, variable_id: str): draft_var_srv = WorkflowDraftVariableService( session=db.session(), @@ -301,10 +332,10 @@ class VariableApi(Resource): }, ) ) - @console_ns.response(200, "Variable updated successfully", _WORKFLOW_DRAFT_VARIABLE_FIELDS) + @console_ns.response(200, "Variable updated successfully", workflow_draft_variable_model) @console_ns.response(404, "Variable not found") @_api_prerequisite - @marshal_with(_WORKFLOW_DRAFT_VARIABLE_FIELDS) + @marshal_with(workflow_draft_variable_model) def patch(self, app_model: App, variable_id: str): # Request payload for file types: # @@ -390,7 +421,7 @@ class VariableResetApi(Resource): @console_ns.doc("reset_variable") @console_ns.doc(description="Reset a workflow variable to its default value") @console_ns.doc(params={"app_id": "Application ID", "variable_id": "Variable ID"}) - @console_ns.response(200, "Variable reset successfully", _WORKFLOW_DRAFT_VARIABLE_FIELDS) + @console_ns.response(200, "Variable reset successfully", workflow_draft_variable_model) @console_ns.response(204, "Variable reset (no content)") @console_ns.response(404, "Variable not found") @_api_prerequisite @@ -416,7 +447,7 @@ class VariableResetApi(Resource): if resetted is None: return Response("", 204) else: - return marshal(resetted, _WORKFLOW_DRAFT_VARIABLE_FIELDS) + return marshal(resetted, workflow_draft_variable_model) def _get_variable_list(app_model: App, node_id) -> WorkflowDraftVariableList: @@ -438,10 +469,10 @@ class ConversationVariableCollectionApi(Resource): @console_ns.doc("get_conversation_variables") @console_ns.doc(description="Get conversation variables for workflow") @console_ns.doc(params={"app_id": "Application ID"}) - @console_ns.response(200, "Conversation variables retrieved successfully", _WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS) + @console_ns.response(200, "Conversation variables retrieved successfully", workflow_draft_variable_list_model) @console_ns.response(404, "Draft workflow not found") @_api_prerequisite - @marshal_with(_WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS) + @marshal_with(workflow_draft_variable_list_model) def get(self, app_model: App): # NOTE(QuantumGhost): Prefill conversation variables into the draft variables table # so their IDs can be returned to the caller. @@ -460,9 +491,9 @@ class SystemVariableCollectionApi(Resource): @console_ns.doc("get_system_variables") @console_ns.doc(description="Get system variables for workflow") @console_ns.doc(params={"app_id": "Application ID"}) - @console_ns.response(200, "System variables retrieved successfully", _WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS) + @console_ns.response(200, "System variables retrieved successfully", workflow_draft_variable_list_model) @_api_prerequisite - @marshal_with(_WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS) + @marshal_with(workflow_draft_variable_list_model) def get(self, app_model: App): return _get_variable_list(app_model, SYSTEM_VARIABLE_NODE_ID) diff --git a/api/controllers/console/app/workflow_run.py b/api/controllers/console/app/workflow_run.py index 51f7445ce0..c016104ce0 100644 --- a/api/controllers/console/app/workflow_run.py +++ b/api/controllers/console/app/workflow_run.py @@ -1,15 +1,20 @@ from typing import cast -from flask_restx import Resource, marshal_with, reqparse +from flask_restx import Resource, fields, marshal_with, reqparse from flask_restx.inputs import int_range from controllers.console import console_ns from controllers.console.app.wraps import get_app_model from controllers.console.wraps import account_initialization_required, setup_required +from fields.end_user_fields import simple_end_user_fields +from fields.member_fields import simple_account_fields from fields.workflow_run_fields import ( + advanced_chat_workflow_run_for_list_fields, advanced_chat_workflow_run_pagination_fields, workflow_run_count_fields, workflow_run_detail_fields, + workflow_run_for_list_fields, + workflow_run_node_execution_fields, workflow_run_node_execution_list_fields, workflow_run_pagination_fields, ) @@ -22,6 +27,71 @@ from services.workflow_run_service import WorkflowRunService # Workflow run status choices for filtering WORKFLOW_RUN_STATUS_CHOICES = ["running", "succeeded", "failed", "stopped", "partial-succeeded"] +# Register models for flask_restx to avoid dict type issues in Swagger +# Register in dependency order: base models first, then dependent models + +# Base models +simple_account_model = console_ns.model("SimpleAccount", simple_account_fields) + +simple_end_user_model = console_ns.model("SimpleEndUser", simple_end_user_fields) + +# Models that depend on simple_account_fields +workflow_run_for_list_fields_copy = workflow_run_for_list_fields.copy() +workflow_run_for_list_fields_copy["created_by_account"] = fields.Nested( + simple_account_model, attribute="created_by_account", allow_null=True +) +workflow_run_for_list_model = console_ns.model("WorkflowRunForList", workflow_run_for_list_fields_copy) + +advanced_chat_workflow_run_for_list_fields_copy = advanced_chat_workflow_run_for_list_fields.copy() +advanced_chat_workflow_run_for_list_fields_copy["created_by_account"] = fields.Nested( + simple_account_model, attribute="created_by_account", allow_null=True +) +advanced_chat_workflow_run_for_list_model = console_ns.model( + "AdvancedChatWorkflowRunForList", advanced_chat_workflow_run_for_list_fields_copy +) + +workflow_run_detail_fields_copy = workflow_run_detail_fields.copy() +workflow_run_detail_fields_copy["created_by_account"] = fields.Nested( + simple_account_model, attribute="created_by_account", allow_null=True +) +workflow_run_detail_fields_copy["created_by_end_user"] = fields.Nested( + simple_end_user_model, attribute="created_by_end_user", allow_null=True +) +workflow_run_detail_model = console_ns.model("WorkflowRunDetail", workflow_run_detail_fields_copy) + +workflow_run_node_execution_fields_copy = workflow_run_node_execution_fields.copy() +workflow_run_node_execution_fields_copy["created_by_account"] = fields.Nested( + simple_account_model, attribute="created_by_account", allow_null=True +) +workflow_run_node_execution_fields_copy["created_by_end_user"] = fields.Nested( + simple_end_user_model, attribute="created_by_end_user", allow_null=True +) +workflow_run_node_execution_model = console_ns.model( + "WorkflowRunNodeExecution", workflow_run_node_execution_fields_copy +) + +# Simple models without nested dependencies +workflow_run_count_model = console_ns.model("WorkflowRunCount", workflow_run_count_fields) + +# Pagination models that depend on list models +advanced_chat_workflow_run_pagination_fields_copy = advanced_chat_workflow_run_pagination_fields.copy() +advanced_chat_workflow_run_pagination_fields_copy["data"] = fields.List( + fields.Nested(advanced_chat_workflow_run_for_list_model), attribute="data" +) +advanced_chat_workflow_run_pagination_model = console_ns.model( + "AdvancedChatWorkflowRunPagination", advanced_chat_workflow_run_pagination_fields_copy +) + +workflow_run_pagination_fields_copy = workflow_run_pagination_fields.copy() +workflow_run_pagination_fields_copy["data"] = fields.List(fields.Nested(workflow_run_for_list_model), attribute="data") +workflow_run_pagination_model = console_ns.model("WorkflowRunPagination", workflow_run_pagination_fields_copy) + +workflow_run_node_execution_list_fields_copy = workflow_run_node_execution_list_fields.copy() +workflow_run_node_execution_list_fields_copy["data"] = fields.List(fields.Nested(workflow_run_node_execution_model)) +workflow_run_node_execution_list_model = console_ns.model( + "WorkflowRunNodeExecutionList", workflow_run_node_execution_list_fields_copy +) + def _parse_workflow_run_list_args(): """ @@ -100,12 +170,12 @@ class AdvancedChatAppWorkflowRunListApi(Resource): @console_ns.doc( params={"triggered_from": "Filter by trigger source (optional): debugging or app-run. Default: debugging"} ) - @console_ns.response(200, "Workflow runs retrieved successfully", advanced_chat_workflow_run_pagination_fields) + @console_ns.response(200, "Workflow runs retrieved successfully", advanced_chat_workflow_run_pagination_model) @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.ADVANCED_CHAT]) - @marshal_with(advanced_chat_workflow_run_pagination_fields) + @marshal_with(advanced_chat_workflow_run_pagination_model) def get(self, app_model: App): """ Get advanced chat app workflow run list @@ -146,12 +216,12 @@ class AdvancedChatAppWorkflowRunCountApi(Resource): @console_ns.doc( params={"triggered_from": "Filter by trigger source (optional): debugging or app-run. Default: debugging"} ) - @console_ns.response(200, "Workflow runs count retrieved successfully", workflow_run_count_fields) + @console_ns.response(200, "Workflow runs count retrieved successfully", workflow_run_count_model) @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.ADVANCED_CHAT]) - @marshal_with(workflow_run_count_fields) + @marshal_with(workflow_run_count_model) def get(self, app_model: App): """ Get advanced chat workflow runs count statistics @@ -188,12 +258,12 @@ class WorkflowRunListApi(Resource): @console_ns.doc( params={"triggered_from": "Filter by trigger source (optional): debugging or app-run. Default: debugging"} ) - @console_ns.response(200, "Workflow runs retrieved successfully", workflow_run_pagination_fields) + @console_ns.response(200, "Workflow runs retrieved successfully", workflow_run_pagination_model) @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @marshal_with(workflow_run_pagination_fields) + @marshal_with(workflow_run_pagination_model) def get(self, app_model: App): """ Get workflow run list @@ -234,12 +304,12 @@ class WorkflowRunCountApi(Resource): @console_ns.doc( params={"triggered_from": "Filter by trigger source (optional): debugging or app-run. Default: debugging"} ) - @console_ns.response(200, "Workflow runs count retrieved successfully", workflow_run_count_fields) + @console_ns.response(200, "Workflow runs count retrieved successfully", workflow_run_count_model) @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @marshal_with(workflow_run_count_fields) + @marshal_with(workflow_run_count_model) def get(self, app_model: App): """ Get workflow runs count statistics @@ -269,13 +339,13 @@ class WorkflowRunDetailApi(Resource): @console_ns.doc("get_workflow_run_detail") @console_ns.doc(description="Get workflow run detail") @console_ns.doc(params={"app_id": "Application ID", "run_id": "Workflow run ID"}) - @console_ns.response(200, "Workflow run detail retrieved successfully", workflow_run_detail_fields) + @console_ns.response(200, "Workflow run detail retrieved successfully", workflow_run_detail_model) @console_ns.response(404, "Workflow run not found") @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @marshal_with(workflow_run_detail_fields) + @marshal_with(workflow_run_detail_model) def get(self, app_model: App, run_id): """ Get workflow run detail @@ -293,13 +363,13 @@ class WorkflowRunNodeExecutionListApi(Resource): @console_ns.doc("get_workflow_run_node_executions") @console_ns.doc(description="Get workflow run node execution list") @console_ns.doc(params={"app_id": "Application ID", "run_id": "Workflow run ID"}) - @console_ns.response(200, "Node executions retrieved successfully", workflow_run_node_execution_list_fields) + @console_ns.response(200, "Node executions retrieved successfully", workflow_run_node_execution_list_model) @console_ns.response(404, "Workflow run not found") @setup_required @login_required @account_initialization_required @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @marshal_with(workflow_run_node_execution_list_fields) + @marshal_with(workflow_run_node_execution_list_model) def get(self, app_model: App, run_id): """ Get workflow run node execution list diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 54761413f4..45bc1fa694 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -8,7 +8,10 @@ from werkzeug.exceptions import Forbidden, NotFound import services from configs import dify_config from controllers.console import console_ns -from controllers.console.apikey import api_key_fields, api_key_list +from controllers.console.apikey import ( + api_key_item_model, + api_key_list_model, +) from controllers.console.app.error import ProviderNotInitializeError from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError from controllers.console.wraps import ( @@ -27,8 +30,22 @@ from core.rag.extractor.entity.datasource_type import DatasourceType from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo, WebsiteInfo from core.rag.retrieval.retrieval_methods import RetrievalMethod from extensions.ext_database import db -from fields.app_fields import related_app_list -from fields.dataset_fields import dataset_detail_fields, dataset_query_detail_fields +from fields.app_fields import app_detail_kernel_fields, related_app_list +from fields.dataset_fields import ( + dataset_detail_fields, + dataset_fields, + dataset_query_detail_fields, + dataset_retrieval_model_fields, + doc_metadata_fields, + external_knowledge_info_fields, + external_retrieval_model_fields, + icon_info_fields, + keyword_setting_fields, + reranking_model_fields, + tag_fields, + vector_setting_fields, + weighted_score_fields, +) from fields.document_fields import document_status_fields from libs.login import current_account_with_tenant, login_required from libs.validators import validate_description_length @@ -38,6 +55,58 @@ from models.provider_ids import ModelProviderID from services.dataset_service import DatasetPermissionService, DatasetService, DocumentService +def _get_or_create_model(model_name: str, field_def): + existing = console_ns.models.get(model_name) + if existing is None: + existing = console_ns.model(model_name, field_def) + return existing + + +# Register models for flask_restx to avoid dict type issues in Swagger +dataset_base_model = _get_or_create_model("DatasetBase", dataset_fields) + +tag_model = _get_or_create_model("Tag", tag_fields) + +keyword_setting_model = _get_or_create_model("DatasetKeywordSetting", keyword_setting_fields) +vector_setting_model = _get_or_create_model("DatasetVectorSetting", vector_setting_fields) + +weighted_score_fields_copy = weighted_score_fields.copy() +weighted_score_fields_copy["keyword_setting"] = fields.Nested(keyword_setting_model) +weighted_score_fields_copy["vector_setting"] = fields.Nested(vector_setting_model) +weighted_score_model = _get_or_create_model("DatasetWeightedScore", weighted_score_fields_copy) + +reranking_model = _get_or_create_model("DatasetRerankingModel", reranking_model_fields) + +dataset_retrieval_model_fields_copy = dataset_retrieval_model_fields.copy() +dataset_retrieval_model_fields_copy["reranking_model"] = fields.Nested(reranking_model) +dataset_retrieval_model_fields_copy["weights"] = fields.Nested(weighted_score_model, allow_null=True) +dataset_retrieval_model = _get_or_create_model("DatasetRetrievalModel", dataset_retrieval_model_fields_copy) + +external_knowledge_info_model = _get_or_create_model("ExternalKnowledgeInfo", external_knowledge_info_fields) + +external_retrieval_model = _get_or_create_model("ExternalRetrievalModel", external_retrieval_model_fields) + +doc_metadata_model = _get_or_create_model("DatasetDocMetadata", doc_metadata_fields) + +icon_info_model = _get_or_create_model("DatasetIconInfo", icon_info_fields) + +dataset_detail_fields_copy = dataset_detail_fields.copy() +dataset_detail_fields_copy["retrieval_model_dict"] = fields.Nested(dataset_retrieval_model) +dataset_detail_fields_copy["tags"] = fields.List(fields.Nested(tag_model)) +dataset_detail_fields_copy["external_knowledge_info"] = fields.Nested(external_knowledge_info_model) +dataset_detail_fields_copy["external_retrieval_model"] = fields.Nested(external_retrieval_model, allow_null=True) +dataset_detail_fields_copy["doc_metadata"] = fields.List(fields.Nested(doc_metadata_model)) +dataset_detail_fields_copy["icon_info"] = fields.Nested(icon_info_model) +dataset_detail_model = _get_or_create_model("DatasetDetail", dataset_detail_fields_copy) + +dataset_query_detail_model = _get_or_create_model("DatasetQueryDetail", dataset_query_detail_fields) + +app_detail_kernel_model = _get_or_create_model("AppDetailKernel", app_detail_kernel_fields) +related_app_list_copy = related_app_list.copy() +related_app_list_copy["data"] = fields.List(fields.Nested(app_detail_kernel_model)) +related_app_list_model = _get_or_create_model("RelatedAppList", related_app_list_copy) + + def _validate_name(name: str) -> str: if not name or len(name) < 1 or len(name) > 40: raise ValueError("Name must be between 1 to 40 characters.") @@ -282,7 +351,7 @@ class DatasetApi(Resource): @console_ns.doc("get_dataset") @console_ns.doc(description="Get dataset details") @console_ns.doc(params={"dataset_id": "Dataset ID"}) - @console_ns.response(200, "Dataset retrieved successfully", dataset_detail_fields) + @console_ns.response(200, "Dataset retrieved successfully", dataset_detail_model) @console_ns.response(404, "Dataset not found") @console_ns.response(403, "Permission denied") @setup_required @@ -342,7 +411,7 @@ class DatasetApi(Resource): }, ) ) - @console_ns.response(200, "Dataset updated successfully", dataset_detail_fields) + @console_ns.response(200, "Dataset updated successfully", dataset_detail_model) @console_ns.response(404, "Dataset not found") @console_ns.response(403, "Permission denied") @setup_required @@ -507,7 +576,7 @@ class DatasetQueryApi(Resource): @console_ns.doc("get_dataset_queries") @console_ns.doc(description="Get dataset query history") @console_ns.doc(params={"dataset_id": "Dataset ID"}) - @console_ns.response(200, "Query history retrieved successfully", dataset_query_detail_fields) + @console_ns.response(200, "Query history retrieved successfully", dataset_query_detail_model) @setup_required @login_required @account_initialization_required @@ -529,7 +598,7 @@ class DatasetQueryApi(Resource): dataset_queries, total = DatasetService.get_dataset_queries(dataset_id=dataset.id, page=page, per_page=limit) response = { - "data": marshal(dataset_queries, dataset_query_detail_fields), + "data": marshal(dataset_queries, dataset_query_detail_model), "has_more": len(dataset_queries) == limit, "limit": limit, "total": total, @@ -653,11 +722,11 @@ class DatasetRelatedAppListApi(Resource): @console_ns.doc("get_dataset_related_apps") @console_ns.doc(description="Get applications related to dataset") @console_ns.doc(params={"dataset_id": "Dataset ID"}) - @console_ns.response(200, "Related apps retrieved successfully", related_app_list) + @console_ns.response(200, "Related apps retrieved successfully", related_app_list_model) @setup_required @login_required @account_initialization_required - @marshal_with(related_app_list) + @marshal_with(related_app_list_model) def get(self, dataset_id): current_user, _ = current_account_with_tenant() dataset_id_str = str(dataset_id) @@ -740,11 +809,11 @@ class DatasetApiKeyApi(Resource): @console_ns.doc("get_dataset_api_keys") @console_ns.doc(description="Get dataset API keys") - @console_ns.response(200, "API keys retrieved successfully", api_key_list) + @console_ns.response(200, "API keys retrieved successfully", api_key_list_model) @setup_required @login_required @account_initialization_required - @marshal_with(api_key_list) + @marshal_with(api_key_list_model) def get(self): _, current_tenant_id = current_account_with_tenant() keys = db.session.scalars( @@ -756,7 +825,7 @@ class DatasetApiKeyApi(Resource): @login_required @is_admin_or_owner_required @account_initialization_required - @marshal_with(api_key_fields) + @marshal_with(api_key_item_model) def post(self): _, current_tenant_id = current_account_with_tenant() diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index b5761c9ada..2663c939bc 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -45,9 +45,11 @@ from core.plugin.impl.exc import PluginDaemonClientSideError from core.rag.extractor.entity.datasource_type import DatasourceType from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo, WebsiteInfo from extensions.ext_database import db +from fields.dataset_fields import dataset_fields from fields.document_fields import ( dataset_and_document_fields, document_fields, + document_metadata_fields, document_status_fields, document_with_segments_fields, ) @@ -61,6 +63,36 @@ from services.entities.knowledge_entities.knowledge_entities import KnowledgeCon logger = logging.getLogger(__name__) +def _get_or_create_model(model_name: str, field_def): + existing = console_ns.models.get(model_name) + if existing is None: + existing = console_ns.model(model_name, field_def) + return existing + + +# Register models for flask_restx to avoid dict type issues in Swagger +dataset_model = _get_or_create_model("Dataset", dataset_fields) + +document_metadata_model = _get_or_create_model("DocumentMetadata", document_metadata_fields) + +document_fields_copy = document_fields.copy() +document_fields_copy["doc_metadata"] = fields.List( + fields.Nested(document_metadata_model), attribute="doc_metadata_details" +) +document_model = _get_or_create_model("Document", document_fields_copy) + +document_with_segments_fields_copy = document_with_segments_fields.copy() +document_with_segments_fields_copy["doc_metadata"] = fields.List( + fields.Nested(document_metadata_model), attribute="doc_metadata_details" +) +document_with_segments_model = _get_or_create_model("DocumentWithSegments", document_with_segments_fields_copy) + +dataset_and_document_fields_copy = dataset_and_document_fields.copy() +dataset_and_document_fields_copy["dataset"] = fields.Nested(dataset_model) +dataset_and_document_fields_copy["documents"] = fields.List(fields.Nested(document_model)) +dataset_and_document_model = _get_or_create_model("DatasetAndDocument", dataset_and_document_fields_copy) + + class DocumentResource(Resource): def get_document(self, dataset_id: str, document_id: str) -> Document: current_user, current_tenant_id = current_account_with_tenant() @@ -169,9 +201,8 @@ class DatasetDocumentListApi(Resource): @setup_required @login_required @account_initialization_required - def get(self, dataset_id): + def get(self, dataset_id: str): current_user, current_tenant_id = current_account_with_tenant() - dataset_id = str(dataset_id) page = request.args.get("page", default=1, type=int) limit = request.args.get("limit", default=20, type=int) search = request.args.get("keyword", default=None, type=str) @@ -276,7 +307,7 @@ class DatasetDocumentListApi(Resource): @setup_required @login_required @account_initialization_required - @marshal_with(dataset_and_document_fields) + @marshal_with(dataset_and_document_model) @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): @@ -370,12 +401,12 @@ class DatasetInitApi(Resource): }, ) ) - @console_ns.response(201, "Dataset initialized successfully", dataset_and_document_fields) + @console_ns.response(201, "Dataset initialized successfully", dataset_and_document_model) @console_ns.response(400, "Invalid request parameters") @setup_required @login_required @account_initialization_required - @marshal_with(dataset_and_document_fields) + @marshal_with(dataset_and_document_model) @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_rate_limit_check("knowledge") def post(self): diff --git a/api/controllers/console/datasets/external.py b/api/controllers/console/datasets/external.py index f48f384e94..950884e496 100644 --- a/api/controllers/console/datasets/external.py +++ b/api/controllers/console/datasets/external.py @@ -6,7 +6,19 @@ import services from controllers.console import console_ns from controllers.console.datasets.error import DatasetNameDuplicateError from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required -from fields.dataset_fields import dataset_detail_fields +from fields.dataset_fields import ( + dataset_detail_fields, + dataset_retrieval_model_fields, + doc_metadata_fields, + external_knowledge_info_fields, + external_retrieval_model_fields, + icon_info_fields, + keyword_setting_fields, + reranking_model_fields, + tag_fields, + vector_setting_fields, + weighted_score_fields, +) from libs.login import current_account_with_tenant, login_required from services.dataset_service import DatasetService from services.external_knowledge_service import ExternalDatasetService @@ -14,6 +26,51 @@ from services.hit_testing_service import HitTestingService from services.knowledge_service import ExternalDatasetTestService +def _get_or_create_model(model_name: str, field_def): + existing = console_ns.models.get(model_name) + if existing is None: + existing = console_ns.model(model_name, field_def) + return existing + + +def _build_dataset_detail_model(): + keyword_setting_model = _get_or_create_model("DatasetKeywordSetting", keyword_setting_fields) + vector_setting_model = _get_or_create_model("DatasetVectorSetting", vector_setting_fields) + + weighted_score_fields_copy = weighted_score_fields.copy() + weighted_score_fields_copy["keyword_setting"] = fields.Nested(keyword_setting_model) + weighted_score_fields_copy["vector_setting"] = fields.Nested(vector_setting_model) + weighted_score_model = _get_or_create_model("DatasetWeightedScore", weighted_score_fields_copy) + + reranking_model = _get_or_create_model("DatasetRerankingModel", reranking_model_fields) + + dataset_retrieval_model_fields_copy = dataset_retrieval_model_fields.copy() + dataset_retrieval_model_fields_copy["reranking_model"] = fields.Nested(reranking_model) + dataset_retrieval_model_fields_copy["weights"] = fields.Nested(weighted_score_model, allow_null=True) + dataset_retrieval_model = _get_or_create_model("DatasetRetrievalModel", dataset_retrieval_model_fields_copy) + + tag_model = _get_or_create_model("Tag", tag_fields) + doc_metadata_model = _get_or_create_model("DatasetDocMetadata", doc_metadata_fields) + external_knowledge_info_model = _get_or_create_model("ExternalKnowledgeInfo", external_knowledge_info_fields) + external_retrieval_model = _get_or_create_model("ExternalRetrievalModel", external_retrieval_model_fields) + icon_info_model = _get_or_create_model("DatasetIconInfo", icon_info_fields) + + dataset_detail_fields_copy = dataset_detail_fields.copy() + dataset_detail_fields_copy["retrieval_model_dict"] = fields.Nested(dataset_retrieval_model) + dataset_detail_fields_copy["tags"] = fields.List(fields.Nested(tag_model)) + dataset_detail_fields_copy["external_knowledge_info"] = fields.Nested(external_knowledge_info_model) + dataset_detail_fields_copy["external_retrieval_model"] = fields.Nested(external_retrieval_model, allow_null=True) + dataset_detail_fields_copy["doc_metadata"] = fields.List(fields.Nested(doc_metadata_model)) + dataset_detail_fields_copy["icon_info"] = fields.Nested(icon_info_model) + return _get_or_create_model("DatasetDetail", dataset_detail_fields_copy) + + +try: + dataset_detail_model = console_ns.models["DatasetDetail"] +except KeyError: + dataset_detail_model = _build_dataset_detail_model() + + def _validate_name(name: str) -> str: if not name or len(name) < 1 or len(name) > 100: raise ValueError("Name must be between 1 to 100 characters.") @@ -194,7 +251,7 @@ class ExternalDatasetCreateApi(Resource): }, ) ) - @console_ns.response(201, "External dataset created successfully", dataset_detail_fields) + @console_ns.response(201, "External dataset created successfully", dataset_detail_model) @console_ns.response(400, "Invalid parameters") @console_ns.response(403, "Permission denied") @setup_required diff --git a/api/controllers/console/extension.py b/api/controllers/console/extension.py index 6f92b9744f..08f29b4655 100644 --- a/api/controllers/console/extension.py +++ b/api/controllers/console/extension.py @@ -9,6 +9,10 @@ from models.api_based_extension import APIBasedExtension from services.api_based_extension_service import APIBasedExtensionService from services.code_based_extension_service import CodeBasedExtensionService +api_based_extension_model = console_ns.model("ApiBasedExtensionModel", api_based_extension_fields) + +api_based_extension_list_model = fields.List(fields.Nested(api_based_extension_model)) + @console_ns.route("/code-based-extension") class CodeBasedExtensionAPI(Resource): @@ -41,11 +45,11 @@ class CodeBasedExtensionAPI(Resource): class APIBasedExtensionAPI(Resource): @console_ns.doc("get_api_based_extensions") @console_ns.doc(description="Get all API-based extensions for current tenant") - @console_ns.response(200, "Success", fields.List(fields.Nested(api_based_extension_fields))) + @console_ns.response(200, "Success", api_based_extension_list_model) @setup_required @login_required @account_initialization_required - @marshal_with(api_based_extension_fields) + @marshal_with(api_based_extension_model) def get(self): _, tenant_id = current_account_with_tenant() return APIBasedExtensionService.get_all_by_tenant_id(tenant_id) @@ -62,11 +66,11 @@ class APIBasedExtensionAPI(Resource): }, ) ) - @console_ns.response(201, "Extension created successfully", api_based_extension_fields) + @console_ns.response(201, "Extension created successfully", api_based_extension_model) @setup_required @login_required @account_initialization_required - @marshal_with(api_based_extension_fields) + @marshal_with(api_based_extension_model) def post(self): args = console_ns.payload _, current_tenant_id = current_account_with_tenant() @@ -86,11 +90,11 @@ class APIBasedExtensionDetailAPI(Resource): @console_ns.doc("get_api_based_extension") @console_ns.doc(description="Get API-based extension by ID") @console_ns.doc(params={"id": "Extension ID"}) - @console_ns.response(200, "Success", api_based_extension_fields) + @console_ns.response(200, "Success", api_based_extension_model) @setup_required @login_required @account_initialization_required - @marshal_with(api_based_extension_fields) + @marshal_with(api_based_extension_model) def get(self, id): api_based_extension_id = str(id) _, tenant_id = current_account_with_tenant() @@ -110,11 +114,11 @@ class APIBasedExtensionDetailAPI(Resource): }, ) ) - @console_ns.response(200, "Extension updated successfully", api_based_extension_fields) + @console_ns.response(200, "Extension updated successfully", api_based_extension_model) @setup_required @login_required @account_initialization_required - @marshal_with(api_based_extension_fields) + @marshal_with(api_based_extension_model) def post(self, id): api_based_extension_id = str(id) _, current_tenant_id = current_account_with_tenant()