diff --git a/.cursorrules b/.cursorrules new file mode 100644 index 0000000000..cdfb8b17a3 --- /dev/null +++ b/.cursorrules @@ -0,0 +1,6 @@ +# Cursor Rules for Dify Project + +## Automated Test Generation + +- Use `web/testing/testing.md` as the canonical instruction set for generating frontend automated tests. +- When proposing or saving tests, re-read that document and follow every requirement. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000000..53afcbda1e --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,12 @@ +# Copilot Instructions + +GitHub Copilot must follow the unified frontend testing requirements documented in `web/testing/testing.md`. + +Key reminders: + +- Generate tests using the mandated tech stack, naming, and code style (AAA pattern, `fireEvent`, descriptive test names, cleans up mocks). +- Cover rendering, prop combinations, and edge cases by default; extend coverage for hooks, routing, async flows, and domain-specific components when applicable. +- Target >95% line and branch coverage and 100% function/statement coverage. +- Apply the project's mocking conventions for i18n, toast notifications, and Next.js utilities. + +Any suggestions from Copilot that conflict with `web/testing/testing.md` should be revised before acceptance. diff --git a/.windsurf/rules/testing.md b/.windsurf/rules/testing.md new file mode 100644 index 0000000000..64fec20cb8 --- /dev/null +++ b/.windsurf/rules/testing.md @@ -0,0 +1,5 @@ +# Windsurf Testing Rules + +- Use `web/testing/testing.md` as the single source of truth for frontend automated testing. +- Honor every requirement in that document when generating or accepting tests. +- When proposing or saving tests, re-read that document and follow every requirement. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fdc414b047..20a7d6c6f6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,6 +77,8 @@ How we prioritize: For setting up the frontend service, please refer to our comprehensive [guide](https://github.com/langgenius/dify/blob/main/web/README.md) in the `web/README.md` file. This document provides detailed instructions to help you set up the frontend environment properly. +**Testing**: All React components must have comprehensive test coverage. See [web/testing/testing.md](https://github.com/langgenius/dify/blob/main/web/testing/testing.md) for the canonical frontend testing guidelines and follow every requirement described there. + #### Backend For setting up the backend service, kindly refer to our detailed [instructions](https://github.com/langgenius/dify/blob/main/api/README.md) in the `api/README.md` file. This document contains step-by-step guidance to help you get the backend up and running smoothly. diff --git a/README.md b/README.md index e5cc05fbc0..09ba1f634b 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,12 @@ Issues closed Discussion posts + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/api/.env.example b/api/.env.example index ba512a668d..50607f5b35 100644 --- a/api/.env.example +++ b/api/.env.example @@ -176,6 +176,7 @@ WEAVIATE_ENDPOINT=http://localhost:8080 WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih WEAVIATE_GRPC_ENABLED=false WEAVIATE_BATCH_SIZE=100 +WEAVIATE_TOKENIZATION=word # OceanBase Vector configuration OCEANBASE_VECTOR_HOST=127.0.0.1 @@ -539,6 +540,7 @@ WORKFLOW_LOG_CLEANUP_BATCH_SIZE=100 # App configuration APP_MAX_EXECUTION_TIME=1200 +APP_DEFAULT_ACTIVE_REQUESTS=0 APP_MAX_ACTIVE_REQUESTS=0 # Celery beat configuration diff --git a/api/.importlinter b/api/.importlinter index 98fe5f50bb..24ece72b30 100644 --- a/api/.importlinter +++ b/api/.importlinter @@ -16,6 +16,7 @@ layers = graph nodes node_events + runtime entities containers = core.workflow diff --git a/api/Dockerfile b/api/Dockerfile index ed61923a40..5bfc2f4463 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -57,7 +57,7 @@ RUN \ # for gmpy2 \ libgmp-dev libmpfr-dev libmpc-dev \ # For Security - expat libldap-2.5-0 perl libsqlite3-0 zlib1g \ + expat libldap-2.5-0=2.5.13+dfsg-5 perl libsqlite3-0=3.40.1-2+deb12u2 zlib1g=1:1.2.13.dfsg-1 \ # install fonts to support the use of tools like pypdfium2 fonts-noto-cjk \ # install a package to improve the accuracy of guessing mime type and file extension @@ -73,7 +73,8 @@ COPY --from=packages ${VIRTUAL_ENV} ${VIRTUAL_ENV} ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" # Download nltk data -RUN python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger')" +RUN mkdir -p /usr/local/share/nltk_data && NLTK_DATA=/usr/local/share/nltk_data python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger'); nltk.download('stopwords')" \ + && chmod -R 755 /usr/local/share/nltk_data ENV TIKTOKEN_CACHE_DIR=/app/api/.tiktoken_cache @@ -86,7 +87,15 @@ COPY . /app/api/ COPY docker/entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh +# Create non-root user and set permissions +RUN groupadd -r -g 1001 dify && \ + useradd -r -u 1001 -g 1001 -s /bin/bash dify && \ + mkdir -p /home/dify && \ + chown -R 1001:1001 /app /home/dify ${TIKTOKEN_CACHE_DIR} /entrypoint.sh + ARG COMMIT_SHA ENV COMMIT_SHA=${COMMIT_SHA} +ENV NLTK_DATA=/usr/local/share/nltk_data +USER 1001 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 7cce3847b4..9c0c48c955 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -73,6 +73,10 @@ class AppExecutionConfig(BaseSettings): description="Maximum allowed execution time for the application in seconds", default=1200, ) + APP_DEFAULT_ACTIVE_REQUESTS: NonNegativeInt = Field( + description="Default number of concurrent active requests per app (0 for unlimited)", + default=0, + ) APP_MAX_ACTIVE_REQUESTS: NonNegativeInt = Field( description="Maximum number of concurrent active requests per app (0 for unlimited)", default=0, diff --git a/api/configs/middleware/vdb/weaviate_config.py b/api/configs/middleware/vdb/weaviate_config.py index aa81c870f6..6f4fccaa7f 100644 --- a/api/configs/middleware/vdb/weaviate_config.py +++ b/api/configs/middleware/vdb/weaviate_config.py @@ -31,3 +31,8 @@ class WeaviateConfig(BaseSettings): description="Number of objects to be processed in a single batch operation (default is 100)", default=100, ) + + WEAVIATE_TOKENIZATION: str | None = Field( + description="Tokenization for Weaviate (default is word)", + default="word", + ) diff --git a/api/controllers/console/app/completion.py b/api/controllers/console/app/completion.py index 031a95e178..2f8429f2ff 100644 --- a/api/controllers/console/app/completion.py +++ b/api/controllers/console/app/completion.py @@ -17,7 +17,6 @@ from controllers.console.app.error import ( from controllers.console.app.wraps import get_app_model from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ( ModelCurrentlyNotSupportError, @@ -32,6 +31,7 @@ from libs.login import current_user, login_required from models import Account from models.model import AppMode from services.app_generate_service import AppGenerateService +from services.app_task_service import AppTaskService from services.errors.llm import InvokeRateLimitError logger = logging.getLogger(__name__) @@ -121,7 +121,13 @@ class CompletionMessageStopApi(Resource): def post(self, app_model, task_id): if not isinstance(current_user, Account): raise ValueError("current_user must be an Account instance") - AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, current_user.id) + + AppTaskService.stop_task( + task_id=task_id, + invoke_from=InvokeFrom.DEBUGGER, + user_id=current_user.id, + app_mode=AppMode.value_of(app_model.mode), + ) return {"result": "success"}, 200 @@ -220,6 +226,12 @@ class ChatMessageStopApi(Resource): def post(self, app_model, task_id): if not isinstance(current_user, Account): raise ValueError("current_user must be an Account instance") - AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, current_user.id) + + AppTaskService.stop_task( + task_id=task_id, + invoke_from=InvokeFrom.DEBUGGER, + user_id=current_user.id, + app_mode=AppMode.value_of(app_model.mode), + ) return {"result": "success"}, 200 diff --git a/api/controllers/console/app/message.py b/api/controllers/console/app/message.py index 7fdf49c3fa..40e4020267 100644 --- a/api/controllers/console/app/message.py +++ b/api/controllers/console/app/message.py @@ -369,6 +369,58 @@ class MessageSuggestedQuestionApi(Resource): return {"data": questions} +# Shared parser for feedback export (used for both documentation and runtime parsing) +feedback_export_parser = ( + console_ns.parser() + .add_argument("from_source", type=str, choices=["user", "admin"], location="args", help="Filter by feedback source") + .add_argument("rating", type=str, choices=["like", "dislike"], location="args", help="Filter by rating") + .add_argument("has_comment", type=bool, location="args", help="Only include feedback with comments") + .add_argument("start_date", type=str, location="args", help="Start date (YYYY-MM-DD)") + .add_argument("end_date", type=str, location="args", help="End date (YYYY-MM-DD)") + .add_argument("format", type=str, choices=["csv", "json"], default="csv", location="args", help="Export format") +) + + +@console_ns.route("/apps//feedbacks/export") +class MessageFeedbackExportApi(Resource): + @console_ns.doc("export_feedbacks") + @console_ns.doc(description="Export user feedback data for Google Sheets") + @console_ns.doc(params={"app_id": "Application ID"}) + @console_ns.expect(feedback_export_parser) + @console_ns.response(200, "Feedback data exported successfully") + @console_ns.response(400, "Invalid parameters") + @console_ns.response(500, "Internal server error") + @get_app_model + @setup_required + @login_required + @account_initialization_required + def get(self, app_model): + args = feedback_export_parser.parse_args() + + # Import the service function + from services.feedback_service import FeedbackService + + try: + export_data = FeedbackService.export_feedbacks( + app_id=app_model.id, + from_source=args.get("from_source"), + rating=args.get("rating"), + has_comment=args.get("has_comment"), + start_date=args.get("start_date"), + end_date=args.get("end_date"), + format_type=args.get("format", "csv"), + ) + + return export_data + + except ValueError as e: + logger.exception("Parameter validation error in feedback export") + return {"error": f"Parameter validation error: {str(e)}"}, 400 + except Exception as e: + logger.exception("Error exporting feedback data") + raise InternalServerError(str(e)) + + @console_ns.route("/apps//messages/") class MessageApi(Resource): @console_ns.doc("get_message") diff --git a/api/controllers/console/app/workflow.py b/api/controllers/console/app/workflow.py index 7b7a8defa5..0082089365 100644 --- a/api/controllers/console/app/workflow.py +++ b/api/controllers/console/app/workflow.py @@ -90,14 +90,20 @@ workflow_pagination_model = console_ns.model("WorkflowPagination", workflow_pagi # Otherwise register it here from fields.end_user_fields import simple_end_user_fields +simple_end_user_model = None try: simple_end_user_model = console_ns.models.get("SimpleEndUser") -except (KeyError, AttributeError): +except AttributeError: + pass +if simple_end_user_model is None: simple_end_user_model = console_ns.model("SimpleEndUser", simple_end_user_fields) +workflow_run_node_execution_model = None try: workflow_run_node_execution_model = console_ns.models.get("WorkflowRunNodeExecution") -except (KeyError, AttributeError): +except AttributeError: + pass +if workflow_run_node_execution_model is None: workflow_run_node_execution_model = console_ns.model("WorkflowRunNodeExecution", workflow_run_node_execution_fields) diff --git a/api/controllers/console/app/workflow_trigger.py b/api/controllers/console/app/workflow_trigger.py index c3ea60ae3a..5d16e4f979 100644 --- a/api/controllers/console/app/workflow_trigger.py +++ b/api/controllers/console/app/workflow_trigger.py @@ -1,14 +1,13 @@ import logging -from flask_restx import Resource, marshal_with, reqparse +from flask import request +from flask_restx import Resource, marshal_with +from pydantic import BaseModel from sqlalchemy import select from sqlalchemy.orm import Session from werkzeug.exceptions import NotFound from configs import dify_config -from controllers.console import console_ns -from controllers.console.app.wraps import get_app_model -from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required from extensions.ext_database import db from fields.workflow_trigger_fields import trigger_fields, triggers_list_fields, webhook_trigger_fields from libs.login import current_user, login_required @@ -16,12 +15,35 @@ from models.enums import AppTriggerStatus from models.model import Account, App, AppMode from models.trigger import AppTrigger, WorkflowWebhookTrigger +from .. import console_ns +from ..app.wraps import get_app_model +from ..wraps import account_initialization_required, edit_permission_required, setup_required + logger = logging.getLogger(__name__) +DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" +class Parser(BaseModel): + node_id: str + + +class ParserEnable(BaseModel): + trigger_id: str + enable_trigger: bool + + +console_ns.schema_model(Parser.__name__, Parser.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0)) + +console_ns.schema_model( + ParserEnable.__name__, ParserEnable.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + + +@console_ns.route("/apps//workflows/triggers/webhook") class WebhookTriggerApi(Resource): """Webhook Trigger API""" + @console_ns.expect(console_ns.models[Parser.__name__]) @setup_required @login_required @account_initialization_required @@ -29,10 +51,9 @@ class WebhookTriggerApi(Resource): @marshal_with(webhook_trigger_fields) def get(self, app_model: App): """Get webhook trigger for a node""" - parser = reqparse.RequestParser().add_argument("node_id", type=str, required=True, help="Node ID is required") - args = parser.parse_args() + args = Parser.model_validate(request.args.to_dict(flat=True)) # type: ignore - node_id = str(args["node_id"]) + node_id = args.node_id with Session(db.engine) as session: # Get webhook trigger for this app and node @@ -51,6 +72,7 @@ class WebhookTriggerApi(Resource): return webhook_trigger +@console_ns.route("/apps//triggers") class AppTriggersApi(Resource): """App Triggers list API""" @@ -90,7 +112,9 @@ class AppTriggersApi(Resource): return {"data": triggers} +@console_ns.route("/apps//trigger-enable") class AppTriggerEnableApi(Resource): + @console_ns.expect(console_ns.models[ParserEnable.__name__], validate=True) @setup_required @login_required @account_initialization_required @@ -99,17 +123,11 @@ class AppTriggerEnableApi(Resource): @marshal_with(trigger_fields) def post(self, app_model: App): """Update app trigger (enable/disable)""" - parser = ( - reqparse.RequestParser() - .add_argument("trigger_id", type=str, required=True, nullable=False, location="json") - .add_argument("enable_trigger", type=bool, required=True, nullable=False, location="json") - ) - args = parser.parse_args() + args = ParserEnable.model_validate(console_ns.payload) assert current_user.current_tenant_id is not None - trigger_id = args["trigger_id"] - + trigger_id = args.trigger_id with Session(db.engine) as session: # Find the trigger using select trigger = session.execute( @@ -124,7 +142,7 @@ class AppTriggerEnableApi(Resource): raise NotFound("Trigger not found") # Update status based on enable_trigger boolean - trigger.status = AppTriggerStatus.ENABLED if args["enable_trigger"] else AppTriggerStatus.DISABLED + trigger.status = AppTriggerStatus.ENABLED if args.enable_trigger else AppTriggerStatus.DISABLED session.commit() session.refresh(trigger) @@ -137,8 +155,3 @@ class AppTriggerEnableApi(Resource): trigger.icon = "" # type: ignore return trigger - - -console_ns.add_resource(WebhookTriggerApi, "/apps//workflows/triggers/webhook") -console_ns.add_resource(AppTriggersApi, "/apps//triggers") -console_ns.add_resource(AppTriggerEnableApi, "/apps//trigger-enable") diff --git a/api/controllers/console/explore/completion.py b/api/controllers/console/explore/completion.py index 9386ecebae..52d6426e7f 100644 --- a/api/controllers/console/explore/completion.py +++ b/api/controllers/console/explore/completion.py @@ -15,7 +15,6 @@ from controllers.console.app.error import ( from controllers.console.explore.error import NotChatAppError, NotCompletionAppError from controllers.console.explore.wraps import InstalledAppResource from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ( ModelCurrentlyNotSupportError, @@ -31,6 +30,7 @@ from libs.login import current_user from models import Account from models.model import AppMode from services.app_generate_service import AppGenerateService +from services.app_task_service import AppTaskService from services.errors.llm import InvokeRateLimitError from .. import console_ns @@ -46,7 +46,7 @@ logger = logging.getLogger(__name__) class CompletionApi(InstalledAppResource): def post(self, installed_app): app_model = installed_app.app - if app_model.mode != "completion": + if app_model.mode != AppMode.COMPLETION: raise NotCompletionAppError() parser = ( @@ -102,12 +102,18 @@ class CompletionApi(InstalledAppResource): class CompletionStopApi(InstalledAppResource): def post(self, installed_app, task_id): app_model = installed_app.app - if app_model.mode != "completion": + if app_model.mode != AppMode.COMPLETION: raise NotCompletionAppError() if not isinstance(current_user, Account): raise ValueError("current_user must be an Account instance") - AppQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id) + + AppTaskService.stop_task( + task_id=task_id, + invoke_from=InvokeFrom.EXPLORE, + user_id=current_user.id, + app_mode=AppMode.value_of(app_model.mode), + ) return {"result": "success"}, 200 @@ -184,6 +190,12 @@ class ChatStopApi(InstalledAppResource): if not isinstance(current_user, Account): raise ValueError("current_user must be an Account instance") - AppQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id) + + AppTaskService.stop_task( + task_id=task_id, + invoke_from=InvokeFrom.EXPLORE, + user_id=current_user.id, + app_mode=app_mode, + ) return {"result": "success"}, 200 diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index 838cd3ee95..b4d1b42657 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -1,8 +1,10 @@ from datetime import datetime +from typing import Literal import pytz from flask import request -from flask_restx import Resource, fields, marshal_with, reqparse +from flask_restx import Resource, fields, marshal_with +from pydantic import BaseModel, Field, field_validator, model_validator from sqlalchemy import select from sqlalchemy.orm import Session @@ -42,20 +44,198 @@ from services.account_service import AccountService from services.billing_service import BillingService from services.errors.account import CurrentPasswordIncorrectError as ServiceCurrentPasswordIncorrectError +DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" -def _init_parser(): - parser = reqparse.RequestParser() - if dify_config.EDITION == "CLOUD": - parser.add_argument("invitation_code", type=str, location="json") - parser.add_argument("interface_language", type=supported_language, required=True, location="json").add_argument( - "timezone", type=timezone, required=True, location="json" - ) - return parser + +class AccountInitPayload(BaseModel): + interface_language: str + timezone: str + invitation_code: str | None = None + + @field_validator("interface_language") + @classmethod + def validate_language(cls, value: str) -> str: + return supported_language(value) + + @field_validator("timezone") + @classmethod + def validate_timezone(cls, value: str) -> str: + return timezone(value) + + +class AccountNamePayload(BaseModel): + name: str = Field(min_length=3, max_length=30) + + +class AccountAvatarPayload(BaseModel): + avatar: str + + +class AccountInterfaceLanguagePayload(BaseModel): + interface_language: str + + @field_validator("interface_language") + @classmethod + def validate_language(cls, value: str) -> str: + return supported_language(value) + + +class AccountInterfaceThemePayload(BaseModel): + interface_theme: Literal["light", "dark"] + + +class AccountTimezonePayload(BaseModel): + timezone: str + + @field_validator("timezone") + @classmethod + def validate_timezone(cls, value: str) -> str: + return timezone(value) + + +class AccountPasswordPayload(BaseModel): + password: str | None = None + new_password: str + repeat_new_password: str + + @model_validator(mode="after") + def check_passwords_match(self) -> "AccountPasswordPayload": + if self.new_password != self.repeat_new_password: + raise RepeatPasswordNotMatchError() + return self + + +class AccountDeletePayload(BaseModel): + token: str + code: str + + +class AccountDeletionFeedbackPayload(BaseModel): + email: str + feedback: str + + @field_validator("email") + @classmethod + def validate_email(cls, value: str) -> str: + return email(value) + + +class EducationActivatePayload(BaseModel): + token: str + institution: str + role: str + + +class EducationAutocompleteQuery(BaseModel): + keywords: str + page: int = 0 + limit: int = 20 + + +class ChangeEmailSendPayload(BaseModel): + email: str + language: str | None = None + phase: str | None = None + token: str | None = None + + @field_validator("email") + @classmethod + def validate_email(cls, value: str) -> str: + return email(value) + + +class ChangeEmailValidityPayload(BaseModel): + email: str + code: str + token: str + + @field_validator("email") + @classmethod + def validate_email(cls, value: str) -> str: + return email(value) + + +class ChangeEmailResetPayload(BaseModel): + new_email: str + token: str + + @field_validator("new_email") + @classmethod + def validate_email(cls, value: str) -> str: + return email(value) + + +class CheckEmailUniquePayload(BaseModel): + email: str + + @field_validator("email") + @classmethod + def validate_email(cls, value: str) -> str: + return email(value) + + +console_ns.schema_model( + AccountInitPayload.__name__, AccountInitPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) +console_ns.schema_model( + AccountNamePayload.__name__, AccountNamePayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) +console_ns.schema_model( + AccountAvatarPayload.__name__, AccountAvatarPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) +console_ns.schema_model( + AccountInterfaceLanguagePayload.__name__, + AccountInterfaceLanguagePayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + AccountInterfaceThemePayload.__name__, + AccountInterfaceThemePayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + AccountTimezonePayload.__name__, + AccountTimezonePayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + AccountPasswordPayload.__name__, + AccountPasswordPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + AccountDeletePayload.__name__, + AccountDeletePayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + AccountDeletionFeedbackPayload.__name__, + AccountDeletionFeedbackPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + EducationActivatePayload.__name__, + EducationActivatePayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + EducationAutocompleteQuery.__name__, + EducationAutocompleteQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + ChangeEmailSendPayload.__name__, + ChangeEmailSendPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + ChangeEmailValidityPayload.__name__, + ChangeEmailValidityPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + ChangeEmailResetPayload.__name__, + ChangeEmailResetPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + CheckEmailUniquePayload.__name__, + CheckEmailUniquePayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) @console_ns.route("/account/init") class AccountInitApi(Resource): - @console_ns.expect(_init_parser()) + @console_ns.expect(console_ns.models[AccountInitPayload.__name__]) @setup_required @login_required def post(self): @@ -64,17 +244,18 @@ class AccountInitApi(Resource): if account.status == "active": raise AccountAlreadyInitedError() - args = _init_parser().parse_args() + payload = console_ns.payload or {} + args = AccountInitPayload.model_validate(payload) if dify_config.EDITION == "CLOUD": - if not args["invitation_code"]: + if not args.invitation_code: raise ValueError("invitation_code is required") # check invitation code invitation_code = ( db.session.query(InvitationCode) .where( - InvitationCode.code == args["invitation_code"], + InvitationCode.code == args.invitation_code, InvitationCode.status == "unused", ) .first() @@ -88,8 +269,8 @@ class AccountInitApi(Resource): invitation_code.used_by_tenant_id = account.current_tenant_id invitation_code.used_by_account_id = account.id - account.interface_language = args["interface_language"] - account.timezone = args["timezone"] + account.interface_language = args.interface_language + account.timezone = args.timezone account.interface_theme = "light" account.status = "active" account.initialized_at = naive_utc_now() @@ -110,137 +291,104 @@ class AccountProfileApi(Resource): return current_user -parser_name = reqparse.RequestParser().add_argument("name", type=str, required=True, location="json") - - @console_ns.route("/account/name") class AccountNameApi(Resource): - @console_ns.expect(parser_name) + @console_ns.expect(console_ns.models[AccountNamePayload.__name__]) @setup_required @login_required @account_initialization_required @marshal_with(account_fields) def post(self): current_user, _ = current_account_with_tenant() - args = parser_name.parse_args() - - # Validate account name length - if len(args["name"]) < 3 or len(args["name"]) > 30: - raise ValueError("Account name must be between 3 and 30 characters.") - - updated_account = AccountService.update_account(current_user, name=args["name"]) + payload = console_ns.payload or {} + args = AccountNamePayload.model_validate(payload) + updated_account = AccountService.update_account(current_user, name=args.name) return updated_account -parser_avatar = reqparse.RequestParser().add_argument("avatar", type=str, required=True, location="json") - - @console_ns.route("/account/avatar") class AccountAvatarApi(Resource): - @console_ns.expect(parser_avatar) + @console_ns.expect(console_ns.models[AccountAvatarPayload.__name__]) @setup_required @login_required @account_initialization_required @marshal_with(account_fields) def post(self): current_user, _ = current_account_with_tenant() - args = parser_avatar.parse_args() + payload = console_ns.payload or {} + args = AccountAvatarPayload.model_validate(payload) - updated_account = AccountService.update_account(current_user, avatar=args["avatar"]) + updated_account = AccountService.update_account(current_user, avatar=args.avatar) return updated_account -parser_interface = reqparse.RequestParser().add_argument( - "interface_language", type=supported_language, required=True, location="json" -) - - @console_ns.route("/account/interface-language") class AccountInterfaceLanguageApi(Resource): - @console_ns.expect(parser_interface) + @console_ns.expect(console_ns.models[AccountInterfaceLanguagePayload.__name__]) @setup_required @login_required @account_initialization_required @marshal_with(account_fields) def post(self): current_user, _ = current_account_with_tenant() - args = parser_interface.parse_args() + payload = console_ns.payload or {} + args = AccountInterfaceLanguagePayload.model_validate(payload) - updated_account = AccountService.update_account(current_user, interface_language=args["interface_language"]) + updated_account = AccountService.update_account(current_user, interface_language=args.interface_language) return updated_account -parser_theme = reqparse.RequestParser().add_argument( - "interface_theme", type=str, choices=["light", "dark"], required=True, location="json" -) - - @console_ns.route("/account/interface-theme") class AccountInterfaceThemeApi(Resource): - @console_ns.expect(parser_theme) + @console_ns.expect(console_ns.models[AccountInterfaceThemePayload.__name__]) @setup_required @login_required @account_initialization_required @marshal_with(account_fields) def post(self): current_user, _ = current_account_with_tenant() - args = parser_theme.parse_args() + payload = console_ns.payload or {} + args = AccountInterfaceThemePayload.model_validate(payload) - updated_account = AccountService.update_account(current_user, interface_theme=args["interface_theme"]) + updated_account = AccountService.update_account(current_user, interface_theme=args.interface_theme) return updated_account -parser_timezone = reqparse.RequestParser().add_argument("timezone", type=str, required=True, location="json") - - @console_ns.route("/account/timezone") class AccountTimezoneApi(Resource): - @console_ns.expect(parser_timezone) + @console_ns.expect(console_ns.models[AccountTimezonePayload.__name__]) @setup_required @login_required @account_initialization_required @marshal_with(account_fields) def post(self): current_user, _ = current_account_with_tenant() - args = parser_timezone.parse_args() + payload = console_ns.payload or {} + args = AccountTimezonePayload.model_validate(payload) - # Validate timezone string, e.g. America/New_York, Asia/Shanghai - if args["timezone"] not in pytz.all_timezones: - raise ValueError("Invalid timezone string.") - - updated_account = AccountService.update_account(current_user, timezone=args["timezone"]) + updated_account = AccountService.update_account(current_user, timezone=args.timezone) return updated_account -parser_pw = ( - reqparse.RequestParser() - .add_argument("password", type=str, required=False, location="json") - .add_argument("new_password", type=str, required=True, location="json") - .add_argument("repeat_new_password", type=str, required=True, location="json") -) - - @console_ns.route("/account/password") class AccountPasswordApi(Resource): - @console_ns.expect(parser_pw) + @console_ns.expect(console_ns.models[AccountPasswordPayload.__name__]) @setup_required @login_required @account_initialization_required @marshal_with(account_fields) def post(self): current_user, _ = current_account_with_tenant() - args = parser_pw.parse_args() - - if args["new_password"] != args["repeat_new_password"]: - raise RepeatPasswordNotMatchError() + payload = console_ns.payload or {} + args = AccountPasswordPayload.model_validate(payload) try: - AccountService.update_account_password(current_user, args["password"], args["new_password"]) + AccountService.update_account_password(current_user, args.password, args.new_password) except ServiceCurrentPasswordIncorrectError: raise CurrentPasswordIncorrectError() @@ -316,25 +464,19 @@ class AccountDeleteVerifyApi(Resource): return {"result": "success", "data": token} -parser_delete = ( - reqparse.RequestParser() - .add_argument("token", type=str, required=True, location="json") - .add_argument("code", type=str, required=True, location="json") -) - - @console_ns.route("/account/delete") class AccountDeleteApi(Resource): - @console_ns.expect(parser_delete) + @console_ns.expect(console_ns.models[AccountDeletePayload.__name__]) @setup_required @login_required @account_initialization_required def post(self): account, _ = current_account_with_tenant() - args = parser_delete.parse_args() + payload = console_ns.payload or {} + args = AccountDeletePayload.model_validate(payload) - if not AccountService.verify_account_deletion_code(args["token"], args["code"]): + if not AccountService.verify_account_deletion_code(args.token, args.code): raise InvalidAccountDeletionCodeError() AccountService.delete_account(account) @@ -342,21 +484,15 @@ class AccountDeleteApi(Resource): return {"result": "success"} -parser_feedback = ( - reqparse.RequestParser() - .add_argument("email", type=str, required=True, location="json") - .add_argument("feedback", type=str, required=True, location="json") -) - - @console_ns.route("/account/delete/feedback") class AccountDeleteUpdateFeedbackApi(Resource): - @console_ns.expect(parser_feedback) + @console_ns.expect(console_ns.models[AccountDeletionFeedbackPayload.__name__]) @setup_required def post(self): - args = parser_feedback.parse_args() + payload = console_ns.payload or {} + args = AccountDeletionFeedbackPayload.model_validate(payload) - BillingService.update_account_deletion_feedback(args["email"], args["feedback"]) + BillingService.update_account_deletion_feedback(args.email, args.feedback) return {"result": "success"} @@ -379,14 +515,6 @@ class EducationVerifyApi(Resource): return BillingService.EducationIdentity.verify(account.id, account.email) -parser_edu = ( - reqparse.RequestParser() - .add_argument("token", type=str, required=True, location="json") - .add_argument("institution", type=str, required=True, location="json") - .add_argument("role", type=str, required=True, location="json") -) - - @console_ns.route("/account/education") class EducationApi(Resource): status_fields = { @@ -396,7 +524,7 @@ class EducationApi(Resource): "allow_refresh": fields.Boolean, } - @console_ns.expect(parser_edu) + @console_ns.expect(console_ns.models[EducationActivatePayload.__name__]) @setup_required @login_required @account_initialization_required @@ -405,9 +533,10 @@ class EducationApi(Resource): def post(self): account, _ = current_account_with_tenant() - args = parser_edu.parse_args() + payload = console_ns.payload or {} + args = EducationActivatePayload.model_validate(payload) - return BillingService.EducationIdentity.activate(account, args["token"], args["institution"], args["role"]) + return BillingService.EducationIdentity.activate(account, args.token, args.institution, args.role) @setup_required @login_required @@ -425,14 +554,6 @@ class EducationApi(Resource): return res -parser_autocomplete = ( - reqparse.RequestParser() - .add_argument("keywords", type=str, required=True, location="args") - .add_argument("page", type=int, required=False, location="args", default=0) - .add_argument("limit", type=int, required=False, location="args", default=20) -) - - @console_ns.route("/account/education/autocomplete") class EducationAutoCompleteApi(Resource): data_fields = { @@ -441,7 +562,7 @@ class EducationAutoCompleteApi(Resource): "has_next": fields.Boolean, } - @console_ns.expect(parser_autocomplete) + @console_ns.expect(console_ns.models[EducationAutocompleteQuery.__name__]) @setup_required @login_required @account_initialization_required @@ -449,46 +570,39 @@ class EducationAutoCompleteApi(Resource): @cloud_edition_billing_enabled @marshal_with(data_fields) def get(self): - args = parser_autocomplete.parse_args() + payload = request.args.to_dict(flat=True) # type: ignore + args = EducationAutocompleteQuery.model_validate(payload) - return BillingService.EducationIdentity.autocomplete(args["keywords"], args["page"], args["limit"]) - - -parser_change_email = ( - reqparse.RequestParser() - .add_argument("email", type=email, required=True, location="json") - .add_argument("language", type=str, required=False, location="json") - .add_argument("phase", type=str, required=False, location="json") - .add_argument("token", type=str, required=False, location="json") -) + return BillingService.EducationIdentity.autocomplete(args.keywords, args.page, args.limit) @console_ns.route("/account/change-email") class ChangeEmailSendEmailApi(Resource): - @console_ns.expect(parser_change_email) + @console_ns.expect(console_ns.models[ChangeEmailSendPayload.__name__]) @enable_change_email @setup_required @login_required @account_initialization_required def post(self): current_user, _ = current_account_with_tenant() - args = parser_change_email.parse_args() + payload = console_ns.payload or {} + args = ChangeEmailSendPayload.model_validate(payload) ip_address = extract_remote_ip(request) if AccountService.is_email_send_ip_limit(ip_address): raise EmailSendIpLimitError() - if args["language"] is not None and args["language"] == "zh-Hans": + if args.language is not None and args.language == "zh-Hans": language = "zh-Hans" else: language = "en-US" account = None - user_email = args["email"] - if args["phase"] is not None and args["phase"] == "new_email": - if args["token"] is None: + user_email = args.email + if args.phase is not None and args.phase == "new_email": + if args.token is None: raise InvalidTokenError() - reset_data = AccountService.get_change_email_data(args["token"]) + reset_data = AccountService.get_change_email_data(args.token) if reset_data is None: raise InvalidTokenError() user_email = reset_data.get("email", "") @@ -497,118 +611,103 @@ class ChangeEmailSendEmailApi(Resource): raise InvalidEmailError() else: with Session(db.engine) as session: - account = session.execute(select(Account).filter_by(email=args["email"])).scalar_one_or_none() + account = session.execute(select(Account).filter_by(email=args.email)).scalar_one_or_none() if account is None: raise AccountNotFound() token = AccountService.send_change_email_email( - account=account, email=args["email"], old_email=user_email, language=language, phase=args["phase"] + account=account, email=args.email, old_email=user_email, language=language, phase=args.phase ) return {"result": "success", "data": token} -parser_validity = ( - reqparse.RequestParser() - .add_argument("email", type=email, required=True, location="json") - .add_argument("code", type=str, required=True, location="json") - .add_argument("token", type=str, required=True, nullable=False, location="json") -) - - @console_ns.route("/account/change-email/validity") class ChangeEmailCheckApi(Resource): - @console_ns.expect(parser_validity) + @console_ns.expect(console_ns.models[ChangeEmailValidityPayload.__name__]) @enable_change_email @setup_required @login_required @account_initialization_required def post(self): - args = parser_validity.parse_args() + payload = console_ns.payload or {} + args = ChangeEmailValidityPayload.model_validate(payload) - user_email = args["email"] + user_email = args.email - is_change_email_error_rate_limit = AccountService.is_change_email_error_rate_limit(args["email"]) + is_change_email_error_rate_limit = AccountService.is_change_email_error_rate_limit(args.email) if is_change_email_error_rate_limit: raise EmailChangeLimitError() - token_data = AccountService.get_change_email_data(args["token"]) + token_data = AccountService.get_change_email_data(args.token) if token_data is None: raise InvalidTokenError() if user_email != token_data.get("email"): raise InvalidEmailError() - if args["code"] != token_data.get("code"): - AccountService.add_change_email_error_rate_limit(args["email"]) + if args.code != token_data.get("code"): + AccountService.add_change_email_error_rate_limit(args.email) raise EmailCodeError() # Verified, revoke the first token - AccountService.revoke_change_email_token(args["token"]) + AccountService.revoke_change_email_token(args.token) # Refresh token data by generating a new token _, new_token = AccountService.generate_change_email_token( - user_email, code=args["code"], old_email=token_data.get("old_email"), additional_data={} + user_email, code=args.code, old_email=token_data.get("old_email"), additional_data={} ) - AccountService.reset_change_email_error_rate_limit(args["email"]) + AccountService.reset_change_email_error_rate_limit(args.email) return {"is_valid": True, "email": token_data.get("email"), "token": new_token} -parser_reset = ( - reqparse.RequestParser() - .add_argument("new_email", type=email, required=True, location="json") - .add_argument("token", type=str, required=True, nullable=False, location="json") -) - - @console_ns.route("/account/change-email/reset") class ChangeEmailResetApi(Resource): - @console_ns.expect(parser_reset) + @console_ns.expect(console_ns.models[ChangeEmailResetPayload.__name__]) @enable_change_email @setup_required @login_required @account_initialization_required @marshal_with(account_fields) def post(self): - args = parser_reset.parse_args() + payload = console_ns.payload or {} + args = ChangeEmailResetPayload.model_validate(payload) - if AccountService.is_account_in_freeze(args["new_email"]): + if AccountService.is_account_in_freeze(args.new_email): raise AccountInFreezeError() - if not AccountService.check_email_unique(args["new_email"]): + if not AccountService.check_email_unique(args.new_email): raise EmailAlreadyInUseError() - reset_data = AccountService.get_change_email_data(args["token"]) + reset_data = AccountService.get_change_email_data(args.token) if not reset_data: raise InvalidTokenError() - AccountService.revoke_change_email_token(args["token"]) + AccountService.revoke_change_email_token(args.token) old_email = reset_data.get("old_email", "") current_user, _ = current_account_with_tenant() if current_user.email != old_email: raise AccountNotFound() - updated_account = AccountService.update_account_email(current_user, email=args["new_email"]) + updated_account = AccountService.update_account_email(current_user, email=args.new_email) AccountService.send_change_email_completed_notify_email( - email=args["new_email"], + email=args.new_email, ) return updated_account -parser_check = reqparse.RequestParser().add_argument("email", type=email, required=True, location="json") - - @console_ns.route("/account/change-email/check-email-unique") class CheckEmailUnique(Resource): - @console_ns.expect(parser_check) + @console_ns.expect(console_ns.models[CheckEmailUniquePayload.__name__]) @setup_required def post(self): - args = parser_check.parse_args() - if AccountService.is_account_in_freeze(args["email"]): + payload = console_ns.payload or {} + args = CheckEmailUniquePayload.model_validate(payload) + if AccountService.is_account_in_freeze(args.email): raise AccountInFreezeError() - if not AccountService.check_email_unique(args["email"]): + if not AccountService.check_email_unique(args.email): raise EmailAlreadyInUseError() return {"result": "success"} diff --git a/api/controllers/console/workspace/members.py b/api/controllers/console/workspace/members.py index f17f8e4bcf..f72d247398 100644 --- a/api/controllers/console/workspace/members.py +++ b/api/controllers/console/workspace/members.py @@ -1,7 +1,8 @@ from urllib import parse from flask import abort, request -from flask_restx import Resource, marshal_with, reqparse +from flask_restx import Resource, marshal_with +from pydantic import BaseModel, Field import services from configs import dify_config @@ -31,6 +32,53 @@ from services.account_service import AccountService, RegisterService, TenantServ from services.errors.account import AccountAlreadyInTenantError from services.feature_service import FeatureService +DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" + + +class MemberInvitePayload(BaseModel): + emails: list[str] = Field(default_factory=list) + role: TenantAccountRole + language: str | None = None + + +class MemberRoleUpdatePayload(BaseModel): + role: str + + +class OwnerTransferEmailPayload(BaseModel): + language: str | None = None + + +class OwnerTransferCheckPayload(BaseModel): + code: str + token: str + + +class OwnerTransferPayload(BaseModel): + token: str + + +console_ns.schema_model( + MemberInvitePayload.__name__, + MemberInvitePayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + MemberRoleUpdatePayload.__name__, + MemberRoleUpdatePayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + OwnerTransferEmailPayload.__name__, + OwnerTransferEmailPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + OwnerTransferCheckPayload.__name__, + OwnerTransferCheckPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) +console_ns.schema_model( + OwnerTransferPayload.__name__, + OwnerTransferPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + @console_ns.route("/workspaces/current/members") class MemberListApi(Resource): @@ -48,29 +96,22 @@ class MemberListApi(Resource): return {"result": "success", "accounts": members}, 200 -parser_invite = ( - reqparse.RequestParser() - .add_argument("emails", type=list, required=True, location="json") - .add_argument("role", type=str, required=True, default="admin", location="json") - .add_argument("language", type=str, required=False, location="json") -) - - @console_ns.route("/workspaces/current/members/invite-email") class MemberInviteEmailApi(Resource): """Invite a new member by email.""" - @console_ns.expect(parser_invite) + @console_ns.expect(console_ns.models[MemberInvitePayload.__name__]) @setup_required @login_required @account_initialization_required @cloud_edition_billing_resource_check("members") def post(self): - args = parser_invite.parse_args() + payload = console_ns.payload or {} + args = MemberInvitePayload.model_validate(payload) - invitee_emails = args["emails"] - invitee_role = args["role"] - interface_language = args["language"] + invitee_emails = args.emails + invitee_role = args.role + interface_language = args.language if not TenantAccountRole.is_non_owner_role(invitee_role): return {"code": "invalid-role", "message": "Invalid role"}, 400 current_user, _ = current_account_with_tenant() @@ -146,20 +187,18 @@ class MemberCancelInviteApi(Resource): }, 200 -parser_update = reqparse.RequestParser().add_argument("role", type=str, required=True, location="json") - - @console_ns.route("/workspaces/current/members//update-role") class MemberUpdateRoleApi(Resource): """Update member role.""" - @console_ns.expect(parser_update) + @console_ns.expect(console_ns.models[MemberRoleUpdatePayload.__name__]) @setup_required @login_required @account_initialization_required def put(self, member_id): - args = parser_update.parse_args() - new_role = args["role"] + payload = console_ns.payload or {} + args = MemberRoleUpdatePayload.model_validate(payload) + new_role = args.role if not TenantAccountRole.is_valid_role(new_role): return {"code": "invalid-role", "message": "Invalid role"}, 400 @@ -197,20 +236,18 @@ class DatasetOperatorMemberListApi(Resource): return {"result": "success", "accounts": members}, 200 -parser_send = reqparse.RequestParser().add_argument("language", type=str, required=False, location="json") - - @console_ns.route("/workspaces/current/members/send-owner-transfer-confirm-email") class SendOwnerTransferEmailApi(Resource): """Send owner transfer email.""" - @console_ns.expect(parser_send) + @console_ns.expect(console_ns.models[OwnerTransferEmailPayload.__name__]) @setup_required @login_required @account_initialization_required @is_allow_transfer_owner def post(self): - args = parser_send.parse_args() + payload = console_ns.payload or {} + args = OwnerTransferEmailPayload.model_validate(payload) ip_address = extract_remote_ip(request) if AccountService.is_email_send_ip_limit(ip_address): raise EmailSendIpLimitError() @@ -221,7 +258,7 @@ class SendOwnerTransferEmailApi(Resource): if not TenantService.is_owner(current_user, current_user.current_tenant): raise NotOwnerError() - if args["language"] is not None and args["language"] == "zh-Hans": + if args.language is not None and args.language == "zh-Hans": language = "zh-Hans" else: language = "en-US" @@ -238,22 +275,16 @@ class SendOwnerTransferEmailApi(Resource): return {"result": "success", "data": token} -parser_owner = ( - reqparse.RequestParser() - .add_argument("code", type=str, required=True, location="json") - .add_argument("token", type=str, required=True, nullable=False, location="json") -) - - @console_ns.route("/workspaces/current/members/owner-transfer-check") class OwnerTransferCheckApi(Resource): - @console_ns.expect(parser_owner) + @console_ns.expect(console_ns.models[OwnerTransferCheckPayload.__name__]) @setup_required @login_required @account_initialization_required @is_allow_transfer_owner def post(self): - args = parser_owner.parse_args() + payload = console_ns.payload or {} + args = OwnerTransferCheckPayload.model_validate(payload) # check if the current user is the owner of the workspace current_user, _ = current_account_with_tenant() if not current_user.current_tenant: @@ -267,41 +298,37 @@ class OwnerTransferCheckApi(Resource): if is_owner_transfer_error_rate_limit: raise OwnerTransferLimitError() - token_data = AccountService.get_owner_transfer_data(args["token"]) + token_data = AccountService.get_owner_transfer_data(args.token) if token_data is None: raise InvalidTokenError() if user_email != token_data.get("email"): raise InvalidEmailError() - if args["code"] != token_data.get("code"): + if args.code != token_data.get("code"): AccountService.add_owner_transfer_error_rate_limit(user_email) raise EmailCodeError() # Verified, revoke the first token - AccountService.revoke_owner_transfer_token(args["token"]) + AccountService.revoke_owner_transfer_token(args.token) # Refresh token data by generating a new token - _, new_token = AccountService.generate_owner_transfer_token(user_email, code=args["code"], additional_data={}) + _, new_token = AccountService.generate_owner_transfer_token(user_email, code=args.code, additional_data={}) AccountService.reset_owner_transfer_error_rate_limit(user_email) return {"is_valid": True, "email": token_data.get("email"), "token": new_token} -parser_owner_transfer = reqparse.RequestParser().add_argument( - "token", type=str, required=True, nullable=False, location="json" -) - - @console_ns.route("/workspaces/current/members//owner-transfer") class OwnerTransfer(Resource): - @console_ns.expect(parser_owner_transfer) + @console_ns.expect(console_ns.models[OwnerTransferPayload.__name__]) @setup_required @login_required @account_initialization_required @is_allow_transfer_owner def post(self, member_id): - args = parser_owner_transfer.parse_args() + payload = console_ns.payload or {} + args = OwnerTransferPayload.model_validate(payload) # check if the current user is the owner of the workspace current_user, _ = current_account_with_tenant() @@ -313,14 +340,14 @@ class OwnerTransfer(Resource): if current_user.id == str(member_id): raise CannotTransferOwnerToSelfError() - transfer_token_data = AccountService.get_owner_transfer_data(args["token"]) + transfer_token_data = AccountService.get_owner_transfer_data(args.token) if not transfer_token_data: raise InvalidTokenError() if transfer_token_data.get("email") != current_user.email: raise InvalidEmailError() - AccountService.revoke_owner_transfer_token(args["token"]) + AccountService.revoke_owner_transfer_token(args.token) member = db.session.get(Account, str(member_id)) if not member: diff --git a/api/controllers/console/workspace/model_providers.py b/api/controllers/console/workspace/model_providers.py index 8ca69121bf..d40748d5e3 100644 --- a/api/controllers/console/workspace/model_providers.py +++ b/api/controllers/console/workspace/model_providers.py @@ -1,31 +1,123 @@ import io +from typing import Any, Literal -from flask import send_file -from flask_restx import Resource, reqparse +from flask import request, send_file +from flask_restx import Resource +from pydantic import BaseModel, Field, field_validator from controllers.console import console_ns from controllers.console.wraps import account_initialization_required, is_admin_or_owner_required, setup_required from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.utils.encoders import jsonable_encoder -from libs.helper import StrLen, uuid_value +from libs.helper import uuid_value from libs.login import current_account_with_tenant, login_required from services.billing_service import BillingService from services.model_provider_service import ModelProviderService -parser_model = reqparse.RequestParser().add_argument( - "model_type", - type=str, - required=False, - nullable=True, - choices=[mt.value for mt in ModelType], - location="args", +DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" + + +class ParserModelList(BaseModel): + model_type: ModelType | None = None + + +class ParserCredentialId(BaseModel): + credential_id: str | None = None + + @field_validator("credential_id") + @classmethod + def validate_optional_credential_id(cls, value: str | None) -> str | None: + if value is None: + return value + return uuid_value(value) + + +class ParserCredentialCreate(BaseModel): + credentials: dict[str, Any] + name: str | None = Field(default=None, max_length=30) + + +class ParserCredentialUpdate(BaseModel): + credential_id: str + credentials: dict[str, Any] + name: str | None = Field(default=None, max_length=30) + + @field_validator("credential_id") + @classmethod + def validate_update_credential_id(cls, value: str) -> str: + return uuid_value(value) + + +class ParserCredentialDelete(BaseModel): + credential_id: str + + @field_validator("credential_id") + @classmethod + def validate_delete_credential_id(cls, value: str) -> str: + return uuid_value(value) + + +class ParserCredentialSwitch(BaseModel): + credential_id: str + + @field_validator("credential_id") + @classmethod + def validate_switch_credential_id(cls, value: str) -> str: + return uuid_value(value) + + +class ParserCredentialValidate(BaseModel): + credentials: dict[str, Any] + + +class ParserPreferredProviderType(BaseModel): + preferred_provider_type: Literal["system", "custom"] + + +console_ns.schema_model( + ParserModelList.__name__, ParserModelList.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + +console_ns.schema_model( + ParserCredentialId.__name__, + ParserCredentialId.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserCredentialCreate.__name__, + ParserCredentialCreate.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserCredentialUpdate.__name__, + ParserCredentialUpdate.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserCredentialDelete.__name__, + ParserCredentialDelete.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserCredentialSwitch.__name__, + ParserCredentialSwitch.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserCredentialValidate.__name__, + ParserCredentialValidate.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserPreferredProviderType.__name__, + ParserPreferredProviderType.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), ) @console_ns.route("/workspaces/current/model-providers") class ModelProviderListApi(Resource): - @console_ns.expect(parser_model) + @console_ns.expect(console_ns.models[ParserModelList.__name__]) @setup_required @login_required @account_initialization_required @@ -33,38 +125,18 @@ class ModelProviderListApi(Resource): _, current_tenant_id = current_account_with_tenant() tenant_id = current_tenant_id - args = parser_model.parse_args() + payload = request.args.to_dict(flat=True) # type: ignore + args = ParserModelList.model_validate(payload) model_provider_service = ModelProviderService() - provider_list = model_provider_service.get_provider_list(tenant_id=tenant_id, model_type=args.get("model_type")) + provider_list = model_provider_service.get_provider_list(tenant_id=tenant_id, model_type=args.model_type) return jsonable_encoder({"data": provider_list}) -parser_cred = reqparse.RequestParser().add_argument( - "credential_id", type=uuid_value, required=False, nullable=True, location="args" -) -parser_post_cred = ( - reqparse.RequestParser() - .add_argument("credentials", type=dict, required=True, nullable=False, location="json") - .add_argument("name", type=StrLen(30), required=False, nullable=True, location="json") -) - -parser_put_cred = ( - reqparse.RequestParser() - .add_argument("credential_id", type=uuid_value, required=True, nullable=False, location="json") - .add_argument("credentials", type=dict, required=True, nullable=False, location="json") - .add_argument("name", type=StrLen(30), required=False, nullable=True, location="json") -) - -parser_delete_cred = reqparse.RequestParser().add_argument( - "credential_id", type=uuid_value, required=True, nullable=False, location="json" -) - - @console_ns.route("/workspaces/current/model-providers//credentials") class ModelProviderCredentialApi(Resource): - @console_ns.expect(parser_cred) + @console_ns.expect(console_ns.models[ParserCredentialId.__name__]) @setup_required @login_required @account_initialization_required @@ -72,23 +144,25 @@ class ModelProviderCredentialApi(Resource): _, current_tenant_id = current_account_with_tenant() tenant_id = current_tenant_id # if credential_id is not provided, return current used credential - args = parser_cred.parse_args() + payload = request.args.to_dict(flat=True) # type: ignore + args = ParserCredentialId.model_validate(payload) model_provider_service = ModelProviderService() credentials = model_provider_service.get_provider_credential( - tenant_id=tenant_id, provider=provider, credential_id=args.get("credential_id") + tenant_id=tenant_id, provider=provider, credential_id=args.credential_id ) return {"credentials": credentials} - @console_ns.expect(parser_post_cred) + @console_ns.expect(console_ns.models[ParserCredentialCreate.__name__]) @setup_required @login_required @is_admin_or_owner_required @account_initialization_required def post(self, provider: str): _, current_tenant_id = current_account_with_tenant() - args = parser_post_cred.parse_args() + payload = console_ns.payload or {} + args = ParserCredentialCreate.model_validate(payload) model_provider_service = ModelProviderService() @@ -96,15 +170,15 @@ class ModelProviderCredentialApi(Resource): model_provider_service.create_provider_credential( tenant_id=current_tenant_id, provider=provider, - credentials=args["credentials"], - credential_name=args["name"], + credentials=args.credentials, + credential_name=args.name, ) except CredentialsValidateFailedError as ex: raise ValueError(str(ex)) return {"result": "success"}, 201 - @console_ns.expect(parser_put_cred) + @console_ns.expect(console_ns.models[ParserCredentialUpdate.__name__]) @setup_required @login_required @is_admin_or_owner_required @@ -112,7 +186,8 @@ class ModelProviderCredentialApi(Resource): def put(self, provider: str): _, current_tenant_id = current_account_with_tenant() - args = parser_put_cred.parse_args() + payload = console_ns.payload or {} + args = ParserCredentialUpdate.model_validate(payload) model_provider_service = ModelProviderService() @@ -120,71 +195,64 @@ class ModelProviderCredentialApi(Resource): model_provider_service.update_provider_credential( tenant_id=current_tenant_id, provider=provider, - credentials=args["credentials"], - credential_id=args["credential_id"], - credential_name=args["name"], + credentials=args.credentials, + credential_id=args.credential_id, + credential_name=args.name, ) except CredentialsValidateFailedError as ex: raise ValueError(str(ex)) return {"result": "success"} - @console_ns.expect(parser_delete_cred) + @console_ns.expect(console_ns.models[ParserCredentialDelete.__name__]) @setup_required @login_required @is_admin_or_owner_required @account_initialization_required def delete(self, provider: str): _, current_tenant_id = current_account_with_tenant() - args = parser_delete_cred.parse_args() + payload = console_ns.payload or {} + args = ParserCredentialDelete.model_validate(payload) model_provider_service = ModelProviderService() model_provider_service.remove_provider_credential( - tenant_id=current_tenant_id, provider=provider, credential_id=args["credential_id"] + tenant_id=current_tenant_id, provider=provider, credential_id=args.credential_id ) return {"result": "success"}, 204 -parser_switch = reqparse.RequestParser().add_argument( - "credential_id", type=str, required=True, nullable=False, location="json" -) - - @console_ns.route("/workspaces/current/model-providers//credentials/switch") class ModelProviderCredentialSwitchApi(Resource): - @console_ns.expect(parser_switch) + @console_ns.expect(console_ns.models[ParserCredentialSwitch.__name__]) @setup_required @login_required @is_admin_or_owner_required @account_initialization_required def post(self, provider: str): _, current_tenant_id = current_account_with_tenant() - args = parser_switch.parse_args() + payload = console_ns.payload or {} + args = ParserCredentialSwitch.model_validate(payload) service = ModelProviderService() service.switch_active_provider_credential( tenant_id=current_tenant_id, provider=provider, - credential_id=args["credential_id"], + credential_id=args.credential_id, ) return {"result": "success"} -parser_validate = reqparse.RequestParser().add_argument( - "credentials", type=dict, required=True, nullable=False, location="json" -) - - @console_ns.route("/workspaces/current/model-providers//credentials/validate") class ModelProviderValidateApi(Resource): - @console_ns.expect(parser_validate) + @console_ns.expect(console_ns.models[ParserCredentialValidate.__name__]) @setup_required @login_required @account_initialization_required def post(self, provider: str): _, current_tenant_id = current_account_with_tenant() - args = parser_validate.parse_args() + payload = console_ns.payload or {} + args = ParserCredentialValidate.model_validate(payload) tenant_id = current_tenant_id @@ -195,7 +263,7 @@ class ModelProviderValidateApi(Resource): try: model_provider_service.validate_provider_credentials( - tenant_id=tenant_id, provider=provider, credentials=args["credentials"] + tenant_id=tenant_id, provider=provider, credentials=args.credentials ) except CredentialsValidateFailedError as ex: result = False @@ -228,19 +296,9 @@ class ModelProviderIconApi(Resource): return send_file(io.BytesIO(icon), mimetype=mimetype) -parser_preferred = reqparse.RequestParser().add_argument( - "preferred_provider_type", - type=str, - required=True, - nullable=False, - choices=["system", "custom"], - location="json", -) - - @console_ns.route("/workspaces/current/model-providers//preferred-provider-type") class PreferredProviderTypeUpdateApi(Resource): - @console_ns.expect(parser_preferred) + @console_ns.expect(console_ns.models[ParserPreferredProviderType.__name__]) @setup_required @login_required @is_admin_or_owner_required @@ -250,11 +308,12 @@ class PreferredProviderTypeUpdateApi(Resource): tenant_id = current_tenant_id - args = parser_preferred.parse_args() + payload = console_ns.payload or {} + args = ParserPreferredProviderType.model_validate(payload) model_provider_service = ModelProviderService() model_provider_service.switch_preferred_provider( - tenant_id=tenant_id, provider=provider, preferred_provider_type=args["preferred_provider_type"] + tenant_id=tenant_id, provider=provider, preferred_provider_type=args.preferred_provider_type ) return {"result": "success"} diff --git a/api/controllers/console/workspace/models.py b/api/controllers/console/workspace/models.py index 2aca73806a..c820a8d1f2 100644 --- a/api/controllers/console/workspace/models.py +++ b/api/controllers/console/workspace/models.py @@ -1,52 +1,172 @@ import logging +from typing import Any, cast -from flask_restx import Resource, reqparse +from flask import request +from flask_restx import Resource +from pydantic import BaseModel, Field, field_validator from controllers.console import console_ns from controllers.console.wraps import account_initialization_required, is_admin_or_owner_required, setup_required from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.utils.encoders import jsonable_encoder -from libs.helper import StrLen, uuid_value +from libs.helper import uuid_value from libs.login import current_account_with_tenant, login_required from services.model_load_balancing_service import ModelLoadBalancingService from services.model_provider_service import ModelProviderService logger = logging.getLogger(__name__) +DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" -parser_get_default = reqparse.RequestParser().add_argument( - "model_type", - type=str, - required=True, - nullable=False, - choices=[mt.value for mt in ModelType], - location="args", +class ParserGetDefault(BaseModel): + model_type: ModelType + + +class ParserPostDefault(BaseModel): + class Inner(BaseModel): + model_type: ModelType + model: str | None = None + provider: str | None = None + + model_settings: list[Inner] + + +console_ns.schema_model( + ParserGetDefault.__name__, ParserGetDefault.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) ) -parser_post_default = reqparse.RequestParser().add_argument( - "model_settings", type=list, required=True, nullable=False, location="json" + +console_ns.schema_model( + ParserPostDefault.__name__, ParserPostDefault.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + + +class ParserDeleteModels(BaseModel): + model: str + model_type: ModelType + + +console_ns.schema_model( + ParserDeleteModels.__name__, ParserDeleteModels.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + + +class LoadBalancingPayload(BaseModel): + configs: list[dict[str, Any]] | None = None + enabled: bool | None = None + + +class ParserPostModels(BaseModel): + model: str + model_type: ModelType + load_balancing: LoadBalancingPayload | None = None + config_from: str | None = None + credential_id: str | None = None + + @field_validator("credential_id") + @classmethod + def validate_credential_id(cls, value: str | None) -> str | None: + if value is None: + return value + return uuid_value(value) + + +class ParserGetCredentials(BaseModel): + model: str + model_type: ModelType + config_from: str | None = None + credential_id: str | None = None + + @field_validator("credential_id") + @classmethod + def validate_get_credential_id(cls, value: str | None) -> str | None: + if value is None: + return value + return uuid_value(value) + + +class ParserCredentialBase(BaseModel): + model: str + model_type: ModelType + + +class ParserCreateCredential(ParserCredentialBase): + name: str | None = Field(default=None, max_length=30) + credentials: dict[str, Any] + + +class ParserUpdateCredential(ParserCredentialBase): + credential_id: str + credentials: dict[str, Any] + name: str | None = Field(default=None, max_length=30) + + @field_validator("credential_id") + @classmethod + def validate_update_credential_id(cls, value: str) -> str: + return uuid_value(value) + + +class ParserDeleteCredential(ParserCredentialBase): + credential_id: str + + @field_validator("credential_id") + @classmethod + def validate_delete_credential_id(cls, value: str) -> str: + return uuid_value(value) + + +class ParserParameter(BaseModel): + model: str + + +console_ns.schema_model( + ParserPostModels.__name__, ParserPostModels.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + +console_ns.schema_model( + ParserGetCredentials.__name__, + ParserGetCredentials.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserCreateCredential.__name__, + ParserCreateCredential.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserUpdateCredential.__name__, + ParserUpdateCredential.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserDeleteCredential.__name__, + ParserDeleteCredential.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserParameter.__name__, ParserParameter.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) ) @console_ns.route("/workspaces/current/default-model") class DefaultModelApi(Resource): - @console_ns.expect(parser_get_default) + @console_ns.expect(console_ns.models[ParserGetDefault.__name__]) @setup_required @login_required @account_initialization_required def get(self): _, tenant_id = current_account_with_tenant() - args = parser_get_default.parse_args() + args = ParserGetDefault.model_validate(request.args.to_dict(flat=True)) # type: ignore model_provider_service = ModelProviderService() default_model_entity = model_provider_service.get_default_model_of_model_type( - tenant_id=tenant_id, model_type=args["model_type"] + tenant_id=tenant_id, model_type=args.model_type ) return jsonable_encoder({"data": default_model_entity}) - @console_ns.expect(parser_post_default) + @console_ns.expect(console_ns.models[ParserPostDefault.__name__]) @setup_required @login_required @is_admin_or_owner_required @@ -54,66 +174,31 @@ class DefaultModelApi(Resource): def post(self): _, tenant_id = current_account_with_tenant() - args = parser_post_default.parse_args() + args = ParserPostDefault.model_validate(console_ns.payload) model_provider_service = ModelProviderService() - model_settings = args["model_settings"] + model_settings = args.model_settings for model_setting in model_settings: - if "model_type" not in model_setting or model_setting["model_type"] not in [mt.value for mt in ModelType]: - raise ValueError("invalid model type") - - if "provider" not in model_setting: + if model_setting.provider is None: continue - if "model" not in model_setting: - raise ValueError("invalid model") - try: model_provider_service.update_default_model_of_model_type( tenant_id=tenant_id, - model_type=model_setting["model_type"], - provider=model_setting["provider"], - model=model_setting["model"], + model_type=model_setting.model_type, + provider=model_setting.provider, + model=cast(str, model_setting.model), ) except Exception as ex: logger.exception( "Failed to update default model, model type: %s, model: %s", - model_setting["model_type"], - model_setting.get("model"), + model_setting.model_type, + model_setting.model, ) raise ex return {"result": "success"} -parser_post_models = ( - reqparse.RequestParser() - .add_argument("model", type=str, required=True, nullable=False, location="json") - .add_argument( - "model_type", - type=str, - required=True, - nullable=False, - choices=[mt.value for mt in ModelType], - location="json", - ) - .add_argument("load_balancing", type=dict, required=False, nullable=True, location="json") - .add_argument("config_from", type=str, required=False, nullable=True, location="json") - .add_argument("credential_id", type=uuid_value, required=False, nullable=True, location="json") -) -parser_delete_models = ( - reqparse.RequestParser() - .add_argument("model", type=str, required=True, nullable=False, location="json") - .add_argument( - "model_type", - type=str, - required=True, - nullable=False, - choices=[mt.value for mt in ModelType], - location="json", - ) -) - - @console_ns.route("/workspaces/current/model-providers//models") class ModelProviderModelApi(Resource): @setup_required @@ -127,7 +212,7 @@ class ModelProviderModelApi(Resource): return jsonable_encoder({"data": models}) - @console_ns.expect(parser_post_models) + @console_ns.expect(console_ns.models[ParserPostModels.__name__]) @setup_required @login_required @is_admin_or_owner_required @@ -135,45 +220,45 @@ class ModelProviderModelApi(Resource): def post(self, provider: str): # To save the model's load balance configs _, tenant_id = current_account_with_tenant() - args = parser_post_models.parse_args() + args = ParserPostModels.model_validate(console_ns.payload) - if args.get("config_from", "") == "custom-model": - if not args.get("credential_id"): + if args.config_from == "custom-model": + if not args.credential_id: raise ValueError("credential_id is required when configuring a custom-model") service = ModelProviderService() service.switch_active_custom_model_credential( tenant_id=tenant_id, provider=provider, - model_type=args["model_type"], - model=args["model"], - credential_id=args["credential_id"], + model_type=args.model_type, + model=args.model, + credential_id=args.credential_id, ) model_load_balancing_service = ModelLoadBalancingService() - if "load_balancing" in args and args["load_balancing"] and "configs" in args["load_balancing"]: + if args.load_balancing and args.load_balancing.configs: # save load balancing configs model_load_balancing_service.update_load_balancing_configs( tenant_id=tenant_id, provider=provider, - model=args["model"], - model_type=args["model_type"], - configs=args["load_balancing"]["configs"], - config_from=args.get("config_from", ""), + model=args.model, + model_type=args.model_type, + configs=args.load_balancing.configs, + config_from=args.config_from or "", ) - if args.get("load_balancing", {}).get("enabled"): + if args.load_balancing.enabled: model_load_balancing_service.enable_model_load_balancing( - tenant_id=tenant_id, provider=provider, model=args["model"], model_type=args["model_type"] + tenant_id=tenant_id, provider=provider, model=args.model, model_type=args.model_type ) else: model_load_balancing_service.disable_model_load_balancing( - tenant_id=tenant_id, provider=provider, model=args["model"], model_type=args["model_type"] + tenant_id=tenant_id, provider=provider, model=args.model, model_type=args.model_type ) return {"result": "success"}, 200 - @console_ns.expect(parser_delete_models) + @console_ns.expect(console_ns.models[ParserDeleteModels.__name__], validate=True) @setup_required @login_required @is_admin_or_owner_required @@ -181,113 +266,53 @@ class ModelProviderModelApi(Resource): def delete(self, provider: str): _, tenant_id = current_account_with_tenant() - args = parser_delete_models.parse_args() + args = ParserDeleteModels.model_validate(console_ns.payload) model_provider_service = ModelProviderService() model_provider_service.remove_model( - tenant_id=tenant_id, provider=provider, model=args["model"], model_type=args["model_type"] + tenant_id=tenant_id, provider=provider, model=args.model, model_type=args.model_type ) return {"result": "success"}, 204 -parser_get_credentials = ( - reqparse.RequestParser() - .add_argument("model", type=str, required=True, nullable=False, location="args") - .add_argument( - "model_type", - type=str, - required=True, - nullable=False, - choices=[mt.value for mt in ModelType], - location="args", - ) - .add_argument("config_from", type=str, required=False, nullable=True, location="args") - .add_argument("credential_id", type=uuid_value, required=False, nullable=True, location="args") -) - - -parser_post_cred = ( - reqparse.RequestParser() - .add_argument("model", type=str, required=True, nullable=False, location="json") - .add_argument( - "model_type", - type=str, - required=True, - nullable=False, - choices=[mt.value for mt in ModelType], - location="json", - ) - .add_argument("name", type=StrLen(30), required=False, nullable=True, location="json") - .add_argument("credentials", type=dict, required=True, nullable=False, location="json") -) -parser_put_cred = ( - reqparse.RequestParser() - .add_argument("model", type=str, required=True, nullable=False, location="json") - .add_argument( - "model_type", - type=str, - required=True, - nullable=False, - choices=[mt.value for mt in ModelType], - location="json", - ) - .add_argument("credential_id", type=uuid_value, required=True, nullable=False, location="json") - .add_argument("credentials", type=dict, required=True, nullable=False, location="json") - .add_argument("name", type=StrLen(30), required=False, nullable=True, location="json") -) -parser_delete_cred = ( - reqparse.RequestParser() - .add_argument("model", type=str, required=True, nullable=False, location="json") - .add_argument( - "model_type", - type=str, - required=True, - nullable=False, - choices=[mt.value for mt in ModelType], - location="json", - ) - .add_argument("credential_id", type=uuid_value, required=True, nullable=False, location="json") -) - - @console_ns.route("/workspaces/current/model-providers//models/credentials") class ModelProviderModelCredentialApi(Resource): - @console_ns.expect(parser_get_credentials) + @console_ns.expect(console_ns.models[ParserGetCredentials.__name__]) @setup_required @login_required @account_initialization_required def get(self, provider: str): _, tenant_id = current_account_with_tenant() - args = parser_get_credentials.parse_args() + args = ParserGetCredentials.model_validate(request.args.to_dict(flat=True)) # type: ignore model_provider_service = ModelProviderService() current_credential = model_provider_service.get_model_credential( tenant_id=tenant_id, provider=provider, - model_type=args["model_type"], - model=args["model"], - credential_id=args.get("credential_id"), + model_type=args.model_type, + model=args.model, + credential_id=args.credential_id, ) model_load_balancing_service = ModelLoadBalancingService() is_load_balancing_enabled, load_balancing_configs = model_load_balancing_service.get_load_balancing_configs( tenant_id=tenant_id, provider=provider, - model=args["model"], - model_type=args["model_type"], - config_from=args.get("config_from", ""), + model=args.model, + model_type=args.model_type, + config_from=args.config_from or "", ) - if args.get("config_from", "") == "predefined-model": + if args.config_from == "predefined-model": available_credentials = model_provider_service.provider_manager.get_provider_available_credentials( tenant_id=tenant_id, provider_name=provider ) else: - model_type = ModelType.value_of(args["model_type"]).to_origin_model_type() + model_type = args.model_type available_credentials = model_provider_service.provider_manager.get_provider_model_available_credentials( - tenant_id=tenant_id, provider_name=provider, model_type=model_type, model_name=args["model"] + tenant_id=tenant_id, provider_name=provider, model_type=model_type, model_name=args.model ) return jsonable_encoder( @@ -304,7 +329,7 @@ class ModelProviderModelCredentialApi(Resource): } ) - @console_ns.expect(parser_post_cred) + @console_ns.expect(console_ns.models[ParserCreateCredential.__name__]) @setup_required @login_required @is_admin_or_owner_required @@ -312,7 +337,7 @@ class ModelProviderModelCredentialApi(Resource): def post(self, provider: str): _, tenant_id = current_account_with_tenant() - args = parser_post_cred.parse_args() + args = ParserCreateCredential.model_validate(console_ns.payload) model_provider_service = ModelProviderService() @@ -320,30 +345,30 @@ class ModelProviderModelCredentialApi(Resource): model_provider_service.create_model_credential( tenant_id=tenant_id, provider=provider, - model=args["model"], - model_type=args["model_type"], - credentials=args["credentials"], - credential_name=args["name"], + model=args.model, + model_type=args.model_type, + credentials=args.credentials, + credential_name=args.name, ) except CredentialsValidateFailedError as ex: logger.exception( "Failed to save model credentials, tenant_id: %s, model: %s, model_type: %s", tenant_id, - args.get("model"), - args.get("model_type"), + args.model, + args.model_type, ) raise ValueError(str(ex)) return {"result": "success"}, 201 - @console_ns.expect(parser_put_cred) + @console_ns.expect(console_ns.models[ParserUpdateCredential.__name__]) @setup_required @login_required @is_admin_or_owner_required @account_initialization_required def put(self, provider: str): _, current_tenant_id = current_account_with_tenant() - args = parser_put_cred.parse_args() + args = ParserUpdateCredential.model_validate(console_ns.payload) model_provider_service = ModelProviderService() @@ -351,106 +376,87 @@ class ModelProviderModelCredentialApi(Resource): model_provider_service.update_model_credential( tenant_id=current_tenant_id, provider=provider, - model_type=args["model_type"], - model=args["model"], - credentials=args["credentials"], - credential_id=args["credential_id"], - credential_name=args["name"], + model_type=args.model_type, + model=args.model, + credentials=args.credentials, + credential_id=args.credential_id, + credential_name=args.name, ) except CredentialsValidateFailedError as ex: raise ValueError(str(ex)) return {"result": "success"} - @console_ns.expect(parser_delete_cred) + @console_ns.expect(console_ns.models[ParserDeleteCredential.__name__]) @setup_required @login_required @is_admin_or_owner_required @account_initialization_required def delete(self, provider: str): _, current_tenant_id = current_account_with_tenant() - args = parser_delete_cred.parse_args() + args = ParserDeleteCredential.model_validate(console_ns.payload) model_provider_service = ModelProviderService() model_provider_service.remove_model_credential( tenant_id=current_tenant_id, provider=provider, - model_type=args["model_type"], - model=args["model"], - credential_id=args["credential_id"], + model_type=args.model_type, + model=args.model, + credential_id=args.credential_id, ) return {"result": "success"}, 204 -parser_switch = ( - reqparse.RequestParser() - .add_argument("model", type=str, required=True, nullable=False, location="json") - .add_argument( - "model_type", - type=str, - required=True, - nullable=False, - choices=[mt.value for mt in ModelType], - location="json", - ) - .add_argument("credential_id", type=str, required=True, nullable=False, location="json") +class ParserSwitch(BaseModel): + model: str + model_type: ModelType + credential_id: str + + +console_ns.schema_model( + ParserSwitch.__name__, ParserSwitch.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) ) @console_ns.route("/workspaces/current/model-providers//models/credentials/switch") class ModelProviderModelCredentialSwitchApi(Resource): - @console_ns.expect(parser_switch) + @console_ns.expect(console_ns.models[ParserSwitch.__name__]) @setup_required @login_required @is_admin_or_owner_required @account_initialization_required def post(self, provider: str): _, current_tenant_id = current_account_with_tenant() - - args = parser_switch.parse_args() + args = ParserSwitch.model_validate(console_ns.payload) service = ModelProviderService() service.add_model_credential_to_model_list( tenant_id=current_tenant_id, provider=provider, - model_type=args["model_type"], - model=args["model"], - credential_id=args["credential_id"], + model_type=args.model_type, + model=args.model, + credential_id=args.credential_id, ) return {"result": "success"} -parser_model_enable_disable = ( - reqparse.RequestParser() - .add_argument("model", type=str, required=True, nullable=False, location="json") - .add_argument( - "model_type", - type=str, - required=True, - nullable=False, - choices=[mt.value for mt in ModelType], - location="json", - ) -) - - @console_ns.route( "/workspaces/current/model-providers//models/enable", endpoint="model-provider-model-enable" ) class ModelProviderModelEnableApi(Resource): - @console_ns.expect(parser_model_enable_disable) + @console_ns.expect(console_ns.models[ParserDeleteModels.__name__]) @setup_required @login_required @account_initialization_required def patch(self, provider: str): _, tenant_id = current_account_with_tenant() - args = parser_model_enable_disable.parse_args() + args = ParserDeleteModels.model_validate(console_ns.payload) model_provider_service = ModelProviderService() model_provider_service.enable_model( - tenant_id=tenant_id, provider=provider, model=args["model"], model_type=args["model_type"] + tenant_id=tenant_id, provider=provider, model=args.model, model_type=args.model_type ) return {"result": "success"} @@ -460,48 +466,43 @@ class ModelProviderModelEnableApi(Resource): "/workspaces/current/model-providers//models/disable", endpoint="model-provider-model-disable" ) class ModelProviderModelDisableApi(Resource): - @console_ns.expect(parser_model_enable_disable) + @console_ns.expect(console_ns.models[ParserDeleteModels.__name__]) @setup_required @login_required @account_initialization_required def patch(self, provider: str): _, tenant_id = current_account_with_tenant() - args = parser_model_enable_disable.parse_args() + args = ParserDeleteModels.model_validate(console_ns.payload) model_provider_service = ModelProviderService() model_provider_service.disable_model( - tenant_id=tenant_id, provider=provider, model=args["model"], model_type=args["model_type"] + tenant_id=tenant_id, provider=provider, model=args.model, model_type=args.model_type ) return {"result": "success"} -parser_validate = ( - reqparse.RequestParser() - .add_argument("model", type=str, required=True, nullable=False, location="json") - .add_argument( - "model_type", - type=str, - required=True, - nullable=False, - choices=[mt.value for mt in ModelType], - location="json", - ) - .add_argument("credentials", type=dict, required=True, nullable=False, location="json") +class ParserValidate(BaseModel): + model: str + model_type: ModelType + credentials: dict + + +console_ns.schema_model( + ParserValidate.__name__, ParserValidate.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) ) @console_ns.route("/workspaces/current/model-providers//models/credentials/validate") class ModelProviderModelValidateApi(Resource): - @console_ns.expect(parser_validate) + @console_ns.expect(console_ns.models[ParserValidate.__name__]) @setup_required @login_required @account_initialization_required def post(self, provider: str): _, tenant_id = current_account_with_tenant() - - args = parser_validate.parse_args() + args = ParserValidate.model_validate(console_ns.payload) model_provider_service = ModelProviderService() @@ -512,9 +513,9 @@ class ModelProviderModelValidateApi(Resource): model_provider_service.validate_model_credentials( tenant_id=tenant_id, provider=provider, - model=args["model"], - model_type=args["model_type"], - credentials=args["credentials"], + model=args.model, + model_type=args.model_type, + credentials=args.credentials, ) except CredentialsValidateFailedError as ex: result = False @@ -528,24 +529,19 @@ class ModelProviderModelValidateApi(Resource): return response -parser_parameter = reqparse.RequestParser().add_argument( - "model", type=str, required=True, nullable=False, location="args" -) - - @console_ns.route("/workspaces/current/model-providers//models/parameter-rules") class ModelProviderModelParameterRuleApi(Resource): - @console_ns.expect(parser_parameter) + @console_ns.expect(console_ns.models[ParserParameter.__name__]) @setup_required @login_required @account_initialization_required def get(self, provider: str): - args = parser_parameter.parse_args() + args = ParserParameter.model_validate(request.args.to_dict(flat=True)) # type: ignore _, tenant_id = current_account_with_tenant() model_provider_service = ModelProviderService() parameter_rules = model_provider_service.get_model_parameter_rules( - tenant_id=tenant_id, provider=provider, model=args["model"] + tenant_id=tenant_id, provider=provider, model=args.model ) return jsonable_encoder({"data": parameter_rules}) diff --git a/api/controllers/console/workspace/plugin.py b/api/controllers/console/workspace/plugin.py index e3345033f8..7e08ea55f9 100644 --- a/api/controllers/console/workspace/plugin.py +++ b/api/controllers/console/workspace/plugin.py @@ -1,7 +1,9 @@ import io +from typing import Literal from flask import request, send_file -from flask_restx import Resource, reqparse +from flask_restx import Resource +from pydantic import BaseModel, Field from werkzeug.exceptions import Forbidden from configs import dify_config @@ -17,6 +19,8 @@ from services.plugin.plugin_parameter_service import PluginParameterService from services.plugin.plugin_permission_service import PluginPermissionService from services.plugin.plugin_service import PluginService +DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" + @console_ns.route("/workspaces/current/plugin/debugging-key") class PluginDebuggingKeyApi(Resource): @@ -37,88 +41,251 @@ class PluginDebuggingKeyApi(Resource): raise ValueError(e) -parser_list = ( - reqparse.RequestParser() - .add_argument("page", type=int, required=False, location="args", default=1) - .add_argument("page_size", type=int, required=False, location="args", default=256) +class ParserList(BaseModel): + page: int = Field(default=1) + page_size: int = Field(default=256) + + +console_ns.schema_model( + ParserList.__name__, ParserList.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) ) @console_ns.route("/workspaces/current/plugin/list") class PluginListApi(Resource): - @console_ns.expect(parser_list) + @console_ns.expect(console_ns.models[ParserList.__name__]) @setup_required @login_required @account_initialization_required def get(self): _, tenant_id = current_account_with_tenant() - args = parser_list.parse_args() + args = ParserList.model_validate(request.args.to_dict(flat=True)) # type: ignore try: - plugins_with_total = PluginService.list_with_total(tenant_id, args["page"], args["page_size"]) + plugins_with_total = PluginService.list_with_total(tenant_id, args.page, args.page_size) except PluginDaemonClientSideError as e: raise ValueError(e) return jsonable_encoder({"plugins": plugins_with_total.list, "total": plugins_with_total.total}) -parser_latest = reqparse.RequestParser().add_argument("plugin_ids", type=list, required=True, location="json") +class ParserLatest(BaseModel): + plugin_ids: list[str] + + +console_ns.schema_model( + ParserLatest.__name__, ParserLatest.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + + +class ParserIcon(BaseModel): + tenant_id: str + filename: str + + +class ParserAsset(BaseModel): + plugin_unique_identifier: str + file_name: str + + +class ParserGithubUpload(BaseModel): + repo: str + version: str + package: str + + +class ParserPluginIdentifiers(BaseModel): + plugin_unique_identifiers: list[str] + + +class ParserGithubInstall(BaseModel): + plugin_unique_identifier: str + repo: str + version: str + package: str + + +class ParserPluginIdentifierQuery(BaseModel): + plugin_unique_identifier: str + + +class ParserTasks(BaseModel): + page: int + page_size: int + + +class ParserMarketplaceUpgrade(BaseModel): + original_plugin_unique_identifier: str + new_plugin_unique_identifier: str + + +class ParserGithubUpgrade(BaseModel): + original_plugin_unique_identifier: str + new_plugin_unique_identifier: str + repo: str + version: str + package: str + + +class ParserUninstall(BaseModel): + plugin_installation_id: str + + +class ParserPermissionChange(BaseModel): + install_permission: TenantPluginPermission.InstallPermission + debug_permission: TenantPluginPermission.DebugPermission + + +class ParserDynamicOptions(BaseModel): + plugin_id: str + provider: str + action: str + parameter: str + credential_id: str | None = None + provider_type: Literal["tool", "trigger"] + + +class PluginPermissionSettingsPayload(BaseModel): + install_permission: TenantPluginPermission.InstallPermission = TenantPluginPermission.InstallPermission.EVERYONE + debug_permission: TenantPluginPermission.DebugPermission = TenantPluginPermission.DebugPermission.EVERYONE + + +class PluginAutoUpgradeSettingsPayload(BaseModel): + strategy_setting: TenantPluginAutoUpgradeStrategy.StrategySetting = ( + TenantPluginAutoUpgradeStrategy.StrategySetting.FIX_ONLY + ) + upgrade_time_of_day: int = 0 + upgrade_mode: TenantPluginAutoUpgradeStrategy.UpgradeMode = TenantPluginAutoUpgradeStrategy.UpgradeMode.EXCLUDE + exclude_plugins: list[str] = Field(default_factory=list) + include_plugins: list[str] = Field(default_factory=list) + + +class ParserPreferencesChange(BaseModel): + permission: PluginPermissionSettingsPayload + auto_upgrade: PluginAutoUpgradeSettingsPayload + + +class ParserExcludePlugin(BaseModel): + plugin_id: str + + +class ParserReadme(BaseModel): + plugin_unique_identifier: str + language: str = Field(default="en-US") + + +console_ns.schema_model( + ParserIcon.__name__, ParserIcon.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + +console_ns.schema_model( + ParserAsset.__name__, ParserAsset.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + +console_ns.schema_model( + ParserGithubUpload.__name__, ParserGithubUpload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + +console_ns.schema_model( + ParserPluginIdentifiers.__name__, + ParserPluginIdentifiers.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserGithubInstall.__name__, ParserGithubInstall.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + +console_ns.schema_model( + ParserPluginIdentifierQuery.__name__, + ParserPluginIdentifierQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserTasks.__name__, ParserTasks.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + +console_ns.schema_model( + ParserMarketplaceUpgrade.__name__, + ParserMarketplaceUpgrade.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserGithubUpgrade.__name__, ParserGithubUpgrade.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + +console_ns.schema_model( + ParserUninstall.__name__, ParserUninstall.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + +console_ns.schema_model( + ParserPermissionChange.__name__, + ParserPermissionChange.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserDynamicOptions.__name__, + ParserDynamicOptions.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserPreferencesChange.__name__, + ParserPreferencesChange.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserExcludePlugin.__name__, + ParserExcludePlugin.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + ParserReadme.__name__, ParserReadme.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) @console_ns.route("/workspaces/current/plugin/list/latest-versions") class PluginListLatestVersionsApi(Resource): - @console_ns.expect(parser_latest) + @console_ns.expect(console_ns.models[ParserLatest.__name__]) @setup_required @login_required @account_initialization_required def post(self): - args = parser_latest.parse_args() + args = ParserLatest.model_validate(console_ns.payload) try: - versions = PluginService.list_latest_versions(args["plugin_ids"]) + versions = PluginService.list_latest_versions(args.plugin_ids) except PluginDaemonClientSideError as e: raise ValueError(e) return jsonable_encoder({"versions": versions}) -parser_ids = reqparse.RequestParser().add_argument("plugin_ids", type=list, required=True, location="json") - - @console_ns.route("/workspaces/current/plugin/list/installations/ids") class PluginListInstallationsFromIdsApi(Resource): - @console_ns.expect(parser_ids) + @console_ns.expect(console_ns.models[ParserLatest.__name__]) @setup_required @login_required @account_initialization_required def post(self): _, tenant_id = current_account_with_tenant() - args = parser_ids.parse_args() + args = ParserLatest.model_validate(console_ns.payload) try: - plugins = PluginService.list_installations_from_ids(tenant_id, args["plugin_ids"]) + plugins = PluginService.list_installations_from_ids(tenant_id, args.plugin_ids) except PluginDaemonClientSideError as e: raise ValueError(e) return jsonable_encoder({"plugins": plugins}) -parser_icon = ( - reqparse.RequestParser() - .add_argument("tenant_id", type=str, required=True, location="args") - .add_argument("filename", type=str, required=True, location="args") -) - - @console_ns.route("/workspaces/current/plugin/icon") class PluginIconApi(Resource): - @console_ns.expect(parser_icon) + @console_ns.expect(console_ns.models[ParserIcon.__name__]) @setup_required def get(self): - args = parser_icon.parse_args() + args = ParserIcon.model_validate(request.args.to_dict(flat=True)) # type: ignore try: - icon_bytes, mimetype = PluginService.get_asset(args["tenant_id"], args["filename"]) + icon_bytes, mimetype = PluginService.get_asset(args.tenant_id, args.filename) except PluginDaemonClientSideError as e: raise ValueError(e) @@ -128,20 +295,16 @@ class PluginIconApi(Resource): @console_ns.route("/workspaces/current/plugin/asset") class PluginAssetApi(Resource): + @console_ns.expect(console_ns.models[ParserAsset.__name__]) @setup_required @login_required @account_initialization_required def get(self): - req = ( - reqparse.RequestParser() - .add_argument("plugin_unique_identifier", type=str, required=True, location="args") - .add_argument("file_name", type=str, required=True, location="args") - ) - args = req.parse_args() + args = ParserAsset.model_validate(request.args.to_dict(flat=True)) # type: ignore _, tenant_id = current_account_with_tenant() try: - binary = PluginService.extract_asset(tenant_id, args["plugin_unique_identifier"], args["file_name"]) + binary = PluginService.extract_asset(tenant_id, args.plugin_unique_identifier, args.file_name) return send_file(io.BytesIO(binary), mimetype="application/octet-stream") except PluginDaemonClientSideError as e: raise ValueError(e) @@ -171,17 +334,9 @@ class PluginUploadFromPkgApi(Resource): return jsonable_encoder(response) -parser_github = ( - reqparse.RequestParser() - .add_argument("repo", type=str, required=True, location="json") - .add_argument("version", type=str, required=True, location="json") - .add_argument("package", type=str, required=True, location="json") -) - - @console_ns.route("/workspaces/current/plugin/upload/github") class PluginUploadFromGithubApi(Resource): - @console_ns.expect(parser_github) + @console_ns.expect(console_ns.models[ParserGithubUpload.__name__]) @setup_required @login_required @account_initialization_required @@ -189,10 +344,10 @@ class PluginUploadFromGithubApi(Resource): def post(self): _, tenant_id = current_account_with_tenant() - args = parser_github.parse_args() + args = ParserGithubUpload.model_validate(console_ns.payload) try: - response = PluginService.upload_pkg_from_github(tenant_id, args["repo"], args["version"], args["package"]) + response = PluginService.upload_pkg_from_github(tenant_id, args.repo, args.version, args.package) except PluginDaemonClientSideError as e: raise ValueError(e) @@ -223,47 +378,28 @@ class PluginUploadFromBundleApi(Resource): return jsonable_encoder(response) -parser_pkg = reqparse.RequestParser().add_argument( - "plugin_unique_identifiers", type=list, required=True, location="json" -) - - @console_ns.route("/workspaces/current/plugin/install/pkg") class PluginInstallFromPkgApi(Resource): - @console_ns.expect(parser_pkg) + @console_ns.expect(console_ns.models[ParserPluginIdentifiers.__name__]) @setup_required @login_required @account_initialization_required @plugin_permission_required(install_required=True) def post(self): _, tenant_id = current_account_with_tenant() - args = parser_pkg.parse_args() - - # check if all plugin_unique_identifiers are valid string - for plugin_unique_identifier in args["plugin_unique_identifiers"]: - if not isinstance(plugin_unique_identifier, str): - raise ValueError("Invalid plugin unique identifier") + args = ParserPluginIdentifiers.model_validate(console_ns.payload) try: - response = PluginService.install_from_local_pkg(tenant_id, args["plugin_unique_identifiers"]) + response = PluginService.install_from_local_pkg(tenant_id, args.plugin_unique_identifiers) except PluginDaemonClientSideError as e: raise ValueError(e) return jsonable_encoder(response) -parser_githubapi = ( - reqparse.RequestParser() - .add_argument("repo", type=str, required=True, location="json") - .add_argument("version", type=str, required=True, location="json") - .add_argument("package", type=str, required=True, location="json") - .add_argument("plugin_unique_identifier", type=str, required=True, location="json") -) - - @console_ns.route("/workspaces/current/plugin/install/github") class PluginInstallFromGithubApi(Resource): - @console_ns.expect(parser_githubapi) + @console_ns.expect(console_ns.models[ParserGithubInstall.__name__]) @setup_required @login_required @account_initialization_required @@ -271,15 +407,15 @@ class PluginInstallFromGithubApi(Resource): def post(self): _, tenant_id = current_account_with_tenant() - args = parser_githubapi.parse_args() + args = ParserGithubInstall.model_validate(console_ns.payload) try: response = PluginService.install_from_github( tenant_id, - args["plugin_unique_identifier"], - args["repo"], - args["version"], - args["package"], + args.plugin_unique_identifier, + args.repo, + args.version, + args.package, ) except PluginDaemonClientSideError as e: raise ValueError(e) @@ -287,14 +423,9 @@ class PluginInstallFromGithubApi(Resource): return jsonable_encoder(response) -parser_marketplace = reqparse.RequestParser().add_argument( - "plugin_unique_identifiers", type=list, required=True, location="json" -) - - @console_ns.route("/workspaces/current/plugin/install/marketplace") class PluginInstallFromMarketplaceApi(Resource): - @console_ns.expect(parser_marketplace) + @console_ns.expect(console_ns.models[ParserPluginIdentifiers.__name__]) @setup_required @login_required @account_initialization_required @@ -302,43 +433,33 @@ class PluginInstallFromMarketplaceApi(Resource): def post(self): _, tenant_id = current_account_with_tenant() - args = parser_marketplace.parse_args() - - # check if all plugin_unique_identifiers are valid string - for plugin_unique_identifier in args["plugin_unique_identifiers"]: - if not isinstance(plugin_unique_identifier, str): - raise ValueError("Invalid plugin unique identifier") + args = ParserPluginIdentifiers.model_validate(console_ns.payload) try: - response = PluginService.install_from_marketplace_pkg(tenant_id, args["plugin_unique_identifiers"]) + response = PluginService.install_from_marketplace_pkg(tenant_id, args.plugin_unique_identifiers) except PluginDaemonClientSideError as e: raise ValueError(e) return jsonable_encoder(response) -parser_pkgapi = reqparse.RequestParser().add_argument( - "plugin_unique_identifier", type=str, required=True, location="args" -) - - @console_ns.route("/workspaces/current/plugin/marketplace/pkg") class PluginFetchMarketplacePkgApi(Resource): - @console_ns.expect(parser_pkgapi) + @console_ns.expect(console_ns.models[ParserPluginIdentifierQuery.__name__]) @setup_required @login_required @account_initialization_required @plugin_permission_required(install_required=True) def get(self): _, tenant_id = current_account_with_tenant() - args = parser_pkgapi.parse_args() + args = ParserPluginIdentifierQuery.model_validate(request.args.to_dict(flat=True)) # type: ignore try: return jsonable_encoder( { "manifest": PluginService.fetch_marketplace_pkg( tenant_id, - args["plugin_unique_identifier"], + args.plugin_unique_identifier, ) } ) @@ -346,14 +467,9 @@ class PluginFetchMarketplacePkgApi(Resource): raise ValueError(e) -parser_fetch = reqparse.RequestParser().add_argument( - "plugin_unique_identifier", type=str, required=True, location="args" -) - - @console_ns.route("/workspaces/current/plugin/fetch-manifest") class PluginFetchManifestApi(Resource): - @console_ns.expect(parser_fetch) + @console_ns.expect(console_ns.models[ParserPluginIdentifierQuery.__name__]) @setup_required @login_required @account_initialization_required @@ -361,30 +477,19 @@ class PluginFetchManifestApi(Resource): def get(self): _, tenant_id = current_account_with_tenant() - args = parser_fetch.parse_args() + args = ParserPluginIdentifierQuery.model_validate(request.args.to_dict(flat=True)) # type: ignore try: return jsonable_encoder( - { - "manifest": PluginService.fetch_plugin_manifest( - tenant_id, args["plugin_unique_identifier"] - ).model_dump() - } + {"manifest": PluginService.fetch_plugin_manifest(tenant_id, args.plugin_unique_identifier).model_dump()} ) except PluginDaemonClientSideError as e: raise ValueError(e) -parser_tasks = ( - reqparse.RequestParser() - .add_argument("page", type=int, required=True, location="args") - .add_argument("page_size", type=int, required=True, location="args") -) - - @console_ns.route("/workspaces/current/plugin/tasks") class PluginFetchInstallTasksApi(Resource): - @console_ns.expect(parser_tasks) + @console_ns.expect(console_ns.models[ParserTasks.__name__]) @setup_required @login_required @account_initialization_required @@ -392,12 +497,10 @@ class PluginFetchInstallTasksApi(Resource): def get(self): _, tenant_id = current_account_with_tenant() - args = parser_tasks.parse_args() + args = ParserTasks.model_validate(request.args.to_dict(flat=True)) # type: ignore try: - return jsonable_encoder( - {"tasks": PluginService.fetch_install_tasks(tenant_id, args["page"], args["page_size"])} - ) + return jsonable_encoder({"tasks": PluginService.fetch_install_tasks(tenant_id, args.page, args.page_size)}) except PluginDaemonClientSideError as e: raise ValueError(e) @@ -462,16 +565,9 @@ class PluginDeleteInstallTaskItemApi(Resource): raise ValueError(e) -parser_marketplace_api = ( - reqparse.RequestParser() - .add_argument("original_plugin_unique_identifier", type=str, required=True, location="json") - .add_argument("new_plugin_unique_identifier", type=str, required=True, location="json") -) - - @console_ns.route("/workspaces/current/plugin/upgrade/marketplace") class PluginUpgradeFromMarketplaceApi(Resource): - @console_ns.expect(parser_marketplace_api) + @console_ns.expect(console_ns.models[ParserMarketplaceUpgrade.__name__]) @setup_required @login_required @account_initialization_required @@ -479,31 +575,21 @@ class PluginUpgradeFromMarketplaceApi(Resource): def post(self): _, tenant_id = current_account_with_tenant() - args = parser_marketplace_api.parse_args() + args = ParserMarketplaceUpgrade.model_validate(console_ns.payload) try: return jsonable_encoder( PluginService.upgrade_plugin_with_marketplace( - tenant_id, args["original_plugin_unique_identifier"], args["new_plugin_unique_identifier"] + tenant_id, args.original_plugin_unique_identifier, args.new_plugin_unique_identifier ) ) except PluginDaemonClientSideError as e: raise ValueError(e) -parser_github_post = ( - reqparse.RequestParser() - .add_argument("original_plugin_unique_identifier", type=str, required=True, location="json") - .add_argument("new_plugin_unique_identifier", type=str, required=True, location="json") - .add_argument("repo", type=str, required=True, location="json") - .add_argument("version", type=str, required=True, location="json") - .add_argument("package", type=str, required=True, location="json") -) - - @console_ns.route("/workspaces/current/plugin/upgrade/github") class PluginUpgradeFromGithubApi(Resource): - @console_ns.expect(parser_github_post) + @console_ns.expect(console_ns.models[ParserGithubUpgrade.__name__]) @setup_required @login_required @account_initialization_required @@ -511,56 +597,44 @@ class PluginUpgradeFromGithubApi(Resource): def post(self): _, tenant_id = current_account_with_tenant() - args = parser_github_post.parse_args() + args = ParserGithubUpgrade.model_validate(console_ns.payload) try: return jsonable_encoder( PluginService.upgrade_plugin_with_github( tenant_id, - args["original_plugin_unique_identifier"], - args["new_plugin_unique_identifier"], - args["repo"], - args["version"], - args["package"], + args.original_plugin_unique_identifier, + args.new_plugin_unique_identifier, + args.repo, + args.version, + args.package, ) ) except PluginDaemonClientSideError as e: raise ValueError(e) -parser_uninstall = reqparse.RequestParser().add_argument( - "plugin_installation_id", type=str, required=True, location="json" -) - - @console_ns.route("/workspaces/current/plugin/uninstall") class PluginUninstallApi(Resource): - @console_ns.expect(parser_uninstall) + @console_ns.expect(console_ns.models[ParserUninstall.__name__]) @setup_required @login_required @account_initialization_required @plugin_permission_required(install_required=True) def post(self): - args = parser_uninstall.parse_args() + args = ParserUninstall.model_validate(console_ns.payload) _, tenant_id = current_account_with_tenant() try: - return {"success": PluginService.uninstall(tenant_id, args["plugin_installation_id"])} + return {"success": PluginService.uninstall(tenant_id, args.plugin_installation_id)} except PluginDaemonClientSideError as e: raise ValueError(e) -parser_change_post = ( - reqparse.RequestParser() - .add_argument("install_permission", type=str, required=True, location="json") - .add_argument("debug_permission", type=str, required=True, location="json") -) - - @console_ns.route("/workspaces/current/plugin/permission/change") class PluginChangePermissionApi(Resource): - @console_ns.expect(parser_change_post) + @console_ns.expect(console_ns.models[ParserPermissionChange.__name__]) @setup_required @login_required @account_initialization_required @@ -570,14 +644,15 @@ class PluginChangePermissionApi(Resource): if not user.is_admin_or_owner: raise Forbidden() - args = parser_change_post.parse_args() - - install_permission = TenantPluginPermission.InstallPermission(args["install_permission"]) - debug_permission = TenantPluginPermission.DebugPermission(args["debug_permission"]) + args = ParserPermissionChange.model_validate(console_ns.payload) tenant_id = current_tenant_id - return {"success": PluginPermissionService.change_permission(tenant_id, install_permission, debug_permission)} + return { + "success": PluginPermissionService.change_permission( + tenant_id, args.install_permission, args.debug_permission + ) + } @console_ns.route("/workspaces/current/plugin/permission/fetch") @@ -605,20 +680,9 @@ class PluginFetchPermissionApi(Resource): ) -parser_dynamic = ( - reqparse.RequestParser() - .add_argument("plugin_id", type=str, required=True, location="args") - .add_argument("provider", type=str, required=True, location="args") - .add_argument("action", type=str, required=True, location="args") - .add_argument("parameter", type=str, required=True, location="args") - .add_argument("credential_id", type=str, required=False, location="args") - .add_argument("provider_type", type=str, required=True, location="args") -) - - @console_ns.route("/workspaces/current/plugin/parameters/dynamic-options") class PluginFetchDynamicSelectOptionsApi(Resource): - @console_ns.expect(parser_dynamic) + @console_ns.expect(console_ns.models[ParserDynamicOptions.__name__]) @setup_required @login_required @is_admin_or_owner_required @@ -627,18 +691,18 @@ class PluginFetchDynamicSelectOptionsApi(Resource): current_user, tenant_id = current_account_with_tenant() user_id = current_user.id - args = parser_dynamic.parse_args() + args = ParserDynamicOptions.model_validate(request.args.to_dict(flat=True)) # type: ignore try: options = PluginParameterService.get_dynamic_select_options( tenant_id=tenant_id, user_id=user_id, - plugin_id=args["plugin_id"], - provider=args["provider"], - action=args["action"], - parameter=args["parameter"], - credential_id=args["credential_id"], - provider_type=args["provider_type"], + plugin_id=args.plugin_id, + provider=args.provider, + action=args.action, + parameter=args.parameter, + credential_id=args.credential_id, + provider_type=args.provider_type, ) except PluginDaemonClientSideError as e: raise ValueError(e) @@ -646,16 +710,9 @@ class PluginFetchDynamicSelectOptionsApi(Resource): return jsonable_encoder({"options": options}) -parser_change = ( - reqparse.RequestParser() - .add_argument("permission", type=dict, required=True, location="json") - .add_argument("auto_upgrade", type=dict, required=True, location="json") -) - - @console_ns.route("/workspaces/current/plugin/preferences/change") class PluginChangePreferencesApi(Resource): - @console_ns.expect(parser_change) + @console_ns.expect(console_ns.models[ParserPreferencesChange.__name__]) @setup_required @login_required @account_initialization_required @@ -664,22 +721,20 @@ class PluginChangePreferencesApi(Resource): if not user.is_admin_or_owner: raise Forbidden() - args = parser_change.parse_args() + args = ParserPreferencesChange.model_validate(console_ns.payload) - permission = args["permission"] + permission = args.permission - install_permission = TenantPluginPermission.InstallPermission(permission.get("install_permission", "everyone")) - debug_permission = TenantPluginPermission.DebugPermission(permission.get("debug_permission", "everyone")) + install_permission = permission.install_permission + debug_permission = permission.debug_permission - auto_upgrade = args["auto_upgrade"] + auto_upgrade = args.auto_upgrade - strategy_setting = TenantPluginAutoUpgradeStrategy.StrategySetting( - auto_upgrade.get("strategy_setting", "fix_only") - ) - upgrade_time_of_day = auto_upgrade.get("upgrade_time_of_day", 0) - upgrade_mode = TenantPluginAutoUpgradeStrategy.UpgradeMode(auto_upgrade.get("upgrade_mode", "exclude")) - exclude_plugins = auto_upgrade.get("exclude_plugins", []) - include_plugins = auto_upgrade.get("include_plugins", []) + strategy_setting = auto_upgrade.strategy_setting + upgrade_time_of_day = auto_upgrade.upgrade_time_of_day + upgrade_mode = auto_upgrade.upgrade_mode + exclude_plugins = auto_upgrade.exclude_plugins + include_plugins = auto_upgrade.include_plugins # set permission set_permission_result = PluginPermissionService.change_permission( @@ -744,12 +799,9 @@ class PluginFetchPreferencesApi(Resource): return jsonable_encoder({"permission": permission_dict, "auto_upgrade": auto_upgrade_dict}) -parser_exclude = reqparse.RequestParser().add_argument("plugin_id", type=str, required=True, location="json") - - @console_ns.route("/workspaces/current/plugin/preferences/autoupgrade/exclude") class PluginAutoUpgradeExcludePluginApi(Resource): - @console_ns.expect(parser_exclude) + @console_ns.expect(console_ns.models[ParserExcludePlugin.__name__]) @setup_required @login_required @account_initialization_required @@ -757,28 +809,20 @@ class PluginAutoUpgradeExcludePluginApi(Resource): # exclude one single plugin _, tenant_id = current_account_with_tenant() - args = parser_exclude.parse_args() + args = ParserExcludePlugin.model_validate(console_ns.payload) - return jsonable_encoder({"success": PluginAutoUpgradeService.exclude_plugin(tenant_id, args["plugin_id"])}) + return jsonable_encoder({"success": PluginAutoUpgradeService.exclude_plugin(tenant_id, args.plugin_id)}) @console_ns.route("/workspaces/current/plugin/readme") class PluginReadmeApi(Resource): + @console_ns.expect(console_ns.models[ParserReadme.__name__]) @setup_required @login_required @account_initialization_required def get(self): _, tenant_id = current_account_with_tenant() - parser = ( - reqparse.RequestParser() - .add_argument("plugin_unique_identifier", type=str, required=True, location="args") - .add_argument("language", type=str, required=False, location="args") - ) - args = parser.parse_args() + args = ParserReadme.model_validate(request.args.to_dict(flat=True)) # type: ignore return jsonable_encoder( - { - "readme": PluginService.fetch_plugin_readme( - tenant_id, args["plugin_unique_identifier"], args.get("language", "en-US") - ) - } + {"readme": PluginService.fetch_plugin_readme(tenant_id, args.plugin_unique_identifier, args.language)} ) diff --git a/api/controllers/console/workspace/trigger_providers.py b/api/controllers/console/workspace/trigger_providers.py index 1bcd80c1a5..69281c6214 100644 --- a/api/controllers/console/workspace/trigger_providers.py +++ b/api/controllers/console/workspace/trigger_providers.py @@ -6,8 +6,6 @@ from sqlalchemy.orm import Session from werkzeug.exceptions import BadRequest, Forbidden from configs import dify_config -from controllers.console import console_ns -from controllers.console.wraps import account_initialization_required, is_admin_or_owner_required, setup_required from controllers.web.error import NotFoundError from core.model_runtime.utils.encoders import jsonable_encoder from core.plugin.entities.plugin_daemon import CredentialType @@ -23,9 +21,13 @@ from services.trigger.trigger_provider_service import TriggerProviderService from services.trigger.trigger_subscription_builder_service import TriggerSubscriptionBuilderService from services.trigger.trigger_subscription_operator_service import TriggerSubscriptionOperatorService +from .. import console_ns +from ..wraps import account_initialization_required, is_admin_or_owner_required, setup_required + logger = logging.getLogger(__name__) +@console_ns.route("/workspaces/current/trigger-provider//icon") class TriggerProviderIconApi(Resource): @setup_required @login_required @@ -38,6 +40,7 @@ class TriggerProviderIconApi(Resource): return TriggerManager.get_trigger_plugin_icon(tenant_id=user.current_tenant_id, provider_id=provider) +@console_ns.route("/workspaces/current/triggers") class TriggerProviderListApi(Resource): @setup_required @login_required @@ -50,6 +53,7 @@ class TriggerProviderListApi(Resource): return jsonable_encoder(TriggerProviderService.list_trigger_providers(user.current_tenant_id)) +@console_ns.route("/workspaces/current/trigger-provider//info") class TriggerProviderInfoApi(Resource): @setup_required @login_required @@ -64,6 +68,7 @@ class TriggerProviderInfoApi(Resource): ) +@console_ns.route("/workspaces/current/trigger-provider//subscriptions/list") class TriggerSubscriptionListApi(Resource): @setup_required @login_required @@ -87,7 +92,16 @@ class TriggerSubscriptionListApi(Resource): raise +parser = reqparse.RequestParser().add_argument( + "credential_type", type=str, required=False, nullable=True, location="json" +) + + +@console_ns.route( + "/workspaces/current/trigger-provider//subscriptions/builder/create", +) class TriggerSubscriptionBuilderCreateApi(Resource): + @console_ns.expect(parser) @setup_required @login_required @is_admin_or_owner_required @@ -97,9 +111,6 @@ class TriggerSubscriptionBuilderCreateApi(Resource): user = current_user assert user.current_tenant_id is not None - parser = reqparse.RequestParser().add_argument( - "credential_type", type=str, required=False, nullable=True, location="json" - ) args = parser.parse_args() try: @@ -116,6 +127,9 @@ class TriggerSubscriptionBuilderCreateApi(Resource): raise +@console_ns.route( + "/workspaces/current/trigger-provider//subscriptions/builder/", +) class TriggerSubscriptionBuilderGetApi(Resource): @setup_required @login_required @@ -127,7 +141,18 @@ class TriggerSubscriptionBuilderGetApi(Resource): ) +parser_api = ( + reqparse.RequestParser() + # The credentials of the subscription builder + .add_argument("credentials", type=dict, required=False, nullable=True, location="json") +) + + +@console_ns.route( + "/workspaces/current/trigger-provider//subscriptions/builder/verify/", +) class TriggerSubscriptionBuilderVerifyApi(Resource): + @console_ns.expect(parser_api) @setup_required @login_required @is_admin_or_owner_required @@ -136,12 +161,8 @@ class TriggerSubscriptionBuilderVerifyApi(Resource): """Verify a subscription instance for a trigger provider""" user = current_user assert user.current_tenant_id is not None - parser = ( - reqparse.RequestParser() - # The credentials of the subscription builder - .add_argument("credentials", type=dict, required=False, nullable=True, location="json") - ) - args = parser.parse_args() + + args = parser_api.parse_args() try: # Use atomic update_and_verify to prevent race conditions @@ -159,7 +180,24 @@ class TriggerSubscriptionBuilderVerifyApi(Resource): raise ValueError(str(e)) from e +parser_update_api = ( + reqparse.RequestParser() + # The name of the subscription builder + .add_argument("name", type=str, required=False, nullable=True, location="json") + # The parameters of the subscription builder + .add_argument("parameters", type=dict, required=False, nullable=True, location="json") + # The properties of the subscription builder + .add_argument("properties", type=dict, required=False, nullable=True, location="json") + # The credentials of the subscription builder + .add_argument("credentials", type=dict, required=False, nullable=True, location="json") +) + + +@console_ns.route( + "/workspaces/current/trigger-provider//subscriptions/builder/update/", +) class TriggerSubscriptionBuilderUpdateApi(Resource): + @console_ns.expect(parser_update_api) @setup_required @login_required @account_initialization_required @@ -169,18 +207,7 @@ class TriggerSubscriptionBuilderUpdateApi(Resource): assert isinstance(user, Account) assert user.current_tenant_id is not None - parser = ( - reqparse.RequestParser() - # The name of the subscription builder - .add_argument("name", type=str, required=False, nullable=True, location="json") - # The parameters of the subscription builder - .add_argument("parameters", type=dict, required=False, nullable=True, location="json") - # The properties of the subscription builder - .add_argument("properties", type=dict, required=False, nullable=True, location="json") - # The credentials of the subscription builder - .add_argument("credentials", type=dict, required=False, nullable=True, location="json") - ) - args = parser.parse_args() + args = parser_update_api.parse_args() try: return jsonable_encoder( TriggerSubscriptionBuilderService.update_trigger_subscription_builder( @@ -200,6 +227,9 @@ class TriggerSubscriptionBuilderUpdateApi(Resource): raise +@console_ns.route( + "/workspaces/current/trigger-provider//subscriptions/builder/logs/", +) class TriggerSubscriptionBuilderLogsApi(Resource): @setup_required @login_required @@ -218,7 +248,11 @@ class TriggerSubscriptionBuilderLogsApi(Resource): raise +@console_ns.route( + "/workspaces/current/trigger-provider//subscriptions/builder/build/", +) class TriggerSubscriptionBuilderBuildApi(Resource): + @console_ns.expect(parser_update_api) @setup_required @login_required @is_admin_or_owner_required @@ -227,18 +261,7 @@ class TriggerSubscriptionBuilderBuildApi(Resource): """Build a subscription instance for a trigger provider""" user = current_user assert user.current_tenant_id is not None - parser = ( - reqparse.RequestParser() - # The name of the subscription builder - .add_argument("name", type=str, required=False, nullable=True, location="json") - # The parameters of the subscription builder - .add_argument("parameters", type=dict, required=False, nullable=True, location="json") - # The properties of the subscription builder - .add_argument("properties", type=dict, required=False, nullable=True, location="json") - # The credentials of the subscription builder - .add_argument("credentials", type=dict, required=False, nullable=True, location="json") - ) - args = parser.parse_args() + args = parser_update_api.parse_args() try: # Use atomic update_and_build to prevent race conditions TriggerSubscriptionBuilderService.update_and_build_builder( @@ -258,6 +281,9 @@ class TriggerSubscriptionBuilderBuildApi(Resource): raise ValueError(str(e)) from e +@console_ns.route( + "/workspaces/current/trigger-provider//subscriptions/delete", +) class TriggerSubscriptionDeleteApi(Resource): @setup_required @login_required @@ -291,6 +317,7 @@ class TriggerSubscriptionDeleteApi(Resource): raise +@console_ns.route("/workspaces/current/trigger-provider//subscriptions/oauth/authorize") class TriggerOAuthAuthorizeApi(Resource): @setup_required @login_required @@ -374,6 +401,7 @@ class TriggerOAuthAuthorizeApi(Resource): raise +@console_ns.route("/oauth/plugin//trigger/callback") class TriggerOAuthCallbackApi(Resource): @setup_required def get(self, provider): @@ -438,6 +466,14 @@ class TriggerOAuthCallbackApi(Resource): return redirect(f"{dify_config.CONSOLE_WEB_URL}/oauth-callback") +parser_oauth_client = ( + reqparse.RequestParser() + .add_argument("client_params", type=dict, required=False, nullable=True, location="json") + .add_argument("enabled", type=bool, required=False, nullable=True, location="json") +) + + +@console_ns.route("/workspaces/current/trigger-provider//oauth/client") class TriggerOAuthClientManageApi(Resource): @setup_required @login_required @@ -484,6 +520,7 @@ class TriggerOAuthClientManageApi(Resource): logger.exception("Error getting OAuth client", exc_info=e) raise + @console_ns.expect(parser_oauth_client) @setup_required @login_required @is_admin_or_owner_required @@ -493,12 +530,7 @@ class TriggerOAuthClientManageApi(Resource): user = current_user assert user.current_tenant_id is not None - parser = ( - reqparse.RequestParser() - .add_argument("client_params", type=dict, required=False, nullable=True, location="json") - .add_argument("enabled", type=bool, required=False, nullable=True, location="json") - ) - args = parser.parse_args() + args = parser_oauth_client.parse_args() try: provider_id = TriggerProviderID(provider) @@ -536,52 +568,3 @@ class TriggerOAuthClientManageApi(Resource): except Exception as e: logger.exception("Error removing OAuth client", exc_info=e) raise - - -# Trigger Subscription -console_ns.add_resource(TriggerProviderIconApi, "/workspaces/current/trigger-provider//icon") -console_ns.add_resource(TriggerProviderListApi, "/workspaces/current/triggers") -console_ns.add_resource(TriggerProviderInfoApi, "/workspaces/current/trigger-provider//info") -console_ns.add_resource( - TriggerSubscriptionListApi, "/workspaces/current/trigger-provider//subscriptions/list" -) -console_ns.add_resource( - TriggerSubscriptionDeleteApi, - "/workspaces/current/trigger-provider//subscriptions/delete", -) - -# Trigger Subscription Builder -console_ns.add_resource( - TriggerSubscriptionBuilderCreateApi, - "/workspaces/current/trigger-provider//subscriptions/builder/create", -) -console_ns.add_resource( - TriggerSubscriptionBuilderGetApi, - "/workspaces/current/trigger-provider//subscriptions/builder/", -) -console_ns.add_resource( - TriggerSubscriptionBuilderUpdateApi, - "/workspaces/current/trigger-provider//subscriptions/builder/update/", -) -console_ns.add_resource( - TriggerSubscriptionBuilderVerifyApi, - "/workspaces/current/trigger-provider//subscriptions/builder/verify/", -) -console_ns.add_resource( - TriggerSubscriptionBuilderBuildApi, - "/workspaces/current/trigger-provider//subscriptions/builder/build/", -) -console_ns.add_resource( - TriggerSubscriptionBuilderLogsApi, - "/workspaces/current/trigger-provider//subscriptions/builder/logs/", -) - - -# OAuth -console_ns.add_resource( - TriggerOAuthAuthorizeApi, "/workspaces/current/trigger-provider//subscriptions/oauth/authorize" -) -console_ns.add_resource(TriggerOAuthCallbackApi, "/oauth/plugin//trigger/callback") -console_ns.add_resource( - TriggerOAuthClientManageApi, "/workspaces/current/trigger-provider//oauth/client" -) diff --git a/api/controllers/console/workspace/workspace.py b/api/controllers/console/workspace/workspace.py index 37c7dc3040..9b76cb7a9c 100644 --- a/api/controllers/console/workspace/workspace.py +++ b/api/controllers/console/workspace/workspace.py @@ -1,7 +1,8 @@ import logging from flask import request -from flask_restx import Resource, fields, inputs, marshal, marshal_with, reqparse +from flask_restx import Resource, fields, marshal, marshal_with +from pydantic import BaseModel, Field from sqlalchemy import select from werkzeug.exceptions import Unauthorized @@ -32,6 +33,45 @@ from services.file_service import FileService from services.workspace_service import WorkspaceService logger = logging.getLogger(__name__) +DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" + + +class WorkspaceListQuery(BaseModel): + page: int = Field(default=1, ge=1, le=99999) + limit: int = Field(default=20, ge=1, le=100) + + +class SwitchWorkspacePayload(BaseModel): + tenant_id: str + + +class WorkspaceCustomConfigPayload(BaseModel): + remove_webapp_brand: bool | None = None + replace_webapp_logo: str | None = None + + +class WorkspaceInfoPayload(BaseModel): + name: str + + +console_ns.schema_model( + WorkspaceListQuery.__name__, WorkspaceListQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0) +) + +console_ns.schema_model( + SwitchWorkspacePayload.__name__, + SwitchWorkspacePayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + WorkspaceCustomConfigPayload.__name__, + WorkspaceCustomConfigPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + +console_ns.schema_model( + WorkspaceInfoPayload.__name__, + WorkspaceInfoPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) provider_fields = { @@ -95,18 +135,15 @@ class TenantListApi(Resource): @console_ns.route("/all-workspaces") class WorkspaceListApi(Resource): + @console_ns.expect(console_ns.models[WorkspaceListQuery.__name__]) @setup_required @admin_required def get(self): - parser = ( - reqparse.RequestParser() - .add_argument("page", type=inputs.int_range(1, 99999), required=False, default=1, location="args") - .add_argument("limit", type=inputs.int_range(1, 100), required=False, default=20, location="args") - ) - args = parser.parse_args() + payload = request.args.to_dict(flat=True) # type: ignore + args = WorkspaceListQuery.model_validate(payload) stmt = select(Tenant).order_by(Tenant.created_at.desc()) - tenants = db.paginate(select=stmt, page=args["page"], per_page=args["limit"], error_out=False) + tenants = db.paginate(select=stmt, page=args.page, per_page=args.limit, error_out=False) has_more = False if tenants.has_next: @@ -115,8 +152,8 @@ class WorkspaceListApi(Resource): return { "data": marshal(tenants.items, workspace_fields), "has_more": has_more, - "limit": args["limit"], - "page": args["page"], + "limit": args.limit, + "page": args.page, "total": tenants.total, }, 200 @@ -150,26 +187,24 @@ class TenantApi(Resource): return WorkspaceService.get_tenant_info(tenant), 200 -parser_switch = reqparse.RequestParser().add_argument("tenant_id", type=str, required=True, location="json") - - @console_ns.route("/workspaces/switch") class SwitchWorkspaceApi(Resource): - @console_ns.expect(parser_switch) + @console_ns.expect(console_ns.models[SwitchWorkspacePayload.__name__]) @setup_required @login_required @account_initialization_required def post(self): current_user, _ = current_account_with_tenant() - args = parser_switch.parse_args() + payload = console_ns.payload or {} + args = SwitchWorkspacePayload.model_validate(payload) # check if tenant_id is valid, 403 if not try: - TenantService.switch_tenant(current_user, args["tenant_id"]) + TenantService.switch_tenant(current_user, args.tenant_id) except Exception: raise AccountNotLinkTenantError("Account not link tenant") - new_tenant = db.session.query(Tenant).get(args["tenant_id"]) # Get new tenant + new_tenant = db.session.query(Tenant).get(args.tenant_id) # Get new tenant if new_tenant is None: raise ValueError("Tenant not found") @@ -178,24 +213,21 @@ class SwitchWorkspaceApi(Resource): @console_ns.route("/workspaces/custom-config") class CustomConfigWorkspaceApi(Resource): + @console_ns.expect(console_ns.models[WorkspaceCustomConfigPayload.__name__]) @setup_required @login_required @account_initialization_required @cloud_edition_billing_resource_check("workspace_custom") def post(self): _, current_tenant_id = current_account_with_tenant() - parser = ( - reqparse.RequestParser() - .add_argument("remove_webapp_brand", type=bool, location="json") - .add_argument("replace_webapp_logo", type=str, location="json") - ) - args = parser.parse_args() + payload = console_ns.payload or {} + args = WorkspaceCustomConfigPayload.model_validate(payload) tenant = db.get_or_404(Tenant, current_tenant_id) custom_config_dict = { - "remove_webapp_brand": args["remove_webapp_brand"], - "replace_webapp_logo": args["replace_webapp_logo"] - if args["replace_webapp_logo"] is not None + "remove_webapp_brand": args.remove_webapp_brand, + "replace_webapp_logo": args.replace_webapp_logo + if args.replace_webapp_logo is not None else tenant.custom_config_dict.get("replace_webapp_logo"), } @@ -245,24 +277,22 @@ class WebappLogoWorkspaceApi(Resource): return {"id": upload_file.id}, 201 -parser_info = reqparse.RequestParser().add_argument("name", type=str, required=True, location="json") - - @console_ns.route("/workspaces/info") class WorkspaceInfoApi(Resource): - @console_ns.expect(parser_info) + @console_ns.expect(console_ns.models[WorkspaceInfoPayload.__name__]) @setup_required @login_required @account_initialization_required # Change workspace name def post(self): _, current_tenant_id = current_account_with_tenant() - args = parser_info.parse_args() + payload = console_ns.payload or {} + args = WorkspaceInfoPayload.model_validate(payload) if not current_tenant_id: raise ValueError("No current tenant") tenant = db.get_or_404(Tenant, current_tenant_id) - tenant.name = args["name"] + tenant.name = args.name db.session.commit() return {"result": "success", "tenant": marshal(WorkspaceService.get_tenant_info(tenant), tenant_fields)} diff --git a/api/controllers/service_api/app/completion.py b/api/controllers/service_api/app/completion.py index 915e7e9416..c5dd919759 100644 --- a/api/controllers/service_api/app/completion.py +++ b/api/controllers/service_api/app/completion.py @@ -17,7 +17,6 @@ from controllers.service_api.app.error import ( ) from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ( ModelCurrentlyNotSupportError, @@ -30,6 +29,7 @@ from libs import helper from libs.helper import uuid_value from models.model import App, AppMode, EndUser from services.app_generate_service import AppGenerateService +from services.app_task_service import AppTaskService from services.errors.app import IsDraftWorkflowError, WorkflowIdFormatError, WorkflowNotFoundError from services.errors.llm import InvokeRateLimitError @@ -88,7 +88,7 @@ class CompletionApi(Resource): This endpoint generates a completion based on the provided inputs and query. Supports both blocking and streaming response modes. """ - if app_model.mode != "completion": + if app_model.mode != AppMode.COMPLETION: raise AppUnavailableError() args = completion_parser.parse_args() @@ -147,10 +147,15 @@ class CompletionStopApi(Resource): @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON, required=True)) def post(self, app_model: App, end_user: EndUser, task_id: str): """Stop a running completion task.""" - if app_model.mode != "completion": + if app_model.mode != AppMode.COMPLETION: raise AppUnavailableError() - AppQueueManager.set_stop_flag(task_id, InvokeFrom.SERVICE_API, end_user.id) + AppTaskService.stop_task( + task_id=task_id, + invoke_from=InvokeFrom.SERVICE_API, + user_id=end_user.id, + app_mode=AppMode.value_of(app_model.mode), + ) return {"result": "success"}, 200 @@ -244,6 +249,11 @@ class ChatStopApi(Resource): if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}: raise NotChatAppError() - AppQueueManager.set_stop_flag(task_id, InvokeFrom.SERVICE_API, end_user.id) + AppTaskService.stop_task( + task_id=task_id, + invoke_from=InvokeFrom.SERVICE_API, + user_id=end_user.id, + app_mode=app_mode, + ) return {"result": "success"}, 200 diff --git a/api/controllers/trigger/webhook.py b/api/controllers/trigger/webhook.py index cec5c3d8ae..22b24271c6 100644 --- a/api/controllers/trigger/webhook.py +++ b/api/controllers/trigger/webhook.py @@ -1,7 +1,7 @@ import logging import time -from flask import jsonify +from flask import jsonify, request from werkzeug.exceptions import NotFound, RequestEntityTooLarge from controllers.trigger import bp @@ -28,8 +28,14 @@ def _prepare_webhook_execution(webhook_id: str, is_debug: bool = False): webhook_data = WebhookService.extract_and_validate_webhook_data(webhook_trigger, node_config) return webhook_trigger, workflow, node_config, webhook_data, None except ValueError as e: - # Fall back to raw extraction for error reporting - webhook_data = WebhookService.extract_webhook_data(webhook_trigger) + # Provide minimal context for error reporting without risking another parse failure + webhook_data = { + "method": request.method, + "headers": dict(request.headers), + "query_params": dict(request.args), + "body": {}, + "files": {}, + } return webhook_trigger, workflow, node_config, webhook_data, str(e) diff --git a/api/controllers/web/completion.py b/api/controllers/web/completion.py index 5e45beffc0..e8a4698375 100644 --- a/api/controllers/web/completion.py +++ b/api/controllers/web/completion.py @@ -17,7 +17,6 @@ from controllers.web.error import ( ) from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError from controllers.web.wraps import WebApiResource -from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ( ModelCurrentlyNotSupportError, @@ -29,6 +28,7 @@ from libs import helper from libs.helper import uuid_value from models.model import AppMode from services.app_generate_service import AppGenerateService +from services.app_task_service import AppTaskService from services.errors.llm import InvokeRateLimitError logger = logging.getLogger(__name__) @@ -64,7 +64,7 @@ class CompletionApi(WebApiResource): } ) def post(self, app_model, end_user): - if app_model.mode != "completion": + if app_model.mode != AppMode.COMPLETION: raise NotCompletionAppError() parser = ( @@ -125,10 +125,15 @@ class CompletionStopApi(WebApiResource): } ) def post(self, app_model, end_user, task_id): - if app_model.mode != "completion": + if app_model.mode != AppMode.COMPLETION: raise NotCompletionAppError() - AppQueueManager.set_stop_flag(task_id, InvokeFrom.WEB_APP, end_user.id) + AppTaskService.stop_task( + task_id=task_id, + invoke_from=InvokeFrom.WEB_APP, + user_id=end_user.id, + app_mode=AppMode.value_of(app_model.mode), + ) return {"result": "success"}, 200 @@ -234,6 +239,11 @@ class ChatStopApi(WebApiResource): if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}: raise NotChatAppError() - AppQueueManager.set_stop_flag(task_id, InvokeFrom.WEB_APP, end_user.id) + AppTaskService.stop_task( + task_id=task_id, + invoke_from=InvokeFrom.WEB_APP, + user_id=end_user.id, + app_mode=app_mode, + ) return {"result": "success"}, 200 diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py index 01c377956b..c98bc1ffdd 100644 --- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py +++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py @@ -62,7 +62,8 @@ from core.app.task_pipeline.message_cycle_manager import MessageCycleManager from core.base.tts import AppGeneratorTTSPublisher, AudioTrunk from core.model_runtime.entities.llm_entities import LLMUsage from core.model_runtime.utils.encoders import jsonable_encoder -from core.ops.ops_trace_manager import TraceQueueManager +from core.ops.entities.trace_entity import TraceTaskName +from core.ops.ops_trace_manager import TraceQueueManager, TraceTask from core.workflow.enums import WorkflowExecutionStatus from core.workflow.nodes import NodeType from core.workflow.repositories.draft_variable_repository import DraftVariableSaverFactory @@ -72,7 +73,7 @@ from extensions.ext_database import db from libs.datetime_utils import naive_utc_now from models import Account, Conversation, EndUser, Message, MessageFile from models.enums import CreatorUserRole -from models.workflow import Workflow +from models.workflow import Workflow, WorkflowNodeExecutionModel logger = logging.getLogger(__name__) @@ -580,7 +581,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport): with self._database_session() as session: # Save message - self._save_message(session=session, graph_runtime_state=resolved_state) + self._save_message(session=session, graph_runtime_state=resolved_state, trace_manager=trace_manager) yield workflow_finish_resp elif event.stopped_by in ( @@ -590,7 +591,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport): # When hitting input-moderation or annotation-reply, the workflow will not start with self._database_session() as session: # Save message - self._save_message(session=session) + self._save_message(session=session, trace_manager=trace_manager) yield self._message_end_to_stream_response() @@ -599,6 +600,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport): event: QueueAdvancedChatMessageEndEvent, *, graph_runtime_state: GraphRuntimeState | None = None, + trace_manager: TraceQueueManager | None = None, **kwargs, ) -> Generator[StreamResponse, None, None]: """Handle advanced chat message end events.""" @@ -616,7 +618,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport): # Save message with self._database_session() as session: - self._save_message(session=session, graph_runtime_state=resolved_state) + self._save_message(session=session, graph_runtime_state=resolved_state, trace_manager=trace_manager) yield self._message_end_to_stream_response() @@ -770,7 +772,13 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport): if self._conversation_name_generate_thread: self._conversation_name_generate_thread.join() - def _save_message(self, *, session: Session, graph_runtime_state: GraphRuntimeState | None = None): + def _save_message( + self, + *, + session: Session, + graph_runtime_state: GraphRuntimeState | None = None, + trace_manager: TraceQueueManager | None = None, + ): message = self._get_message(session=session) # If there are assistant files, remove markdown image links from answer @@ -809,6 +817,14 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport): metadata = self._task_state.metadata.model_dump() message.message_metadata = json.dumps(jsonable_encoder(metadata)) + + # Extract model provider and model_id from workflow node executions for tracing + if message.workflow_run_id: + model_info = self._extract_model_info_from_workflow(session, message.workflow_run_id) + if model_info: + message.model_provider = model_info.get("provider") + message.model_id = model_info.get("model") + message_files = [ MessageFile( message_id=message.id, @@ -826,6 +842,68 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport): ] session.add_all(message_files) + # Trigger MESSAGE_TRACE for tracing integrations + if trace_manager: + trace_manager.add_trace_task( + TraceTask( + TraceTaskName.MESSAGE_TRACE, conversation_id=self._conversation_id, message_id=self._message_id + ) + ) + + def _extract_model_info_from_workflow(self, session: Session, workflow_run_id: str) -> dict[str, str] | None: + """ + Extract model provider and model_id from workflow node executions. + Returns dict with 'provider' and 'model' keys, or None if not found. + """ + try: + # Query workflow node executions for LLM or Agent nodes + stmt = ( + select(WorkflowNodeExecutionModel) + .where(WorkflowNodeExecutionModel.workflow_run_id == workflow_run_id) + .where(WorkflowNodeExecutionModel.node_type.in_(["llm", "agent"])) + .order_by(WorkflowNodeExecutionModel.created_at.desc()) + .limit(1) + ) + node_execution = session.scalar(stmt) + + if not node_execution: + return None + + # Try to extract from execution_metadata for agent nodes + if node_execution.execution_metadata: + try: + metadata = json.loads(node_execution.execution_metadata) + agent_log = metadata.get("agent_log", []) + # Look for the first agent thought with provider info + for log_entry in agent_log: + entry_metadata = log_entry.get("metadata", {}) + provider_str = entry_metadata.get("provider") + if provider_str: + # Parse format like "langgenius/deepseek/deepseek" + parts = provider_str.split("/") + if len(parts) >= 3: + return {"provider": parts[1], "model": parts[2]} + elif len(parts) == 2: + return {"provider": parts[0], "model": parts[1]} + except (json.JSONDecodeError, KeyError, AttributeError) as e: + logger.debug("Failed to parse execution_metadata: %s", e) + + # Try to extract from process_data for llm nodes + if node_execution.process_data: + try: + process_data = json.loads(node_execution.process_data) + provider = process_data.get("model_provider") + model = process_data.get("model_name") + if provider and model: + return {"provider": provider, "model": model} + except (json.JSONDecodeError, KeyError) as e: + logger.debug("Failed to parse process_data: %s", e) + + return None + except Exception as e: + logger.warning("Failed to extract model info from workflow: %s", e) + return None + def _seed_graph_runtime_state_from_queue_manager(self) -> None: """Bootstrap the cached runtime state from the queue manager when present.""" candidate = self._base_task_pipeline.queue_manager.graph_runtime_state diff --git a/api/core/app/apps/base_app_generator.py b/api/core/app/apps/base_app_generator.py index 85be05fb69..1c6ca87925 100644 --- a/api/core/app/apps/base_app_generator.py +++ b/api/core/app/apps/base_app_generator.py @@ -155,8 +155,17 @@ class BaseAppGenerator: f"{variable_entity.variable} in input form must be less than {variable_entity.max_length} files" ) case VariableEntityType.CHECKBOX: - if not isinstance(value, bool): - raise ValueError(f"{variable_entity.variable} in input form must be a valid boolean value") + if isinstance(value, str): + normalized_value = value.strip().lower() + if normalized_value in {"true", "1", "yes", "on"}: + value = True + elif normalized_value in {"false", "0", "no", "off"}: + value = False + elif isinstance(value, (int, float)): + if value == 1: + value = True + elif value == 0: + value = False case _: raise AssertionError("this statement should be unreachable.") diff --git a/api/core/app/apps/workflow/generate_task_pipeline.py b/api/core/app/apps/workflow/generate_task_pipeline.py index 4157870620..842ad545ad 100644 --- a/api/core/app/apps/workflow/generate_task_pipeline.py +++ b/api/core/app/apps/workflow/generate_task_pipeline.py @@ -258,6 +258,10 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport): run_id = self._extract_workflow_run_id(runtime_state) self._workflow_execution_id = run_id + + with self._database_session() as session: + self._save_workflow_app_log(session=session, workflow_run_id=self._workflow_execution_id) + start_resp = self._workflow_response_converter.workflow_start_to_stream_response( task_id=self._application_generate_entity.task_id, workflow_run_id=run_id, @@ -414,9 +418,6 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport): graph_runtime_state=validated_state, ) - with self._database_session() as session: - self._save_workflow_app_log(session=session, workflow_run_id=self._workflow_execution_id) - yield workflow_finish_resp def _handle_workflow_partial_success_event( @@ -437,10 +438,6 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport): graph_runtime_state=validated_state, exceptions_count=event.exceptions_count, ) - - with self._database_session() as session: - self._save_workflow_app_log(session=session, workflow_run_id=self._workflow_execution_id) - yield workflow_finish_resp def _handle_workflow_failed_and_stop_events( @@ -471,10 +468,6 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport): error=error, exceptions_count=exceptions_count, ) - - with self._database_session() as session: - self._save_workflow_app_log(session=session, workflow_run_id=self._workflow_execution_id) - yield workflow_finish_resp def _handle_text_chunk_event( @@ -655,7 +648,6 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport): ) session.add(workflow_app_log) - session.commit() def _text_chunk_to_stream_response( self, text: str, from_variable_selector: list[str] | None = None diff --git a/api/core/app/entities/task_entities.py b/api/core/app/entities/task_entities.py index 79a5e657b3..7692128985 100644 --- a/api/core/app/entities/task_entities.py +++ b/api/core/app/entities/task_entities.py @@ -40,6 +40,9 @@ class EasyUITaskState(TaskState): """ llm_result: LLMResult + first_token_time: float | None = None + last_token_time: float | None = None + is_streaming_response: bool = False class WorkflowTaskState(TaskState): diff --git a/api/core/app/layers/pause_state_persist_layer.py b/api/core/app/layers/pause_state_persist_layer.py index 412eb98dd4..61a3e1baca 100644 --- a/api/core/app/layers/pause_state_persist_layer.py +++ b/api/core/app/layers/pause_state_persist_layer.py @@ -118,6 +118,7 @@ class PauseStatePersistenceLayer(GraphEngineLayer): workflow_run_id=workflow_run_id, state_owner_user_id=self._state_owner_user_id, state=state.dumps(), + pause_reasons=event.reasons, ) def on_graph_end(self, error: Exception | None) -> None: diff --git a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py index da2ebac3bd..c49db9aad1 100644 --- a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py +++ b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py @@ -332,6 +332,12 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline): if not self._task_state.llm_result.prompt_messages: self._task_state.llm_result.prompt_messages = chunk.prompt_messages + # Track streaming response times + if self._task_state.first_token_time is None: + self._task_state.first_token_time = time.perf_counter() + self._task_state.is_streaming_response = True + self._task_state.last_token_time = time.perf_counter() + # handle output moderation chunk should_direct_answer = self._handle_output_moderation_chunk(cast(str, delta_text)) if should_direct_answer: @@ -398,6 +404,18 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline): message.total_price = usage.total_price message.currency = usage.currency self._task_state.llm_result.usage.latency = message.provider_response_latency + + # Add streaming metrics to usage if available + if self._task_state.is_streaming_response and self._task_state.first_token_time: + start_time = self.start_at + first_token_time = self._task_state.first_token_time + last_token_time = self._task_state.last_token_time or first_token_time + usage.time_to_first_token = round(first_token_time - start_time, 3) + usage.time_to_generate = round(last_token_time - first_token_time, 3) + + # Update metadata with the complete usage info + self._task_state.metadata.usage = usage + message.message_metadata = self._task_state.metadata.model_dump_json() if trace_manager: diff --git a/api/core/helper/code_executor/code_executor.py b/api/core/helper/code_executor/code_executor.py index f92278f9e2..73174ed28d 100644 --- a/api/core/helper/code_executor/code_executor.py +++ b/api/core/helper/code_executor/code_executor.py @@ -152,10 +152,5 @@ class CodeExecutor: raise CodeExecutionError(f"Unsupported language {language}") runner, preload = template_transformer.transform_caller(code, inputs) - - try: - response = cls.execute_code(language, preload, runner) - except CodeExecutionError as e: - raise e - + response = cls.execute_code(language, preload, runner) return template_transformer.transform_response(response) diff --git a/api/core/model_runtime/docs/en_US/customizable_model_scale_out.md b/api/core/model_runtime/docs/en_US/customizable_model_scale_out.md deleted file mode 100644 index 245aa4699c..0000000000 --- a/api/core/model_runtime/docs/en_US/customizable_model_scale_out.md +++ /dev/null @@ -1,308 +0,0 @@ -## Custom Integration of Pre-defined Models - -### Introduction - -After completing the vendors integration, the next step is to connect the vendor's models. To illustrate the entire connection process, we will use Xinference as an example to demonstrate a complete vendor integration. - -It is important to note that for custom models, each model connection requires a complete vendor credential. - -Unlike pre-defined models, a custom vendor integration always includes the following two parameters, which do not need to be defined in the vendor YAML file. - -![](images/index/image-3.png) - -As mentioned earlier, vendors do not need to implement validate_provider_credential. The runtime will automatically call the corresponding model layer's validate_credentials to validate the credentials based on the model type and name selected by the user. - -### Writing the Vendor YAML - -First, we need to identify the types of models supported by the vendor we are integrating. - -Currently supported model types are as follows: - -- `llm` Text Generation Models - -- `text_embedding` Text Embedding Models - -- `rerank` Rerank Models - -- `speech2text` Speech-to-Text - -- `tts` Text-to-Speech - -- `moderation` Moderation - -Xinference supports LLM, Text Embedding, and Rerank. So we will start by writing xinference.yaml. - -```yaml -provider: xinference #Define the vendor identifier -label: # Vendor display name, supports both en_US (English) and zh_Hans (Simplified Chinese). If zh_Hans is not set, it will use en_US by default. - en_US: Xorbits Inference -icon_small: # Small icon, refer to other vendors' icons stored in the _assets directory within the vendor implementation directory; follows the same language policy as the label - en_US: icon_s_en.svg -icon_large: # Large icon - en_US: icon_l_en.svg -help: # Help information - title: - en_US: How to deploy Xinference - zh_Hans: 如何部署 Xinference - url: - en_US: https://github.com/xorbitsai/inference -supported_model_types: # Supported model types. Xinference supports LLM, Text Embedding, and Rerank -- llm -- text-embedding -- rerank -configurate_methods: # Since Xinference is a locally deployed vendor with no predefined models, users need to deploy whatever models they need according to Xinference documentation. Thus, it only supports custom models. -- customizable-model -provider_credential_schema: - credential_form_schemas: -``` - -Then, we need to determine what credentials are required to define a model in Xinference. - -- Since it supports three different types of models, we need to specify the model_type to denote the model type. Here is how we can define it: - -```yaml -provider_credential_schema: - credential_form_schemas: - - variable: model_type - type: select - label: - en_US: Model type - zh_Hans: 模型类型 - required: true - options: - - value: text-generation - label: - en_US: Language Model - zh_Hans: 语言模型 - - value: embeddings - label: - en_US: Text Embedding - - value: reranking - label: - en_US: Rerank -``` - -- Next, each model has its own model_name, so we need to define that here: - -```yaml - - variable: model_name - type: text-input - label: - en_US: Model name - zh_Hans: 模型名称 - required: true - placeholder: - zh_Hans: 填写模型名称 - en_US: Input model name -``` - -- Specify the Xinference local deployment address: - -```yaml - - variable: server_url - label: - zh_Hans: 服务器 URL - en_US: Server url - type: text-input - required: true - placeholder: - zh_Hans: 在此输入 Xinference 的服务器地址，如 https://example.com/xxx - en_US: Enter the url of your Xinference, for example https://example.com/xxx -``` - -- Each model has a unique model_uid, so we also need to define that here: - -```yaml - - variable: model_uid - label: - zh_Hans: 模型 UID - en_US: Model uid - type: text-input - required: true - placeholder: - zh_Hans: 在此输入您的 Model UID - en_US: Enter the model uid -``` - -Now, we have completed the basic definition of the vendor. - -### Writing the Model Code - -Next, let's take the `llm` type as an example and write `xinference.llm.llm.py`. - -In `llm.py`, create a Xinference LLM class, we name it `XinferenceAILargeLanguageModel` (this can be arbitrary), inheriting from the `__base.large_language_model.LargeLanguageModel` base class, and implement the following methods: - -- LLM Invocation - -Implement the core method for LLM invocation, supporting both stream and synchronous responses. - -```python -def _invoke(self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, - stream: bool = True, user: Optional[str] = None) \ - -> Union[LLMResult, Generator]: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param model_parameters: model parameters - :param tools: tools for tool usage - :param stop: stop words - :param stream: is the response a stream - :param user: unique user id - :return: full response or stream response chunk generator result - """ -``` - -When implementing, ensure to use two functions to return data separately for synchronous and stream responses. This is important because Python treats functions containing the `yield` keyword as generator functions, mandating them to return `Generator` types. Here’s an example (note that the example uses simplified parameters; in real implementation, use the parameter list as defined above): - -```python -def _invoke(self, stream: bool, **kwargs) \ - -> Union[LLMResult, Generator]: - if stream: - return self._handle_stream_response(**kwargs) - return self._handle_sync_response(**kwargs) - -def _handle_stream_response(self, **kwargs) -> Generator: - for chunk in response: - yield chunk -def _handle_sync_response(self, **kwargs) -> LLMResult: - return LLMResult(**response) -``` - -- Pre-compute Input Tokens - -If the model does not provide an interface for pre-computing tokens, you can return 0 directly. - -```python -def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage],tools: Optional[list[PromptMessageTool]] = None) -> int: - """ - Get number of tokens for given prompt messages - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param tools: tools for tool usage - :return: token count - """ -``` - -Sometimes, you might not want to return 0 directly. In such cases, you can use `self._get_num_tokens_by_gpt2(text: str)` to get pre-computed tokens and ensure environment variable `PLUGIN_BASED_TOKEN_COUNTING_ENABLED` is set to `true`, This method is provided by the `AIModel` base class, and it uses GPT2's Tokenizer for calculation. However, it should be noted that this is only a substitute and may not be fully accurate. - -- Model Credentials Validation - -Similar to vendor credentials validation, this method validates individual model credentials. - -```python -def validate_credentials(self, model: str, credentials: dict) -> None: - """ - Validate model credentials - - :param model: model name - :param credentials: model credentials - :return: None - """ -``` - -- Model Parameter Schema - -Unlike custom types, since the YAML file does not define which parameters a model supports, we need to dynamically generate the model parameter schema. - -For instance, Xinference supports `max_tokens`, `temperature`, and `top_p` parameters. - -However, some vendors may support different parameters for different models. For example, the `OpenLLM` vendor supports `top_k`, but not all models provided by this vendor support `top_k`. Let's say model A supports `top_k` but model B does not. In such cases, we need to dynamically generate the model parameter schema, as illustrated below: - -```python - def get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]: - """ - used to define customizable model schema - """ - rules = [ - ParameterRule( - name='temperature', type=ParameterType.FLOAT, - use_template='temperature', - label=I18nObject( - zh_Hans='温度', en_US='Temperature' - ) - ), - ParameterRule( - name='top_p', type=ParameterType.FLOAT, - use_template='top_p', - label=I18nObject( - zh_Hans='Top P', en_US='Top P' - ) - ), - ParameterRule( - name='max_tokens', type=ParameterType.INT, - use_template='max_tokens', - min=1, - default=512, - label=I18nObject( - zh_Hans='最大生成长度', en_US='Max Tokens' - ) - ) - ] - - # if model is A, add top_k to rules - if model == 'A': - rules.append( - ParameterRule( - name='top_k', type=ParameterType.INT, - use_template='top_k', - min=1, - default=50, - label=I18nObject( - zh_Hans='Top K', en_US='Top K' - ) - ) - ) - - """ - some NOT IMPORTANT code here - """ - - entity = AIModelEntity( - model=model, - label=I18nObject( - en_US=model - ), - fetch_from=FetchFrom.CUSTOMIZABLE_MODEL, - model_type=model_type, - model_properties={ - ModelPropertyKey.MODE: ModelType.LLM, - }, - parameter_rules=rules - ) - - return entity -``` - -- Exception Error Mapping - -When a model invocation error occurs, it should be mapped to the runtime's specified `InvokeError` type, enabling Dify to handle different errors appropriately. - -Runtime Errors: - -- `InvokeConnectionError` Connection error during invocation -- `InvokeServerUnavailableError` Service provider unavailable -- `InvokeRateLimitError` Rate limit reached -- `InvokeAuthorizationError` Authorization failure -- `InvokeBadRequestError` Invalid request parameters - -```python - @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: - """ - Map model invoke error to unified error - The key is the error type thrown to the caller - The value is the error type thrown by the model, - which needs to be converted into a unified error type for the caller. - - :return: Invoke error mapping - """ -``` - -For interface method details, see: [Interfaces](./interfaces.md). For specific implementations, refer to: [llm.py](https://github.com/langgenius/dify-runtime/blob/main/lib/model_providers/anthropic/llm/llm.py). diff --git a/api/core/model_runtime/docs/en_US/images/index/image-1.png b/api/core/model_runtime/docs/en_US/images/index/image-1.png deleted file mode 100644 index b158d44b29..0000000000 Binary files a/api/core/model_runtime/docs/en_US/images/index/image-1.png and /dev/null differ diff --git a/api/core/model_runtime/docs/en_US/images/index/image-2.png b/api/core/model_runtime/docs/en_US/images/index/image-2.png deleted file mode 100644 index c70cd3da5e..0000000000 Binary files a/api/core/model_runtime/docs/en_US/images/index/image-2.png and /dev/null differ diff --git a/api/core/model_runtime/docs/en_US/images/index/image-20231210143654461.png b/api/core/model_runtime/docs/en_US/images/index/image-20231210143654461.png deleted file mode 100644 index 2e234f6c21..0000000000 Binary files a/api/core/model_runtime/docs/en_US/images/index/image-20231210143654461.png and /dev/null differ diff --git a/api/core/model_runtime/docs/en_US/images/index/image-20231210144229650.png b/api/core/model_runtime/docs/en_US/images/index/image-20231210144229650.png deleted file mode 100644 index 742c1ba808..0000000000 Binary files a/api/core/model_runtime/docs/en_US/images/index/image-20231210144229650.png and /dev/null differ diff --git a/api/core/model_runtime/docs/en_US/images/index/image-20231210144814617.png b/api/core/model_runtime/docs/en_US/images/index/image-20231210144814617.png deleted file mode 100644 index b28aba83c9..0000000000 Binary files a/api/core/model_runtime/docs/en_US/images/index/image-20231210144814617.png and /dev/null differ diff --git a/api/core/model_runtime/docs/en_US/images/index/image-20231210151548521.png b/api/core/model_runtime/docs/en_US/images/index/image-20231210151548521.png deleted file mode 100644 index 0d88bf4bda..0000000000 Binary files a/api/core/model_runtime/docs/en_US/images/index/image-20231210151548521.png and /dev/null differ diff --git a/api/core/model_runtime/docs/en_US/images/index/image-20231210151628992.png b/api/core/model_runtime/docs/en_US/images/index/image-20231210151628992.png deleted file mode 100644 index a07aaebd2f..0000000000 Binary files a/api/core/model_runtime/docs/en_US/images/index/image-20231210151628992.png and /dev/null differ diff --git a/api/core/model_runtime/docs/en_US/images/index/image-20231210165243632.png b/api/core/model_runtime/docs/en_US/images/index/image-20231210165243632.png deleted file mode 100644 index 18ec605e83..0000000000 Binary files a/api/core/model_runtime/docs/en_US/images/index/image-20231210165243632.png and /dev/null differ diff --git a/api/core/model_runtime/docs/en_US/images/index/image-3.png b/api/core/model_runtime/docs/en_US/images/index/image-3.png deleted file mode 100644 index bf0b9a7f47..0000000000 Binary files a/api/core/model_runtime/docs/en_US/images/index/image-3.png and /dev/null differ diff --git a/api/core/model_runtime/docs/en_US/images/index/image.png b/api/core/model_runtime/docs/en_US/images/index/image.png deleted file mode 100644 index eb63d107e1..0000000000 Binary files a/api/core/model_runtime/docs/en_US/images/index/image.png and /dev/null differ diff --git a/api/core/model_runtime/docs/en_US/interfaces.md b/api/core/model_runtime/docs/en_US/interfaces.md deleted file mode 100644 index 9a8c2ec942..0000000000 --- a/api/core/model_runtime/docs/en_US/interfaces.md +++ /dev/null @@ -1,701 +0,0 @@ -# Interface Methods - -This section describes the interface methods and parameter explanations that need to be implemented by providers and various model types. - -## Provider - -Inherit the `__base.model_provider.ModelProvider` base class and implement the following interfaces: - -```python -def validate_provider_credentials(self, credentials: dict) -> None: - """ - Validate provider credentials - You can choose any validate_credentials method of model type or implement validate method by yourself, - such as: get model list api - - if validate failed, raise exception - - :param credentials: provider credentials, credentials form defined in `provider_credential_schema`. - """ -``` - -- `credentials` (object) Credential information - - The parameters of credential information are defined by the `provider_credential_schema` in the provider's YAML configuration file. Inputs such as `api_key` are included. - -If verification fails, throw the `errors.validate.CredentialsValidateFailedError` error. - -## Model - -Models are divided into 5 different types, each inheriting from different base classes and requiring the implementation of different methods. - -All models need to uniformly implement the following 2 methods: - -- Model Credential Verification - - Similar to provider credential verification, this step involves verification for an individual model. - - ```python - def validate_credentials(self, model: str, credentials: dict) -> None: - """ - Validate model credentials - - :param model: model name - :param credentials: model credentials - :return: - """ - ``` - - Parameters: - - - `model` (string) Model name - - - `credentials` (object) Credential information - - The parameters of credential information are defined by either the `provider_credential_schema` or `model_credential_schema` in the provider's YAML configuration file. Inputs such as `api_key` are included. - - If verification fails, throw the `errors.validate.CredentialsValidateFailedError` error. - -- Invocation Error Mapping Table - - When there is an exception in model invocation, it needs to be mapped to the `InvokeError` type specified by Runtime. This facilitates Dify's ability to handle different errors with appropriate follow-up actions. - - Runtime Errors: - - - `InvokeConnectionError` Invocation connection error - - `InvokeServerUnavailableError` Invocation service provider unavailable - - `InvokeRateLimitError` Invocation reached rate limit - - `InvokeAuthorizationError` Invocation authorization failure - - `InvokeBadRequestError` Invocation parameter error - - ```python - @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: - """ - Map model invoke error to unified error - The key is the error type thrown to the caller - The value is the error type thrown by the model, - which needs to be converted into a unified error type for the caller. - - :return: Invoke error mapping - """ - ``` - -​ You can refer to OpenAI's `_invoke_error_mapping` for an example. - -### LLM - -Inherit the `__base.large_language_model.LargeLanguageModel` base class and implement the following interfaces: - -- LLM Invocation - - Implement the core method for LLM invocation, which can support both streaming and synchronous returns. - - ```python - def _invoke(self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, stop: Optional[List[str]] = None, - stream: bool = True, user: Optional[str] = None) \ - -> Union[LLMResult, Generator]: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param model_parameters: model parameters - :param tools: tools for tool calling - :param stop: stop words - :param stream: is stream response - :param user: unique user id - :return: full response or stream response chunk generator result - """ - ``` - - - Parameters: - - - `model` (string) Model name - - - `credentials` (object) Credential information - - The parameters of credential information are defined by either the `provider_credential_schema` or `model_credential_schema` in the provider's YAML configuration file. Inputs such as `api_key` are included. - - - `prompt_messages` (array\[[PromptMessage](#PromptMessage)\]) List of prompts - - If the model is of the `Completion` type, the list only needs to include one [UserPromptMessage](#UserPromptMessage) element; - - If the model is of the `Chat` type, it requires a list of elements such as [SystemPromptMessage](#SystemPromptMessage), [UserPromptMessage](#UserPromptMessage), [AssistantPromptMessage](#AssistantPromptMessage), [ToolPromptMessage](#ToolPromptMessage) depending on the message. - - - `model_parameters` (object) Model parameters - - The model parameters are defined by the `parameter_rules` in the model's YAML configuration. - - - `tools` (array\[[PromptMessageTool](#PromptMessageTool)\]) [optional] List of tools, equivalent to the `function` in `function calling`. - - That is, the tool list for tool calling. - - - `stop` (array[string]) [optional] Stop sequences - - The model output will stop before the string defined by the stop sequence. - - - `stream` (bool) Whether to output in a streaming manner, default is True - - Streaming output returns Generator\[[LLMResultChunk](#LLMResultChunk)\], non-streaming output returns [LLMResult](#LLMResult). - - - `user` (string) [optional] Unique identifier of the user - - This can help the provider monitor and detect abusive behavior. - - - Returns - - Streaming output returns Generator\[[LLMResultChunk](#LLMResultChunk)\], non-streaming output returns [LLMResult](#LLMResult). - -- Pre-calculating Input Tokens - - If the model does not provide a pre-calculated tokens interface, you can directly return 0. - - ```python - def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None) -> int: - """ - Get number of tokens for given prompt messages - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param tools: tools for tool calling - :return: - """ - ``` - - For parameter explanations, refer to the above section on `LLM Invocation`. - -- Fetch Custom Model Schema [Optional] - - ```python - def get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]: - """ - Get customizable model schema - - :param model: model name - :param credentials: model credentials - :return: model schema - """ - ``` - - When the provider supports adding custom LLMs, this method can be implemented to allow custom models to fetch model schema. The default return null. - -### TextEmbedding - -Inherit the `__base.text_embedding_model.TextEmbeddingModel` base class and implement the following interfaces: - -- Embedding Invocation - - ```python - def _invoke(self, model: str, credentials: dict, - texts: list[str], user: Optional[str] = None) \ - -> TextEmbeddingResult: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param texts: texts to embed - :param user: unique user id - :return: embeddings result - """ - ``` - - - Parameters: - - - `model` (string) Model name - - - `credentials` (object) Credential information - - The parameters of credential information are defined by either the `provider_credential_schema` or `model_credential_schema` in the provider's YAML configuration file. Inputs such as `api_key` are included. - - - `texts` (array[string]) List of texts, capable of batch processing - - - `user` (string) [optional] Unique identifier of the user - - This can help the provider monitor and detect abusive behavior. - - - Returns: - - [TextEmbeddingResult](#TextEmbeddingResult) entity. - -- Pre-calculating Tokens - - ```python - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: - """ - Get number of tokens for given prompt messages - - :param model: model name - :param credentials: model credentials - :param texts: texts to embed - :return: - """ - ``` - - For parameter explanations, refer to the above section on `Embedding Invocation`. - -### Rerank - -Inherit the `__base.rerank_model.RerankModel` base class and implement the following interfaces: - -- Rerank Invocation - - ```python - def _invoke(self, model: str, credentials: dict, - query: str, docs: list[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, - user: Optional[str] = None) \ - -> RerankResult: - """ - Invoke rerank model - - :param model: model name - :param credentials: model credentials - :param query: search query - :param docs: docs for reranking - :param score_threshold: score threshold - :param top_n: top n - :param user: unique user id - :return: rerank result - """ - ``` - - - Parameters: - - - `model` (string) Model name - - - `credentials` (object) Credential information - - The parameters of credential information are defined by either the `provider_credential_schema` or `model_credential_schema` in the provider's YAML configuration file. Inputs such as `api_key` are included. - - - `query` (string) Query request content - - - `docs` (array[string]) List of segments to be reranked - - - `score_threshold` (float) [optional] Score threshold - - - `top_n` (int) [optional] Select the top n segments - - - `user` (string) [optional] Unique identifier of the user - - This can help the provider monitor and detect abusive behavior. - - - Returns: - - [RerankResult](#RerankResult) entity. - -### Speech2text - -Inherit the `__base.speech2text_model.Speech2TextModel` base class and implement the following interfaces: - -- Invoke Invocation - - ```python - def _invoke(self, model: str, credentials: dict, file: IO[bytes], user: Optional[str] = None) -> str: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param file: audio file - :param user: unique user id - :return: text for given audio file - """ - ``` - - - Parameters: - - - `model` (string) Model name - - - `credentials` (object) Credential information - - The parameters of credential information are defined by either the `provider_credential_schema` or `model_credential_schema` in the provider's YAML configuration file. Inputs such as `api_key` are included. - - - `file` (File) File stream - - - `user` (string) [optional] Unique identifier of the user - - This can help the provider monitor and detect abusive behavior. - - - Returns: - - The string after speech-to-text conversion. - -### Text2speech - -Inherit the `__base.text2speech_model.Text2SpeechModel` base class and implement the following interfaces: - -- Invoke Invocation - - ```python - def _invoke(self, model: str, credentials: dict, content_text: str, streaming: bool, user: Optional[str] = None): - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param content_text: text content to be translated - :param streaming: output is streaming - :param user: unique user id - :return: translated audio file - """ - ``` - - - Parameters： - - - `model` (string) Model name - - - `credentials` (object) Credential information - - The parameters of credential information are defined by either the `provider_credential_schema` or `model_credential_schema` in the provider's YAML configuration file. Inputs such as `api_key` are included. - - - `content_text` (string) The text content that needs to be converted - - - `streaming` (bool) Whether to stream output - - - `user` (string) [optional] Unique identifier of the user - - This can help the provider monitor and detect abusive behavior. - - - Returns： - - Text converted speech stream. - -### Moderation - -Inherit the `__base.moderation_model.ModerationModel` base class and implement the following interfaces: - -- Invoke Invocation - - ```python - def _invoke(self, model: str, credentials: dict, - text: str, user: Optional[str] = None) \ - -> bool: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param text: text to moderate - :param user: unique user id - :return: false if text is safe, true otherwise - """ - ``` - - - Parameters: - - - `model` (string) Model name - - - `credentials` (object) Credential information - - The parameters of credential information are defined by either the `provider_credential_schema` or `model_credential_schema` in the provider's YAML configuration file. Inputs such as `api_key` are included. - - - `text` (string) Text content - - - `user` (string) [optional] Unique identifier of the user - - This can help the provider monitor and detect abusive behavior. - - - Returns: - - False indicates that the input text is safe, True indicates otherwise. - -## Entities - -### PromptMessageRole - -Message role - -```python -class PromptMessageRole(Enum): - """ - Enum class for prompt message. - """ - SYSTEM = "system" - USER = "user" - ASSISTANT = "assistant" - TOOL = "tool" -``` - -### PromptMessageContentType - -Message content types, divided into text and image. - -```python -class PromptMessageContentType(Enum): - """ - Enum class for prompt message content type. - """ - TEXT = 'text' - IMAGE = 'image' -``` - -### PromptMessageContent - -Message content base class, used only for parameter declaration and cannot be initialized. - -```python -class PromptMessageContent(BaseModel): - """ - Model class for prompt message content. - """ - type: PromptMessageContentType - data: str -``` - -Currently, two types are supported: text and image. It's possible to simultaneously input text and multiple images. - -You need to initialize `TextPromptMessageContent` and `ImagePromptMessageContent` separately for input. - -### TextPromptMessageContent - -```python -class TextPromptMessageContent(PromptMessageContent): - """ - Model class for text prompt message content. - """ - type: PromptMessageContentType = PromptMessageContentType.TEXT -``` - -If inputting a combination of text and images, the text needs to be constructed into this entity as part of the `content` list. - -### ImagePromptMessageContent - -```python -class ImagePromptMessageContent(PromptMessageContent): - """ - Model class for image prompt message content. - """ - class DETAIL(Enum): - LOW = 'low' - HIGH = 'high' - - type: PromptMessageContentType = PromptMessageContentType.IMAGE - detail: DETAIL = DETAIL.LOW # Resolution -``` - -If inputting a combination of text and images, the images need to be constructed into this entity as part of the `content` list. - -`data` can be either a `url` or a `base64` encoded string of the image. - -### PromptMessage - -The base class for all Role message bodies, used only for parameter declaration and cannot be initialized. - -```python -class PromptMessage(BaseModel): - """ - Model class for prompt message. - """ - role: PromptMessageRole - content: Optional[str | list[PromptMessageContent]] = None # Supports two types: string and content list. The content list is designed to meet the needs of multimodal inputs. For more details, see the PromptMessageContent explanation. - name: Optional[str] = None -``` - -### UserPromptMessage - -UserMessage message body, representing a user's message. - -```python -class UserPromptMessage(PromptMessage): - """ - Model class for user prompt message. - """ - role: PromptMessageRole = PromptMessageRole.USER -``` - -### AssistantPromptMessage - -Represents a message returned by the model, typically used for `few-shots` or inputting chat history. - -```python -class AssistantPromptMessage(PromptMessage): - """ - Model class for assistant prompt message. - """ - class ToolCall(BaseModel): - """ - Model class for assistant prompt message tool call. - """ - class ToolCallFunction(BaseModel): - """ - Model class for assistant prompt message tool call function. - """ - name: str # tool name - arguments: str # tool arguments - - id: str # Tool ID, effective only in OpenAI tool calls. It's the unique ID for tool invocation and the same tool can be called multiple times. - type: str # default: function - function: ToolCallFunction # tool call information - - role: PromptMessageRole = PromptMessageRole.ASSISTANT - tool_calls: list[ToolCall] = [] # The result of tool invocation in response from the model (returned only when tools are input and the model deems it necessary to invoke a tool). -``` - -Where `tool_calls` are the list of `tool calls` returned by the model after invoking the model with the `tools` input. - -### SystemPromptMessage - -Represents system messages, usually used for setting system commands given to the model. - -```python -class SystemPromptMessage(PromptMessage): - """ - Model class for system prompt message. - """ - role: PromptMessageRole = PromptMessageRole.SYSTEM -``` - -### ToolPromptMessage - -Represents tool messages, used for conveying the results of a tool execution to the model for the next step of processing. - -```python -class ToolPromptMessage(PromptMessage): - """ - Model class for tool prompt message. - """ - role: PromptMessageRole = PromptMessageRole.TOOL - tool_call_id: str # Tool invocation ID. If OpenAI tool call is not supported, the name of the tool can also be inputted. -``` - -The base class's `content` takes in the results of tool execution. - -### PromptMessageTool - -```python -class PromptMessageTool(BaseModel): - """ - Model class for prompt message tool. - """ - name: str - description: str - parameters: dict -``` - -______________________________________________________________________ - -### LLMResult - -```python -class LLMResult(BaseModel): - """ - Model class for llm result. - """ - model: str # Actual used modele - prompt_messages: list[PromptMessage] # prompt messages - message: AssistantPromptMessage # response message - usage: LLMUsage # usage info - system_fingerprint: Optional[str] = None # request fingerprint, refer to OpenAI definition -``` - -### LLMResultChunkDelta - -In streaming returns, each iteration contains the `delta` entity. - -```python -class LLMResultChunkDelta(BaseModel): - """ - Model class for llm result chunk delta. - """ - index: int - message: AssistantPromptMessage # response message - usage: Optional[LLMUsage] = None # usage info - finish_reason: Optional[str] = None # finish reason, only the last one returns -``` - -### LLMResultChunk - -Each iteration entity in streaming returns. - -```python -class LLMResultChunk(BaseModel): - """ - Model class for llm result chunk. - """ - model: str # Actual used modele - prompt_messages: list[PromptMessage] # prompt messages - system_fingerprint: Optional[str] = None # request fingerprint, refer to OpenAI definition - delta: LLMResultChunkDelta -``` - -### LLMUsage - -```python -class LLMUsage(ModelUsage): - """ - Model class for LLM usage. - """ - prompt_tokens: int # Tokens used for prompt - prompt_unit_price: Decimal # Unit price for prompt - prompt_price_unit: Decimal # Price unit for prompt, i.e., the unit price based on how many tokens - prompt_price: Decimal # Cost for prompt - completion_tokens: int # Tokens used for response - completion_unit_price: Decimal # Unit price for response - completion_price_unit: Decimal # Price unit for response, i.e., the unit price based on how many tokens - completion_price: Decimal # Cost for response - total_tokens: int # Total number of tokens used - total_price: Decimal # Total cost - currency: str # Currency unit - latency: float # Request latency (s) -``` - -______________________________________________________________________ - -### TextEmbeddingResult - -```python -class TextEmbeddingResult(BaseModel): - """ - Model class for text embedding result. - """ - model: str # Actual model used - embeddings: list[list[float]] # List of embedding vectors, corresponding to the input texts list - usage: EmbeddingUsage # Usage information -``` - -### EmbeddingUsage - -```python -class EmbeddingUsage(ModelUsage): - """ - Model class for embedding usage. - """ - tokens: int # Number of tokens used - total_tokens: int # Total number of tokens used - unit_price: Decimal # Unit price - price_unit: Decimal # Price unit, i.e., the unit price based on how many tokens - total_price: Decimal # Total cost - currency: str # Currency unit - latency: float # Request latency (s) -``` - -______________________________________________________________________ - -### RerankResult - -```python -class RerankResult(BaseModel): - """ - Model class for rerank result. - """ - model: str # Actual model used - docs: list[RerankDocument] # Reranked document list -``` - -### RerankDocument - -```python -class RerankDocument(BaseModel): - """ - Model class for rerank document. - """ - index: int # original index - text: str - score: float -``` diff --git a/api/core/model_runtime/docs/en_US/predefined_model_scale_out.md b/api/core/model_runtime/docs/en_US/predefined_model_scale_out.md deleted file mode 100644 index 97968e9988..0000000000 --- a/api/core/model_runtime/docs/en_US/predefined_model_scale_out.md +++ /dev/null @@ -1,176 +0,0 @@ -## Predefined Model Integration - -After completing the vendor integration, the next step is to integrate the models from the vendor. - -First, we need to determine the type of model to be integrated and create the corresponding model type `module` under the respective vendor's directory. - -Currently supported model types are: - -- `llm` Text Generation Model -- `text_embedding` Text Embedding Model -- `rerank` Rerank Model -- `speech2text` Speech-to-Text -- `tts` Text-to-Speech -- `moderation` Moderation - -Continuing with `Anthropic` as an example, `Anthropic` only supports LLM, so create a `module` named `llm` under `model_providers.anthropic`. - -For predefined models, we first need to create a YAML file named after the model under the `llm` `module`, such as `claude-2.1.yaml`. - -### Prepare Model YAML - -```yaml -model: claude-2.1 # Model identifier -# Display name of the model, which can be set to en_US English or zh_Hans Chinese. If zh_Hans is not set, it will default to en_US. -# This can also be omitted, in which case the model identifier will be used as the label -label: - en_US: claude-2.1 -model_type: llm # Model type, claude-2.1 is an LLM -features: # Supported features, agent-thought supports Agent reasoning, vision supports image understanding -- agent-thought -model_properties: # Model properties - mode: chat # LLM mode, complete for text completion models, chat for conversation models - context_size: 200000 # Maximum context size -parameter_rules: # Parameter rules for the model call; only LLM requires this -- name: temperature # Parameter variable name - # Five default configuration templates are provided: temperature/top_p/max_tokens/presence_penalty/frequency_penalty - # The template variable name can be set directly in use_template, which will use the default configuration in entities.defaults.PARAMETER_RULE_TEMPLATE - # Additional configuration parameters will override the default configuration if set - use_template: temperature -- name: top_p - use_template: top_p -- name: top_k - label: # Display name of the parameter - zh_Hans: 取样数量 - en_US: Top k - type: int # Parameter type, supports float/int/string/boolean - help: # Help information, describing the parameter's function - zh_Hans: 仅从每个后续标记的前 K 个选项中采样。 - en_US: Only sample from the top K options for each subsequent token. - required: false # Whether the parameter is mandatory; can be omitted -- name: max_tokens_to_sample - use_template: max_tokens - default: 4096 # Default value of the parameter - min: 1 # Minimum value of the parameter, applicable to float/int only - max: 4096 # Maximum value of the parameter, applicable to float/int only -pricing: # Pricing information - input: '8.00' # Input unit price, i.e., prompt price - output: '24.00' # Output unit price, i.e., response content price - unit: '0.000001' # Price unit, meaning the above prices are per 100K - currency: USD # Price currency -``` - -It is recommended to prepare all model configurations before starting the implementation of the model code. - -You can also refer to the YAML configuration information under the corresponding model type directories of other vendors in the `model_providers` directory. For the complete YAML rules, refer to: [Schema](schema.md#aimodelentity). - -### Implement the Model Call Code - -Next, create a Python file named `llm.py` under the `llm` `module` to write the implementation code. - -Create an Anthropic LLM class named `AnthropicLargeLanguageModel` (or any other name), inheriting from the `__base.large_language_model.LargeLanguageModel` base class, and implement the following methods: - -- LLM Call - -Implement the core method for calling the LLM, supporting both streaming and synchronous responses. - -```python - def _invoke(self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, - stream: bool = True, user: Optional[str] = None) \ - -> Union[LLMResult, Generator]: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param model_parameters: model parameters - :param tools: tools for tool calling - :param stop: stop words - :param stream: is stream response - :param user: unique user id - :return: full response or stream response chunk generator result - """ -``` - -Ensure to use two functions for returning data, one for synchronous returns and the other for streaming returns, because Python identifies functions containing the `yield` keyword as generator functions, fixing the return type to `Generator`. Thus, synchronous and streaming returns need to be implemented separately, as shown below (note that the example uses simplified parameters, for actual implementation follow the above parameter list): - -```python - def _invoke(self, stream: bool, **kwargs) \ - -> Union[LLMResult, Generator]: - if stream: - return self._handle_stream_response(**kwargs) - return self._handle_sync_response(**kwargs) - - def _handle_stream_response(self, **kwargs) -> Generator: - for chunk in response: - yield chunk - def _handle_sync_response(self, **kwargs) -> LLMResult: - return LLMResult(**response) -``` - -- Pre-compute Input Tokens - -If the model does not provide an interface to precompute tokens, return 0 directly. - -```python - def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None) -> int: - """ - Get number of tokens for given prompt messages - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param tools: tools for tool calling - :return: - """ -``` - -- Validate Model Credentials - -Similar to vendor credential validation, but specific to a single model. - -```python - def validate_credentials(self, model: str, credentials: dict) -> None: - """ - Validate model credentials - - :param model: model name - :param credentials: model credentials - :return: - """ -``` - -- Map Invoke Errors - -When a model call fails, map it to a specific `InvokeError` type as required by Runtime, allowing Dify to handle different errors accordingly. - -Runtime Errors: - -- `InvokeConnectionError` Connection error - -- `InvokeServerUnavailableError` Service provider unavailable - -- `InvokeRateLimitError` Rate limit reached - -- `InvokeAuthorizationError` Authorization failed - -- `InvokeBadRequestError` Parameter error - -```python - @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: - """ - Map model invoke error to unified error - The key is the error type thrown to the caller - The value is the error type thrown by the model, - which needs to be converted into a unified error type for the caller. - - :return: Invoke error mapping - """ -``` - -For interface method explanations, see: [Interfaces](./interfaces.md). For detailed implementation, refer to: [llm.py](https://github.com/langgenius/dify-runtime/blob/main/lib/model_providers/anthropic/llm/llm.py). diff --git a/api/core/model_runtime/docs/en_US/provider_scale_out.md b/api/core/model_runtime/docs/en_US/provider_scale_out.md deleted file mode 100644 index c38c7c0f0c..0000000000 --- a/api/core/model_runtime/docs/en_US/provider_scale_out.md +++ /dev/null @@ -1,266 +0,0 @@ -## Adding a New Provider - -Providers support three types of model configuration methods: - -- `predefined-model` Predefined model - - This indicates that users only need to configure the unified provider credentials to use the predefined models under the provider. - -- `customizable-model` Customizable model - - Users need to add credential configurations for each model. - -- `fetch-from-remote` Fetch from remote - - This is consistent with the `predefined-model` configuration method. Only unified provider credentials need to be configured, and models are obtained from the provider through credential information. - -These three configuration methods **can coexist**, meaning a provider can support `predefined-model` + `customizable-model` or `predefined-model` + `fetch-from-remote`, etc. In other words, configuring the unified provider credentials allows the use of predefined and remotely fetched models, and if new models are added, they can be used in addition to the custom models. - -## Getting Started - -Adding a new provider starts with determining the English identifier of the provider, such as `anthropic`, and using this identifier to create a `module` in `model_providers`. - -Under this `module`, we first need to prepare the provider's YAML configuration. - -### Preparing Provider YAML - -Here, using `Anthropic` as an example, we preset the provider's basic information, supported model types, configuration methods, and credential rules. - -```YAML -provider: anthropic # Provider identifier -label: # Provider display name, can be set in en_US English and zh_Hans Chinese, zh_Hans will default to en_US if not set. - en_US: Anthropic -icon_small: # Small provider icon, stored in the _assets directory under the corresponding provider implementation directory, same language strategy as label - en_US: icon_s_en.png -icon_large: # Large provider icon, stored in the _assets directory under the corresponding provider implementation directory, same language strategy as label - en_US: icon_l_en.png -supported_model_types: # Supported model types, Anthropic only supports LLM -- llm -configurate_methods: # Supported configuration methods, Anthropic only supports predefined models -- predefined-model -provider_credential_schema: # Provider credential rules, as Anthropic only supports predefined models, unified provider credential rules need to be defined - credential_form_schemas: # List of credential form items - - variable: anthropic_api_key # Credential parameter variable name - label: # Display name - en_US: API Key - type: secret-input # Form type, here secret-input represents an encrypted information input box, showing masked information when editing. - required: true # Whether required - placeholder: # Placeholder information - zh_Hans: Enter your API Key here - en_US: Enter your API Key - - variable: anthropic_api_url - label: - en_US: API URL - type: text-input # Form type, here text-input represents a text input box - required: false - placeholder: - zh_Hans: Enter your API URL here - en_US: Enter your API URL -``` - -You can also refer to the YAML configuration information under other provider directories in `model_providers`. The complete YAML rules are available at: [Schema](schema.md#provider). - -### Implementing Provider Code - -Providers need to inherit the `__base.model_provider.ModelProvider` base class and implement the `validate_provider_credentials` method for unified provider credential verification. For reference, see [AnthropicProvider](https://github.com/langgenius/dify-runtime/blob/main/lib/model_providers/anthropic/anthropic.py). - -> If the provider is the type of `customizable-model`, there is no need to implement the `validate_provider_credentials` method. - -```python -def validate_provider_credentials(self, credentials: dict) -> None: - """ - Validate provider credentials - You can choose any validate_credentials method of model type or implement validate method by yourself, - such as: get model list api - - if validate failed, raise exception - - :param credentials: provider credentials, credentials form defined in `provider_credential_schema`. - """ -``` - -Of course, you can also preliminarily reserve the implementation of `validate_provider_credentials` and directly reuse it after the model credential verification method is implemented. - -______________________________________________________________________ - -### Adding Models - -After the provider integration is complete, the next step is to integrate models under the provider. - -First, we need to determine the type of the model to be integrated and create a `module` for the corresponding model type in the provider's directory. - -The currently supported model types are as follows: - -- `llm` Text generation model -- `text_embedding` Text Embedding model -- `rerank` Rerank model -- `speech2text` Speech to text -- `tts` Text to speech -- `moderation` Moderation - -Continuing with `Anthropic` as an example, since `Anthropic` only supports LLM, we create a `module` named `llm` in `model_providers.anthropic`. - -For predefined models, we first need to create a YAML file named after the model, such as `claude-2.1.yaml`, under the `llm` `module`. - -#### Preparing Model YAML - -```yaml -model: claude-2.1 # Model identifier -# Model display name, can be set in en_US English and zh_Hans Chinese, zh_Hans will default to en_US if not set. -# Alternatively, if the label is not set, use the model identifier content. -label: - en_US: claude-2.1 -model_type: llm # Model type, claude-2.1 is an LLM -features: # Supported features, agent-thought for Agent reasoning, vision for image understanding -- agent-thought -model_properties: # Model properties - mode: chat # LLM mode, complete for text completion model, chat for dialogue model - context_size: 200000 # Maximum supported context size -parameter_rules: # Model invocation parameter rules, only required for LLM -- name: temperature # Invocation parameter variable name - # Default preset with 5 variable content configuration templates: temperature/top_p/max_tokens/presence_penalty/frequency_penalty - # Directly set the template variable name in use_template, which will use the default configuration in entities.defaults.PARAMETER_RULE_TEMPLATE - # If additional configuration parameters are set, they will override the default configuration - use_template: temperature -- name: top_p - use_template: top_p -- name: top_k - label: # Invocation parameter display name - zh_Hans: Sampling quantity - en_US: Top k - type: int # Parameter type, supports float/int/string/boolean - help: # Help information, describing the role of the parameter - zh_Hans: Only sample from the top K options for each subsequent token. - en_US: Only sample from the top K options for each subsequent token. - required: false # Whether required, can be left unset -- name: max_tokens_to_sample - use_template: max_tokens - default: 4096 # Default parameter value - min: 1 # Minimum parameter value, only applicable for float/int - max: 4096 # Maximum parameter value, only applicable for float/int -pricing: # Pricing information - input: '8.00' # Input price, i.e., Prompt price - output: '24.00' # Output price, i.e., returned content price - unit: '0.000001' # Pricing unit, i.e., the above prices are per 100K - currency: USD # Currency -``` - -It is recommended to prepare all model configurations before starting the implementation of the model code. - -Similarly, you can also refer to the YAML configuration information for corresponding model types of other providers in the `model_providers` directory. The complete YAML rules can be found at: [Schema](schema.md#AIModel). - -#### Implementing Model Invocation Code - -Next, you need to create a python file named `llm.py` under the `llm` `module` to write the implementation code. - -In `llm.py`, create an Anthropic LLM class, which we name `AnthropicLargeLanguageModel` (arbitrarily), inheriting the `__base.large_language_model.LargeLanguageModel` base class, and implement the following methods: - -- LLM Invocation - - Implement the core method for LLM invocation, which can support both streaming and synchronous returns. - - ```python - def _invoke(self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, - stream: bool = True, user: Optional[str] = None) \ - -> Union[LLMResult, Generator]: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param model_parameters: model parameters - :param tools: tools for tool calling - :param stop: stop words - :param stream: is stream response - :param user: unique user id - :return: full response or stream response chunk generator result - """ - ``` - -- Pre-calculating Input Tokens - - If the model does not provide a pre-calculated tokens interface, you can directly return 0. - - ```python - def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None) -> int: - """ - Get number of tokens for given prompt messages - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param tools: tools for tool calling - :return: - """ - ``` - -- Model Credential Verification - - Similar to provider credential verification, this step involves verification for an individual model. - - ```python - def validate_credentials(self, model: str, credentials: dict) -> None: - """ - Validate model credentials - - :param model: model name - :param credentials: model credentials - :return: - """ - ``` - -- Invocation Error Mapping Table - - When there is an exception in model invocation, it needs to be mapped to the `InvokeError` type specified by Runtime. This facilitates Dify's ability to handle different errors with appropriate follow-up actions. - - Runtime Errors: - - - `InvokeConnectionError` Invocation connection error - - `InvokeServerUnavailableError` Invocation service provider unavailable - - `InvokeRateLimitError` Invocation reached rate limit - - `InvokeAuthorizationError` Invocation authorization failure - - `InvokeBadRequestError` Invocation parameter error - - ```python - @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: - """ - Map model invoke error to unified error - The key is the error type thrown to the caller - The value is the error type thrown by the model, - which needs to be converted into a unified error type for the caller. - - :return: Invoke error mapping - """ - ``` - -For details on the interface methods, see: [Interfaces](interfaces.md). For specific implementations, refer to: [llm.py](https://github.com/langgenius/dify-runtime/blob/main/lib/model_providers/anthropic/llm/llm.py). - -### Testing - -To ensure the availability of integrated providers/models, each method written needs corresponding integration test code in the `tests` directory. - -Continuing with `Anthropic` as an example: - -Before writing test code, you need to first add the necessary credential environment variables for the test provider in `.env.example`, such as: `ANTHROPIC_API_KEY`. - -Before execution, copy `.env.example` to `.env` and then execute. - -#### Writing Test Code - -Create a `module` with the same name as the provider in the `tests` directory: `anthropic`, and continue to create `test_provider.py` and test py files for the corresponding model types within this module, as shown below: - -```shell -. -├── __init__.py -├── anthropic -│   ├── __init__.py -│   ├── test_llm.py # LLM Testing -│   └── test_provider.py # Provider Testing -``` - -Write test code for all the various cases implemented above and submit the code after passing the tests. diff --git a/api/core/model_runtime/docs/en_US/schema.md b/api/core/model_runtime/docs/en_US/schema.md deleted file mode 100644 index 1cea4127f4..0000000000 --- a/api/core/model_runtime/docs/en_US/schema.md +++ /dev/null @@ -1,208 +0,0 @@ -# Configuration Rules - -- Provider rules are based on the [Provider](#Provider) entity. -- Model rules are based on the [AIModelEntity](#AIModelEntity) entity. - -> All entities mentioned below are based on `Pydantic BaseModel` and can be found in the `entities` module. - -### Provider - -- `provider` (string) Provider identifier, e.g., `openai` -- `label` (object) Provider display name, i18n, with `en_US` English and `zh_Hans` Chinese language settings - - `zh_Hans` (string) [optional] Chinese label name, if `zh_Hans` is not set, `en_US` will be used by default. - - `en_US` (string) English label name -- `description` (object) Provider description, i18n - - `zh_Hans` (string) [optional] Chinese description - - `en_US` (string) English description -- `icon_small` (string) [optional] Small provider ICON, stored in the `_assets` directory under the corresponding provider implementation directory, with the same language strategy as `label` - - `zh_Hans` (string) Chinese ICON - - `en_US` (string) English ICON -- `icon_large` (string) [optional] Large provider ICON, stored in the `_assets` directory under the corresponding provider implementation directory, with the same language strategy as `label` - - `zh_Hans` (string) Chinese ICON - - `en_US` (string) English ICON -- `background` (string) [optional] Background color value, e.g., #FFFFFF, if empty, the default frontend color value will be displayed. -- `help` (object) [optional] help information - - `title` (object) help title, i18n - - `zh_Hans` (string) [optional] Chinese title - - `en_US` (string) English title - - `url` (object) help link, i18n - - `zh_Hans` (string) [optional] Chinese link - - `en_US` (string) English link -- `supported_model_types` (array\[[ModelType](#ModelType)\]) Supported model types -- `configurate_methods` (array\[[ConfigurateMethod](#ConfigurateMethod)\]) Configuration methods -- `provider_credential_schema` ([ProviderCredentialSchema](#ProviderCredentialSchema)) Provider credential specification -- `model_credential_schema` ([ModelCredentialSchema](#ModelCredentialSchema)) Model credential specification - -### AIModelEntity - -- `model` (string) Model identifier, e.g., `gpt-3.5-turbo` -- `label` (object) [optional] Model display name, i18n, with `en_US` English and `zh_Hans` Chinese language settings - - `zh_Hans` (string) [optional] Chinese label name - - `en_US` (string) English label name -- `model_type` ([ModelType](#ModelType)) Model type -- `features` (array\[[ModelFeature](#ModelFeature)\]) [optional] Supported feature list -- `model_properties` (object) Model properties - - `mode` ([LLMMode](#LLMMode)) Mode (available for model type `llm`) - - `context_size` (int) Context size (available for model types `llm`, `text-embedding`) - - `max_chunks` (int) Maximum number of chunks (available for model types `text-embedding`, `moderation`) - - `file_upload_limit` (int) Maximum file upload limit, in MB (available for model type `speech2text`) - - `supported_file_extensions` (string) Supported file extension formats, e.g., mp3, mp4 (available for model type `speech2text`) - - `default_voice` (string) default voice, e.g.：alloy,echo,fable,onyx,nova,shimmer（available for model type `tts`） - - `voices` (list) List of available voice.（available for model type `tts`） - - `mode` (string) voice model.（available for model type `tts`） - - `name` (string) voice model display name.（available for model type `tts`） - - `language` (string) the voice model supports languages.（available for model type `tts`） - - `word_limit` (int) Single conversion word limit, paragraph-wise by default（available for model type `tts`） - - `audio_type` (string) Support audio file extension format, e.g.：mp3,wav（available for model type `tts`） - - `max_workers` (int) Number of concurrent workers supporting text and audio conversion（available for model type`tts`） - - `max_characters_per_chunk` (int) Maximum characters per chunk (available for model type `moderation`) -- `parameter_rules` (array\[[ParameterRule](#ParameterRule)\]) [optional] Model invocation parameter rules -- `pricing` ([PriceConfig](#PriceConfig)) [optional] Pricing information -- `deprecated` (bool) Whether deprecated. If deprecated, the model will no longer be displayed in the list, but those already configured can continue to be used. Default False. - -### ModelType - -- `llm` Text generation model -- `text-embedding` Text Embedding model -- `rerank` Rerank model -- `speech2text` Speech to text -- `tts` Text to speech -- `moderation` Moderation - -### ConfigurateMethod - -- `predefined-model` Predefined model - - Indicates that users can use the predefined models under the provider by configuring the unified provider credentials. - -- `customizable-model` Customizable model - - Users need to add credential configuration for each model. - -- `fetch-from-remote` Fetch from remote - - Consistent with the `predefined-model` configuration method, only unified provider credentials need to be configured, and models are obtained from the provider through credential information. - -### ModelFeature - -- `agent-thought` Agent reasoning, generally over 70B with thought chain capability. -- `vision` Vision, i.e., image understanding. -- `tool-call` -- `multi-tool-call` -- `stream-tool-call` - -### FetchFrom - -- `predefined-model` Predefined model -- `fetch-from-remote` Remote model - -### LLMMode - -- `complete` Text completion -- `chat` Dialogue - -### ParameterRule - -- `name` (string) Actual model invocation parameter name - -- `use_template` (string) [optional] Using template - - By default, 5 variable content configuration templates are preset: - - - `temperature` - - `top_p` - - `frequency_penalty` - - `presence_penalty` - - `max_tokens` - - In use_template, you can directly set the template variable name, which will use the default configuration in entities.defaults.PARAMETER_RULE_TEMPLATE - No need to set any parameters other than `name` and `use_template`. If additional configuration parameters are set, they will override the default configuration. - Refer to `openai/llm/gpt-3.5-turbo.yaml`. - -- `label` (object) [optional] Label, i18n - - - `zh_Hans`(string) [optional] Chinese label name - - `en_US` (string) English label name - -- `type`(string) [optional] Parameter type - - - `int` Integer - - `float` Float - - `string` String - - `boolean` Boolean - -- `help` (string) [optional] Help information - - - `zh_Hans` (string) [optional] Chinese help information - - `en_US` (string) English help information - -- `required` (bool) Required, default False. - -- `default`(int/float/string/bool) [optional] Default value - -- `min`(int/float) [optional] Minimum value, applicable only to numeric types - -- `max`(int/float) [optional] Maximum value, applicable only to numeric types - -- `precision`(int) [optional] Precision, number of decimal places to keep, applicable only to numeric types - -- `options` (array[string]) [optional] Dropdown option values, applicable only when `type` is `string`, if not set or null, option values are not restricted - -### PriceConfig - -- `input` (float) Input price, i.e., Prompt price -- `output` (float) Output price, i.e., returned content price -- `unit` (float) Pricing unit, e.g., if the price is measured in 1M tokens, the corresponding token amount for the unit price is `0.000001`. -- `currency` (string) Currency unit - -### ProviderCredentialSchema - -- `credential_form_schemas` (array\[[CredentialFormSchema](#CredentialFormSchema)\]) Credential form standard - -### ModelCredentialSchema - -- `model` (object) Model identifier, variable name defaults to `model` - - `label` (object) Model form item display name - - `en_US` (string) English - - `zh_Hans`(string) [optional] Chinese - - `placeholder` (object) Model prompt content - - `en_US`(string) English - - `zh_Hans`(string) [optional] Chinese -- `credential_form_schemas` (array\[[CredentialFormSchema](#CredentialFormSchema)\]) Credential form standard - -### CredentialFormSchema - -- `variable` (string) Form item variable name -- `label` (object) Form item label name - - `en_US`(string) English - - `zh_Hans` (string) [optional] Chinese -- `type` ([FormType](#FormType)) Form item type -- `required` (bool) Whether required -- `default`(string) Default value -- `options` (array\[[FormOption](#FormOption)\]) Specific property of form items of type `select` or `radio`, defining dropdown content -- `placeholder`(object) Specific property of form items of type `text-input`, placeholder content - - `en_US`(string) English - - `zh_Hans` (string) [optional] Chinese -- `max_length` (int) Specific property of form items of type `text-input`, defining maximum input length, 0 for no limit. -- `show_on` (array\[[FormShowOnObject](#FormShowOnObject)\]) Displayed when other form item values meet certain conditions, displayed always if empty. - -### FormType - -- `text-input` Text input component -- `secret-input` Password input component -- `select` Single-choice dropdown -- `radio` Radio component -- `switch` Switch component, only supports `true` and `false` values - -### FormOption - -- `label` (object) Label - - `en_US`(string) English - - `zh_Hans`(string) [optional] Chinese -- `value` (string) Dropdown option value -- `show_on` (array\[[FormShowOnObject](#FormShowOnObject)\]) Displayed when other form item values meet certain conditions, displayed always if empty. - -### FormShowOnObject - -- `variable` (string) Variable name of other form items -- `value` (string) Variable value of other form items diff --git a/api/core/model_runtime/docs/zh_Hans/customizable_model_scale_out.md b/api/core/model_runtime/docs/zh_Hans/customizable_model_scale_out.md deleted file mode 100644 index 825f9349d7..0000000000 --- a/api/core/model_runtime/docs/zh_Hans/customizable_model_scale_out.md +++ /dev/null @@ -1,304 +0,0 @@ -## 自定义预定义模型接入 - -### 介绍 - -供应商集成完成后，接下来为供应商下模型的接入，为了帮助理解整个接入过程，我们以`Xinference`为例，逐步完成一个完整的供应商接入。 - -需要注意的是，对于自定义模型，每一个模型的接入都需要填写一个完整的供应商凭据。 - -而不同于预定义模型，自定义供应商接入时永远会拥有如下两个参数，不需要在供应商 yaml 中定义。 - -![Alt text](images/index/image-3.png) - -在前文中，我们已经知道了供应商无需实现`validate_provider_credential`，Runtime 会自行根据用户在此选择的模型类型和模型名称调用对应的模型层的`validate_credentials`来进行验证。 - -### 编写供应商 yaml - -我们首先要确定，接入的这个供应商支持哪些类型的模型。 - -当前支持模型类型如下： - -- `llm` 文本生成模型 -- `text_embedding` 文本 Embedding 模型 -- `rerank` Rerank 模型 -- `speech2text` 语音转文字 -- `tts` 文字转语音 -- `moderation` 审查 - -`Xinference`支持`LLM`和`Text Embedding`和 Rerank，那么我们开始编写`xinference.yaml`。 - -```yaml -provider: xinference #确定供应商标识 -label: # 供应商展示名称，可设置 en_US 英文、zh_Hans 中文两种语言，zh_Hans 不设置将默认使用 en_US。 - en_US: Xorbits Inference -icon_small: # 小图标，可以参考其他供应商的图标，存储在对应供应商实现目录下的 _assets 目录，中英文策略同 label - en_US: icon_s_en.svg -icon_large: # 大图标 - en_US: icon_l_en.svg -help: # 帮助 - title: - en_US: How to deploy Xinference - zh_Hans: 如何部署 Xinference - url: - en_US: https://github.com/xorbitsai/inference -supported_model_types: # 支持的模型类型，Xinference 同时支持 LLM/Text Embedding/Rerank -- llm -- text-embedding -- rerank -configurate_methods: # 因为 Xinference 为本地部署的供应商，并且没有预定义模型，需要用什么模型需要根据 Xinference 的文档自己部署，所以这里只支持自定义模型 -- customizable-model -provider_credential_schema: - credential_form_schemas: -``` - -随后，我们需要思考在 Xinference 中定义一个模型需要哪些凭据 - -- 它支持三种不同的模型，因此，我们需要有`model_type`来指定这个模型的类型，它有三种类型，所以我们这么编写 - -```yaml -provider_credential_schema: - credential_form_schemas: - - variable: model_type - type: select - label: - en_US: Model type - zh_Hans: 模型类型 - required: true - options: - - value: text-generation - label: - en_US: Language Model - zh_Hans: 语言模型 - - value: embeddings - label: - en_US: Text Embedding - - value: reranking - label: - en_US: Rerank -``` - -- 每一个模型都有自己的名称`model_name`，因此需要在这里定义 - -```yaml - - variable: model_name - type: text-input - label: - en_US: Model name - zh_Hans: 模型名称 - required: true - placeholder: - zh_Hans: 填写模型名称 - en_US: Input model name -``` - -- 填写 Xinference 本地部署的地址 - -```yaml - - variable: server_url - label: - zh_Hans: 服务器 URL - en_US: Server url - type: text-input - required: true - placeholder: - zh_Hans: 在此输入 Xinference 的服务器地址，如 https://example.com/xxx - en_US: Enter the url of your Xinference, for example https://example.com/xxx -``` - -- 每个模型都有唯一的 model_uid，因此需要在这里定义 - -```yaml - - variable: model_uid - label: - zh_Hans: 模型 UID - en_US: Model uid - type: text-input - required: true - placeholder: - zh_Hans: 在此输入您的 Model UID - en_US: Enter the model uid -``` - -现在，我们就完成了供应商的基础定义。 - -### 编写模型代码 - -然后我们以`llm`类型为例，编写`xinference.llm.llm.py` - -在 `llm.py` 中创建一个 Xinference LLM 类，我们取名为 `XinferenceAILargeLanguageModel`（随意），继承 `__base.large_language_model.LargeLanguageModel` 基类，实现以下几个方法： - -- LLM 调用 - - 实现 LLM 调用的核心方法，可同时支持流式和同步返回。 - - ```python - def _invoke(self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, - stream: bool = True, user: Optional[str] = None) \ - -> Union[LLMResult, Generator]: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param model_parameters: model parameters - :param tools: tools for tool calling - :param stop: stop words - :param stream: is stream response - :param user: unique user id - :return: full response or stream response chunk generator result - """ - ``` - - 在实现时，需要注意使用两个函数来返回数据，分别用于处理同步返回和流式返回，因为 Python 会将函数中包含 `yield` 关键字的函数识别为生成器函数，返回的数据类型固定为 `Generator`，因此同步和流式返回需要分别实现，就像下面这样（注意下面例子使用了简化参数，实际实现时需要按照上面的参数列表进行实现）： - - ```python - def _invoke(self, stream: bool, **kwargs) \ - -> Union[LLMResult, Generator]: - if stream: - return self._handle_stream_response(**kwargs) - return self._handle_sync_response(**kwargs) - - def _handle_stream_response(self, **kwargs) -> Generator: - for chunk in response: - yield chunk - def _handle_sync_response(self, **kwargs) -> LLMResult: - return LLMResult(**response) - ``` - -- 预计算输入 tokens - - 若模型未提供预计算 tokens 接口，可直接返回 0。 - - ```python - def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None) -> int: - """ - Get number of tokens for given prompt messages - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param tools: tools for tool calling - :return: - """ - ``` - - 有时候，也许你不需要直接返回 0，所以你可以使用`self._get_num_tokens_by_gpt2(text: str)`来获取预计算的 tokens，并确保环境变量`PLUGIN_BASED_TOKEN_COUNTING_ENABLED`设置为`true`，这个方法位于`AIModel`基类中，它会使用 GPT2 的 Tokenizer 进行计算，但是只能作为替代方法，并不完全准确。 - -- 模型凭据校验 - - 与供应商凭据校验类似，这里针对单个模型进行校验。 - - ```python - def validate_credentials(self, model: str, credentials: dict) -> None: - """ - Validate model credentials - - :param model: model name - :param credentials: model credentials - :return: - """ - ``` - -- 模型参数 Schema - - 与自定义类型不同，由于没有在 yaml 文件中定义一个模型支持哪些参数，因此，我们需要动态时间模型参数的 Schema。 - - 如 Xinference 支持`max_tokens` `temperature` `top_p` 这三个模型参数。 - - 但是有的供应商根据不同的模型支持不同的参数，如供应商`OpenLLM`支持`top_k`，但是并不是这个供应商提供的所有模型都支持`top_k`，我们这里举例 A 模型支持`top_k`，B 模型不支持`top_k`，那么我们需要在这里动态生成模型参数的 Schema，如下所示： - - ```python - def get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]: - """ - used to define customizable model schema - """ - rules = [ - ParameterRule( - name='temperature', type=ParameterType.FLOAT, - use_template='temperature', - label=I18nObject( - zh_Hans='温度', en_US='Temperature' - ) - ), - ParameterRule( - name='top_p', type=ParameterType.FLOAT, - use_template='top_p', - label=I18nObject( - zh_Hans='Top P', en_US='Top P' - ) - ), - ParameterRule( - name='max_tokens', type=ParameterType.INT, - use_template='max_tokens', - min=1, - default=512, - label=I18nObject( - zh_Hans='最大生成长度', en_US='Max Tokens' - ) - ) - ] - - # if model is A, add top_k to rules - if model == 'A': - rules.append( - ParameterRule( - name='top_k', type=ParameterType.INT, - use_template='top_k', - min=1, - default=50, - label=I18nObject( - zh_Hans='Top K', en_US='Top K' - ) - ) - ) - - """ - some NOT IMPORTANT code here - """ - - entity = AIModelEntity( - model=model, - label=I18nObject( - en_US=model - ), - fetch_from=FetchFrom.CUSTOMIZABLE_MODEL, - model_type=model_type, - model_properties={ - ModelPropertyKey.MODE: ModelType.LLM, - }, - parameter_rules=rules - ) - - return entity - ``` - -- 调用异常错误映射表 - - 当模型调用异常时需要映射到 Runtime 指定的 `InvokeError` 类型，方便 Dify 针对不同错误做不同后续处理。 - - Runtime Errors: - - - `InvokeConnectionError` 调用连接错误 - - `InvokeServerUnavailableError ` 调用服务方不可用 - - `InvokeRateLimitError ` 调用达到限额 - - `InvokeAuthorizationError` 调用鉴权失败 - - `InvokeBadRequestError ` 调用传参有误 - - ```python - @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: - """ - Map model invoke error to unified error - The key is the error type thrown to the caller - The value is the error type thrown by the model, - which needs to be converted into a unified error type for the caller. - - :return: Invoke error mapping - """ - ``` - -接口方法说明见：[Interfaces](./interfaces.md)，具体实现可参考：[llm.py](https://github.com/langgenius/dify-runtime/blob/main/lib/model_providers/anthropic/llm/llm.py)。 diff --git a/api/core/model_runtime/docs/zh_Hans/images/index/image-1.png b/api/core/model_runtime/docs/zh_Hans/images/index/image-1.png deleted file mode 100644 index b158d44b29..0000000000 Binary files a/api/core/model_runtime/docs/zh_Hans/images/index/image-1.png and /dev/null differ diff --git a/api/core/model_runtime/docs/zh_Hans/images/index/image-2.png b/api/core/model_runtime/docs/zh_Hans/images/index/image-2.png deleted file mode 100644 index c70cd3da5e..0000000000 Binary files a/api/core/model_runtime/docs/zh_Hans/images/index/image-2.png and /dev/null differ diff --git a/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210143654461.png b/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210143654461.png deleted file mode 100644 index f1c30158dd..0000000000 Binary files a/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210143654461.png and /dev/null differ diff --git a/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210144229650.png b/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210144229650.png deleted file mode 100644 index 742c1ba808..0000000000 Binary files a/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210144229650.png and /dev/null differ diff --git a/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210144814617.png b/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210144814617.png deleted file mode 100644 index b28aba83c9..0000000000 Binary files a/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210144814617.png and /dev/null differ diff --git a/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210151548521.png b/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210151548521.png deleted file mode 100644 index 0d88bf4bda..0000000000 Binary files a/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210151548521.png and /dev/null differ diff --git a/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210151628992.png b/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210151628992.png deleted file mode 100644 index a07aaebd2f..0000000000 Binary files a/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210151628992.png and /dev/null differ diff --git a/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210165243632.png b/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210165243632.png deleted file mode 100644 index 18ec605e83..0000000000 Binary files a/api/core/model_runtime/docs/zh_Hans/images/index/image-20231210165243632.png and /dev/null differ diff --git a/api/core/model_runtime/docs/zh_Hans/images/index/image-3.png b/api/core/model_runtime/docs/zh_Hans/images/index/image-3.png deleted file mode 100644 index bf0b9a7f47..0000000000 Binary files a/api/core/model_runtime/docs/zh_Hans/images/index/image-3.png and /dev/null differ diff --git a/api/core/model_runtime/docs/zh_Hans/images/index/image.png b/api/core/model_runtime/docs/zh_Hans/images/index/image.png deleted file mode 100644 index eb63d107e1..0000000000 Binary files a/api/core/model_runtime/docs/zh_Hans/images/index/image.png and /dev/null differ diff --git a/api/core/model_runtime/docs/zh_Hans/interfaces.md b/api/core/model_runtime/docs/zh_Hans/interfaces.md deleted file mode 100644 index 8eeeee9ff9..0000000000 --- a/api/core/model_runtime/docs/zh_Hans/interfaces.md +++ /dev/null @@ -1,744 +0,0 @@ -# 接口方法 - -这里介绍供应商和各模型类型需要实现的接口方法和参数说明。 - -## 供应商 - -继承 `__base.model_provider.ModelProvider` 基类，实现以下接口： - -```python -def validate_provider_credentials(self, credentials: dict) -> None: - """ - Validate provider credentials - You can choose any validate_credentials method of model type or implement validate method by yourself, - such as: get model list api - - if validate failed, raise exception - - :param credentials: provider credentials, credentials form defined in `provider_credential_schema`. - """ -``` - -- `credentials` (object) 凭据信息 - - 凭据信息的参数由供应商 YAML 配置文件的 `provider_credential_schema` 定义，传入如：`api_key` 等。 - -验证失败请抛出 `errors.validate.CredentialsValidateFailedError` 错误。 - -**注：预定义模型需完整实现该接口，自定义模型供应商只需要如下简单实现即可** - -```python -class XinferenceProvider(Provider): - def validate_provider_credentials(self, credentials: dict) -> None: - pass -``` - -## 模型 - -模型分为 5 种不同的模型类型，不同模型类型继承的基类不同，需要实现的方法也不同。 - -### 通用接口 - -所有模型均需要统一实现下面 2 个方法： - -- 模型凭据校验 - - 与供应商凭据校验类似，这里针对单个模型进行校验。 - - ```python - def validate_credentials(self, model: str, credentials: dict) -> None: - """ - Validate model credentials - - :param model: model name - :param credentials: model credentials - :return: - """ - ``` - - 参数： - - - `model` (string) 模型名称 - - - `credentials` (object) 凭据信息 - - 凭据信息的参数由供应商 YAML 配置文件的 `provider_credential_schema` 或 `model_credential_schema` 定义，传入如：`api_key` 等。 - - 验证失败请抛出 `errors.validate.CredentialsValidateFailedError` 错误。 - -- 调用异常错误映射表 - - 当模型调用异常时需要映射到 Runtime 指定的 `InvokeError` 类型，方便 Dify 针对不同错误做不同后续处理。 - - Runtime Errors: - - - `InvokeConnectionError` 调用连接错误 - - `InvokeServerUnavailableError ` 调用服务方不可用 - - `InvokeRateLimitError ` 调用达到限额 - - `InvokeAuthorizationError` 调用鉴权失败 - - `InvokeBadRequestError ` 调用传参有误 - - ```python - @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: - """ - Map model invoke error to unified error - The key is the error type thrown to the caller - The value is the error type thrown by the model, - which needs to be converted into a unified error type for the caller. - - :return: Invoke error mapping - """ - ``` - - 也可以直接抛出对应 Errors，并做如下定义，这样在之后的调用中可以直接抛出`InvokeConnectionError`等异常。 - - ```python - @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: - return { - InvokeConnectionError: [ - InvokeConnectionError - ], - InvokeServerUnavailableError: [ - InvokeServerUnavailableError - ], - InvokeRateLimitError: [ - InvokeRateLimitError - ], - InvokeAuthorizationError: [ - InvokeAuthorizationError - ], - InvokeBadRequestError: [ - InvokeBadRequestError - ], - } - ``` - -​ 可参考 OpenAI `_invoke_error_mapping`。 - -### LLM - -继承 `__base.large_language_model.LargeLanguageModel` 基类，实现以下接口： - -- LLM 调用 - - 实现 LLM 调用的核心方法，可同时支持流式和同步返回。 - - ```python - def _invoke(self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, - stream: bool = True, user: Optional[str] = None) \ - -> Union[LLMResult, Generator]: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param model_parameters: model parameters - :param tools: tools for tool calling - :param stop: stop words - :param stream: is stream response - :param user: unique user id - :return: full response or stream response chunk generator result - """ - ``` - - - 参数： - - - `model` (string) 模型名称 - - - `credentials` (object) 凭据信息 - - 凭据信息的参数由供应商 YAML 配置文件的 `provider_credential_schema` 或 `model_credential_schema` 定义，传入如：`api_key` 等。 - - - `prompt_messages` (array\[[PromptMessage](#PromptMessage)\]) Prompt 列表 - - 若模型为 `Completion` 类型，则列表只需要传入一个 [UserPromptMessage](#UserPromptMessage) 元素即可； - - 若模型为 `Chat` 类型，需要根据消息不同传入 [SystemPromptMessage](#SystemPromptMessage), [UserPromptMessage](#UserPromptMessage), [AssistantPromptMessage](#AssistantPromptMessage), [ToolPromptMessage](#ToolPromptMessage) 元素列表 - - - `model_parameters` (object) 模型参数 - - 模型参数由模型 YAML 配置的 `parameter_rules` 定义。 - - - `tools` (array\[[PromptMessageTool](#PromptMessageTool)\]) [optional] 工具列表，等同于 `function calling` 中的 `function`。 - - 即传入 tool calling 的工具列表。 - - - `stop` (array[string]) [optional] 停止序列 - - 模型返回将在停止序列定义的字符串之前停止输出。 - - - `stream` (bool) 是否流式输出，默认 True - - 流式输出返回 Generator\[[LLMResultChunk](#LLMResultChunk)\]，非流式输出返回 [LLMResult](#LLMResult)。 - - - `user` (string) [optional] 用户的唯一标识符 - - 可以帮助供应商监控和检测滥用行为。 - - - 返回 - - 流式输出返回 Generator\[[LLMResultChunk](#LLMResultChunk)\]，非流式输出返回 [LLMResult](#LLMResult)。 - -- 预计算输入 tokens - - 若模型未提供预计算 tokens 接口，可直接返回 0。 - - ```python - def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None) -> int: - """ - Get number of tokens for given prompt messages - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param tools: tools for tool calling - :return: - """ - ``` - - 参数说明见上述 `LLM 调用`。 - - 该接口需要根据对应`model`选择合适的`tokenizer`进行计算，如果对应模型没有提供`tokenizer`，可以使用`AIModel`基类中的`_get_num_tokens_by_gpt2(text: str)`方法进行计算。 - -- 获取自定义模型规则 [可选] - - ```python - def get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]: - """ - Get customizable model schema - - :param model: model name - :param credentials: model credentials - :return: model schema - """ - ``` - -​当供应商支持增加自定义 LLM 时，可实现此方法让自定义模型可获取模型规则，默认返回 None。 - -对于`OpenAI`供应商下的大部分微调模型，可以通过其微调模型名称获取到其基类模型，如`gpt-3.5-turbo-1106`，然后返回基类模型的预定义参数规则，参考[openai](https://github.com/langgenius/dify/blob/feat/model-runtime/api/core/model_runtime/model_providers/openai/llm/llm.py#L801) -的具体实现 - -### TextEmbedding - -继承 `__base.text_embedding_model.TextEmbeddingModel` 基类，实现以下接口： - -- Embedding 调用 - - ```python - def _invoke(self, model: str, credentials: dict, - texts: list[str], user: Optional[str] = None) \ - -> TextEmbeddingResult: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param texts: texts to embed - :param user: unique user id - :return: embeddings result - """ - ``` - - - 参数： - - - `model` (string) 模型名称 - - - `credentials` (object) 凭据信息 - - 凭据信息的参数由供应商 YAML 配置文件的 `provider_credential_schema` 或 `model_credential_schema` 定义，传入如：`api_key` 等。 - - - `texts` (array[string]) 文本列表，可批量处理 - - - `user` (string) [optional] 用户的唯一标识符 - - 可以帮助供应商监控和检测滥用行为。 - - - 返回： - - [TextEmbeddingResult](#TextEmbeddingResult) 实体。 - -- 预计算 tokens - - ```python - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: - """ - Get number of tokens for given prompt messages - - :param model: model name - :param credentials: model credentials - :param texts: texts to embed - :return: - """ - ``` - - 参数说明见上述 `Embedding 调用`。 - - 同上述`LargeLanguageModel`，该接口需要根据对应`model`选择合适的`tokenizer`进行计算，如果对应模型没有提供`tokenizer`，可以使用`AIModel`基类中的`_get_num_tokens_by_gpt2(text: str)`方法进行计算。 - -### Rerank - -继承 `__base.rerank_model.RerankModel` 基类，实现以下接口： - -- rerank 调用 - - ```python - def _invoke(self, model: str, credentials: dict, - query: str, docs: list[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, - user: Optional[str] = None) \ - -> RerankResult: - """ - Invoke rerank model - - :param model: model name - :param credentials: model credentials - :param query: search query - :param docs: docs for reranking - :param score_threshold: score threshold - :param top_n: top n - :param user: unique user id - :return: rerank result - """ - ``` - - - 参数： - - - `model` (string) 模型名称 - - - `credentials` (object) 凭据信息 - - 凭据信息的参数由供应商 YAML 配置文件的 `provider_credential_schema` 或 `model_credential_schema` 定义，传入如：`api_key` 等。 - - - `query` (string) 查询请求内容 - - - `docs` (array[string]) 需要重排的分段列表 - - - `score_threshold` (float) [optional] Score 阈值 - - - `top_n` (int) [optional] 取前 n 个分段 - - - `user` (string) [optional] 用户的唯一标识符 - - 可以帮助供应商监控和检测滥用行为。 - - - 返回： - - [RerankResult](#RerankResult) 实体。 - -### Speech2text - -继承 `__base.speech2text_model.Speech2TextModel` 基类，实现以下接口： - -- Invoke 调用 - - ```python - def _invoke(self, model: str, credentials: dict, - file: IO[bytes], user: Optional[str] = None) \ - -> str: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param file: audio file - :param user: unique user id - :return: text for given audio file - """ - ``` - - - 参数： - - - `model` (string) 模型名称 - - - `credentials` (object) 凭据信息 - - 凭据信息的参数由供应商 YAML 配置文件的 `provider_credential_schema` 或 `model_credential_schema` 定义，传入如：`api_key` 等。 - - - `file` (File) 文件流 - - - `user` (string) [optional] 用户的唯一标识符 - - 可以帮助供应商监控和检测滥用行为。 - - - 返回： - - 语音转换后的字符串。 - -### Text2speech - -继承 `__base.text2speech_model.Text2SpeechModel` 基类，实现以下接口： - -- Invoke 调用 - - ```python - def _invoke(self, model: str, credentials: dict, content_text: str, streaming: bool, user: Optional[str] = None): - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param content_text: text content to be translated - :param streaming: output is streaming - :param user: unique user id - :return: translated audio file - """ - ``` - - - 参数： - - - `model` (string) 模型名称 - - - `credentials` (object) 凭据信息 - - 凭据信息的参数由供应商 YAML 配置文件的 `provider_credential_schema` 或 `model_credential_schema` 定义，传入如：`api_key` 等。 - - - `content_text` (string) 需要转换的文本内容 - - - `streaming` (bool) 是否进行流式输出 - - - `user` (string) [optional] 用户的唯一标识符 - - 可以帮助供应商监控和检测滥用行为。 - - - 返回： - - 文本转换后的语音流。 - -### Moderation - -继承 `__base.moderation_model.ModerationModel` 基类，实现以下接口： - -- Invoke 调用 - - ```python - def _invoke(self, model: str, credentials: dict, - text: str, user: Optional[str] = None) \ - -> bool: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param text: text to moderate - :param user: unique user id - :return: false if text is safe, true otherwise - """ - ``` - - - 参数： - - - `model` (string) 模型名称 - - - `credentials` (object) 凭据信息 - - 凭据信息的参数由供应商 YAML 配置文件的 `provider_credential_schema` 或 `model_credential_schema` 定义，传入如：`api_key` 等。 - - - `text` (string) 文本内容 - - - `user` (string) [optional] 用户的唯一标识符 - - 可以帮助供应商监控和检测滥用行为。 - - - 返回： - - False 代表传入的文本安全，True 则反之。 - -## 实体 - -### PromptMessageRole - -消息角色 - -```python -class PromptMessageRole(Enum): - """ - Enum class for prompt message. - """ - SYSTEM = "system" - USER = "user" - ASSISTANT = "assistant" - TOOL = "tool" -``` - -### PromptMessageContentType - -消息内容类型，分为纯文本和图片。 - -```python -class PromptMessageContentType(Enum): - """ - Enum class for prompt message content type. - """ - TEXT = 'text' - IMAGE = 'image' -``` - -### PromptMessageContent - -消息内容基类，仅作为参数声明用，不可初始化。 - -```python -class PromptMessageContent(BaseModel): - """ - Model class for prompt message content. - """ - type: PromptMessageContentType - data: str # 内容数据 -``` - -当前支持文本和图片两种类型，可支持同时传入文本和多图。 - -需要分别初始化 `TextPromptMessageContent` 和 `ImagePromptMessageContent` 传入。 - -### TextPromptMessageContent - -```python -class TextPromptMessageContent(PromptMessageContent): - """ - Model class for text prompt message content. - """ - type: PromptMessageContentType = PromptMessageContentType.TEXT -``` - -若传入图文，其中文字需要构造此实体作为 `content` 列表中的一部分。 - -### ImagePromptMessageContent - -```python -class ImagePromptMessageContent(PromptMessageContent): - """ - Model class for image prompt message content. - """ - class DETAIL(Enum): - LOW = 'low' - HIGH = 'high' - - type: PromptMessageContentType = PromptMessageContentType.IMAGE - detail: DETAIL = DETAIL.LOW # 分辨率 -``` - -若传入图文，其中图片需要构造此实体作为 `content` 列表中的一部分 - -`data` 可以为 `url` 或者图片 `base64` 加密后的字符串。 - -### PromptMessage - -所有 Role 消息体的基类，仅作为参数声明用，不可初始化。 - -```python -class PromptMessage(BaseModel): - """ - Model class for prompt message. - """ - role: PromptMessageRole # 消息角色 - content: Optional[str | list[PromptMessageContent]] = None # 支持两种类型，字符串和内容列表，内容列表是为了满足多模态的需要，可详见 PromptMessageContent 说明。 - name: Optional[str] = None # 名称，可选。 -``` - -### UserPromptMessage - -UserMessage 消息体，代表用户消息。 - -```python -class UserPromptMessage(PromptMessage): - """ - Model class for user prompt message. - """ - role: PromptMessageRole = PromptMessageRole.USER -``` - -### AssistantPromptMessage - -代表模型返回消息，通常用于 `few-shots` 或聊天历史传入。 - -```python -class AssistantPromptMessage(PromptMessage): - """ - Model class for assistant prompt message. - """ - class ToolCall(BaseModel): - """ - Model class for assistant prompt message tool call. - """ - class ToolCallFunction(BaseModel): - """ - Model class for assistant prompt message tool call function. - """ - name: str # 工具名称 - arguments: str # 工具参数 - - id: str # 工具 ID，仅在 OpenAI tool call 生效，为工具调用的唯一 ID，同一个工具可以调用多次 - type: str # 默认 function - function: ToolCallFunction # 工具调用信息 - - role: PromptMessageRole = PromptMessageRole.ASSISTANT - tool_calls: list[ToolCall] = [] # 模型回复的工具调用结果（仅当传入 tools，并且模型认为需要调用工具时返回） -``` - -其中 `tool_calls` 为调用模型传入 `tools` 后，由模型返回的 `tool call` 列表。 - -### SystemPromptMessage - -代表系统消息，通常用于设定给模型的系统指令。 - -```python -class SystemPromptMessage(PromptMessage): - """ - Model class for system prompt message. - """ - role: PromptMessageRole = PromptMessageRole.SYSTEM -``` - -### ToolPromptMessage - -代表工具消息，用于工具执行后将结果交给模型进行下一步计划。 - -```python -class ToolPromptMessage(PromptMessage): - """ - Model class for tool prompt message. - """ - role: PromptMessageRole = PromptMessageRole.TOOL - tool_call_id: str # 工具调用 ID，若不支持 OpenAI tool call，也可传入工具名称 -``` - -基类的 `content` 传入工具执行结果。 - -### PromptMessageTool - -```python -class PromptMessageTool(BaseModel): - """ - Model class for prompt message tool. - """ - name: str # 工具名称 - description: str # 工具描述 - parameters: dict # 工具参数 dict -``` - -______________________________________________________________________ - -### LLMResult - -```python -class LLMResult(BaseModel): - """ - Model class for llm result. - """ - model: str # 实际使用模型 - prompt_messages: list[PromptMessage] # prompt 消息列表 - message: AssistantPromptMessage # 回复消息 - usage: LLMUsage # 使用的 tokens 及费用信息 - system_fingerprint: Optional[str] = None # 请求指纹，可参考 OpenAI 该参数定义 -``` - -### LLMResultChunkDelta - -流式返回中每个迭代内部 `delta` 实体 - -```python -class LLMResultChunkDelta(BaseModel): - """ - Model class for llm result chunk delta. - """ - index: int # 序号 - message: AssistantPromptMessage # 回复消息 - usage: Optional[LLMUsage] = None # 使用的 tokens 及费用信息，仅最后一条返回 - finish_reason: Optional[str] = None # 结束原因，仅最后一条返回 -``` - -### LLMResultChunk - -流式返回中每个迭代实体 - -```python -class LLMResultChunk(BaseModel): - """ - Model class for llm result chunk. - """ - model: str # 实际使用模型 - prompt_messages: list[PromptMessage] # prompt 消息列表 - system_fingerprint: Optional[str] = None # 请求指纹，可参考 OpenAI 该参数定义 - delta: LLMResultChunkDelta # 每个迭代存在变化的内容 -``` - -### LLMUsage - -```python -class LLMUsage(ModelUsage): - """ - Model class for llm usage. - """ - prompt_tokens: int # prompt 使用 tokens - prompt_unit_price: Decimal # prompt 单价 - prompt_price_unit: Decimal # prompt 价格单位，即单价基于多少 tokens - prompt_price: Decimal # prompt 费用 - completion_tokens: int # 回复使用 tokens - completion_unit_price: Decimal # 回复单价 - completion_price_unit: Decimal # 回复价格单位，即单价基于多少 tokens - completion_price: Decimal # 回复费用 - total_tokens: int # 总使用 token 数 - total_price: Decimal # 总费用 - currency: str # 货币单位 - latency: float # 请求耗时 (s) -``` - -______________________________________________________________________ - -### TextEmbeddingResult - -```python -class TextEmbeddingResult(BaseModel): - """ - Model class for text embedding result. - """ - model: str # 实际使用模型 - embeddings: list[list[float]] # embedding 向量列表，对应传入的 texts 列表 - usage: EmbeddingUsage # 使用信息 -``` - -### EmbeddingUsage - -```python -class EmbeddingUsage(ModelUsage): - """ - Model class for embedding usage. - """ - tokens: int # 使用 token 数 - total_tokens: int # 总使用 token 数 - unit_price: Decimal # 单价 - price_unit: Decimal # 价格单位，即单价基于多少 tokens - total_price: Decimal # 总费用 - currency: str # 货币单位 - latency: float # 请求耗时 (s) -``` - -______________________________________________________________________ - -### RerankResult - -```python -class RerankResult(BaseModel): - """ - Model class for rerank result. - """ - model: str # 实际使用模型 - docs: list[RerankDocument] # 重排后的分段列表 -``` - -### RerankDocument - -```python -class RerankDocument(BaseModel): - """ - Model class for rerank document. - """ - index: int # 原序号 - text: str # 分段文本内容 - score: float # 分数 -``` diff --git a/api/core/model_runtime/docs/zh_Hans/predefined_model_scale_out.md b/api/core/model_runtime/docs/zh_Hans/predefined_model_scale_out.md deleted file mode 100644 index cd4de51ef7..0000000000 --- a/api/core/model_runtime/docs/zh_Hans/predefined_model_scale_out.md +++ /dev/null @@ -1,172 +0,0 @@ -## 预定义模型接入 - -供应商集成完成后，接下来为供应商下模型的接入。 - -我们首先需要确定接入模型的类型，并在对应供应商的目录下创建对应模型类型的 `module`。 - -当前支持模型类型如下： - -- `llm` 文本生成模型 -- `text_embedding` 文本 Embedding 模型 -- `rerank` Rerank 模型 -- `speech2text` 语音转文字 -- `tts` 文字转语音 -- `moderation` 审查 - -依旧以 `Anthropic` 为例，`Anthropic` 仅支持 LLM，因此在 `model_providers.anthropic` 创建一个 `llm` 为名称的 `module`。 - -对于预定义的模型，我们首先需要在 `llm` `module` 下创建以模型名为文件名称的 YAML 文件，如：`claude-2.1.yaml`。 - -### 准备模型 YAML - -```yaml -model: claude-2.1 # 模型标识 -# 模型展示名称，可设置 en_US 英文、zh_Hans 中文两种语言，zh_Hans 不设置将默认使用 en_US。 -# 也可不设置 label，则使用 model 标识内容。 -label: - en_US: claude-2.1 -model_type: llm # 模型类型，claude-2.1 为 LLM -features: # 支持功能，agent-thought 为支持 Agent 推理，vision 为支持图片理解 -- agent-thought -model_properties: # 模型属性 - mode: chat # LLM 模式，complete 文本补全模型，chat 对话模型 - context_size: 200000 # 支持最大上下文大小 -parameter_rules: # 模型调用参数规则，仅 LLM 需要提供 -- name: temperature # 调用参数变量名 - # 默认预置了 5 种变量内容配置模板，temperature/top_p/max_tokens/presence_penalty/frequency_penalty - # 可在 use_template 中直接设置模板变量名，将会使用 entities.defaults.PARAMETER_RULE_TEMPLATE 中的默认配置 - # 若设置了额外的配置参数，将覆盖默认配置 - use_template: temperature -- name: top_p - use_template: top_p -- name: top_k - label: # 调用参数展示名称 - zh_Hans: 取样数量 - en_US: Top k - type: int # 参数类型，支持 float/int/string/boolean - help: # 帮助信息，描述参数作用 - zh_Hans: 仅从每个后续标记的前 K 个选项中采样。 - en_US: Only sample from the top K options for each subsequent token. - required: false # 是否必填，可不设置 -- name: max_tokens_to_sample - use_template: max_tokens - default: 4096 # 参数默认值 - min: 1 # 参数最小值，仅 float/int 可用 - max: 4096 # 参数最大值，仅 float/int 可用 -pricing: # 价格信息 - input: '8.00' # 输入单价，即 Prompt 单价 - output: '24.00' # 输出单价，即返回内容单价 - unit: '0.000001' # 价格单位，即上述价格为每 100K 的单价 - currency: USD # 价格货币 -``` - -建议将所有模型配置都准备完毕后再开始模型代码的实现。 - -同样，也可以参考 `model_providers` 目录下其他供应商对应模型类型目录下的 YAML 配置信息，完整的 YAML 规则见：[Schema](schema.md#aimodelentity)。 - -### 实现模型调用代码 - -接下来需要在 `llm` `module` 下创建一个同名的 python 文件 `llm.py` 来编写代码实现。 - -在 `llm.py` 中创建一个 Anthropic LLM 类，我们取名为 `AnthropicLargeLanguageModel`（随意），继承 `__base.large_language_model.LargeLanguageModel` 基类，实现以下几个方法： - -- LLM 调用 - - 实现 LLM 调用的核心方法，可同时支持流式和同步返回。 - - ```python - def _invoke(self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, - stream: bool = True, user: Optional[str] = None) \ - -> Union[LLMResult, Generator]: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param model_parameters: model parameters - :param tools: tools for tool calling - :param stop: stop words - :param stream: is stream response - :param user: unique user id - :return: full response or stream response chunk generator result - """ - ``` - - 在实现时，需要注意使用两个函数来返回数据，分别用于处理同步返回和流式返回，因为 Python 会将函数中包含 `yield` 关键字的函数识别为生成器函数，返回的数据类型固定为 `Generator`，因此同步和流式返回需要分别实现，就像下面这样（注意下面例子使用了简化参数，实际实现时需要按照上面的参数列表进行实现）： - - ```python - def _invoke(self, stream: bool, **kwargs) \ - -> Union[LLMResult, Generator]: - if stream: - return self._handle_stream_response(**kwargs) - return self._handle_sync_response(**kwargs) - - def _handle_stream_response(self, **kwargs) -> Generator: - for chunk in response: - yield chunk - def _handle_sync_response(self, **kwargs) -> LLMResult: - return LLMResult(**response) - ``` - -- 预计算输入 tokens - - 若模型未提供预计算 tokens 接口，可直接返回 0。 - - ```python - def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None) -> int: - """ - Get number of tokens for given prompt messages - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param tools: tools for tool calling - :return: - """ - ``` - -- 模型凭据校验 - - 与供应商凭据校验类似，这里针对单个模型进行校验。 - - ```python - def validate_credentials(self, model: str, credentials: dict) -> None: - """ - Validate model credentials - - :param model: model name - :param credentials: model credentials - :return: - """ - ``` - -- 调用异常错误映射表 - - 当模型调用异常时需要映射到 Runtime 指定的 `InvokeError` 类型，方便 Dify 针对不同错误做不同后续处理。 - - Runtime Errors: - - - `InvokeConnectionError` 调用连接错误 - - `InvokeServerUnavailableError ` 调用服务方不可用 - - `InvokeRateLimitError ` 调用达到限额 - - `InvokeAuthorizationError` 调用鉴权失败 - - `InvokeBadRequestError ` 调用传参有误 - - ```python - @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: - """ - Map model invoke error to unified error - The key is the error type thrown to the caller - The value is the error type thrown by the model, - which needs to be converted into a unified error type for the caller. - - :return: Invoke error mapping - """ - ``` - -接口方法说明见：[Interfaces](./interfaces.md)，具体实现可参考：[llm.py](https://github.com/langgenius/dify-runtime/blob/main/lib/model_providers/anthropic/llm/llm.py)。 diff --git a/api/core/model_runtime/docs/zh_Hans/provider_scale_out.md b/api/core/model_runtime/docs/zh_Hans/provider_scale_out.md deleted file mode 100644 index de48b0d11a..0000000000 --- a/api/core/model_runtime/docs/zh_Hans/provider_scale_out.md +++ /dev/null @@ -1,192 +0,0 @@ -## 增加新供应商 - -供应商支持三种模型配置方式： - -- `predefined-model ` 预定义模型 - - 表示用户只需要配置统一的供应商凭据即可使用供应商下的预定义模型。 - -- `customizable-model` 自定义模型 - - 用户需要新增每个模型的凭据配置，如 Xinference，它同时支持 LLM 和 Text Embedding，但是每个模型都有唯一的**model_uid**，如果想要将两者同时接入，就需要为每个模型配置一个**model_uid**。 - -- `fetch-from-remote` 从远程获取 - - 与 `predefined-model` 配置方式一致，只需要配置统一的供应商凭据即可，模型通过凭据信息从供应商获取。 - - 如 OpenAI，我们可以基于 gpt-turbo-3.5 来 Fine Tune 多个模型，而他们都位于同一个**api_key**下，当配置为 `fetch-from-remote` 时，开发者只需要配置统一的**api_key**即可让 DifyRuntime 获取到开发者所有的微调模型并接入 Dify。 - -这三种配置方式**支持共存**，即存在供应商支持 `predefined-model` + `customizable-model` 或 `predefined-model` + `fetch-from-remote` 等，也就是配置了供应商统一凭据可以使用预定义模型和从远程获取的模型，若新增了模型，则可以在此基础上额外使用自定义的模型。 - -## 开始 - -### 介绍 - -#### 名词解释 - -- `module`: 一个`module`即为一个 Python Package，或者通俗一点，称为一个文件夹，里面包含了一个`__init__.py`文件，以及其他的`.py`文件。 - -#### 步骤 - -新增一个供应商主要分为几步，这里简单列出，帮助大家有一个大概的认识，具体的步骤会在下面详细介绍。 - -- 创建供应商 yaml 文件，根据[ProviderSchema](./schema.md#provider)编写 -- 创建供应商代码，实现一个`class`。 -- 根据模型类型，在供应商`module`下创建对应的模型类型 `module`，如`llm`或`text_embedding`。 -- 根据模型类型，在对应的模型`module`下创建同名的代码文件，如`llm.py`，并实现一个`class`。 -- 如果有预定义模型，根据模型名称创建同名的 yaml 文件在模型`module`下，如`claude-2.1.yaml`，根据[AIModelEntity](./schema.md#aimodelentity)编写。 -- 编写测试代码，确保功能可用。 - -### 开始吧 - -增加一个新的供应商需要先确定供应商的英文标识，如 `anthropic`，使用该标识在 `model_providers` 创建以此为名称的 `module`。 - -在此 `module` 下，我们需要先准备供应商的 YAML 配置。 - -#### 准备供应商 YAML - -此处以 `Anthropic` 为例，预设了供应商基础信息、支持的模型类型、配置方式、凭据规则。 - -```YAML -provider: anthropic # 供应商标识 -label: # 供应商展示名称，可设置 en_US 英文、zh_Hans 中文两种语言，zh_Hans 不设置将默认使用 en_US。 - en_US: Anthropic -icon_small: # 供应商小图标，存储在对应供应商实现目录下的 _assets 目录，中英文策略同 label - en_US: icon_s_en.png -icon_large: # 供应商大图标，存储在对应供应商实现目录下的 _assets 目录，中英文策略同 label - en_US: icon_l_en.png -supported_model_types: # 支持的模型类型，Anthropic 仅支持 LLM -- llm -configurate_methods: # 支持的配置方式，Anthropic 仅支持预定义模型 -- predefined-model -provider_credential_schema: # 供应商凭据规则，由于 Anthropic 仅支持预定义模型，则需要定义统一供应商凭据规则 - credential_form_schemas: # 凭据表单项列表 - - variable: anthropic_api_key # 凭据参数变量名 - label: # 展示名称 - en_US: API Key - type: secret-input # 表单类型，此处 secret-input 代表加密信息输入框，编辑时只展示屏蔽后的信息。 - required: true # 是否必填 - placeholder: # PlaceHolder 信息 - zh_Hans: 在此输入您的 API Key - en_US: Enter your API Key - - variable: anthropic_api_url - label: - en_US: API URL - type: text-input # 表单类型，此处 text-input 代表文本输入框 - required: false - placeholder: - zh_Hans: 在此输入您的 API URL - en_US: Enter your API URL -``` - -如果接入的供应商提供自定义模型，比如`OpenAI`提供微调模型，那么我们就需要添加[`model_credential_schema`](./schema.md#modelcredentialschema)，以`OpenAI`为例： - -```yaml -model_credential_schema: - model: # 微调模型名称 - label: - en_US: Model Name - zh_Hans: 模型名称 - placeholder: - en_US: Enter your model name - zh_Hans: 输入模型名称 - credential_form_schemas: - - variable: openai_api_key - label: - en_US: API Key - type: secret-input - required: true - placeholder: - zh_Hans: 在此输入您的 API Key - en_US: Enter your API Key - - variable: openai_organization - label: - zh_Hans: 组织 ID - en_US: Organization - type: text-input - required: false - placeholder: - zh_Hans: 在此输入您的组织 ID - en_US: Enter your Organization ID - - variable: openai_api_base - label: - zh_Hans: API Base - en_US: API Base - type: text-input - required: false - placeholder: - zh_Hans: 在此输入您的 API Base - en_US: Enter your API Base -``` - -也可以参考 `model_providers` 目录下其他供应商目录下的 YAML 配置信息，完整的 YAML 规则见：[Schema](schema.md#provider)。 - -#### 实现供应商代码 - -我们需要在`model_providers`下创建一个同名的 python 文件，如`anthropic.py`，并实现一个`class`，继承`__base.provider.Provider`基类，如`AnthropicProvider`。 - -##### 自定义模型供应商 - -当供应商为 Xinference 等自定义模型供应商时，可跳过该步骤，仅创建一个空的`XinferenceProvider`类即可，并实现一个空的`validate_provider_credentials`方法，该方法并不会被实际使用，仅用作避免抽象类无法实例化。 - -```python -class XinferenceProvider(Provider): - def validate_provider_credentials(self, credentials: dict) -> None: - pass -``` - -##### 预定义模型供应商 - -供应商需要继承 `__base.model_provider.ModelProvider` 基类，实现 `validate_provider_credentials` 供应商统一凭据校验方法即可，可参考 [AnthropicProvider](https://github.com/langgenius/dify-runtime/blob/main/lib/model_providers/anthropic/anthropic.py)。 - -```python -def validate_provider_credentials(self, credentials: dict) -> None: - """ - Validate provider credentials - You can choose any validate_credentials method of model type or implement validate method by yourself, - such as: get model list api - - if validate failed, raise exception - - :param credentials: provider credentials, credentials form defined in `provider_credential_schema`. - """ -``` - -当然也可以先预留 `validate_provider_credentials` 实现，在模型凭据校验方法实现后直接复用。 - -#### 增加模型 - -#### [增加预定义模型 👈🏻](./predefined_model_scale_out.md) - -对于预定义模型，我们可以通过简单定义一个 yaml，并通过实现调用代码来接入。 - -#### [增加自定义模型 👈🏻](./customizable_model_scale_out.md) - -对于自定义模型，我们只需要实现调用代码即可接入，但是它需要处理的参数可能会更加复杂。 - -______________________________________________________________________ - -### 测试 - -为了保证接入供应商/模型的可用性，编写后的每个方法均需要在 `tests` 目录中编写对应的集成测试代码。 - -依旧以 `Anthropic` 为例。 - -在编写测试代码前，需要先在 `.env.example` 新增测试供应商所需要的凭据环境变量，如：`ANTHROPIC_API_KEY`。 - -在执行前需要将 `.env.example` 复制为 `.env` 再执行。 - -#### 编写测试代码 - -在 `tests` 目录下创建供应商同名的 `module`: `anthropic`，继续在此模块中创建 `test_provider.py` 以及对应模型类型的 test py 文件，如下所示： - -```shell -. -├── __init__.py -├── anthropic -│   ├── __init__.py -│   ├── test_llm.py # LLM 测试 -│   └── test_provider.py # 供应商测试 -``` - -针对上面实现的代码的各种情况进行测试代码编写，并测试通过后提交代码。 diff --git a/api/core/model_runtime/docs/zh_Hans/schema.md b/api/core/model_runtime/docs/zh_Hans/schema.md deleted file mode 100644 index e68cb500e1..0000000000 --- a/api/core/model_runtime/docs/zh_Hans/schema.md +++ /dev/null @@ -1,209 +0,0 @@ -# 配置规则 - -- 供应商规则基于 [Provider](#Provider) 实体。 - -- 模型规则基于 [AIModelEntity](#AIModelEntity) 实体。 - -> 以下所有实体均基于 `Pydantic BaseModel`，可在 `entities` 模块中找到对应实体。 - -### Provider - -- `provider` (string) 供应商标识，如：`openai` -- `label` (object) 供应商展示名称，i18n，可设置 `en_US` 英文、`zh_Hans` 中文两种语言 - - `zh_Hans ` (string) [optional] 中文标签名，`zh_Hans` 不设置将默认使用 `en_US`。 - - `en_US` (string) 英文标签名 -- `description` (object) [optional] 供应商描述，i18n - - `zh_Hans` (string) [optional] 中文描述 - - `en_US` (string) 英文描述 -- `icon_small` (string) [optional] 供应商小 ICON，存储在对应供应商实现目录下的 `_assets` 目录，中英文策略同 `label` - - `zh_Hans` (string) [optional] 中文 ICON - - `en_US` (string) 英文 ICON -- `icon_large` (string) [optional] 供应商大 ICON，存储在对应供应商实现目录下的 \_assets 目录，中英文策略同 label - - `zh_Hans `(string) [optional] 中文 ICON - - `en_US` (string) 英文 ICON -- `background` (string) [optional] 背景颜色色值，例：#FFFFFF，为空则展示前端默认色值。 -- `help` (object) [optional] 帮助信息 - - `title` (object) 帮助标题，i18n - - `zh_Hans` (string) [optional] 中文标题 - - `en_US` (string) 英文标题 - - `url` (object) 帮助链接，i18n - - `zh_Hans` (string) [optional] 中文链接 - - `en_US` (string) 英文链接 -- `supported_model_types` (array\[[ModelType](#ModelType)\]) 支持的模型类型 -- `configurate_methods` (array\[[ConfigurateMethod](#ConfigurateMethod)\]) 配置方式 -- `provider_credential_schema` ([ProviderCredentialSchema](#ProviderCredentialSchema)) 供应商凭据规格 -- `model_credential_schema` ([ModelCredentialSchema](#ModelCredentialSchema)) 模型凭据规格 - -### AIModelEntity - -- `model` (string) 模型标识，如：`gpt-3.5-turbo` -- `label` (object) [optional] 模型展示名称，i18n，可设置 `en_US` 英文、`zh_Hans` 中文两种语言 - - `zh_Hans `(string) [optional] 中文标签名 - - `en_US` (string) 英文标签名 -- `model_type` ([ModelType](#ModelType)) 模型类型 -- `features` (array\[[ModelFeature](#ModelFeature)\]) [optional] 支持功能列表 -- `model_properties` (object) 模型属性 - - `mode` ([LLMMode](#LLMMode)) 模式 (模型类型 `llm` 可用) - - `context_size` (int) 上下文大小 (模型类型 `llm` `text-embedding` 可用) - - `max_chunks` (int) 最大分块数量 (模型类型 `text-embedding ` `moderation` 可用) - - `file_upload_limit` (int) 文件最大上传限制，单位：MB。（模型类型 `speech2text` 可用） - - `supported_file_extensions` (string) 支持文件扩展格式，如：mp3,mp4（模型类型 `speech2text` 可用） - - `default_voice` (string) 缺省音色，必选：alloy,echo,fable,onyx,nova,shimmer（模型类型 `tts` 可用） - - `voices` (list) 可选音色列表。 - - `mode` (string) 音色模型。（模型类型 `tts` 可用） - - `name` (string) 音色模型显示名称。（模型类型 `tts` 可用） - - `language` (string) 音色模型支持语言。（模型类型 `tts` 可用） - - `word_limit` (int) 单次转换字数限制，默认按段落分段（模型类型 `tts` 可用） - - `audio_type` (string) 支持音频文件扩展格式，如：mp3,wav（模型类型 `tts` 可用） - - `max_workers` (int) 支持文字音频转换并发任务数（模型类型 `tts` 可用） - - `max_characters_per_chunk` (int) 每块最大字符数 (模型类型 `moderation` 可用) -- `parameter_rules` (array\[[ParameterRule](#ParameterRule)\]) [optional] 模型调用参数规则 -- `pricing` ([PriceConfig](#PriceConfig)) [optional] 价格信息 -- `deprecated` (bool) 是否废弃。若废弃，模型列表将不再展示，但已经配置的可以继续使用，默认 False。 - -### ModelType - -- `llm` 文本生成模型 -- `text-embedding` 文本 Embedding 模型 -- `rerank` Rerank 模型 -- `speech2text` 语音转文字 -- `tts` 文字转语音 -- `moderation` 审查 - -### ConfigurateMethod - -- `predefined-model ` 预定义模型 - - 表示用户只需要配置统一的供应商凭据即可使用供应商下的预定义模型。 - -- `customizable-model` 自定义模型 - - 用户需要新增每个模型的凭据配置。 - -- `fetch-from-remote` 从远程获取 - - 与 `predefined-model` 配置方式一致，只需要配置统一的供应商凭据即可，模型通过凭据信息从供应商获取。 - -### ModelFeature - -- `agent-thought` Agent 推理，一般超过 70B 有思维链能力。 -- `vision` 视觉，即：图像理解。 -- `tool-call` 工具调用 -- `multi-tool-call` 多工具调用 -- `stream-tool-call` 流式工具调用 - -### FetchFrom - -- `predefined-model` 预定义模型 -- `fetch-from-remote` 远程模型 - -### LLMMode - -- `completion` 文本补全 -- `chat` 对话 - -### ParameterRule - -- `name` (string) 调用模型实际参数名 - -- `use_template` (string) [optional] 使用模板 - - 默认预置了 5 种变量内容配置模板： - - - `temperature` - - `top_p` - - `frequency_penalty` - - `presence_penalty` - - `max_tokens` - - 可在 use_template 中直接设置模板变量名，将会使用 entities.defaults.PARAMETER_RULE_TEMPLATE 中的默认配置 - 不用设置除 `name` 和 `use_template` 之外的所有参数，若设置了额外的配置参数，将覆盖默认配置。 - 可参考 `openai/llm/gpt-3.5-turbo.yaml`。 - -- `label` (object) [optional] 标签，i18n - - - `zh_Hans`(string) [optional] 中文标签名 - - `en_US` (string) 英文标签名 - -- `type`(string) [optional] 参数类型 - - - `int` 整数 - - `float` 浮点数 - - `string` 字符串 - - `boolean` 布尔型 - -- `help` (string) [optional] 帮助信息 - - - `zh_Hans` (string) [optional] 中文帮助信息 - - `en_US` (string) 英文帮助信息 - -- `required` (bool) 是否必填，默认 False。 - -- `default`(int/float/string/bool) [optional] 默认值 - -- `min`(int/float) [optional] 最小值，仅数字类型适用 - -- `max`(int/float) [optional] 最大值，仅数字类型适用 - -- `precision`(int) [optional] 精度，保留小数位数，仅数字类型适用 - -- `options` (array[string]) [optional] 下拉选项值，仅当 `type` 为 `string` 时适用，若不设置或为 null 则不限制选项值 - -### PriceConfig - -- `input` (float) 输入单价，即 Prompt 单价 -- `output` (float) 输出单价，即返回内容单价 -- `unit` (float) 价格单位，如以 1M tokens 计价，则单价对应的单位 token 数为 `0.000001` -- `currency` (string) 货币单位 - -### ProviderCredentialSchema - -- `credential_form_schemas` (array\[[CredentialFormSchema](#CredentialFormSchema)\]) 凭据表单规范 - -### ModelCredentialSchema - -- `model` (object) 模型标识，变量名默认 `model` - - `label` (object) 模型表单项展示名称 - - `en_US` (string) 英文 - - `zh_Hans`(string) [optional] 中文 - - `placeholder` (object) 模型提示内容 - - `en_US`(string) 英文 - - `zh_Hans`(string) [optional] 中文 -- `credential_form_schemas` (array\[[CredentialFormSchema](#CredentialFormSchema)\]) 凭据表单规范 - -### CredentialFormSchema - -- `variable` (string) 表单项变量名 -- `label` (object) 表单项标签名 - - `en_US`(string) 英文 - - `zh_Hans` (string) [optional] 中文 -- `type` ([FormType](#FormType)) 表单项类型 -- `required` (bool) 是否必填 -- `default`(string) 默认值 -- `options` (array\[[FormOption](#FormOption)\]) 表单项为 `select` 或 `radio` 专有属性，定义下拉内容 -- `placeholder`(object) 表单项为 `text-input `专有属性，表单项 PlaceHolder - - `en_US`(string) 英文 - - `zh_Hans` (string) [optional] 中文 -- `max_length` (int) 表单项为`text-input`专有属性，定义输入最大长度，0 为不限制。 -- `show_on` (array\[[FormShowOnObject](#FormShowOnObject)\]) 当其他表单项值符合条件时显示，为空则始终显示。 - -### FormType - -- `text-input` 文本输入组件 -- `secret-input` 密码输入组件 -- `select` 单选下拉 -- `radio` Radio 组件 -- `switch` 开关组件，仅支持 `true` 和 `false` - -### FormOption - -- `label` (object) 标签 - - `en_US`(string) 英文 - - `zh_Hans`(string) [optional] 中文 -- `value` (string) 下拉选项值 -- `show_on` (array\[[FormShowOnObject](#FormShowOnObject)\]) 当其他表单项值符合条件时显示，为空则始终显示。 - -### FormShowOnObject - -- `variable` (string) 其他表单项变量名 -- `value` (string) 其他表单项变量值 diff --git a/api/core/ops/tencent_trace/span_builder.py b/api/core/ops/tencent_trace/span_builder.py index 26e8779e3e..db92e9b8bd 100644 --- a/api/core/ops/tencent_trace/span_builder.py +++ b/api/core/ops/tencent_trace/span_builder.py @@ -222,6 +222,59 @@ class TencentSpanBuilder: links=links, ) + @staticmethod + def build_message_llm_span( + trace_info: MessageTraceInfo, trace_id: int, parent_span_id: int, user_id: str + ) -> SpanData: + """Build LLM span for message traces with detailed LLM attributes.""" + status = Status(StatusCode.OK) + if trace_info.error: + status = Status(StatusCode.ERROR, trace_info.error) + + # Extract model information from `metadata`` or `message_data` + trace_metadata = trace_info.metadata or {} + message_data = trace_info.message_data or {} + + model_provider = trace_metadata.get("ls_provider") or ( + message_data.get("model_provider", "") if isinstance(message_data, dict) else "" + ) + model_name = trace_metadata.get("ls_model_name") or ( + message_data.get("model_id", "") if isinstance(message_data, dict) else "" + ) + + inputs_str = str(trace_info.inputs or "") + outputs_str = str(trace_info.outputs or "") + + attributes = { + GEN_AI_SESSION_ID: trace_metadata.get("conversation_id", ""), + GEN_AI_USER_ID: str(user_id), + GEN_AI_SPAN_KIND: GenAISpanKind.GENERATION.value, + GEN_AI_FRAMEWORK: "dify", + GEN_AI_MODEL_NAME: str(model_name), + GEN_AI_PROVIDER: str(model_provider), + GEN_AI_USAGE_INPUT_TOKENS: str(trace_info.message_tokens or 0), + GEN_AI_USAGE_OUTPUT_TOKENS: str(trace_info.answer_tokens or 0), + GEN_AI_USAGE_TOTAL_TOKENS: str(trace_info.total_tokens or 0), + GEN_AI_PROMPT: inputs_str, + GEN_AI_COMPLETION: outputs_str, + INPUT_VALUE: inputs_str, + OUTPUT_VALUE: outputs_str, + } + + if trace_info.is_streaming_request: + attributes[GEN_AI_IS_STREAMING_REQUEST] = "true" + + return SpanData( + trace_id=trace_id, + parent_span_id=parent_span_id, + span_id=TencentTraceUtils.convert_to_span_id(trace_info.message_id, "llm"), + name="GENERATION", + start_time=TencentSpanBuilder._get_time_nanoseconds(trace_info.start_time), + end_time=TencentSpanBuilder._get_time_nanoseconds(trace_info.end_time), + attributes=attributes, + status=status, + ) + @staticmethod def build_tool_span(trace_info: ToolTraceInfo, trace_id: int, parent_span_id: int) -> SpanData: """Build tool span.""" diff --git a/api/core/ops/tencent_trace/tencent_trace.py b/api/core/ops/tencent_trace/tencent_trace.py index 9b3df86e16..3d176da97a 100644 --- a/api/core/ops/tencent_trace/tencent_trace.py +++ b/api/core/ops/tencent_trace/tencent_trace.py @@ -107,9 +107,13 @@ class TencentDataTrace(BaseTraceInstance): links.append(TencentTraceUtils.create_link(trace_info.trace_id)) message_span = TencentSpanBuilder.build_message_span(trace_info, trace_id, str(user_id), links) - self.trace_client.add_span(message_span) + # Add LLM child span with detailed attributes + parent_span_id = TencentTraceUtils.convert_to_span_id(trace_info.message_id, "message") + llm_span = TencentSpanBuilder.build_message_llm_span(trace_info, trace_id, parent_span_id, str(user_id)) + self.trace_client.add_span(llm_span) + self._record_message_llm_metrics(trace_info) # Record trace duration for entry span diff --git a/api/core/rag/datasource/keyword/jieba/jieba_keyword_table_handler.py b/api/core/rag/datasource/keyword/jieba/jieba_keyword_table_handler.py index 81619570f9..57a60e6970 100644 --- a/api/core/rag/datasource/keyword/jieba/jieba_keyword_table_handler.py +++ b/api/core/rag/datasource/keyword/jieba/jieba_keyword_table_handler.py @@ -1,20 +1,110 @@ import re +from operator import itemgetter from typing import cast class JiebaKeywordTableHandler: def __init__(self): + from core.rag.datasource.keyword.jieba.stopwords import STOPWORDS + + tfidf = self._load_tfidf_extractor() + tfidf.stop_words = STOPWORDS # type: ignore[attr-defined] + self._tfidf = tfidf + + def _load_tfidf_extractor(self): + """ + Load jieba TFIDF extractor with fallback strategy. + + Loading Flow: + ┌─────────────────────────────────────────────────────────────────────┐ + │ jieba.analyse.default_tfidf │ + │ exists? │ + └─────────────────────────────────────────────────────────────────────┘ + │ │ + YES NO + │ │ + ▼ ▼ + ┌──────────────────┐ ┌──────────────────────────────────┐ + │ Return default │ │ jieba.analyse.TFIDF exists? │ + │ TFIDF │ └──────────────────────────────────┘ + └──────────────────┘ │ │ + YES NO + │ │ + │ ▼ + │ ┌────────────────────────────┐ + │ │ Try import from │ + │ │ jieba.analyse.tfidf.TFIDF │ + │ └────────────────────────────┘ + │ │ │ + │ SUCCESS FAILED + │ │ │ + ▼ ▼ ▼ + ┌────────────────────────┐ ┌─────────────────┐ + │ Instantiate TFIDF() │ │ Build fallback │ + │ & cache to default │ │ _SimpleTFIDF │ + └────────────────────────┘ └─────────────────┘ + """ import jieba.analyse # type: ignore + tfidf = getattr(jieba.analyse, "default_tfidf", None) + if tfidf is not None: + return tfidf + + tfidf_class = getattr(jieba.analyse, "TFIDF", None) + if tfidf_class is None: + try: + from jieba.analyse.tfidf import TFIDF # type: ignore + + tfidf_class = TFIDF + except Exception: + tfidf_class = None + + if tfidf_class is not None: + tfidf = tfidf_class() + jieba.analyse.default_tfidf = tfidf # type: ignore[attr-defined] + return tfidf + + return self._build_fallback_tfidf() + + @staticmethod + def _build_fallback_tfidf(): + """Fallback lightweight TFIDF for environments missing jieba's TFIDF.""" + import jieba # type: ignore + from core.rag.datasource.keyword.jieba.stopwords import STOPWORDS - jieba.analyse.default_tfidf.stop_words = STOPWORDS # type: ignore + class _SimpleTFIDF: + def __init__(self): + self.stop_words = STOPWORDS + self._lcut = getattr(jieba, "lcut", None) + + def extract_tags(self, sentence: str, top_k: int | None = 20, **kwargs): + # Basic frequency-based keyword extraction as a fallback when TF-IDF is unavailable. + top_k = kwargs.pop("topK", top_k) + cut = getattr(jieba, "cut", None) + if self._lcut: + tokens = self._lcut(sentence) + elif callable(cut): + tokens = list(cut(sentence)) + else: + tokens = re.findall(r"\w+", sentence) + + words = [w for w in tokens if w and w not in self.stop_words] + freq: dict[str, int] = {} + for w in words: + freq[w] = freq.get(w, 0) + 1 + + sorted_words = sorted(freq.items(), key=itemgetter(1), reverse=True) + if top_k is not None: + sorted_words = sorted_words[:top_k] + + return [item[0] for item in sorted_words] + + return _SimpleTFIDF() def extract_keywords(self, text: str, max_keywords_per_chunk: int | None = 10) -> set[str]: """Extract keywords with JIEBA tfidf.""" - import jieba.analyse # type: ignore - - keywords = jieba.analyse.extract_tags( + keywords = self._tfidf.extract_tags( sentence=text, topK=max_keywords_per_chunk, ) diff --git a/api/core/rag/datasource/vdb/oracle/oraclevector.py b/api/core/rag/datasource/vdb/oracle/oraclevector.py index d289cde9e4..d82ab89a34 100644 --- a/api/core/rag/datasource/vdb/oracle/oraclevector.py +++ b/api/core/rag/datasource/vdb/oracle/oraclevector.py @@ -302,8 +302,7 @@ class OracleVector(BaseVector): nltk.data.find("tokenizers/punkt") nltk.data.find("corpora/stopwords") except LookupError: - nltk.download("punkt") - nltk.download("stopwords") + raise LookupError("Unable to find the required NLTK data package: punkt and stopwords") e_str = re.sub(r"[^\w ]", "", query) all_tokens = nltk.word_tokenize(e_str) stop_words = stopwords.words("english") diff --git a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py index 591de01669..2c7bc592c0 100644 --- a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py +++ b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py @@ -167,13 +167,18 @@ class WeaviateVector(BaseVector): try: if not self._client.collections.exists(self._collection_name): + tokenization = ( + wc.Tokenization(dify_config.WEAVIATE_TOKENIZATION) + if dify_config.WEAVIATE_TOKENIZATION + else wc.Tokenization.WORD + ) self._client.collections.create( name=self._collection_name, properties=[ wc.Property( name=Field.TEXT_KEY.value, data_type=wc.DataType.TEXT, - tokenization=wc.Tokenization.WORD, + tokenization=tokenization, ), wc.Property(name="document_id", data_type=wc.DataType.TEXT), wc.Property(name="doc_id", data_type=wc.DataType.TEXT), diff --git a/api/core/tools/entities/tool_bundle.py b/api/core/tools/entities/tool_bundle.py index eba20b07f0..10710c4376 100644 --- a/api/core/tools/entities/tool_bundle.py +++ b/api/core/tools/entities/tool_bundle.py @@ -1,4 +1,6 @@ -from pydantic import BaseModel +from collections.abc import Mapping + +from pydantic import BaseModel, Field from core.tools.entities.tool_entities import ToolParameter @@ -25,3 +27,5 @@ class ApiToolBundle(BaseModel): icon: str | None = None # openapi operation openapi: dict + # output schema + output_schema: Mapping[str, object] = Field(default_factory=dict) diff --git a/api/core/tools/utils/workflow_configuration_sync.py b/api/core/tools/utils/workflow_configuration_sync.py index d16d6fc576..188da0c32d 100644 --- a/api/core/tools/utils/workflow_configuration_sync.py +++ b/api/core/tools/utils/workflow_configuration_sync.py @@ -3,6 +3,7 @@ from typing import Any from core.app.app_config.entities import VariableEntity from core.tools.entities.tool_entities import WorkflowToolParameterConfiguration +from core.workflow.nodes.base.entities import OutputVariableEntity class WorkflowToolConfigurationUtils: @@ -24,6 +25,31 @@ class WorkflowToolConfigurationUtils: return [VariableEntity.model_validate(variable) for variable in start_node.get("data", {}).get("variables", [])] + @classmethod + def get_workflow_graph_output(cls, graph: Mapping[str, Any]) -> Sequence[OutputVariableEntity]: + """ + get workflow graph output + """ + nodes = graph.get("nodes", []) + outputs_by_variable: dict[str, OutputVariableEntity] = {} + variable_order: list[str] = [] + + for node in nodes: + if node.get("data", {}).get("type") != "end": + continue + + for output in node.get("data", {}).get("outputs", []): + entity = OutputVariableEntity.model_validate(output) + variable = entity.variable + + if variable not in variable_order: + variable_order.append(variable) + + # Later end nodes override duplicated variable definitions. + outputs_by_variable[variable] = entity + + return [outputs_by_variable[variable] for variable in variable_order] + @classmethod def check_is_synced( cls, variables: list[VariableEntity], tool_configurations: list[WorkflowToolParameterConfiguration] diff --git a/api/core/tools/workflow_as_tool/provider.py b/api/core/tools/workflow_as_tool/provider.py index c8e91413cd..4852e9d2d8 100644 --- a/api/core/tools/workflow_as_tool/provider.py +++ b/api/core/tools/workflow_as_tool/provider.py @@ -141,6 +141,7 @@ class WorkflowToolProviderController(ToolProviderController): form=parameter.form, llm_description=parameter.description, required=variable.required, + default=variable.default, options=options, placeholder=I18nObject(en_US="", zh_Hans=""), ) @@ -161,6 +162,20 @@ class WorkflowToolProviderController(ToolProviderController): else: raise ValueError("variable not found") + # get output schema from workflow + outputs = WorkflowToolConfigurationUtils.get_workflow_graph_output(graph) + + reserved_keys = {"json", "text", "files"} + + properties = {} + for output in outputs: + if output.variable not in reserved_keys: + properties[output.variable] = { + "type": output.value_type, + "description": "", + } + output_schema = {"type": "object", "properties": properties} + return WorkflowTool( workflow_as_tool_id=db_provider.id, entity=ToolEntity( @@ -176,6 +191,7 @@ class WorkflowToolProviderController(ToolProviderController): llm=db_provider.description, ), parameters=workflow_tool_parameters, + output_schema=output_schema, ), runtime=ToolRuntime( tenant_id=db_provider.tenant_id, diff --git a/api/core/tools/workflow_as_tool/tool.py b/api/core/tools/workflow_as_tool/tool.py index 5703c19c88..1751b45d9b 100644 --- a/api/core/tools/workflow_as_tool/tool.py +++ b/api/core/tools/workflow_as_tool/tool.py @@ -114,6 +114,11 @@ class WorkflowTool(Tool): for file in files: yield self.create_file_message(file) # type: ignore + # traverse `outputs` field and create variable messages + for key, value in outputs.items(): + if key not in {"text", "json", "files"}: + yield self.create_variable_message(variable_name=key, variable_value=value) + self._latest_usage = self._derive_usage_from_result(data) yield self.create_text_message(json.dumps(outputs, ensure_ascii=False)) diff --git a/api/core/trigger/entities/entities.py b/api/core/trigger/entities/entities.py index 49e24fe8b8..89824481b5 100644 --- a/api/core/trigger/entities/entities.py +++ b/api/core/trigger/entities/entities.py @@ -71,6 +71,11 @@ class TriggerProviderIdentity(BaseModel): icon_dark: str | None = Field(default=None, description="The dark icon of the trigger provider") tags: list[str] = Field(default_factory=list, description="The tags of the trigger provider") + @field_validator("tags", mode="before") + @classmethod + def validate_tags(cls, v: list[str] | None) -> list[str]: + return v or [] + class EventIdentity(BaseModel): """ diff --git a/api/core/workflow/entities/__init__.py b/api/core/workflow/entities/__init__.py index f4ce9052e0..be70e467a0 100644 --- a/api/core/workflow/entities/__init__.py +++ b/api/core/workflow/entities/__init__.py @@ -1,17 +1,11 @@ -from ..runtime.graph_runtime_state import GraphRuntimeState -from ..runtime.variable_pool import VariablePool from .agent import AgentNodeStrategyInit from .graph_init_params import GraphInitParams from .workflow_execution import WorkflowExecution from .workflow_node_execution import WorkflowNodeExecution -from .workflow_pause import WorkflowPauseEntity __all__ = [ "AgentNodeStrategyInit", "GraphInitParams", - "GraphRuntimeState", - "VariablePool", "WorkflowExecution", "WorkflowNodeExecution", - "WorkflowPauseEntity", ] diff --git a/api/core/workflow/entities/pause_reason.py b/api/core/workflow/entities/pause_reason.py index 16ad3d639d..c6655b7eab 100644 --- a/api/core/workflow/entities/pause_reason.py +++ b/api/core/workflow/entities/pause_reason.py @@ -1,49 +1,26 @@ from enum import StrEnum, auto -from typing import Annotated, Any, ClassVar, TypeAlias +from typing import Annotated, Literal, TypeAlias -from pydantic import BaseModel, Discriminator, Tag +from pydantic import BaseModel, Field -class _PauseReasonType(StrEnum): +class PauseReasonType(StrEnum): HUMAN_INPUT_REQUIRED = auto() SCHEDULED_PAUSE = auto() -class _PauseReasonBase(BaseModel): - TYPE: ClassVar[_PauseReasonType] +class HumanInputRequired(BaseModel): + TYPE: Literal[PauseReasonType.HUMAN_INPUT_REQUIRED] = PauseReasonType.HUMAN_INPUT_REQUIRED + + form_id: str + # The identifier of the human input node causing the pause. + node_id: str -class HumanInputRequired(_PauseReasonBase): - TYPE = _PauseReasonType.HUMAN_INPUT_REQUIRED - - -class SchedulingPause(_PauseReasonBase): - TYPE = _PauseReasonType.SCHEDULED_PAUSE +class SchedulingPause(BaseModel): + TYPE: Literal[PauseReasonType.SCHEDULED_PAUSE] = PauseReasonType.SCHEDULED_PAUSE message: str -def _get_pause_reason_discriminator(v: Any) -> _PauseReasonType | None: - if isinstance(v, _PauseReasonBase): - return v.TYPE - elif isinstance(v, dict): - reason_type_str = v.get("TYPE") - if reason_type_str is None: - return None - try: - reason_type = _PauseReasonType(reason_type_str) - except ValueError: - return None - return reason_type - else: - # return None if the discriminator value isn't found - return None - - -PauseReason: TypeAlias = Annotated[ - ( - Annotated[HumanInputRequired, Tag(_PauseReasonType.HUMAN_INPUT_REQUIRED)] - | Annotated[SchedulingPause, Tag(_PauseReasonType.SCHEDULED_PAUSE)] - ), - Discriminator(_get_pause_reason_discriminator), -] +PauseReason: TypeAlias = Annotated[HumanInputRequired | SchedulingPause, Field(discriminator="TYPE")] diff --git a/api/core/workflow/graph_engine/domain/graph_execution.py b/api/core/workflow/graph_engine/domain/graph_execution.py index 3d587d6691..9ca607458f 100644 --- a/api/core/workflow/graph_engine/domain/graph_execution.py +++ b/api/core/workflow/graph_engine/domain/graph_execution.py @@ -42,7 +42,7 @@ class GraphExecutionState(BaseModel): completed: bool = Field(default=False) aborted: bool = Field(default=False) paused: bool = Field(default=False) - pause_reason: PauseReason | None = Field(default=None) + pause_reasons: list[PauseReason] = Field(default_factory=list) error: GraphExecutionErrorState | None = Field(default=None) exceptions_count: int = Field(default=0) node_executions: list[NodeExecutionState] = Field(default_factory=list[NodeExecutionState]) @@ -107,7 +107,7 @@ class GraphExecution: completed: bool = False aborted: bool = False paused: bool = False - pause_reason: PauseReason | None = None + pause_reasons: list[PauseReason] = field(default_factory=list) error: Exception | None = None node_executions: dict[str, NodeExecution] = field(default_factory=dict[str, NodeExecution]) exceptions_count: int = 0 @@ -137,10 +137,8 @@ class GraphExecution: raise RuntimeError("Cannot pause execution that has completed") if self.aborted: raise RuntimeError("Cannot pause execution that has been aborted") - if self.paused: - return self.paused = True - self.pause_reason = reason + self.pause_reasons.append(reason) def fail(self, error: Exception) -> None: """Mark the graph execution as failed.""" @@ -195,7 +193,7 @@ class GraphExecution: completed=self.completed, aborted=self.aborted, paused=self.paused, - pause_reason=self.pause_reason, + pause_reasons=self.pause_reasons, error=_serialize_error(self.error), exceptions_count=self.exceptions_count, node_executions=node_states, @@ -221,7 +219,7 @@ class GraphExecution: self.completed = state.completed self.aborted = state.aborted self.paused = state.paused - self.pause_reason = state.pause_reason + self.pause_reasons = state.pause_reasons self.error = _deserialize_error(state.error) self.exceptions_count = state.exceptions_count self.node_executions = { diff --git a/api/core/workflow/graph_engine/event_management/event_manager.py b/api/core/workflow/graph_engine/event_management/event_manager.py index 689cf53cf0..71043b9a43 100644 --- a/api/core/workflow/graph_engine/event_management/event_manager.py +++ b/api/core/workflow/graph_engine/event_management/event_manager.py @@ -110,7 +110,13 @@ class EventManager: """ with self._lock.write_lock(): self._events.append(event) - self._notify_layers(event) + + # NOTE: `_notify_layers` is intentionally called outside the critical section + # to minimize lock contention and avoid blocking other readers or writers. + # + # The public `notify_layers` method also does not use a write lock, + # so protecting `_notify_layers` with a lock here is unnecessary. + self._notify_layers(event) def _get_new_events(self, start_index: int) -> list[GraphEngineEvent]: """ diff --git a/api/core/workflow/graph_engine/graph_engine.py b/api/core/workflow/graph_engine/graph_engine.py index 98e1a20044..a4b2df2a8c 100644 --- a/api/core/workflow/graph_engine/graph_engine.py +++ b/api/core/workflow/graph_engine/graph_engine.py @@ -232,7 +232,7 @@ class GraphEngine: self._graph_execution.start() else: self._graph_execution.paused = False - self._graph_execution.pause_reason = None + self._graph_execution.pause_reasons = [] start_event = GraphRunStartedEvent() self._event_manager.notify_layers(start_event) @@ -246,11 +246,11 @@ class GraphEngine: # Handle completion if self._graph_execution.is_paused: - pause_reason = self._graph_execution.pause_reason - assert pause_reason is not None, "pause_reason should not be None when execution is paused." + pause_reasons = self._graph_execution.pause_reasons + assert pause_reasons, "pause_reasons should not be empty when execution is paused." # Ensure we have a valid PauseReason for the event paused_event = GraphRunPausedEvent( - reason=pause_reason, + reasons=pause_reasons, outputs=self._graph_runtime_state.outputs, ) self._event_manager.notify_layers(paused_event) diff --git a/api/core/workflow/graph_events/graph.py b/api/core/workflow/graph_events/graph.py index 9faafc3173..5d10a76c15 100644 --- a/api/core/workflow/graph_events/graph.py +++ b/api/core/workflow/graph_events/graph.py @@ -45,8 +45,7 @@ class GraphRunAbortedEvent(BaseGraphEvent): class GraphRunPausedEvent(BaseGraphEvent): """Event emitted when a graph run is paused by user command.""" - # reason: str | None = Field(default=None, description="reason for pause") - reason: PauseReason = Field(..., description="reason for pause") + reasons: list[PauseReason] = Field(description="reason for pause", default_factory=list) outputs: dict[str, object] = Field( default_factory=dict, description="Outputs available to the client while the run is paused.", diff --git a/api/core/workflow/nodes/agent/agent_node.py b/api/core/workflow/nodes/agent/agent_node.py index 626ef1df7b..7248f9b1d5 100644 --- a/api/core/workflow/nodes/agent/agent_node.py +++ b/api/core/workflow/nodes/agent/agent_node.py @@ -26,7 +26,6 @@ from core.tools.tool_manager import ToolManager from core.tools.utils.message_transformer import ToolFileMessageTransformer from core.variables.segments import ArrayFileSegment, StringSegment from core.workflow.enums import ( - ErrorStrategy, NodeType, SystemVariableKey, WorkflowNodeExecutionMetadataKey, @@ -40,7 +39,6 @@ from core.workflow.node_events import ( StreamCompletedEvent, ) from core.workflow.nodes.agent.entities import AgentNodeData, AgentOldVersionModelFeatures, ParamsAutoGenerated -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser from core.workflow.runtime import VariablePool @@ -66,7 +64,7 @@ if TYPE_CHECKING: from core.plugin.entities.request import InvokeCredentials -class AgentNode(Node): +class AgentNode(Node[AgentNodeData]): """ Agent Node """ @@ -74,27 +72,6 @@ class AgentNode(Node): node_type = NodeType.AGENT _node_data: AgentNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = AgentNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/answer/answer_node.py b/api/core/workflow/nodes/answer/answer_node.py index 86174c7ea6..0fe40db786 100644 --- a/api/core/workflow/nodes/answer/answer_node.py +++ b/api/core/workflow/nodes/answer/answer_node.py @@ -2,42 +2,20 @@ from collections.abc import Mapping, Sequence from typing import Any from core.variables import ArrayFileSegment, FileSegment, Segment -from core.workflow.enums import ErrorStrategy, NodeExecutionType, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeExecutionType, NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult from core.workflow.nodes.answer.entities import AnswerNodeData -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.base.template import Template from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser -class AnswerNode(Node): +class AnswerNode(Node[AnswerNodeData]): node_type = NodeType.ANSWER execution_type = NodeExecutionType.RESPONSE _node_data: AnswerNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = AnswerNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/base/entities.py b/api/core/workflow/nodes/base/entities.py index 94b0d1d8bc..e816e16d74 100644 --- a/api/core/workflow/nodes/base/entities.py +++ b/api/core/workflow/nodes/base/entities.py @@ -5,7 +5,7 @@ from collections.abc import Sequence from enum import StrEnum from typing import Any, Union -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, field_validator, model_validator from core.workflow.enums import ErrorStrategy @@ -35,6 +35,45 @@ class VariableSelector(BaseModel): value_selector: Sequence[str] +class OutputVariableType(StrEnum): + STRING = "string" + NUMBER = "number" + INTEGER = "integer" + SECRET = "secret" + BOOLEAN = "boolean" + OBJECT = "object" + FILE = "file" + ARRAY = "array" + ARRAY_STRING = "array[string]" + ARRAY_NUMBER = "array[number]" + ARRAY_OBJECT = "array[object]" + ARRAY_BOOLEAN = "array[boolean]" + ARRAY_FILE = "array[file]" + ANY = "any" + ARRAY_ANY = "array[any]" + + +class OutputVariableEntity(BaseModel): + """ + Output Variable Entity. + """ + + variable: str + value_type: OutputVariableType + value_selector: Sequence[str] + + @field_validator("value_type", mode="before") + @classmethod + def normalize_value_type(cls, v: Any) -> Any: + """ + Normalize value_type to handle case-insensitive array types. + Converts 'Array[...]' to 'array[...]' for backward compatibility. + """ + if isinstance(v, str) and v.startswith("Array["): + return v.lower() + return v + + class DefaultValueType(StrEnum): STRING = "string" NUMBER = "number" diff --git a/api/core/workflow/nodes/base/node.py b/api/core/workflow/nodes/base/node.py index eda030699a..bbdd3099da 100644 --- a/api/core/workflow/nodes/base/node.py +++ b/api/core/workflow/nodes/base/node.py @@ -2,7 +2,7 @@ import logging from abc import abstractmethod from collections.abc import Generator, Mapping, Sequence from functools import singledispatchmethod -from typing import Any, ClassVar +from typing import Any, ClassVar, Generic, TypeVar, cast, get_args, get_origin from uuid import uuid4 from core.app.entities.app_invoke_entities import InvokeFrom @@ -49,12 +49,121 @@ from models.enums import UserFrom from .entities import BaseNodeData, RetryConfig +NodeDataT = TypeVar("NodeDataT", bound=BaseNodeData) + logger = logging.getLogger(__name__) -class Node: +class Node(Generic[NodeDataT]): node_type: ClassVar["NodeType"] execution_type: NodeExecutionType = NodeExecutionType.EXECUTABLE + _node_data_type: ClassVar[type[BaseNodeData]] = BaseNodeData + + def __init_subclass__(cls, **kwargs: Any) -> None: + """ + Automatically extract and validate the node data type from the generic parameter. + + When a subclass is defined as `class MyNode(Node[MyNodeData])`, this method: + 1. Inspects `__orig_bases__` to find the `Node[T]` parameterization + 2. Extracts `T` (e.g., `MyNodeData`) from the generic argument + 3. Validates that `T` is a proper `BaseNodeData` subclass + 4. Stores it in `_node_data_type` for automatic hydration in `__init__` + + This eliminates the need for subclasses to manually implement boilerplate + accessor methods like `_get_title()`, `_get_error_strategy()`, etc. + + How it works: + :: + + class CodeNode(Node[CodeNodeData]): + │ │ + │ └─────────────────────────────────┐ + │ │ + ▼ ▼ + ┌─────────────────────────────┐ ┌─────────────────────────────────┐ + │ __orig_bases__ = ( │ │ CodeNodeData(BaseNodeData) │ + │ Node[CodeNodeData], │ │ title: str │ + │ ) │ │ desc: str | None │ + └──────────────┬──────────────┘ │ ... │ + │ └─────────────────────────────────┘ + ▼ ▲ + ┌─────────────────────────────┐ │ + │ get_origin(base) -> Node │ │ + │ get_args(base) -> ( │ │ + │ CodeNodeData, │ ──────────────────────┘ + │ ) │ + └──────────────┬──────────────┘ + │ + ▼ + ┌─────────────────────────────┐ + │ Validate: │ + │ - Is it a type? │ + │ - Is it a BaseNodeData │ + │ subclass? │ + └──────────────┬──────────────┘ + │ + ▼ + ┌─────────────────────────────┐ + │ cls._node_data_type = │ + │ CodeNodeData │ + └─────────────────────────────┘ + + Later, in __init__: + :: + + config["data"] ──► _hydrate_node_data() ──► _node_data_type.model_validate() + │ + ▼ + CodeNodeData instance + (stored in self._node_data) + + Example: + class CodeNode(Node[CodeNodeData]): # CodeNodeData is auto-extracted + node_type = NodeType.CODE + # No need to implement _get_title, _get_error_strategy, etc. + """ + super().__init_subclass__(**kwargs) + + if cls is Node: + return + + node_data_type = cls._extract_node_data_type_from_generic() + + if node_data_type is None: + raise TypeError(f"{cls.__name__} must inherit from Node[T] with a BaseNodeData subtype") + + cls._node_data_type = node_data_type + + @classmethod + def _extract_node_data_type_from_generic(cls) -> type[BaseNodeData] | None: + """ + Extract the node data type from the generic parameter `Node[T]`. + + Inspects `__orig_bases__` to find the `Node[T]` parameterization and extracts `T`. + + Returns: + The extracted BaseNodeData subtype, or None if not found. + + Raises: + TypeError: If the generic argument is invalid (not exactly one argument, + or not a BaseNodeData subtype). + """ + # __orig_bases__ contains the original generic bases before type erasure. + # For `class CodeNode(Node[CodeNodeData])`, this would be `(Node[CodeNodeData],)`. + for base in getattr(cls, "__orig_bases__", ()): # type: ignore[attr-defined] + origin = get_origin(base) # Returns `Node` for `Node[CodeNodeData]` + if origin is Node: + args = get_args(base) # Returns `(CodeNodeData,)` for `Node[CodeNodeData]` + if len(args) != 1: + raise TypeError(f"{cls.__name__} must specify exactly one node data generic argument") + + candidate = args[0] + if not isinstance(candidate, type) or not issubclass(candidate, BaseNodeData): + raise TypeError(f"{cls.__name__} must parameterize Node with a BaseNodeData subtype") + + return candidate + + return None def __init__( self, @@ -63,6 +172,7 @@ class Node: graph_init_params: "GraphInitParams", graph_runtime_state: "GraphRuntimeState", ) -> None: + self._graph_init_params = graph_init_params self.id = id self.tenant_id = graph_init_params.tenant_id self.app_id = graph_init_params.app_id @@ -83,8 +193,24 @@ class Node: self._node_execution_id: str = "" self._start_at = naive_utc_now() - @abstractmethod - def init_node_data(self, data: Mapping[str, Any]) -> None: ... + raw_node_data = config.get("data") or {} + if not isinstance(raw_node_data, Mapping): + raise ValueError("Node config data must be a mapping.") + + self._node_data: NodeDataT = self._hydrate_node_data(raw_node_data) + + self.post_init() + + def post_init(self) -> None: + """Optional hook for subclasses requiring extra initialization.""" + return + + @property + def graph_init_params(self) -> "GraphInitParams": + return self._graph_init_params + + def _hydrate_node_data(self, data: Mapping[str, Any]) -> NodeDataT: + return cast(NodeDataT, self._node_data_type.model_validate(data)) @abstractmethod def _run(self) -> NodeRunResult | Generator[NodeEventBase, None, None]: @@ -273,38 +399,29 @@ class Node: def retry(self) -> bool: return False - # Abstract methods that subclasses must implement to provide access - # to BaseNodeData properties in a type-safe way - - @abstractmethod def _get_error_strategy(self) -> ErrorStrategy | None: """Get the error strategy for this node.""" - ... + return self._node_data.error_strategy - @abstractmethod def _get_retry_config(self) -> RetryConfig: """Get the retry configuration for this node.""" - ... + return self._node_data.retry_config - @abstractmethod def _get_title(self) -> str: """Get the node title.""" - ... + return self._node_data.title - @abstractmethod def _get_description(self) -> str | None: """Get the node description.""" - ... + return self._node_data.desc - @abstractmethod def _get_default_value_dict(self) -> dict[str, Any]: """Get the default values dictionary for this node.""" - ... + return self._node_data.default_value_dict - @abstractmethod def get_base_node_data(self) -> BaseNodeData: """Get the BaseNodeData object for this node.""" - ... + return self._node_data # Public interface properties that delegate to abstract methods @property @@ -332,6 +449,11 @@ class Node: """Get the default values dictionary for this node.""" return self._get_default_value_dict() + @property + def node_data(self) -> NodeDataT: + """Typed access to this node's configuration data.""" + return self._node_data + def _convert_node_run_result_to_graph_node_event(self, result: NodeRunResult) -> GraphNodeEventBase: match result.status: case WorkflowNodeExecutionStatus.FAILED: diff --git a/api/core/workflow/nodes/code/code_node.py b/api/core/workflow/nodes/code/code_node.py index c87cbf9628..4c64f45f04 100644 --- a/api/core/workflow/nodes/code/code_node.py +++ b/api/core/workflow/nodes/code/code_node.py @@ -9,9 +9,8 @@ from core.helper.code_executor.javascript.javascript_code_provider import Javasc from core.helper.code_executor.python3.python3_code_provider import Python3CodeProvider from core.variables.segments import ArrayFileSegment from core.variables.types import SegmentType -from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.code.entities import CodeNodeData @@ -22,32 +21,11 @@ from .exc import ( ) -class CodeNode(Node): +class CodeNode(Node[CodeNodeData]): node_type = NodeType.CODE _node_data: CodeNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = CodeNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def get_default_config(cls, filters: Mapping[str, object] | None = None) -> Mapping[str, object]: """ diff --git a/api/core/workflow/nodes/datasource/datasource_node.py b/api/core/workflow/nodes/datasource/datasource_node.py index 34c1db9468..d8718222f8 100644 --- a/api/core/workflow/nodes/datasource/datasource_node.py +++ b/api/core/workflow/nodes/datasource/datasource_node.py @@ -20,9 +20,8 @@ from core.plugin.impl.exc import PluginDaemonClientSideError from core.variables.segments import ArrayAnySegment from core.variables.variables import ArrayAnyVariable from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus -from core.workflow.enums import ErrorStrategy, NodeExecutionType, NodeType, SystemVariableKey +from core.workflow.enums import NodeExecutionType, NodeType, SystemVariableKey from core.workflow.node_events import NodeRunResult, StreamChunkEvent, StreamCompletedEvent -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser from core.workflow.nodes.tool.exc import ToolFileError @@ -38,7 +37,7 @@ from .entities import DatasourceNodeData from .exc import DatasourceNodeError, DatasourceParameterError -class DatasourceNode(Node): +class DatasourceNode(Node[DatasourceNodeData]): """ Datasource Node """ @@ -47,27 +46,6 @@ class DatasourceNode(Node): node_type = NodeType.DATASOURCE execution_type = NodeExecutionType.ROOT - def init_node_data(self, data: Mapping[str, Any]) -> None: - self._node_data = DatasourceNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - def _run(self) -> Generator: """ Run the datasource node diff --git a/api/core/workflow/nodes/document_extractor/node.py b/api/core/workflow/nodes/document_extractor/node.py index 12cd7e2bd9..17f09e69a2 100644 --- a/api/core/workflow/nodes/document_extractor/node.py +++ b/api/core/workflow/nodes/document_extractor/node.py @@ -25,9 +25,8 @@ from core.file import File, FileTransferMethod, file_manager from core.helper import ssrf_proxy from core.variables import ArrayFileSegment from core.variables.segments import ArrayStringSegment, FileSegment -from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from .entities import DocumentExtractorNodeData @@ -36,7 +35,7 @@ from .exc import DocumentExtractorError, FileDownloadError, TextExtractionError, logger = logging.getLogger(__name__) -class DocumentExtractorNode(Node): +class DocumentExtractorNode(Node[DocumentExtractorNodeData]): """ Extracts text content from various file types. Supports plain text, PDF, and DOC/DOCX files. @@ -46,27 +45,6 @@ class DocumentExtractorNode(Node): _node_data: DocumentExtractorNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = DocumentExtractorNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/end/end_node.py b/api/core/workflow/nodes/end/end_node.py index 7ec74084d0..e188a5616b 100644 --- a/api/core/workflow/nodes/end/end_node.py +++ b/api/core/workflow/nodes/end/end_node.py @@ -1,41 +1,16 @@ -from collections.abc import Mapping -from typing import Any - -from core.workflow.enums import ErrorStrategy, NodeExecutionType, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeExecutionType, NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.base.template import Template from core.workflow.nodes.end.entities import EndNodeData -class EndNode(Node): +class EndNode(Node[EndNodeData]): node_type = NodeType.END execution_type = NodeExecutionType.RESPONSE _node_data: EndNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = EndNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/end/entities.py b/api/core/workflow/nodes/end/entities.py index 79a6928bc6..87a221b5f6 100644 --- a/api/core/workflow/nodes/end/entities.py +++ b/api/core/workflow/nodes/end/entities.py @@ -1,7 +1,6 @@ from pydantic import BaseModel, Field -from core.workflow.nodes.base import BaseNodeData -from core.workflow.nodes.base.entities import VariableSelector +from core.workflow.nodes.base.entities import BaseNodeData, OutputVariableEntity class EndNodeData(BaseNodeData): @@ -9,7 +8,7 @@ class EndNodeData(BaseNodeData): END Node Data. """ - outputs: list[VariableSelector] + outputs: list[OutputVariableEntity] class EndStreamParam(BaseModel): diff --git a/api/core/workflow/nodes/http_request/node.py b/api/core/workflow/nodes/http_request/node.py index 152d3cc562..3114bc3758 100644 --- a/api/core/workflow/nodes/http_request/node.py +++ b/api/core/workflow/nodes/http_request/node.py @@ -7,10 +7,10 @@ from configs import dify_config from core.file import File, FileTransferMethod from core.tools.tool_file_manager import ToolFileManager from core.variables.segments import ArrayFileSegment -from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult from core.workflow.nodes.base import variable_template_parser -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig, VariableSelector +from core.workflow.nodes.base.entities import VariableSelector from core.workflow.nodes.base.node import Node from core.workflow.nodes.http_request.executor import Executor from factories import file_factory @@ -31,32 +31,11 @@ HTTP_REQUEST_DEFAULT_TIMEOUT = HttpRequestNodeTimeout( logger = logging.getLogger(__name__) -class HttpRequestNode(Node): +class HttpRequestNode(Node[HttpRequestNodeData]): node_type = NodeType.HTTP_REQUEST _node_data: HttpRequestNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = HttpRequestNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def get_default_config(cls, filters: Mapping[str, object] | None = None) -> Mapping[str, object]: return { diff --git a/api/core/workflow/nodes/human_input/human_input_node.py b/api/core/workflow/nodes/human_input/human_input_node.py index 2d6d9760af..db2df68f46 100644 --- a/api/core/workflow/nodes/human_input/human_input_node.py +++ b/api/core/workflow/nodes/human_input/human_input_node.py @@ -2,15 +2,14 @@ from collections.abc import Mapping from typing import Any from core.workflow.entities.pause_reason import HumanInputRequired -from core.workflow.enums import ErrorStrategy, NodeExecutionType, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeExecutionType, NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult, PauseRequestedEvent -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from .entities import HumanInputNodeData -class HumanInputNode(Node): +class HumanInputNode(Node[HumanInputNodeData]): node_type = NodeType.HUMAN_INPUT execution_type = NodeExecutionType.BRANCH @@ -28,31 +27,10 @@ class HumanInputNode(Node): _node_data: HumanInputNodeData - def init_node_data(self, data: Mapping[str, Any]) -> None: - self._node_data = HumanInputNodeData(**data) - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - def _run(self): # type: ignore[override] if self._is_completion_ready(): branch_handle = self._resolve_branch_selection() @@ -65,7 +43,8 @@ class HumanInputNode(Node): return self._pause_generator() def _pause_generator(self): - yield PauseRequestedEvent(reason=HumanInputRequired()) + # TODO(QuantumGhost): yield a real form id. + yield PauseRequestedEvent(reason=HumanInputRequired(form_id="test_form_id", node_id=self.id)) def _is_completion_ready(self) -> bool: """Determine whether all required inputs are satisfied.""" diff --git a/api/core/workflow/nodes/if_else/if_else_node.py b/api/core/workflow/nodes/if_else/if_else_node.py index 165e529714..f4c6e1e190 100644 --- a/api/core/workflow/nodes/if_else/if_else_node.py +++ b/api/core/workflow/nodes/if_else/if_else_node.py @@ -3,9 +3,8 @@ from typing import Any, Literal from typing_extensions import deprecated -from core.workflow.enums import ErrorStrategy, NodeExecutionType, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeExecutionType, NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.if_else.entities import IfElseNodeData from core.workflow.runtime import VariablePool @@ -13,33 +12,12 @@ from core.workflow.utils.condition.entities import Condition from core.workflow.utils.condition.processor import ConditionProcessor -class IfElseNode(Node): +class IfElseNode(Node[IfElseNodeData]): node_type = NodeType.IF_ELSE execution_type = NodeExecutionType.BRANCH _node_data: IfElseNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = IfElseNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/iteration/iteration_node.py b/api/core/workflow/nodes/iteration/iteration_node.py index 63e0932a98..9d0a9d48f7 100644 --- a/api/core/workflow/nodes/iteration/iteration_node.py +++ b/api/core/workflow/nodes/iteration/iteration_node.py @@ -14,7 +14,6 @@ from core.variables.segments import ArrayAnySegment, ArraySegment from core.variables.variables import VariableUnion from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID from core.workflow.enums import ( - ErrorStrategy, NodeExecutionType, NodeType, WorkflowNodeExecutionMetadataKey, @@ -36,7 +35,6 @@ from core.workflow.node_events import ( StreamCompletedEvent, ) from core.workflow.nodes.base import LLMUsageTrackingMixin -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.iteration.entities import ErrorHandleMode, IterationNodeData from core.workflow.runtime import VariablePool @@ -60,7 +58,7 @@ logger = logging.getLogger(__name__) EmptyArraySegment = NewType("EmptyArraySegment", ArraySegment) -class IterationNode(LLMUsageTrackingMixin, Node): +class IterationNode(LLMUsageTrackingMixin, Node[IterationNodeData]): """ Iteration Node. """ @@ -69,27 +67,6 @@ class IterationNode(LLMUsageTrackingMixin, Node): execution_type = NodeExecutionType.CONTAINER _node_data: IterationNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = IterationNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def get_default_config(cls, filters: Mapping[str, object] | None = None) -> Mapping[str, object]: return { diff --git a/api/core/workflow/nodes/iteration/iteration_start_node.py b/api/core/workflow/nodes/iteration/iteration_start_node.py index 90b7f4539b..9767bd8d59 100644 --- a/api/core/workflow/nodes/iteration/iteration_start_node.py +++ b/api/core/workflow/nodes/iteration/iteration_start_node.py @@ -1,14 +1,10 @@ -from collections.abc import Mapping -from typing import Any - -from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.iteration.entities import IterationStartNodeData -class IterationStartNode(Node): +class IterationStartNode(Node[IterationStartNodeData]): """ Iteration Start Node. """ @@ -17,27 +13,6 @@ class IterationStartNode(Node): _node_data: IterationStartNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = IterationStartNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/knowledge_index/knowledge_index_node.py b/api/core/workflow/nodes/knowledge_index/knowledge_index_node.py index 2ba1e5e1c5..c222bd9712 100644 --- a/api/core/workflow/nodes/knowledge_index/knowledge_index_node.py +++ b/api/core/workflow/nodes/knowledge_index/knowledge_index_node.py @@ -10,9 +10,8 @@ from core.app.entities.app_invoke_entities import InvokeFrom from core.rag.index_processor.index_processor_factory import IndexProcessorFactory from core.rag.retrieval.retrieval_methods import RetrievalMethod from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus -from core.workflow.enums import ErrorStrategy, NodeExecutionType, NodeType, SystemVariableKey +from core.workflow.enums import NodeExecutionType, NodeType, SystemVariableKey from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.base.template import Template from core.workflow.runtime import VariablePool @@ -35,32 +34,11 @@ default_retrieval_model = { } -class KnowledgeIndexNode(Node): +class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]): _node_data: KnowledgeIndexNodeData node_type = NodeType.KNOWLEDGE_INDEX execution_type = NodeExecutionType.RESPONSE - def init_node_data(self, data: Mapping[str, Any]) -> None: - self._node_data = KnowledgeIndexNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - def _run(self) -> NodeRunResult: # type: ignore node_data = self._node_data variable_pool = self.graph_runtime_state.variable_pool diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index e8ee44d5a9..99bb058c4b 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -30,14 +30,12 @@ from core.variables import ( from core.variables.segments import ArrayObjectSegment from core.workflow.entities import GraphInitParams from core.workflow.enums import ( - ErrorStrategy, NodeType, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus, ) from core.workflow.node_events import ModelInvokeCompletedEvent, NodeRunResult from core.workflow.nodes.base import LLMUsageTrackingMixin -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.knowledge_retrieval.template_prompts import ( METADATA_FILTER_ASSISTANT_PROMPT_1, @@ -82,7 +80,7 @@ default_retrieval_model = { } -class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node): +class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeData]): node_type = NodeType.KNOWLEDGE_RETRIEVAL _node_data: KnowledgeRetrievalNodeData @@ -118,27 +116,6 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node): ) self._llm_file_saver = llm_file_saver - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = KnowledgeRetrievalNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls): return "1" diff --git a/api/core/workflow/nodes/list_operator/node.py b/api/core/workflow/nodes/list_operator/node.py index 180eb2ad90..ab63951082 100644 --- a/api/core/workflow/nodes/list_operator/node.py +++ b/api/core/workflow/nodes/list_operator/node.py @@ -1,12 +1,11 @@ -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Sequence from typing import Any, TypeAlias, TypeVar from core.file import File from core.variables import ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment from core.variables.segments import ArrayAnySegment, ArrayBooleanSegment, ArraySegment -from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from .entities import FilterOperator, ListOperatorNodeData, Order @@ -35,32 +34,11 @@ def _negation(filter_: Callable[[_T], bool]) -> Callable[[_T], bool]: return wrapper -class ListOperatorNode(Node): +class ListOperatorNode(Node[ListOperatorNodeData]): node_type = NodeType.LIST_OPERATOR _node_data: ListOperatorNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = ListOperatorNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" @@ -229,6 +207,8 @@ def _get_file_extract_string_func(*, key: str) -> Callable[[File], str]: return lambda x: x.transfer_method case "url": return lambda x: x.remote_url or "" + case "related_id": + return lambda x: x.related_id or "" case _: raise InvalidKeyError(f"Invalid key: {key}") @@ -299,7 +279,7 @@ def _get_boolean_filter_func(*, condition: FilterOperator, value: bool) -> Calla def _get_file_filter_func(*, key: str, condition: str, value: str | Sequence[str]) -> Callable[[File], bool]: extract_func: Callable[[File], Any] - if key in {"name", "extension", "mime_type", "url"} and isinstance(value, str): + if key in {"name", "extension", "mime_type", "url", "related_id"} and isinstance(value, str): extract_func = _get_file_extract_string_func(key=key) return lambda x: _get_string_filter_func(condition=condition, value=value)(extract_func(x)) if key in {"type", "transfer_method"}: @@ -358,7 +338,7 @@ def _ge(value: int | float) -> Callable[[int | float], bool]: def _order_file(*, order: Order, order_by: str = "", array: Sequence[File]): extract_func: Callable[[File], Any] - if order_by in {"name", "type", "extension", "mime_type", "transfer_method", "url"}: + if order_by in {"name", "type", "extension", "mime_type", "transfer_method", "url", "related_id"}: extract_func = _get_file_extract_string_func(key=order_by) return sorted(array, key=lambda x: extract_func(x), reverse=order == Order.DESC) elif order_by == "size": diff --git a/api/core/workflow/nodes/llm/node.py b/api/core/workflow/nodes/llm/node.py index 06c9beaed2..44a9ed95d9 100644 --- a/api/core/workflow/nodes/llm/node.py +++ b/api/core/workflow/nodes/llm/node.py @@ -55,7 +55,6 @@ from core.variables import ( from core.workflow.constants import SYSTEM_VARIABLE_NODE_ID from core.workflow.entities import GraphInitParams from core.workflow.enums import ( - ErrorStrategy, NodeType, SystemVariableKey, WorkflowNodeExecutionMetadataKey, @@ -69,7 +68,7 @@ from core.workflow.node_events import ( StreamChunkEvent, StreamCompletedEvent, ) -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig, VariableSelector +from core.workflow.nodes.base.entities import VariableSelector from core.workflow.nodes.base.node import Node from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser from core.workflow.runtime import VariablePool @@ -100,7 +99,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -class LLMNode(Node): +class LLMNode(Node[LLMNodeData]): node_type = NodeType.LLM _node_data: LLMNodeData @@ -139,27 +138,6 @@ class LLMNode(Node): ) self._llm_file_saver = llm_file_saver - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = LLMNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/loop/loop_end_node.py b/api/core/workflow/nodes/loop/loop_end_node.py index e5bce1230c..bdcae5c6fb 100644 --- a/api/core/workflow/nodes/loop/loop_end_node.py +++ b/api/core/workflow/nodes/loop/loop_end_node.py @@ -1,14 +1,10 @@ -from collections.abc import Mapping -from typing import Any - -from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.loop.entities import LoopEndNodeData -class LoopEndNode(Node): +class LoopEndNode(Node[LoopEndNodeData]): """ Loop End Node. """ @@ -17,27 +13,6 @@ class LoopEndNode(Node): _node_data: LoopEndNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = LoopEndNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/loop/loop_node.py b/api/core/workflow/nodes/loop/loop_node.py index 60baed1ed5..ce7245952c 100644 --- a/api/core/workflow/nodes/loop/loop_node.py +++ b/api/core/workflow/nodes/loop/loop_node.py @@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Any, Literal, cast from core.model_runtime.entities.llm_entities import LLMUsage from core.variables import Segment, SegmentType from core.workflow.enums import ( - ErrorStrategy, NodeExecutionType, NodeType, WorkflowNodeExecutionMetadataKey, @@ -29,7 +28,6 @@ from core.workflow.node_events import ( StreamCompletedEvent, ) from core.workflow.nodes.base import LLMUsageTrackingMixin -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.loop.entities import LoopNodeData, LoopVariableData from core.workflow.utils.condition.processor import ConditionProcessor @@ -42,7 +40,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -class LoopNode(LLMUsageTrackingMixin, Node): +class LoopNode(LLMUsageTrackingMixin, Node[LoopNodeData]): """ Loop Node. """ @@ -51,27 +49,6 @@ class LoopNode(LLMUsageTrackingMixin, Node): _node_data: LoopNodeData execution_type = NodeExecutionType.CONTAINER - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = LoopNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/loop/loop_start_node.py b/api/core/workflow/nodes/loop/loop_start_node.py index e065dc90a0..f9df4fa3a6 100644 --- a/api/core/workflow/nodes/loop/loop_start_node.py +++ b/api/core/workflow/nodes/loop/loop_start_node.py @@ -1,14 +1,10 @@ -from collections.abc import Mapping -from typing import Any - -from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.loop.entities import LoopStartNodeData -class LoopStartNode(Node): +class LoopStartNode(Node[LoopStartNodeData]): """ Loop Start Node. """ @@ -17,27 +13,6 @@ class LoopStartNode(Node): _node_data: LoopStartNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = LoopStartNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/node_factory.py b/api/core/workflow/nodes/node_factory.py index 84f63d57eb..5fc363257b 100644 --- a/api/core/workflow/nodes/node_factory.py +++ b/api/core/workflow/nodes/node_factory.py @@ -69,17 +69,9 @@ class DifyNodeFactory(NodeFactory): raise ValueError(f"No latest version class found for node type: {node_type}") # Create node instance - node_instance = node_class( + return node_class( id=node_id, config=node_config, graph_init_params=self.graph_init_params, graph_runtime_state=self.graph_runtime_state, ) - - # Initialize node with provided data - node_data = node_config.get("data", {}) - if not is_str_dict(node_data): - raise ValueError(f"Node {node_id} missing data information") - node_instance.init_node_data(node_data) - - return node_instance diff --git a/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py b/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py index e250650fef..e053e6c4a3 100644 --- a/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py +++ b/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py @@ -27,10 +27,9 @@ from core.prompt.entities.advanced_prompt_entities import ChatModelMessage, Comp from core.prompt.simple_prompt_transform import ModelMode from core.prompt.utils.prompt_message_util import PromptMessageUtil from core.variables.types import ArrayValidation, SegmentType -from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeType, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult from core.workflow.nodes.base import variable_template_parser -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.llm import ModelConfig, llm_utils from core.workflow.runtime import VariablePool @@ -84,7 +83,7 @@ def extract_json(text): return None -class ParameterExtractorNode(Node): +class ParameterExtractorNode(Node[ParameterExtractorNodeData]): """ Parameter Extractor Node. """ @@ -93,27 +92,6 @@ class ParameterExtractorNode(Node): _node_data: ParameterExtractorNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = ParameterExtractorNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - _model_instance: ModelInstance | None = None _model_config: ModelConfigWithCredentialsEntity | None = None diff --git a/api/core/workflow/nodes/question_classifier/question_classifier_node.py b/api/core/workflow/nodes/question_classifier/question_classifier_node.py index 948a1cead7..36a692d109 100644 --- a/api/core/workflow/nodes/question_classifier/question_classifier_node.py +++ b/api/core/workflow/nodes/question_classifier/question_classifier_node.py @@ -13,14 +13,13 @@ from core.prompt.simple_prompt_transform import ModelMode from core.prompt.utils.prompt_message_util import PromptMessageUtil from core.workflow.entities import GraphInitParams from core.workflow.enums import ( - ErrorStrategy, NodeExecutionType, NodeType, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus, ) from core.workflow.node_events import ModelInvokeCompletedEvent, NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig, VariableSelector +from core.workflow.nodes.base.entities import VariableSelector from core.workflow.nodes.base.node import Node from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser from core.workflow.nodes.llm import LLMNode, LLMNodeChatModelMessage, LLMNodeCompletionModelPromptTemplate, llm_utils @@ -44,7 +43,7 @@ if TYPE_CHECKING: from core.workflow.runtime import GraphRuntimeState -class QuestionClassifierNode(Node): +class QuestionClassifierNode(Node[QuestionClassifierNodeData]): node_type = NodeType.QUESTION_CLASSIFIER execution_type = NodeExecutionType.BRANCH @@ -78,27 +77,6 @@ class QuestionClassifierNode(Node): ) self._llm_file_saver = llm_file_saver - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = QuestionClassifierNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls): return "1" diff --git a/api/core/workflow/nodes/start/start_node.py b/api/core/workflow/nodes/start/start_node.py index 3b134be1a1..634d6abd09 100644 --- a/api/core/workflow/nodes/start/start_node.py +++ b/api/core/workflow/nodes/start/start_node.py @@ -1,41 +1,16 @@ -from collections.abc import Mapping -from typing import Any - from core.workflow.constants import SYSTEM_VARIABLE_NODE_ID -from core.workflow.enums import ErrorStrategy, NodeExecutionType, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeExecutionType, NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.start.entities import StartNodeData -class StartNode(Node): +class StartNode(Node[StartNodeData]): node_type = NodeType.START execution_type = NodeExecutionType.ROOT _node_data: StartNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = StartNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/template_transform/template_transform_node.py b/api/core/workflow/nodes/template_transform/template_transform_node.py index 254a8318b5..917680c428 100644 --- a/api/core/workflow/nodes/template_transform/template_transform_node.py +++ b/api/core/workflow/nodes/template_transform/template_transform_node.py @@ -3,41 +3,19 @@ from typing import Any from configs import dify_config from core.helper.code_executor.code_executor import CodeExecutionError, CodeExecutor, CodeLanguage -from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.template_transform.entities import TemplateTransformNodeData MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH = dify_config.TEMPLATE_TRANSFORM_MAX_LENGTH -class TemplateTransformNode(Node): +class TemplateTransformNode(Node[TemplateTransformNodeData]): node_type = NodeType.TEMPLATE_TRANSFORM _node_data: TemplateTransformNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = TemplateTransformNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def get_default_config(cls, filters: Mapping[str, object] | None = None) -> Mapping[str, object]: """ diff --git a/api/core/workflow/nodes/tool/tool_node.py b/api/core/workflow/nodes/tool/tool_node.py index 799ad9b92f..2a92292781 100644 --- a/api/core/workflow/nodes/tool/tool_node.py +++ b/api/core/workflow/nodes/tool/tool_node.py @@ -16,14 +16,12 @@ from core.tools.workflow_as_tool.tool import WorkflowTool from core.variables.segments import ArrayAnySegment, ArrayFileSegment from core.variables.variables import ArrayAnyVariable from core.workflow.enums import ( - ErrorStrategy, NodeType, SystemVariableKey, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus, ) from core.workflow.node_events import NodeEventBase, NodeRunResult, StreamChunkEvent, StreamCompletedEvent -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser from extensions.ext_database import db @@ -42,7 +40,7 @@ if TYPE_CHECKING: from core.workflow.runtime import VariablePool -class ToolNode(Node): +class ToolNode(Node[ToolNodeData]): """ Tool Node """ @@ -51,9 +49,6 @@ class ToolNode(Node): _node_data: ToolNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = ToolNodeData.model_validate(data) - @classmethod def version(cls) -> str: return "1" @@ -329,7 +324,15 @@ class ToolNode(Node): json.append(message.message.json_object) elif message.type == ToolInvokeMessage.MessageType.LINK: assert isinstance(message.message, ToolInvokeMessage.TextMessage) - stream_text = f"Link: {message.message.text}\n" + + # Check if this LINK message is a file link + file_obj = (message.meta or {}).get("file") + if isinstance(file_obj, File): + files.append(file_obj) + stream_text = f"File: {message.message.text}\n" + else: + stream_text = f"Link: {message.message.text}\n" + text += stream_text yield StreamChunkEvent( selector=[node_id, "text"], @@ -490,24 +493,6 @@ class ToolNode(Node): return result - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @property def retry(self) -> bool: return self._node_data.retry_config.retry_enabled diff --git a/api/core/workflow/nodes/trigger_plugin/trigger_event_node.py b/api/core/workflow/nodes/trigger_plugin/trigger_event_node.py index c4c2ff87db..d745c06522 100644 --- a/api/core/workflow/nodes/trigger_plugin/trigger_event_node.py +++ b/api/core/workflow/nodes/trigger_plugin/trigger_event_node.py @@ -1,43 +1,18 @@ from collections.abc import Mapping -from typing import Any from core.workflow.constants import SYSTEM_VARIABLE_NODE_ID from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus -from core.workflow.enums import ErrorStrategy, NodeExecutionType, NodeType +from core.workflow.enums import NodeExecutionType, NodeType from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from .entities import TriggerEventNodeData -class TriggerEventNode(Node): +class TriggerEventNode(Node[TriggerEventNodeData]): node_type = NodeType.TRIGGER_PLUGIN execution_type = NodeExecutionType.ROOT - _node_data: TriggerEventNodeData - - def init_node_data(self, data: Mapping[str, Any]) -> None: - self._node_data = TriggerEventNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def get_default_config(cls, filters: Mapping[str, object] | None = None) -> Mapping[str, object]: return { diff --git a/api/core/workflow/nodes/trigger_schedule/trigger_schedule_node.py b/api/core/workflow/nodes/trigger_schedule/trigger_schedule_node.py index 98a841d1be..fb5c8a4dce 100644 --- a/api/core/workflow/nodes/trigger_schedule/trigger_schedule_node.py +++ b/api/core/workflow/nodes/trigger_schedule/trigger_schedule_node.py @@ -1,42 +1,17 @@ from collections.abc import Mapping -from typing import Any from core.workflow.constants import SYSTEM_VARIABLE_NODE_ID from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus -from core.workflow.enums import ErrorStrategy, NodeExecutionType, NodeType +from core.workflow.enums import NodeExecutionType, NodeType from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.trigger_schedule.entities import TriggerScheduleNodeData -class TriggerScheduleNode(Node): +class TriggerScheduleNode(Node[TriggerScheduleNodeData]): node_type = NodeType.TRIGGER_SCHEDULE execution_type = NodeExecutionType.ROOT - _node_data: TriggerScheduleNodeData - - def init_node_data(self, data: Mapping[str, Any]) -> None: - self._node_data = TriggerScheduleNodeData(**data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/trigger_webhook/node.py b/api/core/workflow/nodes/trigger_webhook/node.py index 15009f90d0..4bc6a82349 100644 --- a/api/core/workflow/nodes/trigger_webhook/node.py +++ b/api/core/workflow/nodes/trigger_webhook/node.py @@ -3,41 +3,17 @@ from typing import Any from core.workflow.constants import SYSTEM_VARIABLE_NODE_ID from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus -from core.workflow.enums import ErrorStrategy, NodeExecutionType, NodeType +from core.workflow.enums import NodeExecutionType, NodeType from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from .entities import ContentType, WebhookData -class TriggerWebhookNode(Node): +class TriggerWebhookNode(Node[WebhookData]): node_type = NodeType.TRIGGER_WEBHOOK execution_type = NodeExecutionType.ROOT - _node_data: WebhookData - - def init_node_data(self, data: Mapping[str, Any]) -> None: - self._node_data = WebhookData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def get_default_config(cls, filters: Mapping[str, object] | None = None) -> Mapping[str, object]: return { diff --git a/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py b/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py index 0ac0d3d858..679e001e79 100644 --- a/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py +++ b/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py @@ -1,40 +1,17 @@ from collections.abc import Mapping -from typing import Any from core.variables.segments import Segment -from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.variable_aggregator.entities import VariableAssignerNodeData -class VariableAggregatorNode(Node): +class VariableAggregatorNode(Node[VariableAssignerNodeData]): node_type = NodeType.VARIABLE_AGGREGATOR _node_data: VariableAssignerNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = VariableAssignerNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - @classmethod def version(cls) -> str: return "1" diff --git a/api/core/workflow/nodes/variable_assigner/v1/node.py b/api/core/workflow/nodes/variable_assigner/v1/node.py index 3a0793f092..f07b5760fd 100644 --- a/api/core/workflow/nodes/variable_assigner/v1/node.py +++ b/api/core/workflow/nodes/variable_assigner/v1/node.py @@ -5,9 +5,8 @@ from core.variables import SegmentType, Variable from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID from core.workflow.conversation_variable_updater import ConversationVariableUpdater from core.workflow.entities import GraphInitParams -from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.variable_assigner.common import helpers as common_helpers from core.workflow.nodes.variable_assigner.common.exc import VariableOperatorNodeError @@ -22,33 +21,12 @@ if TYPE_CHECKING: _CONV_VAR_UPDATER_FACTORY: TypeAlias = Callable[[], ConversationVariableUpdater] -class VariableAssignerNode(Node): +class VariableAssignerNode(Node[VariableAssignerData]): node_type = NodeType.VARIABLE_ASSIGNER _conv_var_updater_factory: _CONV_VAR_UPDATER_FACTORY _node_data: VariableAssignerData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = VariableAssignerData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - def __init__( self, id: str, diff --git a/api/core/workflow/nodes/variable_assigner/v2/node.py b/api/core/workflow/nodes/variable_assigner/v2/node.py index f15924d78f..e7150393d5 100644 --- a/api/core/workflow/nodes/variable_assigner/v2/node.py +++ b/api/core/workflow/nodes/variable_assigner/v2/node.py @@ -7,9 +7,8 @@ from core.variables import SegmentType, Variable from core.variables.consts import SELECTORS_LENGTH from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID from core.workflow.conversation_variable_updater import ConversationVariableUpdater -from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus from core.workflow.node_events import NodeRunResult -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig from core.workflow.nodes.base.node import Node from core.workflow.nodes.variable_assigner.common import helpers as common_helpers from core.workflow.nodes.variable_assigner.common.exc import VariableOperatorNodeError @@ -51,32 +50,11 @@ def _source_mapping_from_item(mapping: MutableMapping[str, Sequence[str]], node_ mapping[key] = selector -class VariableAssignerNode(Node): +class VariableAssignerNode(Node[VariableAssignerNodeData]): node_type = NodeType.VARIABLE_ASSIGNER _node_data: VariableAssignerNodeData - def init_node_data(self, data: Mapping[str, Any]): - self._node_data = VariableAssignerNodeData.model_validate(data) - - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._node_data.error_strategy - - def _get_retry_config(self) -> RetryConfig: - return self._node_data.retry_config - - def _get_title(self) -> str: - return self._node_data.title - - def _get_description(self) -> str | None: - return self._node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._node_data - def blocks_variable_output(self, variable_selectors: set[tuple[str, ...]]) -> bool: """ Check if this Variable Assigner node blocks the output of specific variables. diff --git a/api/core/workflow/runtime/graph_runtime_state.py b/api/core/workflow/runtime/graph_runtime_state.py index 0fbc8ab23e..1561b789df 100644 --- a/api/core/workflow/runtime/graph_runtime_state.py +++ b/api/core/workflow/runtime/graph_runtime_state.py @@ -10,6 +10,7 @@ from typing import Any, Protocol from pydantic.json import pydantic_encoder from core.model_runtime.entities.llm_entities import LLMUsage +from core.workflow.entities.pause_reason import PauseReason from core.workflow.runtime.variable_pool import VariablePool @@ -46,7 +47,11 @@ class ReadyQueueProtocol(Protocol): class GraphExecutionProtocol(Protocol): - """Structural interface for graph execution aggregate.""" + """Structural interface for graph execution aggregate. + + Defines the minimal set of attributes and methods required from a GraphExecution entity + for runtime orchestration and state management. + """ workflow_id: str started: bool @@ -54,6 +59,7 @@ class GraphExecutionProtocol(Protocol): aborted: bool error: Exception | None exceptions_count: int + pause_reasons: list[PauseReason] def start(self) -> None: """Transition execution into the running state.""" diff --git a/api/core/workflow/workflow_entry.py b/api/core/workflow/workflow_entry.py index a6c6784e39..d4ec29518a 100644 --- a/api/core/workflow/workflow_entry.py +++ b/api/core/workflow/workflow_entry.py @@ -159,7 +159,6 @@ class WorkflowEntry: graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - node.init_node_data(node_config_data) try: # variable selector to variable mapping @@ -303,7 +302,6 @@ class WorkflowEntry: graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - node.init_node_data(node_data) try: # variable selector to variable mapping diff --git a/api/extensions/ext_storage.py b/api/extensions/ext_storage.py index a609f13dbc..6df0879694 100644 --- a/api/extensions/ext_storage.py +++ b/api/extensions/ext_storage.py @@ -112,7 +112,7 @@ class Storage: def exists(self, filename): return self.storage_runner.exists(filename) - def delete(self, filename): + def delete(self, filename: str): return self.storage_runner.delete(filename) def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]: diff --git a/api/factories/file_factory.py b/api/factories/file_factory.py index 2316e45179..737a79f2b0 100644 --- a/api/factories/file_factory.py +++ b/api/factories/file_factory.py @@ -1,5 +1,6 @@ import mimetypes import os +import re import urllib.parse import uuid from collections.abc import Callable, Mapping, Sequence @@ -268,15 +269,47 @@ def _build_from_remote_url( def _extract_filename(url_path: str, content_disposition: str | None) -> str | None: - filename = None + filename: str | None = None # Try to extract from Content-Disposition header first if content_disposition: - _, params = parse_options_header(content_disposition) - # RFC 5987 https://datatracker.ietf.org/doc/html/rfc5987: filename* takes precedence over filename - filename = params.get("filename*") or params.get("filename") + # Manually extract filename* parameter since parse_options_header doesn't support it + filename_star_match = re.search(r"filename\*=([^;]+)", content_disposition) + if filename_star_match: + raw_star = filename_star_match.group(1).strip() + # Remove trailing quotes if present + raw_star = raw_star.removesuffix('"') + # format: charset'lang'value + try: + parts = raw_star.split("'", 2) + charset = (parts[0] or "utf-8").lower() if len(parts) >= 1 else "utf-8" + value = parts[2] if len(parts) == 3 else parts[-1] + filename = urllib.parse.unquote(value, encoding=charset, errors="replace") + except Exception: + # Fallback: try to extract value after the last single quote + if "''" in raw_star: + filename = urllib.parse.unquote(raw_star.split("''")[-1]) + else: + filename = urllib.parse.unquote(raw_star) + + if not filename: + # Fallback to regular filename parameter + _, params = parse_options_header(content_disposition) + raw = params.get("filename") + if raw: + # Strip surrounding quotes and percent-decode if present + if len(raw) >= 2 and raw[0] == raw[-1] == '"': + raw = raw[1:-1] + filename = urllib.parse.unquote(raw) # Fallback to URL path if no filename from header if not filename: - filename = os.path.basename(url_path) + candidate = os.path.basename(url_path) + filename = urllib.parse.unquote(candidate) if candidate else None + # Defense-in-depth: ensure basename only + if filename: + filename = os.path.basename(filename) + # Return None if filename is empty or only whitespace + if not filename or not filename.strip(): + filename = None return filename or None diff --git a/api/migrations/versions/2025_11_18_1859-7bb281b7a422_add_workflow_pause_reasons_table.py b/api/migrations/versions/2025_11_18_1859-7bb281b7a422_add_workflow_pause_reasons_table.py new file mode 100644 index 0000000000..8478820999 --- /dev/null +++ b/api/migrations/versions/2025_11_18_1859-7bb281b7a422_add_workflow_pause_reasons_table.py @@ -0,0 +1,41 @@ +"""Add workflow_pauses_reasons table + +Revision ID: 7bb281b7a422 +Revises: 09cfdda155d1 +Create Date: 2025-11-18 18:59:26.999572 + +""" + +from alembic import op +import models as models +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "7bb281b7a422" +down_revision = "09cfdda155d1" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "workflow_pause_reasons", + sa.Column("id", models.types.StringUUID(), nullable=False), + sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), + sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), + + sa.Column("pause_id", models.types.StringUUID(), nullable=False), + sa.Column("type_", sa.String(20), nullable=False), + sa.Column("form_id", sa.String(length=36), nullable=False), + sa.Column("node_id", sa.String(length=255), nullable=False), + sa.Column("message", sa.String(length=255), nullable=False), + + sa.PrimaryKeyConstraint("id", name=op.f("workflow_pause_reasons_pkey")), + ) + with op.batch_alter_table("workflow_pause_reasons", schema=None) as batch_op: + batch_op.create_index(batch_op.f("workflow_pause_reasons_pause_id_idx"), ["pause_id"], unique=False) + + +def downgrade(): + op.drop_table("workflow_pause_reasons") diff --git a/api/models/account.py b/api/models/account.py index b1dafed0ed..420e6adc6c 100644 --- a/api/models/account.py +++ b/api/models/account.py @@ -88,7 +88,9 @@ class Account(UserMixin, TypeBase): __tablename__ = "accounts" __table_args__ = (sa.PrimaryKeyConstraint("id", name="account_pkey"), sa.Index("account_email_idx", "email")) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) name: Mapped[str] = mapped_column(String(255)) email: Mapped[str] = mapped_column(String(255)) password: Mapped[str | None] = mapped_column(String(255), default=None) @@ -235,7 +237,9 @@ class Tenant(TypeBase): __tablename__ = "tenants" __table_args__ = (sa.PrimaryKeyConstraint("id", name="tenant_pkey"),) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) name: Mapped[str] = mapped_column(String(255)) encrypt_public_key: Mapped[str | None] = mapped_column(LongText, default=None) plan: Mapped[str] = mapped_column(String(255), server_default=sa.text("'basic'"), default="basic") @@ -275,7 +279,9 @@ class TenantAccountJoin(TypeBase): sa.UniqueConstraint("tenant_id", "account_id", name="unique_tenant_account_join"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID) account_id: Mapped[str] = mapped_column(StringUUID) current: Mapped[bool] = mapped_column(sa.Boolean, server_default=sa.text("false"), default=False) @@ -297,7 +303,9 @@ class AccountIntegrate(TypeBase): sa.UniqueConstraint("provider", "open_id", name="unique_provider_open_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) account_id: Mapped[str] = mapped_column(StringUUID) provider: Mapped[str] = mapped_column(String(16)) open_id: Mapped[str] = mapped_column(String(255)) @@ -348,7 +356,9 @@ class TenantPluginPermission(TypeBase): sa.UniqueConstraint("tenant_id", name="unique_tenant_plugin"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) install_permission: Mapped[InstallPermission] = mapped_column( String(16), nullable=False, server_default="everyone", default=InstallPermission.EVERYONE @@ -375,7 +385,9 @@ class TenantPluginAutoUpgradeStrategy(TypeBase): sa.UniqueConstraint("tenant_id", name="unique_tenant_plugin_auto_upgrade_strategy"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) strategy_setting: Mapped[StrategySetting] = mapped_column( String(16), nullable=False, server_default="fix_only", default=StrategySetting.FIX_ONLY diff --git a/api/models/api_based_extension.py b/api/models/api_based_extension.py index 99d33908f8..b5acab5a75 100644 --- a/api/models/api_based_extension.py +++ b/api/models/api_based_extension.py @@ -24,7 +24,9 @@ class APIBasedExtension(TypeBase): sa.Index("api_based_extension_tenant_idx", "tenant_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) name: Mapped[str] = mapped_column(String(255), nullable=False) api_endpoint: Mapped[str] = mapped_column(String(255), nullable=False) diff --git a/api/models/dataset.py b/api/models/dataset.py index 2ea6d98b5f..e072711b82 100644 --- a/api/models/dataset.py +++ b/api/models/dataset.py @@ -920,7 +920,12 @@ class AppDatasetJoin(TypeBase): ) id: Mapped[str] = mapped_column( - StringUUID, primary_key=True, nullable=False, default=lambda: str(uuid4()), init=False + StringUUID, + primary_key=True, + nullable=False, + insert_default=lambda: str(uuid4()), + default_factory=lambda: str(uuid4()), + init=False, ) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -941,7 +946,12 @@ class DatasetQuery(TypeBase): ) id: Mapped[str] = mapped_column( - StringUUID, primary_key=True, nullable=False, default=lambda: str(uuid4()), init=False + StringUUID, + primary_key=True, + nullable=False, + insert_default=lambda: str(uuid4()), + default_factory=lambda: str(uuid4()), + init=False, ) dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False) content: Mapped[str] = mapped_column(LongText, nullable=False) @@ -961,7 +971,13 @@ class DatasetKeywordTable(TypeBase): sa.Index("dataset_keyword_table_dataset_id_idx", "dataset_id"), ) - id: Mapped[str] = mapped_column(StringUUID, primary_key=True, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, + primary_key=True, + insert_default=lambda: str(uuid4()), + default_factory=lambda: str(uuid4()), + init=False, + ) dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False, unique=True) keyword_table: Mapped[str] = mapped_column(LongText, nullable=False) data_source_type: Mapped[str] = mapped_column( @@ -1012,7 +1028,13 @@ class Embedding(TypeBase): sa.Index("created_at_idx", "created_at"), ) - id: Mapped[str] = mapped_column(StringUUID, primary_key=True, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, + primary_key=True, + insert_default=lambda: str(uuid4()), + default_factory=lambda: str(uuid4()), + init=False, + ) model_name: Mapped[str] = mapped_column( String(255), nullable=False, server_default=sa.text("'text-embedding-ada-002'") ) @@ -1037,7 +1059,13 @@ class DatasetCollectionBinding(TypeBase): sa.Index("provider_model_name_idx", "provider_name", "model_name"), ) - id: Mapped[str] = mapped_column(StringUUID, primary_key=True, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, + primary_key=True, + insert_default=lambda: str(uuid4()), + default_factory=lambda: str(uuid4()), + init=False, + ) provider_name: Mapped[str] = mapped_column(String(255), nullable=False) model_name: Mapped[str] = mapped_column(String(255), nullable=False) type: Mapped[str] = mapped_column(String(40), server_default=sa.text("'dataset'"), nullable=False) @@ -1073,7 +1101,13 @@ class Whitelist(TypeBase): sa.PrimaryKeyConstraint("id", name="whitelists_pkey"), sa.Index("whitelists_tenant_idx", "tenant_id"), ) - id: Mapped[str] = mapped_column(StringUUID, primary_key=True, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, + primary_key=True, + insert_default=lambda: str(uuid4()), + default_factory=lambda: str(uuid4()), + init=False, + ) tenant_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True) category: Mapped[str] = mapped_column(String(255), nullable=False) created_at: Mapped[datetime] = mapped_column( @@ -1090,7 +1124,13 @@ class DatasetPermission(TypeBase): sa.Index("idx_dataset_permissions_tenant_id", "tenant_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), primary_key=True, init=False) + id: Mapped[str] = mapped_column( + StringUUID, + insert_default=lambda: str(uuid4()), + default_factory=lambda: str(uuid4()), + primary_key=True, + init=False, + ) dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False) account_id: Mapped[str] = mapped_column(StringUUID, nullable=False) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -1110,7 +1150,13 @@ class ExternalKnowledgeApis(TypeBase): sa.Index("external_knowledge_apis_name_idx", "name"), ) - id: Mapped[str] = mapped_column(StringUUID, nullable=False, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, + nullable=False, + insert_default=lambda: str(uuid4()), + default_factory=lambda: str(uuid4()), + init=False, + ) name: Mapped[str] = mapped_column(String(255), nullable=False) description: Mapped[str] = mapped_column(String(255), nullable=False) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -1167,7 +1213,13 @@ class ExternalKnowledgeBindings(TypeBase): sa.Index("external_knowledge_bindings_external_knowledge_api_idx", "external_knowledge_api_id"), ) - id: Mapped[str] = mapped_column(StringUUID, nullable=False, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, + nullable=False, + insert_default=lambda: str(uuid4()), + default_factory=lambda: str(uuid4()), + init=False, + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) external_knowledge_api_id: Mapped[str] = mapped_column(StringUUID, nullable=False) dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -1191,7 +1243,9 @@ class DatasetAutoDisableLog(TypeBase): sa.Index("dataset_auto_disable_log_created_atx", "created_at"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False) document_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -1209,7 +1263,9 @@ class RateLimitLog(TypeBase): sa.Index("rate_limit_log_operation_idx", "operation"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) subscription_plan: Mapped[str] = mapped_column(String(255), nullable=False) operation: Mapped[str] = mapped_column(String(255), nullable=False) @@ -1226,7 +1282,9 @@ class DatasetMetadata(TypeBase): sa.Index("dataset_metadata_dataset_idx", "dataset_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False) type: Mapped[str] = mapped_column(String(255), nullable=False) @@ -1255,7 +1313,9 @@ class DatasetMetadataBinding(TypeBase): sa.Index("dataset_metadata_binding_document_idx", "document_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False) metadata_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -1270,7 +1330,9 @@ class PipelineBuiltInTemplate(TypeBase): __tablename__ = "pipeline_built_in_templates" __table_args__ = (sa.PrimaryKeyConstraint("id", name="pipeline_built_in_template_pkey"),) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) name: Mapped[str] = mapped_column(sa.String(255), nullable=False) description: Mapped[str] = mapped_column(LongText, nullable=False) chunk_structure: Mapped[str] = mapped_column(sa.String(255), nullable=False) @@ -1300,7 +1362,9 @@ class PipelineCustomizedTemplate(TypeBase): sa.Index("pipeline_customized_template_tenant_idx", "tenant_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) name: Mapped[str] = mapped_column(sa.String(255), nullable=False) description: Mapped[str] = mapped_column(LongText, nullable=False) @@ -1335,7 +1399,9 @@ class Pipeline(TypeBase): __tablename__ = "pipelines" __table_args__ = (sa.PrimaryKeyConstraint("id", name="pipeline_pkey"),) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) name: Mapped[str] = mapped_column(sa.String(255), nullable=False) description: Mapped[str] = mapped_column(LongText, nullable=False, default=sa.text("''")) @@ -1368,7 +1434,9 @@ class DocumentPipelineExecutionLog(TypeBase): sa.Index("document_pipeline_execution_logs_document_id_idx", "document_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) pipeline_id: Mapped[str] = mapped_column(StringUUID, nullable=False) document_id: Mapped[str] = mapped_column(StringUUID, nullable=False) datasource_type: Mapped[str] = mapped_column(sa.String(255), nullable=False) @@ -1385,7 +1453,9 @@ class PipelineRecommendedPlugin(TypeBase): __tablename__ = "pipeline_recommended_plugins" __table_args__ = (sa.PrimaryKeyConstraint("id", name="pipeline_recommended_plugin_pkey"),) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) plugin_id: Mapped[str] = mapped_column(LongText, nullable=False) provider_name: Mapped[str] = mapped_column(LongText, nullable=False) position: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0) diff --git a/api/models/model.py b/api/models/model.py index fb084d1dc6..1731ff5699 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -572,7 +572,9 @@ class InstalledApp(TypeBase): sa.UniqueConstraint("tenant_id", "app_id", name="unique_tenant_app"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) app_owner_tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -606,7 +608,9 @@ class OAuthProviderApp(TypeBase): sa.Index("oauth_provider_app_client_id_idx", "client_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) app_icon: Mapped[str] = mapped_column(String(255), nullable=False) client_id: Mapped[str] = mapped_column(String(255), nullable=False) client_secret: Mapped[str] = mapped_column(String(255), nullable=False) @@ -1251,9 +1255,13 @@ class Message(Base): "id": self.id, "app_id": self.app_id, "conversation_id": self.conversation_id, + "model_provider": self.model_provider, "model_id": self.model_id, "inputs": self.inputs, "query": self.query, + "message_tokens": self.message_tokens, + "answer_tokens": self.answer_tokens, + "provider_response_latency": self.provider_response_latency, "total_price": self.total_price, "message": self.message, "answer": self.answer, @@ -1275,8 +1283,12 @@ class Message(Base): id=data["id"], app_id=data["app_id"], conversation_id=data["conversation_id"], + model_provider=data.get("model_provider"), model_id=data["model_id"], inputs=data["inputs"], + message_tokens=data.get("message_tokens", 0), + answer_tokens=data.get("answer_tokens", 0), + provider_response_latency=data.get("provider_response_latency", 0.0), total_price=data["total_price"], query=data["query"], message=data["message"], @@ -1303,7 +1315,9 @@ class MessageFeedback(TypeBase): sa.Index("message_feedback_conversation_idx", "conversation_id", "from_source", "rating"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) conversation_id: Mapped[str] = mapped_column(StringUUID, nullable=False) message_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -1352,7 +1366,9 @@ class MessageFile(TypeBase): sa.Index("message_file_created_by_idx", "created_by"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) message_id: Mapped[str] = mapped_column(StringUUID, nullable=False) type: Mapped[str] = mapped_column(String(255), nullable=False) transfer_method: Mapped[FileTransferMethod] = mapped_column(String(255), nullable=False) @@ -1444,7 +1460,9 @@ class AppAnnotationSetting(TypeBase): sa.Index("app_annotation_settings_app_idx", "app_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) score_threshold: Mapped[float] = mapped_column(Float, nullable=False, server_default=sa.text("0")) collection_binding_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -1480,7 +1498,9 @@ class OperationLog(TypeBase): sa.Index("operation_log_account_action_idx", "tenant_id", "account_id", "action"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) account_id: Mapped[str] = mapped_column(StringUUID, nullable=False) action: Mapped[str] = mapped_column(String(255), nullable=False) @@ -1546,7 +1566,9 @@ class AppMCPServer(TypeBase): sa.UniqueConstraint("tenant_id", "app_id", name="unique_app_mcp_server_tenant_app_id"), sa.UniqueConstraint("server_code", name="unique_app_mcp_server_server_code"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) name: Mapped[str] = mapped_column(String(255), nullable=False) @@ -1756,7 +1778,9 @@ class ApiRequest(TypeBase): sa.Index("api_request_token_idx", "tenant_id", "api_token_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) api_token_id: Mapped[str] = mapped_column(StringUUID, nullable=False) path: Mapped[str] = mapped_column(String(255), nullable=False) @@ -1775,7 +1799,9 @@ class MessageChain(TypeBase): sa.Index("message_chain_message_id_idx", "message_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) message_id: Mapped[str] = mapped_column(StringUUID, nullable=False) type: Mapped[str] = mapped_column(String(255), nullable=False) input: Mapped[str | None] = mapped_column(LongText, nullable=True) @@ -1906,7 +1932,9 @@ class DatasetRetrieverResource(TypeBase): sa.Index("dataset_retriever_resource_message_id_idx", "message_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) message_id: Mapped[str] = mapped_column(StringUUID, nullable=False) position: Mapped[int] = mapped_column(sa.Integer, nullable=False) dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -1938,7 +1966,9 @@ class Tag(TypeBase): TAG_TYPE_LIST = ["knowledge", "app"] - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True) type: Mapped[str] = mapped_column(String(16), nullable=False) name: Mapped[str] = mapped_column(String(255), nullable=False) @@ -1956,7 +1986,9 @@ class TagBinding(TypeBase): sa.Index("tag_bind_tag_id_idx", "tag_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True) tag_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True) target_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True) @@ -1973,7 +2005,9 @@ class TraceAppConfig(TypeBase): sa.Index("trace_app_config_app_id_idx", "app_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) tracing_provider: Mapped[str | None] = mapped_column(String(255), nullable=True) tracing_config: Mapped[dict | None] = mapped_column(sa.JSON, nullable=True) diff --git a/api/models/oauth.py b/api/models/oauth.py index 2fce67c998..1db2552469 100644 --- a/api/models/oauth.py +++ b/api/models/oauth.py @@ -17,7 +17,9 @@ class DatasourceOauthParamConfig(TypeBase): sa.UniqueConstraint("plugin_id", "provider", name="datasource_oauth_config_datasource_id_provider_idx"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) plugin_id: Mapped[str] = mapped_column(sa.String(255), nullable=False) provider: Mapped[str] = mapped_column(sa.String(255), nullable=False) system_credentials: Mapped[dict] = mapped_column(AdjustedJSON, nullable=False) @@ -30,7 +32,9 @@ class DatasourceProvider(TypeBase): sa.UniqueConstraint("tenant_id", "plugin_id", "provider", "name", name="datasource_provider_unique_name"), sa.Index("datasource_provider_auth_type_provider_idx", "tenant_id", "plugin_id", "provider"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) name: Mapped[str] = mapped_column(sa.String(255), nullable=False) provider: Mapped[str] = mapped_column(sa.String(128), nullable=False) @@ -60,7 +64,9 @@ class DatasourceOauthTenantParamConfig(TypeBase): sa.UniqueConstraint("tenant_id", "plugin_id", "provider", name="datasource_oauth_tenant_config_unique"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) provider: Mapped[str] = mapped_column(sa.String(255), nullable=False) plugin_id: Mapped[str] = mapped_column(sa.String(255), nullable=False) diff --git a/api/models/provider.py b/api/models/provider.py index 577e098a2e..2afd8c5329 100644 --- a/api/models/provider.py +++ b/api/models/provider.py @@ -58,7 +58,13 @@ class Provider(TypeBase): ), ) - id: Mapped[str] = mapped_column(StringUUID, primary_key=True, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, + primary_key=True, + insert_default=lambda: str(uuidv7()), + default_factory=lambda: str(uuidv7()), + init=False, + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) provider_name: Mapped[str] = mapped_column(String(255), nullable=False) provider_type: Mapped[str] = mapped_column( @@ -132,7 +138,9 @@ class ProviderModel(TypeBase): ), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) provider_name: Mapped[str] = mapped_column(String(255), nullable=False) model_name: Mapped[str] = mapped_column(String(255), nullable=False) @@ -173,7 +181,9 @@ class TenantDefaultModel(TypeBase): sa.Index("tenant_default_model_tenant_id_provider_type_idx", "tenant_id", "provider_name", "model_type"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) provider_name: Mapped[str] = mapped_column(String(255), nullable=False) model_name: Mapped[str] = mapped_column(String(255), nullable=False) @@ -193,7 +203,9 @@ class TenantPreferredModelProvider(TypeBase): sa.Index("tenant_preferred_model_provider_tenant_provider_idx", "tenant_id", "provider_name"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) provider_name: Mapped[str] = mapped_column(String(255), nullable=False) preferred_provider_type: Mapped[str] = mapped_column(String(40), nullable=False) @@ -212,7 +224,9 @@ class ProviderOrder(TypeBase): sa.Index("provider_order_tenant_provider_idx", "tenant_id", "provider_name"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) provider_name: Mapped[str] = mapped_column(String(255), nullable=False) account_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -245,7 +259,9 @@ class ProviderModelSetting(TypeBase): sa.Index("provider_model_setting_tenant_provider_model_idx", "tenant_id", "provider_name", "model_type"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) provider_name: Mapped[str] = mapped_column(String(255), nullable=False) model_name: Mapped[str] = mapped_column(String(255), nullable=False) @@ -273,7 +289,9 @@ class LoadBalancingModelConfig(TypeBase): sa.Index("load_balancing_model_config_tenant_provider_model_idx", "tenant_id", "provider_name", "model_type"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) provider_name: Mapped[str] = mapped_column(String(255), nullable=False) model_name: Mapped[str] = mapped_column(String(255), nullable=False) @@ -302,7 +320,9 @@ class ProviderCredential(TypeBase): sa.Index("provider_credential_tenant_provider_idx", "tenant_id", "provider_name"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) provider_name: Mapped[str] = mapped_column(String(255), nullable=False) credential_name: Mapped[str] = mapped_column(String(255), nullable=False) @@ -332,7 +352,9 @@ class ProviderModelCredential(TypeBase): ), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) provider_name: Mapped[str] = mapped_column(String(255), nullable=False) model_name: Mapped[str] = mapped_column(String(255), nullable=False) diff --git a/api/models/source.py b/api/models/source.py index f093048c00..a8addbe342 100644 --- a/api/models/source.py +++ b/api/models/source.py @@ -18,7 +18,9 @@ class DataSourceOauthBinding(TypeBase): adjusted_json_index("source_info_idx", "source_info"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) access_token: Mapped[str] = mapped_column(String(255), nullable=False) provider: Mapped[str] = mapped_column(String(255), nullable=False) @@ -44,7 +46,9 @@ class DataSourceApiKeyAuthBinding(TypeBase): sa.Index("data_source_api_key_auth_binding_provider_idx", "provider"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) category: Mapped[str] = mapped_column(String(255), nullable=False) provider: Mapped[str] = mapped_column(String(255), nullable=False) diff --git a/api/models/task.py b/api/models/task.py index 539945b251..d98d99ca2c 100644 --- a/api/models/task.py +++ b/api/models/task.py @@ -24,7 +24,8 @@ class CeleryTask(TypeBase): result: Mapped[bytes | None] = mapped_column(BinaryData, nullable=True, default=None) date_done: Mapped[datetime | None] = mapped_column( DateTime, - default=naive_utc_now, + insert_default=naive_utc_now, + default=None, onupdate=naive_utc_now, nullable=True, ) @@ -47,4 +48,6 @@ class CeleryTaskSet(TypeBase): ) taskset_id: Mapped[str] = mapped_column(String(155), unique=True) result: Mapped[bytes | None] = mapped_column(BinaryData, nullable=True, default=None) - date_done: Mapped[datetime | None] = mapped_column(DateTime, default=naive_utc_now, nullable=True) + date_done: Mapped[datetime | None] = mapped_column( + DateTime, insert_default=naive_utc_now, default=None, nullable=True + ) diff --git a/api/models/tools.py b/api/models/tools.py index 0ef261e90b..4d1ec60780 100644 --- a/api/models/tools.py +++ b/api/models/tools.py @@ -32,7 +32,9 @@ class ToolOAuthSystemClient(TypeBase): sa.UniqueConstraint("plugin_id", "provider", name="tool_oauth_system_client_plugin_id_provider_idx"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) plugin_id: Mapped[str] = mapped_column(String(512), nullable=False) provider: Mapped[str] = mapped_column(String(255), nullable=False) # oauth params of the tool provider @@ -47,7 +49,9 @@ class ToolOAuthTenantClient(TypeBase): sa.UniqueConstraint("tenant_id", "plugin_id", "provider", name="unique_tool_oauth_tenant_client"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) # tenant id tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) plugin_id: Mapped[str] = mapped_column(String(255), nullable=False) @@ -73,7 +77,9 @@ class BuiltinToolProvider(TypeBase): ) # id of the tool provider - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) name: Mapped[str] = mapped_column( String(256), nullable=False, @@ -175,7 +181,9 @@ class ApiToolProvider(TypeBase): sa.UniqueConstraint("name", "tenant_id", name="unique_api_tool_provider"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) # name of the api provider name: Mapped[str] = mapped_column( String(255), @@ -247,7 +255,9 @@ class ToolLabelBinding(TypeBase): sa.UniqueConstraint("tool_id", "label_name", name="unique_tool_label_bind"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) # tool id tool_id: Mapped[str] = mapped_column(String(64), nullable=False) # tool type @@ -268,7 +278,9 @@ class WorkflowToolProvider(TypeBase): sa.UniqueConstraint("tenant_id", "app_id", name="unique_workflow_tool_provider_app_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) # name of the workflow provider name: Mapped[str] = mapped_column(String(255), nullable=False) # label of the workflow provider @@ -334,7 +346,9 @@ class MCPToolProvider(TypeBase): sa.UniqueConstraint("tenant_id", "server_identifier", name="unique_mcp_provider_server_identifier"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) # name of the mcp provider name: Mapped[str] = mapped_column(String(40), nullable=False) # server identifier of the mcp provider @@ -415,7 +429,9 @@ class ToolModelInvoke(TypeBase): __tablename__ = "tool_model_invokes" __table_args__ = (sa.PrimaryKeyConstraint("id", name="tool_model_invoke_pkey"),) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) # who invoke this tool user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # tenant id @@ -468,7 +484,9 @@ class ToolConversationVariables(TypeBase): sa.Index("conversation_id_idx", "conversation_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) # conversation user id user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # tenant id @@ -505,7 +523,9 @@ class ToolFile(TypeBase): sa.Index("tool_file_conversation_id_idx", "conversation_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) # conversation user id user_id: Mapped[str] = mapped_column(StringUUID) # tenant id @@ -536,7 +556,9 @@ class DeprecatedPublishedAppTool(TypeBase): sa.UniqueConstraint("app_id", "user_id", name="unique_published_app_tool"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) # id of the app app_id: Mapped[str] = mapped_column(StringUUID, ForeignKey("apps.id"), nullable=False) diff --git a/api/models/trigger.py b/api/models/trigger.py index 088e797f82..87e2a5ccfc 100644 --- a/api/models/trigger.py +++ b/api/models/trigger.py @@ -41,7 +41,9 @@ class TriggerSubscription(TypeBase): UniqueConstraint("tenant_id", "provider_id", "name", name="unique_trigger_provider"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) name: Mapped[str] = mapped_column(String(255), nullable=False, comment="Subscription instance name") tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -111,7 +113,9 @@ class TriggerOAuthSystemClient(TypeBase): sa.UniqueConstraint("plugin_id", "provider", name="trigger_oauth_system_client_plugin_id_provider_idx"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) plugin_id: Mapped[str] = mapped_column(String(255), nullable=False) provider: Mapped[str] = mapped_column(String(255), nullable=False) # oauth params of the trigger provider @@ -136,7 +140,9 @@ class TriggerOAuthTenantClient(TypeBase): sa.UniqueConstraint("tenant_id", "plugin_id", "provider", name="unique_trigger_oauth_tenant_client"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) # tenant id tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) plugin_id: Mapped[str] = mapped_column(String(255), nullable=False) @@ -202,7 +208,9 @@ class WorkflowTriggerLog(TypeBase): sa.Index("workflow_trigger_log_workflow_id_idx", "workflow_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) workflow_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -294,7 +302,9 @@ class WorkflowWebhookTrigger(TypeBase): sa.UniqueConstraint("webhook_id", name="uniq_webhook_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) node_id: Mapped[str] = mapped_column(String(64), nullable=False) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -351,7 +361,9 @@ class WorkflowPluginTrigger(TypeBase): sa.UniqueConstraint("app_id", "node_id", name="uniq_app_node_subscription"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) node_id: Mapped[str] = mapped_column(String(64), nullable=False) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -395,7 +407,9 @@ class AppTrigger(TypeBase): sa.Index("app_trigger_tenant_app_idx", "tenant_id", "app_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) node_id: Mapped[str | None] = mapped_column(String(64), nullable=False) @@ -443,7 +457,13 @@ class WorkflowSchedulePlan(TypeBase): sa.Index("workflow_schedule_plan_next_idx", "next_run_at"), ) - id: Mapped[str] = mapped_column(StringUUID, primary_key=True, default=lambda: str(uuidv7()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, + primary_key=True, + insert_default=lambda: str(uuidv7()), + default_factory=lambda: str(uuidv7()), + init=False, + ) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) node_id: Mapped[str] = mapped_column(String(64), nullable=False) tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) diff --git a/api/models/web.py b/api/models/web.py index 4f0bf7c7da..b2832aa163 100644 --- a/api/models/web.py +++ b/api/models/web.py @@ -18,7 +18,9 @@ class SavedMessage(TypeBase): sa.Index("saved_message_message_idx", "app_id", "message_id", "created_by_role", "created_by"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) message_id: Mapped[str] = mapped_column(StringUUID, nullable=False) created_by_role: Mapped[str] = mapped_column(String(255), nullable=False, server_default=sa.text("'end_user'")) @@ -42,7 +44,9 @@ class PinnedConversation(TypeBase): sa.Index("pinned_conversation_conversation_idx", "app_id", "conversation_id", "created_by_role", "created_by"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) conversation_id: Mapped[str] = mapped_column(StringUUID) created_by_role: Mapped[str] = mapped_column( diff --git a/api/models/workflow.py b/api/models/workflow.py index f206a6a870..42ee8a1f2b 100644 --- a/api/models/workflow.py +++ b/api/models/workflow.py @@ -29,6 +29,7 @@ from core.workflow.constants import ( CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID, ) +from core.workflow.entities.pause_reason import HumanInputRequired, PauseReason, PauseReasonType, SchedulingPause from core.workflow.enums import NodeType from extensions.ext_storage import Storage from factories.variable_factory import TypeMismatchError, build_segment_with_type @@ -1102,7 +1103,9 @@ class WorkflowAppLog(TypeBase): sa.Index("workflow_app_log_workflow_run_id_idx", "workflow_run_id"), ) - id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()), init=False) + id: Mapped[str] = mapped_column( + StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False + ) tenant_id: Mapped[str] = mapped_column(StringUUID) app_id: Mapped[str] = mapped_column(StringUUID) workflow_id: Mapped[str] = mapped_column(StringUUID, nullable=False) @@ -1728,3 +1731,68 @@ class WorkflowPause(DefaultFieldsMixin, Base): primaryjoin="WorkflowPause.workflow_run_id == WorkflowRun.id", back_populates="pause", ) + + +class WorkflowPauseReason(DefaultFieldsMixin, Base): + __tablename__ = "workflow_pause_reasons" + + # `pause_id` represents the identifier of the pause, + # correspond to the `id` field of `WorkflowPause`. + pause_id: Mapped[str] = mapped_column(StringUUID, nullable=False, index=True) + + type_: Mapped[PauseReasonType] = mapped_column(EnumText(PauseReasonType), nullable=False) + + # form_id is not empty if and if only type_ == PauseReasonType.HUMAN_INPUT_REQUIRED + # + form_id: Mapped[str] = mapped_column( + String(36), + nullable=False, + default="", + ) + + # message records the text description of this pause reason. For example, + # "The workflow has been paused due to scheduling." + # + # Empty message means that this pause reason is not speified. + message: Mapped[str] = mapped_column( + String(255), + nullable=False, + default="", + ) + + # `node_id` is the identifier of node causing the pasue, correspond to + # `Node.id`. Empty `node_id` means that this pause reason is not caused by any specific node + # (E.G. time slicing pauses.) + node_id: Mapped[str] = mapped_column( + String(255), + nullable=False, + default="", + ) + + # Relationship to WorkflowPause + pause: Mapped[WorkflowPause] = orm.relationship( + foreign_keys=[pause_id], + # require explicit preloading. + lazy="raise", + uselist=False, + primaryjoin="WorkflowPauseReason.pause_id == WorkflowPause.id", + ) + + @classmethod + def from_entity(cls, pause_reason: PauseReason) -> "WorkflowPauseReason": + if isinstance(pause_reason, HumanInputRequired): + return cls( + type_=PauseReasonType.HUMAN_INPUT_REQUIRED, form_id=pause_reason.form_id, node_id=pause_reason.node_id + ) + elif isinstance(pause_reason, SchedulingPause): + return cls(type_=PauseReasonType.SCHEDULED_PAUSE, message=pause_reason.message, node_id="") + else: + raise AssertionError(f"Unknown pause reason type: {pause_reason}") + + def to_entity(self) -> PauseReason: + if self.type_ == PauseReasonType.HUMAN_INPUT_REQUIRED: + return HumanInputRequired(form_id=self.form_id, node_id=self.node_id) + elif self.type_ == PauseReasonType.SCHEDULED_PAUSE: + return SchedulingPause(message=self.message) + else: + raise AssertionError(f"Unknown pause reason type: {self.type_}") diff --git a/api/pyproject.toml b/api/pyproject.toml index da421f5fc8..a31fd758cc 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dify-api" -version = "1.10.0" +version = "1.10.1" requires-python = ">=3.11,<3.13" dependencies = [ diff --git a/api/repositories/api_workflow_run_repository.py b/api/repositories/api_workflow_run_repository.py index 21fd57cd22..fd547c78ba 100644 --- a/api/repositories/api_workflow_run_repository.py +++ b/api/repositories/api_workflow_run_repository.py @@ -38,11 +38,12 @@ from collections.abc import Sequence from datetime import datetime from typing import Protocol -from core.workflow.entities.workflow_pause import WorkflowPauseEntity +from core.workflow.entities.pause_reason import PauseReason from core.workflow.repositories.workflow_execution_repository import WorkflowExecutionRepository from libs.infinite_scroll_pagination import InfiniteScrollPagination from models.enums import WorkflowRunTriggeredFrom from models.workflow import WorkflowRun +from repositories.entities.workflow_pause import WorkflowPauseEntity from repositories.types import ( AverageInteractionStats, DailyRunsStats, @@ -257,6 +258,7 @@ class APIWorkflowRunRepository(WorkflowExecutionRepository, Protocol): workflow_run_id: str, state_owner_user_id: str, state: str, + pause_reasons: Sequence[PauseReason], ) -> WorkflowPauseEntity: """ Create a new workflow pause state. diff --git a/api/core/workflow/entities/workflow_pause.py b/api/repositories/entities/workflow_pause.py similarity index 77% rename from api/core/workflow/entities/workflow_pause.py rename to api/repositories/entities/workflow_pause.py index 2f31c1ff53..b970f39816 100644 --- a/api/core/workflow/entities/workflow_pause.py +++ b/api/repositories/entities/workflow_pause.py @@ -7,8 +7,11 @@ and don't contain implementation details like tenant_id, app_id, etc. """ from abc import ABC, abstractmethod +from collections.abc import Sequence from datetime import datetime +from core.workflow.entities.pause_reason import PauseReason + class WorkflowPauseEntity(ABC): """ @@ -59,3 +62,15 @@ class WorkflowPauseEntity(ABC): the pause is not resumed yet. """ pass + + @abstractmethod + def get_pause_reasons(self) -> Sequence[PauseReason]: + """ + Retrieve detailed reasons for this pause. + + Returns a sequence of `PauseReason` objects describing the specific nodes and + reasons for which the workflow execution was paused. + This information is related to, but distinct from, the `PauseReason` type + defined in `api/core/workflow/entities/pause_reason.py`. + """ + ... diff --git a/api/repositories/sqlalchemy_api_workflow_run_repository.py b/api/repositories/sqlalchemy_api_workflow_run_repository.py index eb2a32d764..b172c6a3ac 100644 --- a/api/repositories/sqlalchemy_api_workflow_run_repository.py +++ b/api/repositories/sqlalchemy_api_workflow_run_repository.py @@ -31,7 +31,7 @@ from sqlalchemy import and_, delete, func, null, or_, select from sqlalchemy.engine import CursorResult from sqlalchemy.orm import Session, selectinload, sessionmaker -from core.workflow.entities.workflow_pause import WorkflowPauseEntity +from core.workflow.entities.pause_reason import HumanInputRequired, PauseReason, SchedulingPause from core.workflow.enums import WorkflowExecutionStatus from extensions.ext_storage import storage from libs.datetime_utils import naive_utc_now @@ -41,8 +41,9 @@ from libs.time_parser import get_time_threshold from libs.uuid_utils import uuidv7 from models.enums import WorkflowRunTriggeredFrom from models.workflow import WorkflowPause as WorkflowPauseModel -from models.workflow import WorkflowRun +from models.workflow import WorkflowPauseReason, WorkflowRun from repositories.api_workflow_run_repository import APIWorkflowRunRepository +from repositories.entities.workflow_pause import WorkflowPauseEntity from repositories.types import ( AverageInteractionStats, DailyRunsStats, @@ -318,6 +319,7 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository): workflow_run_id: str, state_owner_user_id: str, state: str, + pause_reasons: Sequence[PauseReason], ) -> WorkflowPauseEntity: """ Create a new workflow pause state. @@ -371,6 +373,25 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository): pause_model.workflow_run_id = workflow_run.id pause_model.state_object_key = state_obj_key pause_model.created_at = naive_utc_now() + pause_reason_models = [] + for reason in pause_reasons: + if isinstance(reason, HumanInputRequired): + # TODO(QuantumGhost): record node_id for `WorkflowPauseReason` + pause_reason_model = WorkflowPauseReason( + pause_id=pause_model.id, + type_=reason.TYPE, + form_id=reason.form_id, + ) + elif isinstance(reason, SchedulingPause): + pause_reason_model = WorkflowPauseReason( + pause_id=pause_model.id, + type_=reason.TYPE, + message=reason.message, + ) + else: + raise AssertionError(f"unkown reason type: {type(reason)}") + + pause_reason_models.append(pause_reason_model) # Update workflow run status workflow_run.status = WorkflowExecutionStatus.PAUSED @@ -378,10 +399,16 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository): # Save everything in a transaction session.add(pause_model) session.add(workflow_run) + session.add_all(pause_reason_models) logger.info("Created workflow pause %s for workflow run %s", pause_model.id, workflow_run_id) - return _PrivateWorkflowPauseEntity.from_models(pause_model) + return _PrivateWorkflowPauseEntity(pause_model=pause_model, reason_models=pause_reason_models) + + def _get_reasons_by_pause_id(self, session: Session, pause_id: str): + reason_stmt = select(WorkflowPauseReason).where(WorkflowPauseReason.pause_id == pause_id) + pause_reason_models = session.scalars(reason_stmt).all() + return pause_reason_models def get_workflow_pause( self, @@ -413,8 +440,16 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository): pause_model = workflow_run.pause if pause_model is None: return None + pause_reason_models = self._get_reasons_by_pause_id(session, pause_model.id) - return _PrivateWorkflowPauseEntity.from_models(pause_model) + human_input_form: list[Any] = [] + # TODO(QuantumGhost): query human_input_forms model and rebuild PauseReason + + return _PrivateWorkflowPauseEntity( + pause_model=pause_model, + reason_models=pause_reason_models, + human_input_form=human_input_form, + ) def resume_workflow_pause( self, @@ -466,6 +501,8 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository): if pause_model.resumed_at is not None: raise _WorkflowRunError(f"Cannot resume an already resumed pause, pause_id={pause_model.id}") + pause_reasons = self._get_reasons_by_pause_id(session, pause_model.id) + # Mark as resumed pause_model.resumed_at = naive_utc_now() workflow_run.pause_id = None # type: ignore @@ -476,7 +513,7 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository): logger.info("Resumed workflow pause %s for workflow run %s", pause_model.id, workflow_run_id) - return _PrivateWorkflowPauseEntity.from_models(pause_model) + return _PrivateWorkflowPauseEntity(pause_model=pause_model, reason_models=pause_reasons) def delete_workflow_pause( self, @@ -815,26 +852,13 @@ class _PrivateWorkflowPauseEntity(WorkflowPauseEntity): self, *, pause_model: WorkflowPauseModel, + reason_models: Sequence[WorkflowPauseReason], + human_input_form: Sequence = (), ) -> None: self._pause_model = pause_model + self._reason_models = reason_models self._cached_state: bytes | None = None - - @classmethod - def from_models(cls, workflow_pause_model) -> "_PrivateWorkflowPauseEntity": - """ - Create a _PrivateWorkflowPauseEntity from database models. - - Args: - workflow_pause_model: The WorkflowPause database model - upload_file_model: The UploadFile database model - - Returns: - _PrivateWorkflowPauseEntity: The constructed entity - - Raises: - ValueError: If required model attributes are missing - """ - return cls(pause_model=workflow_pause_model) + self._human_input_form = human_input_form @property def id(self) -> str: @@ -867,3 +891,6 @@ class _PrivateWorkflowPauseEntity(WorkflowPauseEntity): @property def resumed_at(self) -> datetime | None: return self._pause_model.resumed_at + + def get_pause_reasons(self) -> Sequence[PauseReason]: + return [reason.to_entity() for reason in self._reason_models] diff --git a/api/services/account_service.py b/api/services/account_service.py index 13c3993fb5..ac6d1bde77 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -1352,7 +1352,7 @@ class RegisterService: @classmethod def invite_new_member( - cls, tenant: Tenant, email: str, language: str, role: str = "normal", inviter: Account | None = None + cls, tenant: Tenant, email: str, language: str | None, role: str = "normal", inviter: Account | None = None ) -> str: if not inviter: raise ValueError("Inviter is required") diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py index 15fefd6116..1dd6faea5d 100644 --- a/api/services/app_dsl_service.py +++ b/api/services/app_dsl_service.py @@ -550,7 +550,7 @@ class AppDslService: "app": { "name": app_model.name, "mode": app_model.mode, - "icon": "🤖" if app_model.icon_type == "image" else app_model.icon, + "icon": app_model.icon if app_model.icon_type == "image" else "🤖", "icon_background": "#FFEAD5" if app_model.icon_type == "image" else app_model.icon_background, "description": app_model.description, "use_icon_as_answer_icon": app_model.use_icon_as_answer_icon, diff --git a/api/services/app_generate_service.py b/api/services/app_generate_service.py index bb1ea742d0..dc85929b98 100644 --- a/api/services/app_generate_service.py +++ b/api/services/app_generate_service.py @@ -135,7 +135,7 @@ class AppGenerateService: Returns: The maximum number of active requests allowed """ - app_limit = app.max_active_requests or 0 + app_limit = app.max_active_requests or dify_config.APP_DEFAULT_ACTIVE_REQUESTS config_limit = dify_config.APP_MAX_ACTIVE_REQUESTS # Filter out infinite (0) values and return the minimum, or 0 if both are infinite diff --git a/api/services/app_task_service.py b/api/services/app_task_service.py new file mode 100644 index 0000000000..01874b3f9f --- /dev/null +++ b/api/services/app_task_service.py @@ -0,0 +1,45 @@ +"""Service for managing application task operations. + +This service provides centralized logic for task control operations +like stopping tasks, handling both legacy Redis flag mechanism and +new GraphEngine command channel mechanism. +""" + +from core.app.apps.base_app_queue_manager import AppQueueManager +from core.app.entities.app_invoke_entities import InvokeFrom +from core.workflow.graph_engine.manager import GraphEngineManager +from models.model import AppMode + + +class AppTaskService: + """Service for managing application task operations.""" + + @staticmethod + def stop_task( + task_id: str, + invoke_from: InvokeFrom, + user_id: str, + app_mode: AppMode, + ) -> None: + """Stop a running task. + + This method handles stopping tasks using both mechanisms: + 1. Legacy Redis flag mechanism (for backward compatibility) + 2. New GraphEngine command channel (for workflow-based apps) + + Args: + task_id: The task ID to stop + invoke_from: The source of the invoke (e.g., DEBUGGER, WEB_APP, SERVICE_API) + user_id: The user ID requesting the stop + app_mode: The application mode (CHAT, AGENT_CHAT, ADVANCED_CHAT, WORKFLOW, etc.) + + Returns: + None + """ + # Legacy mechanism: Set stop flag in Redis + AppQueueManager.set_stop_flag(task_id, invoke_from, user_id) + + # New mechanism: Send stop command via GraphEngine for workflow-based apps + # This ensures proper workflow status recording in the persistence layer + if app_mode in (AppMode.ADVANCED_CHAT, AppMode.WORKFLOW): + GraphEngineManager.send_stop_command(task_id) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index abfb4baeec..2bec61963c 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -1375,6 +1375,11 @@ class DocumentService: document.name = name db.session.add(document) + if document.data_source_info_dict: + db.session.query(UploadFile).where( + UploadFile.id == document.data_source_info_dict["upload_file_id"] + ).update({UploadFile.name: name}) + db.session.commit() return document diff --git a/api/services/feedback_service.py b/api/services/feedback_service.py new file mode 100644 index 0000000000..2bc965f6ba --- /dev/null +++ b/api/services/feedback_service.py @@ -0,0 +1,185 @@ +import csv +import io +import json +from datetime import datetime + +from flask import Response +from sqlalchemy import or_ + +from extensions.ext_database import db +from models.model import Account, App, Conversation, Message, MessageFeedback + + +class FeedbackService: + @staticmethod + def export_feedbacks( + app_id: str, + from_source: str | None = None, + rating: str | None = None, + has_comment: bool | None = None, + start_date: str | None = None, + end_date: str | None = None, + format_type: str = "csv", + ): + """ + Export feedback data with message details for analysis + + Args: + app_id: Application ID + from_source: Filter by feedback source ('user' or 'admin') + rating: Filter by rating ('like' or 'dislike') + has_comment: Only include feedback with comments + start_date: Start date filter (YYYY-MM-DD) + end_date: End date filter (YYYY-MM-DD) + format_type: Export format ('csv' or 'json') + """ + + # Validate format early to avoid hitting DB when unnecessary + fmt = (format_type or "csv").lower() + if fmt not in {"csv", "json"}: + raise ValueError(f"Unsupported format: {format_type}") + + # Build base query + query = ( + db.session.query(MessageFeedback, Message, Conversation, App, Account) + .join(Message, MessageFeedback.message_id == Message.id) + .join(Conversation, MessageFeedback.conversation_id == Conversation.id) + .join(App, MessageFeedback.app_id == App.id) + .outerjoin(Account, MessageFeedback.from_account_id == Account.id) + .where(MessageFeedback.app_id == app_id) + ) + + # Apply filters + if from_source: + query = query.filter(MessageFeedback.from_source == from_source) + + if rating: + query = query.filter(MessageFeedback.rating == rating) + + if has_comment is not None: + if has_comment: + query = query.filter(MessageFeedback.content.isnot(None), MessageFeedback.content != "") + else: + query = query.filter(or_(MessageFeedback.content.is_(None), MessageFeedback.content == "")) + + if start_date: + try: + start_dt = datetime.strptime(start_date, "%Y-%m-%d") + query = query.filter(MessageFeedback.created_at >= start_dt) + except ValueError: + raise ValueError(f"Invalid start_date format: {start_date}. Use YYYY-MM-DD") + + if end_date: + try: + end_dt = datetime.strptime(end_date, "%Y-%m-%d") + query = query.filter(MessageFeedback.created_at <= end_dt) + except ValueError: + raise ValueError(f"Invalid end_date format: {end_date}. Use YYYY-MM-DD") + + # Order by creation date (newest first) + query = query.order_by(MessageFeedback.created_at.desc()) + + # Execute query + results = query.all() + + # Prepare data for export + export_data = [] + for feedback, message, conversation, app, account in results: + # Get the user query from the message + user_query = message.query or message.inputs.get("query", "") if message.inputs else "" + + # Format the feedback data + feedback_record = { + "feedback_id": str(feedback.id), + "app_name": app.name, + "app_id": str(app.id), + "conversation_id": str(conversation.id), + "conversation_name": conversation.name or "", + "message_id": str(message.id), + "user_query": user_query, + "ai_response": message.answer[:500] + "..." + if len(message.answer) > 500 + else message.answer, # Truncate long responses + "feedback_rating": "👍" if feedback.rating == "like" else "👎", + "feedback_rating_raw": feedback.rating, + "feedback_comment": feedback.content or "", + "feedback_source": feedback.from_source, + "feedback_date": feedback.created_at.strftime("%Y-%m-%d %H:%M:%S"), + "message_date": message.created_at.strftime("%Y-%m-%d %H:%M:%S"), + "from_account_name": account.name if account else "", + "from_end_user_id": str(feedback.from_end_user_id) if feedback.from_end_user_id else "", + "has_comment": "Yes" if feedback.content and feedback.content.strip() else "No", + } + export_data.append(feedback_record) + + # Export based on format + if fmt == "csv": + return FeedbackService._export_csv(export_data, app_id) + else: # fmt == "json" + return FeedbackService._export_json(export_data, app_id) + + @staticmethod + def _export_csv(data, app_id): + """Export data as CSV""" + if not data: + pass # allow empty CSV with headers only + + # Create CSV in memory + output = io.StringIO() + + # Define headers + headers = [ + "feedback_id", + "app_name", + "app_id", + "conversation_id", + "conversation_name", + "message_id", + "user_query", + "ai_response", + "feedback_rating", + "feedback_rating_raw", + "feedback_comment", + "feedback_source", + "feedback_date", + "message_date", + "from_account_name", + "from_end_user_id", + "has_comment", + ] + + writer = csv.DictWriter(output, fieldnames=headers) + writer.writeheader() + writer.writerows(data) + + # Create response without requiring app context + response = Response(output.getvalue(), mimetype="text/csv; charset=utf-8-sig") + response.headers["Content-Disposition"] = ( + f"attachment; filename=dify_feedback_export_{app_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv" + ) + + return response + + @staticmethod + def _export_json(data, app_id): + """Export data as JSON""" + response_data = { + "export_info": { + "app_id": app_id, + "export_date": datetime.now().isoformat(), + "total_records": len(data), + "data_source": "dify_feedback_export", + }, + "feedback_data": data, + } + + # Create response without requiring app context + response = Response( + json.dumps(response_data, ensure_ascii=False, indent=2), + mimetype="application/json; charset=utf-8", + ) + response.headers["Content-Disposition"] = ( + f"attachment; filename=dify_feedback_export_{app_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + ) + + return response diff --git a/api/services/file_service.py b/api/services/file_service.py index b0c5a32c9f..1980cd8d59 100644 --- a/api/services/file_service.py +++ b/api/services/file_service.py @@ -3,8 +3,8 @@ import os import uuid from typing import Literal, Union -from sqlalchemy import Engine -from sqlalchemy.orm import sessionmaker +from sqlalchemy import Engine, select +from sqlalchemy.orm import Session, sessionmaker from werkzeug.exceptions import NotFound from configs import dify_config @@ -29,7 +29,7 @@ PREVIEW_WORDS_LIMIT = 3000 class FileService: - _session_maker: sessionmaker + _session_maker: sessionmaker[Session] def __init__(self, session_factory: sessionmaker | Engine | None = None): if isinstance(session_factory, Engine): @@ -236,11 +236,10 @@ class FileService: return content.decode("utf-8") def delete_file(self, file_id: str): - with self._session_maker(expire_on_commit=False) as session: - upload_file: UploadFile | None = session.query(UploadFile).where(UploadFile.id == file_id).first() + with self._session_maker() as session, session.begin(): + upload_file = session.scalar(select(UploadFile).where(UploadFile.id == file_id)) - if not upload_file: - return - storage.delete(upload_file.key) - session.delete(upload_file) - session.commit() + if not upload_file: + return + storage.delete(upload_file.key) + session.delete(upload_file) diff --git a/api/services/rag_pipeline/pipeline_generate_service.py b/api/services/rag_pipeline/pipeline_generate_service.py index e6cee64df6..f397b28283 100644 --- a/api/services/rag_pipeline/pipeline_generate_service.py +++ b/api/services/rag_pipeline/pipeline_generate_service.py @@ -53,10 +53,11 @@ class PipelineGenerateService: @staticmethod def _get_max_active_requests(app_model: App) -> int: - max_active_requests = app_model.max_active_requests - if max_active_requests is None: - max_active_requests = int(dify_config.APP_MAX_ACTIVE_REQUESTS) - return max_active_requests + app_limit = app_model.max_active_requests or dify_config.APP_DEFAULT_ACTIVE_REQUESTS + config_limit = dify_config.APP_MAX_ACTIVE_REQUESTS + # Filter out infinite (0) values and return the minimum, or 0 if both are infinite + limits = [limit for limit in [app_limit, config_limit] if limit > 0] + return min(limits) if limits else 0 @classmethod def generate_single_iteration( diff --git a/api/services/tools/tools_transform_service.py b/api/services/tools/tools_transform_service.py index 3e976234ba..81872e3ebc 100644 --- a/api/services/tools/tools_transform_service.py +++ b/api/services/tools/tools_transform_service.py @@ -405,6 +405,7 @@ class ToolTransformService: name=tool.operation_id or "", label=I18nObject(en_US=tool.operation_id, zh_Hans=tool.operation_id), description=I18nObject(en_US=tool.summary or "", zh_Hans=tool.summary or ""), + output_schema=tool.output_schema, parameters=tool.parameters, labels=labels or [], ) diff --git a/api/services/tools/workflow_tools_manage_service.py b/api/services/tools/workflow_tools_manage_service.py index 5413725798..b743cc1105 100644 --- a/api/services/tools/workflow_tools_manage_service.py +++ b/api/services/tools/workflow_tools_manage_service.py @@ -291,6 +291,10 @@ class WorkflowToolManageService: if len(workflow_tools) == 0: raise ValueError(f"Tool {db_tool.id} not found") + tool_entity = workflow_tools[0].entity + # get output schema from workflow tool entity + output_schema = tool_entity.output_schema + return { "name": db_tool.name, "label": db_tool.label, @@ -299,6 +303,7 @@ class WorkflowToolManageService: "icon": json.loads(db_tool.icon), "description": db_tool.description, "parameters": jsonable_encoder(db_tool.parameter_configurations), + "output_schema": output_schema, "tool": ToolTransformService.convert_tool_entity_to_api_entity( tool=tool.get_tools(db_tool.tenant_id)[0], labels=ToolLabelManager.get_tool_labels(tool), diff --git a/api/services/trigger/webhook_service.py b/api/services/trigger/webhook_service.py index 6e0ee7a191..4b3e1330fd 100644 --- a/api/services/trigger/webhook_service.py +++ b/api/services/trigger/webhook_service.py @@ -5,6 +5,7 @@ import secrets from collections.abc import Mapping from typing import Any +import orjson from flask import request from pydantic import BaseModel from sqlalchemy import select @@ -169,7 +170,7 @@ class WebhookService: - method: HTTP method - headers: Request headers - query_params: Query parameters as strings - - body: Request body (varies by content type) + - body: Request body (varies by content type; JSON parsing errors raise ValueError) - files: Uploaded files (if any) """ cls._validate_content_length() @@ -255,14 +256,21 @@ class WebhookService: Returns: tuple: (body_data, files_data) where: - - body_data: Parsed JSON content or empty dict if parsing fails + - body_data: Parsed JSON content - files_data: Empty dict (JSON requests don't contain files) + + Raises: + ValueError: If JSON parsing fails """ + raw_body = request.get_data(cache=True) + if not raw_body or raw_body.strip() == b"": + return {}, {} + try: - body = request.get_json() or {} - except Exception: - logger.warning("Failed to parse JSON body") - body = {} + body = orjson.loads(raw_body) + except orjson.JSONDecodeError as exc: + logger.warning("Failed to parse JSON body: %s", exc) + raise ValueError(f"Invalid JSON body: {exc}") from exc return body, {} @classmethod diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index b6764f1fa7..b45a167b73 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -15,7 +15,7 @@ from core.file import File from core.repositories import DifyCoreRepositoryFactory from core.variables import Variable from core.variables.variables import VariableUnion -from core.workflow.entities import VariablePool, WorkflowNodeExecution +from core.workflow.entities import WorkflowNodeExecution from core.workflow.enums import ErrorStrategy, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus from core.workflow.errors import WorkflowNodeRunFailedError from core.workflow.graph_events import GraphNodeEventBase, NodeRunFailedEvent, NodeRunSucceededEvent @@ -24,6 +24,7 @@ from core.workflow.nodes import NodeType from core.workflow.nodes.base.node import Node from core.workflow.nodes.node_mapping import LATEST_VERSION, NODE_TYPE_CLASSES_MAPPING from core.workflow.nodes.start.entities import StartNodeData +from core.workflow.runtime import VariablePool from core.workflow.system_variable import SystemVariable from core.workflow.workflow_entry import WorkflowEntry from enums.cloud_plan import CloudPlan diff --git a/api/tests/integration_tests/.env.example b/api/tests/integration_tests/.env.example index e4c534f046..e508ceef66 100644 --- a/api/tests/integration_tests/.env.example +++ b/api/tests/integration_tests/.env.example @@ -62,6 +62,7 @@ WEAVIATE_ENDPOINT=http://localhost:8080 WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih WEAVIATE_GRPC_ENABLED=false WEAVIATE_BATCH_SIZE=100 +WEAVIATE_TOKENIZATION=word # Upload configuration @@ -174,6 +175,7 @@ MAX_VARIABLE_SIZE=204800 # App configuration APP_MAX_EXECUTION_TIME=1200 +APP_DEFAULT_ACTIVE_REQUESTS=0 APP_MAX_ACTIVE_REQUESTS=0 # Celery beat configuration diff --git a/api/tests/integration_tests/controllers/console/app/test_feedback_api_basic.py b/api/tests/integration_tests/controllers/console/app/test_feedback_api_basic.py new file mode 100644 index 0000000000..b164e4f887 --- /dev/null +++ b/api/tests/integration_tests/controllers/console/app/test_feedback_api_basic.py @@ -0,0 +1,106 @@ +"""Basic integration tests for Feedback API endpoints.""" + +import uuid + +from flask.testing import FlaskClient + + +class TestFeedbackApiBasic: + """Basic tests for feedback API endpoints.""" + + def test_feedback_export_endpoint_exists(self, test_client: FlaskClient, auth_header): + """Test that feedback export endpoint exists and handles basic requests.""" + + app_id = str(uuid.uuid4()) + + # Test endpoint exists (even if it fails, it should return 500 or 403, not 404) + response = test_client.get( + f"/console/api/apps/{app_id}/feedbacks/export", headers=auth_header, query_string={"format": "csv"} + ) + + # Should not return 404 (endpoint exists) + assert response.status_code != 404 + + # Should return authentication or permission error + assert response.status_code in [401, 403, 500] # 500 if app doesn't exist, 403 if no permission + + def test_feedback_summary_endpoint_exists(self, test_client: FlaskClient, auth_header): + """Test that feedback summary endpoint exists and handles basic requests.""" + + app_id = str(uuid.uuid4()) + + # Test endpoint exists + response = test_client.get(f"/console/api/apps/{app_id}/feedbacks/summary", headers=auth_header) + + # Should not return 404 (endpoint exists) + assert response.status_code != 404 + + # Should return authentication or permission error + assert response.status_code in [401, 403, 500] + + def test_feedback_export_invalid_format(self, test_client: FlaskClient, auth_header): + """Test feedback export endpoint with invalid format parameter.""" + + app_id = str(uuid.uuid4()) + + # Test with invalid format + response = test_client.get( + f"/console/api/apps/{app_id}/feedbacks/export", + headers=auth_header, + query_string={"format": "invalid_format"}, + ) + + # Should not return 404 + assert response.status_code != 404 + + def test_feedback_export_with_filters(self, test_client: FlaskClient, auth_header): + """Test feedback export endpoint with various filter parameters.""" + + app_id = str(uuid.uuid4()) + + # Test with various filter combinations + filter_params = [ + {"from_source": "user"}, + {"rating": "like"}, + {"has_comment": True}, + {"start_date": "2024-01-01"}, + {"end_date": "2024-12-31"}, + {"format": "json"}, + { + "from_source": "admin", + "rating": "dislike", + "has_comment": True, + "start_date": "2024-01-01", + "end_date": "2024-12-31", + "format": "csv", + }, + ] + + for params in filter_params: + response = test_client.get( + f"/console/api/apps/{app_id}/feedbacks/export", headers=auth_header, query_string=params + ) + + # Should not return 404 + assert response.status_code != 404 + + def test_feedback_export_invalid_dates(self, test_client: FlaskClient, auth_header): + """Test feedback export endpoint with invalid date formats.""" + + app_id = str(uuid.uuid4()) + + # Test with invalid date formats + invalid_dates = [ + {"start_date": "invalid-date"}, + {"end_date": "not-a-date"}, + {"start_date": "2024-13-01"}, # Invalid month + {"end_date": "2024-12-32"}, # Invalid day + ] + + for params in invalid_dates: + response = test_client.get( + f"/console/api/apps/{app_id}/feedbacks/export", headers=auth_header, query_string=params + ) + + # Should not return 404 + assert response.status_code != 404 diff --git a/api/tests/integration_tests/controllers/console/app/test_feedback_export_api.py b/api/tests/integration_tests/controllers/console/app/test_feedback_export_api.py new file mode 100644 index 0000000000..0f8b42e98b --- /dev/null +++ b/api/tests/integration_tests/controllers/console/app/test_feedback_export_api.py @@ -0,0 +1,334 @@ +"""Integration tests for Feedback Export API endpoints.""" + +import json +import uuid +from datetime import datetime +from types import SimpleNamespace +from unittest import mock + +import pytest +from flask.testing import FlaskClient + +from controllers.console.app import message as message_api +from controllers.console.app import wraps +from libs.datetime_utils import naive_utc_now +from models import App, Tenant +from models.account import Account, TenantAccountJoin, TenantAccountRole +from models.model import AppMode, MessageFeedback +from services.feedback_service import FeedbackService + + +class TestFeedbackExportApi: + """Test feedback export API endpoints.""" + + @pytest.fixture + def mock_app_model(self): + """Create a mock App model for testing.""" + app = App() + app.id = str(uuid.uuid4()) + app.mode = AppMode.CHAT + app.tenant_id = str(uuid.uuid4()) + app.status = "normal" + app.name = "Test App" + return app + + @pytest.fixture + def mock_account(self, monkeypatch: pytest.MonkeyPatch): + """Create a mock Account for testing.""" + account = Account( + name="Test User", + email="test@example.com", + ) + account.last_active_at = naive_utc_now() + account.created_at = naive_utc_now() + account.updated_at = naive_utc_now() + account.id = str(uuid.uuid4()) + + # Create mock tenant + tenant = Tenant(name="Test Tenant") + tenant.id = str(uuid.uuid4()) + + mock_session_instance = mock.Mock() + + mock_tenant_join = TenantAccountJoin(role=TenantAccountRole.OWNER) + monkeypatch.setattr(mock_session_instance, "scalar", mock.Mock(return_value=mock_tenant_join)) + + mock_scalars_result = mock.Mock() + mock_scalars_result.one.return_value = tenant + monkeypatch.setattr(mock_session_instance, "scalars", mock.Mock(return_value=mock_scalars_result)) + + mock_session_context = mock.Mock() + mock_session_context.__enter__.return_value = mock_session_instance + monkeypatch.setattr("models.account.Session", lambda _, expire_on_commit: mock_session_context) + + account.current_tenant = tenant + return account + + @pytest.fixture + def sample_feedback_data(self): + """Create sample feedback data for testing.""" + app_id = str(uuid.uuid4()) + conversation_id = str(uuid.uuid4()) + message_id = str(uuid.uuid4()) + + # Mock feedback data + user_feedback = MessageFeedback( + id=str(uuid.uuid4()), + app_id=app_id, + conversation_id=conversation_id, + message_id=message_id, + rating="like", + from_source="user", + content=None, + from_end_user_id=str(uuid.uuid4()), + from_account_id=None, + created_at=naive_utc_now(), + ) + + admin_feedback = MessageFeedback( + id=str(uuid.uuid4()), + app_id=app_id, + conversation_id=conversation_id, + message_id=message_id, + rating="dislike", + from_source="admin", + content="The response was not helpful", + from_end_user_id=None, + from_account_id=str(uuid.uuid4()), + created_at=naive_utc_now(), + ) + + # Mock message and conversation + mock_message = SimpleNamespace( + id=message_id, + conversation_id=conversation_id, + query="What is the weather today?", + answer="It's sunny and 25 degrees outside.", + inputs={"query": "What is the weather today?"}, + created_at=naive_utc_now(), + ) + + mock_conversation = SimpleNamespace(id=conversation_id, name="Weather Conversation", app_id=app_id) + + mock_app = SimpleNamespace(id=app_id, name="Weather App") + + return { + "user_feedback": user_feedback, + "admin_feedback": admin_feedback, + "message": mock_message, + "conversation": mock_conversation, + "app": mock_app, + } + + @pytest.mark.parametrize( + ("role", "status"), + [ + (TenantAccountRole.OWNER, 200), + (TenantAccountRole.ADMIN, 200), + (TenantAccountRole.EDITOR, 200), + (TenantAccountRole.NORMAL, 403), + (TenantAccountRole.DATASET_OPERATOR, 403), + ], + ) + def test_feedback_export_permissions( + self, + test_client: FlaskClient, + auth_header, + monkeypatch, + mock_app_model, + mock_account, + role: TenantAccountRole, + status: int, + ): + """Test feedback export endpoint permissions.""" + + # Setup mocks + mock_load_app_model = mock.Mock(return_value=mock_app_model) + monkeypatch.setattr(wraps, "_load_app_model", mock_load_app_model) + + mock_export_feedbacks = mock.Mock(return_value="mock csv response") + monkeypatch.setattr(FeedbackService, "export_feedbacks", mock_export_feedbacks) + + monkeypatch.setattr(message_api, "current_user", mock_account) + + # Set user role + mock_account.role = role + + response = test_client.get( + f"/console/api/apps/{mock_app_model.id}/feedbacks/export", + headers=auth_header, + query_string={"format": "csv"}, + ) + + assert response.status_code == status + + if status == 200: + mock_export_feedbacks.assert_called_once() + + def test_feedback_export_csv_format( + self, test_client: FlaskClient, auth_header, monkeypatch, mock_app_model, mock_account, sample_feedback_data + ): + """Test feedback export in CSV format.""" + + # Setup mocks + mock_load_app_model = mock.Mock(return_value=mock_app_model) + monkeypatch.setattr(wraps, "_load_app_model", mock_load_app_model) + + # Create mock CSV response + mock_csv_content = ( + "feedback_id,app_name,conversation_id,user_query,ai_response,feedback_rating,feedback_comment\n" + ) + mock_csv_content += f"{sample_feedback_data['user_feedback'].id},{sample_feedback_data['app'].name}," + mock_csv_content += f"{sample_feedback_data['conversation'].id},{sample_feedback_data['message'].query}," + mock_csv_content += f"{sample_feedback_data['message'].answer},👍,\n" + + mock_response = mock.Mock() + mock_response.headers = {"Content-Type": "text/csv; charset=utf-8-sig"} + mock_response.data = mock_csv_content.encode("utf-8") + + mock_export_feedbacks = mock.Mock(return_value=mock_response) + monkeypatch.setattr(FeedbackService, "export_feedbacks", mock_export_feedbacks) + + monkeypatch.setattr(message_api, "current_user", mock_account) + + response = test_client.get( + f"/console/api/apps/{mock_app_model.id}/feedbacks/export", + headers=auth_header, + query_string={"format": "csv", "from_source": "user"}, + ) + + assert response.status_code == 200 + assert "text/csv" in response.content_type + + def test_feedback_export_json_format( + self, test_client: FlaskClient, auth_header, monkeypatch, mock_app_model, mock_account, sample_feedback_data + ): + """Test feedback export in JSON format.""" + + # Setup mocks + mock_load_app_model = mock.Mock(return_value=mock_app_model) + monkeypatch.setattr(wraps, "_load_app_model", mock_load_app_model) + + mock_json_response = { + "export_info": { + "app_id": mock_app_model.id, + "export_date": datetime.now().isoformat(), + "total_records": 2, + "data_source": "dify_feedback_export", + }, + "feedback_data": [ + { + "feedback_id": sample_feedback_data["user_feedback"].id, + "feedback_rating": "👍", + "feedback_rating_raw": "like", + "feedback_comment": "", + } + ], + } + + mock_response = mock.Mock() + mock_response.headers = {"Content-Type": "application/json; charset=utf-8"} + mock_response.data = json.dumps(mock_json_response).encode("utf-8") + + mock_export_feedbacks = mock.Mock(return_value=mock_response) + monkeypatch.setattr(FeedbackService, "export_feedbacks", mock_export_feedbacks) + + monkeypatch.setattr(message_api, "current_user", mock_account) + + response = test_client.get( + f"/console/api/apps/{mock_app_model.id}/feedbacks/export", + headers=auth_header, + query_string={"format": "json"}, + ) + + assert response.status_code == 200 + assert "application/json" in response.content_type + + def test_feedback_export_with_filters( + self, test_client: FlaskClient, auth_header, monkeypatch, mock_app_model, mock_account + ): + """Test feedback export with various filters.""" + + # Setup mocks + mock_load_app_model = mock.Mock(return_value=mock_app_model) + monkeypatch.setattr(wraps, "_load_app_model", mock_load_app_model) + + mock_export_feedbacks = mock.Mock(return_value="mock filtered response") + monkeypatch.setattr(FeedbackService, "export_feedbacks", mock_export_feedbacks) + + monkeypatch.setattr(message_api, "current_user", mock_account) + + # Test with multiple filters + response = test_client.get( + f"/console/api/apps/{mock_app_model.id}/feedbacks/export", + headers=auth_header, + query_string={ + "from_source": "user", + "rating": "dislike", + "has_comment": True, + "start_date": "2024-01-01", + "end_date": "2024-12-31", + "format": "csv", + }, + ) + + assert response.status_code == 200 + + # Verify service was called with correct parameters + mock_export_feedbacks.assert_called_once_with( + app_id=mock_app_model.id, + from_source="user", + rating="dislike", + has_comment=True, + start_date="2024-01-01", + end_date="2024-12-31", + format_type="csv", + ) + + def test_feedback_export_invalid_date_format( + self, test_client: FlaskClient, auth_header, monkeypatch, mock_app_model, mock_account + ): + """Test feedback export with invalid date format.""" + + # Setup mocks + mock_load_app_model = mock.Mock(return_value=mock_app_model) + monkeypatch.setattr(wraps, "_load_app_model", mock_load_app_model) + + # Mock the service to raise ValueError for invalid date + mock_export_feedbacks = mock.Mock(side_effect=ValueError("Invalid date format")) + monkeypatch.setattr(FeedbackService, "export_feedbacks", mock_export_feedbacks) + + monkeypatch.setattr(message_api, "current_user", mock_account) + + response = test_client.get( + f"/console/api/apps/{mock_app_model.id}/feedbacks/export", + headers=auth_header, + query_string={"start_date": "invalid-date", "format": "csv"}, + ) + + assert response.status_code == 400 + response_json = response.get_json() + assert "Parameter validation error" in response_json["error"] + + def test_feedback_export_server_error( + self, test_client: FlaskClient, auth_header, monkeypatch, mock_app_model, mock_account + ): + """Test feedback export with server error.""" + + # Setup mocks + mock_load_app_model = mock.Mock(return_value=mock_app_model) + monkeypatch.setattr(wraps, "_load_app_model", mock_load_app_model) + + # Mock the service to raise an exception + mock_export_feedbacks = mock.Mock(side_effect=Exception("Database connection failed")) + monkeypatch.setattr(FeedbackService, "export_feedbacks", mock_export_feedbacks) + + monkeypatch.setattr(message_api, "current_user", mock_account) + + response = test_client.get( + f"/console/api/apps/{mock_app_model.id}/feedbacks/export", + headers=auth_header, + query_string={"format": "csv"}, + ) + + assert response.status_code == 500 diff --git a/api/tests/integration_tests/workflow/nodes/test_code.py b/api/tests/integration_tests/workflow/nodes/test_code.py index 78878cdeef..e421e4ff36 100644 --- a/api/tests/integration_tests/workflow/nodes/test_code.py +++ b/api/tests/integration_tests/workflow/nodes/test_code.py @@ -69,10 +69,6 @@ def init_code_node(code_config: dict): graph_runtime_state=graph_runtime_state, ) - # Initialize node data - if "data" in code_config: - node.init_node_data(code_config["data"]) - return node diff --git a/api/tests/integration_tests/workflow/nodes/test_http.py b/api/tests/integration_tests/workflow/nodes/test_http.py index 2367990d3e..e75258a2a2 100644 --- a/api/tests/integration_tests/workflow/nodes/test_http.py +++ b/api/tests/integration_tests/workflow/nodes/test_http.py @@ -65,10 +65,6 @@ def init_http_node(config: dict): graph_runtime_state=graph_runtime_state, ) - # Initialize node data - if "data" in config: - node.init_node_data(config["data"]) - return node @@ -709,10 +705,6 @@ def test_nested_object_variable_selector(setup_http_mock): graph_runtime_state=graph_runtime_state, ) - # Initialize node data - if "data" in graph_config["nodes"][1]: - node.init_node_data(graph_config["nodes"][1]["data"]) - result = node._run() assert result.process_data is not None data = result.process_data.get("request", "") diff --git a/api/tests/integration_tests/workflow/nodes/test_llm.py b/api/tests/integration_tests/workflow/nodes/test_llm.py index 3b16c3920b..d268c5da22 100644 --- a/api/tests/integration_tests/workflow/nodes/test_llm.py +++ b/api/tests/integration_tests/workflow/nodes/test_llm.py @@ -82,10 +82,6 @@ def init_llm_node(config: dict) -> LLMNode: graph_runtime_state=graph_runtime_state, ) - # Initialize node data - if "data" in config: - node.init_node_data(config["data"]) - return node diff --git a/api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py b/api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py index 9d9102cee2..654db59bec 100644 --- a/api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py +++ b/api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py @@ -85,7 +85,6 @@ def init_parameter_extractor_node(config: dict): graph_init_params=init_params, graph_runtime_state=graph_runtime_state, ) - node.init_node_data(config.get("data", {})) return node diff --git a/api/tests/integration_tests/workflow/nodes/test_template_transform.py b/api/tests/integration_tests/workflow/nodes/test_template_transform.py index 285387b817..3bcb9a3a34 100644 --- a/api/tests/integration_tests/workflow/nodes/test_template_transform.py +++ b/api/tests/integration_tests/workflow/nodes/test_template_transform.py @@ -82,7 +82,6 @@ def test_execute_code(setup_code_executor_mock): graph_init_params=init_params, graph_runtime_state=graph_runtime_state, ) - node.init_node_data(config.get("data", {})) # execute node result = node._run() diff --git a/api/tests/integration_tests/workflow/nodes/test_tool.py b/api/tests/integration_tests/workflow/nodes/test_tool.py index 8dd8150b1c..d666f0ebe2 100644 --- a/api/tests/integration_tests/workflow/nodes/test_tool.py +++ b/api/tests/integration_tests/workflow/nodes/test_tool.py @@ -62,7 +62,6 @@ def init_tool_node(config: dict): graph_init_params=init_params, graph_runtime_state=graph_runtime_state, ) - node.init_node_data(config.get("data", {})) return node diff --git a/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py b/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py index bec3517d66..72469ad646 100644 --- a/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py +++ b/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py @@ -319,7 +319,7 @@ class TestPauseStatePersistenceLayerTestContainers: # Create pause event event = GraphRunPausedEvent( - reason=SchedulingPause(message="test pause"), + reasons=[SchedulingPause(message="test pause")], outputs={"intermediate": "result"}, ) @@ -381,7 +381,7 @@ class TestPauseStatePersistenceLayerTestContainers: command_channel = _TestCommandChannelImpl() layer.initialize(graph_runtime_state, command_channel) - event = GraphRunPausedEvent(reason=SchedulingPause(message="test pause")) + event = GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")]) # Act - Save pause state layer.on_event(event) @@ -390,6 +390,7 @@ class TestPauseStatePersistenceLayerTestContainers: pause_entity = self.workflow_run_service._workflow_run_repo.get_workflow_pause(self.test_workflow_run_id) assert pause_entity is not None assert pause_entity.workflow_execution_id == self.test_workflow_run_id + assert pause_entity.get_pause_reasons() == event.reasons state_bytes = pause_entity.get_state() resumption_context = WorkflowResumptionContext.loads(state_bytes.decode()) @@ -414,7 +415,7 @@ class TestPauseStatePersistenceLayerTestContainers: command_channel = _TestCommandChannelImpl() layer.initialize(graph_runtime_state, command_channel) - event = GraphRunPausedEvent(reason=SchedulingPause(message="test pause")) + event = GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")]) # Act layer.on_event(event) @@ -448,7 +449,7 @@ class TestPauseStatePersistenceLayerTestContainers: command_channel = _TestCommandChannelImpl() layer.initialize(graph_runtime_state, command_channel) - event = GraphRunPausedEvent(reason=SchedulingPause(message="test pause")) + event = GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")]) # Act layer.on_event(event) @@ -514,7 +515,7 @@ class TestPauseStatePersistenceLayerTestContainers: command_channel = _TestCommandChannelImpl() layer.initialize(graph_runtime_state, command_channel) - event = GraphRunPausedEvent(reason=SchedulingPause(message="test pause")) + event = GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")]) # Act layer.on_event(event) @@ -570,7 +571,7 @@ class TestPauseStatePersistenceLayerTestContainers: layer = self._create_pause_state_persistence_layer() # Don't initialize - graph_runtime_state should not be set - event = GraphRunPausedEvent(reason=SchedulingPause(message="test pause")) + event = GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")]) # Act & Assert - Should raise AttributeError with pytest.raises(AttributeError): diff --git a/api/tests/test_containers_integration_tests/services/test_api_based_extension_service.py b/api/tests/test_containers_integration_tests/services/test_api_based_extension_service.py index 2cea24d085..8c8be2e670 100644 --- a/api/tests/test_containers_integration_tests/services/test_api_based_extension_service.py +++ b/api/tests/test_containers_integration_tests/services/test_api_based_extension_service.py @@ -295,9 +295,13 @@ class TestAPIBasedExtensionService: original_name = created_extension.name original_endpoint = created_extension.api_endpoint - # Update the extension + # Update the extension with guaranteed different values new_name = fake.company() + # Ensure new endpoint is different from original new_endpoint = f"https://{fake.domain_name()}/api" + # If by chance they're the same, generate a new one + while new_endpoint == original_endpoint: + new_endpoint = f"https://{fake.domain_name()}/api" new_api_key = fake.password(length=20) created_extension.name = new_name diff --git a/api/tests/test_containers_integration_tests/services/test_app_generate_service.py b/api/tests/test_containers_integration_tests/services/test_app_generate_service.py index 0f9ed94017..476f58585d 100644 --- a/api/tests/test_containers_integration_tests/services/test_app_generate_service.py +++ b/api/tests/test_containers_integration_tests/services/test_app_generate_service.py @@ -82,6 +82,7 @@ class TestAppGenerateService: # Setup dify_config mock returns mock_dify_config.BILLING_ENABLED = False mock_dify_config.APP_MAX_ACTIVE_REQUESTS = 100 + mock_dify_config.APP_DEFAULT_ACTIVE_REQUESTS = 100 mock_dify_config.APP_DAILY_RATE_LIMIT = 1000 mock_global_dify_config.BILLING_ENABLED = False diff --git a/api/tests/test_containers_integration_tests/services/test_feedback_service.py b/api/tests/test_containers_integration_tests/services/test_feedback_service.py new file mode 100644 index 0000000000..60919dff0d --- /dev/null +++ b/api/tests/test_containers_integration_tests/services/test_feedback_service.py @@ -0,0 +1,386 @@ +"""Unit tests for FeedbackService.""" + +import json +from datetime import datetime +from types import SimpleNamespace +from unittest import mock + +import pytest + +from extensions.ext_database import db +from models.model import App, Conversation, Message +from services.feedback_service import FeedbackService + + +class TestFeedbackService: + """Test FeedbackService methods.""" + + @pytest.fixture + def mock_db_session(self, monkeypatch): + """Mock database session.""" + mock_session = mock.Mock() + monkeypatch.setattr(db, "session", mock_session) + return mock_session + + @pytest.fixture + def sample_data(self): + """Create sample data for testing.""" + app_id = "test-app-id" + + # Create mock models + app = App(id=app_id, name="Test App") + + conversation = Conversation(id="test-conversation-id", app_id=app_id, name="Test Conversation") + + message = Message( + id="test-message-id", + conversation_id="test-conversation-id", + query="What is AI?", + answer="AI is artificial intelligence.", + inputs={"query": "What is AI?"}, + created_at=datetime(2024, 1, 1, 10, 0, 0), + ) + + # Use SimpleNamespace to avoid ORM model constructor issues + user_feedback = SimpleNamespace( + id="user-feedback-id", + app_id=app_id, + conversation_id="test-conversation-id", + message_id="test-message-id", + rating="like", + from_source="user", + content="Great answer!", + from_end_user_id="user-123", + from_account_id=None, + from_account=None, # Mock account object + created_at=datetime(2024, 1, 1, 10, 5, 0), + ) + + admin_feedback = SimpleNamespace( + id="admin-feedback-id", + app_id=app_id, + conversation_id="test-conversation-id", + message_id="test-message-id", + rating="dislike", + from_source="admin", + content="Could be more detailed", + from_end_user_id=None, + from_account_id="admin-456", + from_account=SimpleNamespace(name="Admin User"), # Mock account object + created_at=datetime(2024, 1, 1, 10, 10, 0), + ) + + return { + "app": app, + "conversation": conversation, + "message": message, + "user_feedback": user_feedback, + "admin_feedback": admin_feedback, + } + + def test_export_feedbacks_csv_format(self, mock_db_session, sample_data): + """Test exporting feedback data in CSV format.""" + + # Setup mock query result + mock_query = mock.Mock() + mock_query.join.return_value = mock_query + mock_query.outerjoin.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.order_by.return_value = mock_query + mock_query.all.return_value = [ + ( + sample_data["user_feedback"], + sample_data["message"], + sample_data["conversation"], + sample_data["app"], + sample_data["user_feedback"].from_account, + ) + ] + + mock_db_session.query.return_value = mock_query + + # Test CSV export + result = FeedbackService.export_feedbacks(app_id=sample_data["app"].id, format_type="csv") + + # Verify response structure + assert hasattr(result, "headers") + assert "text/csv" in result.headers["Content-Type"] + assert "attachment" in result.headers["Content-Disposition"] + + # Check CSV content + csv_content = result.get_data(as_text=True) + # Verify essential headers exist (order may include additional columns) + assert "feedback_id" in csv_content + assert "app_name" in csv_content + assert "conversation_id" in csv_content + assert sample_data["app"].name in csv_content + assert sample_data["message"].query in csv_content + + def test_export_feedbacks_json_format(self, mock_db_session, sample_data): + """Test exporting feedback data in JSON format.""" + + # Setup mock query result + mock_query = mock.Mock() + mock_query.join.return_value = mock_query + mock_query.outerjoin.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.order_by.return_value = mock_query + mock_query.all.return_value = [ + ( + sample_data["admin_feedback"], + sample_data["message"], + sample_data["conversation"], + sample_data["app"], + sample_data["admin_feedback"].from_account, + ) + ] + + mock_db_session.query.return_value = mock_query + + # Test JSON export + result = FeedbackService.export_feedbacks(app_id=sample_data["app"].id, format_type="json") + + # Verify response structure + assert hasattr(result, "headers") + assert "application/json" in result.headers["Content-Type"] + assert "attachment" in result.headers["Content-Disposition"] + + # Check JSON content + json_content = json.loads(result.get_data(as_text=True)) + assert "export_info" in json_content + assert "feedback_data" in json_content + assert json_content["export_info"]["app_id"] == sample_data["app"].id + assert json_content["export_info"]["total_records"] == 1 + + def test_export_feedbacks_with_filters(self, mock_db_session, sample_data): + """Test exporting feedback with various filters.""" + + # Setup mock query result + mock_query = mock.Mock() + mock_query.join.return_value = mock_query + mock_query.outerjoin.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.order_by.return_value = mock_query + mock_query.all.return_value = [ + ( + sample_data["admin_feedback"], + sample_data["message"], + sample_data["conversation"], + sample_data["app"], + sample_data["admin_feedback"].from_account, + ) + ] + + mock_db_session.query.return_value = mock_query + + # Test with filters + result = FeedbackService.export_feedbacks( + app_id=sample_data["app"].id, + from_source="admin", + rating="dislike", + has_comment=True, + start_date="2024-01-01", + end_date="2024-12-31", + format_type="csv", + ) + + # Verify filters were applied + assert mock_query.filter.called + filter_calls = mock_query.filter.call_args_list + # At least three filter invocations are expected (source, rating, comment) + assert len(filter_calls) >= 3 + + def test_export_feedbacks_no_data(self, mock_db_session, sample_data): + """Test exporting feedback when no data exists.""" + + # Setup mock query result with no data + mock_query = mock.Mock() + mock_query.join.return_value = mock_query + mock_query.outerjoin.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.order_by.return_value = mock_query + mock_query.all.return_value = [] + + mock_db_session.query.return_value = mock_query + + result = FeedbackService.export_feedbacks(app_id=sample_data["app"].id, format_type="csv") + + # Should return an empty CSV with headers only + assert hasattr(result, "headers") + assert "text/csv" in result.headers["Content-Type"] + csv_content = result.get_data(as_text=True) + # Headers should exist (order can include additional columns) + assert "feedback_id" in csv_content + assert "app_name" in csv_content + assert "conversation_id" in csv_content + # No data rows expected + assert len([line for line in csv_content.strip().splitlines() if line.strip()]) == 1 + + def test_export_feedbacks_invalid_date_format(self, mock_db_session, sample_data): + """Test exporting feedback with invalid date format.""" + + # Test with invalid start_date + with pytest.raises(ValueError, match="Invalid start_date format"): + FeedbackService.export_feedbacks(app_id=sample_data["app"].id, start_date="invalid-date-format") + + # Test with invalid end_date + with pytest.raises(ValueError, match="Invalid end_date format"): + FeedbackService.export_feedbacks(app_id=sample_data["app"].id, end_date="invalid-date-format") + + def test_export_feedbacks_invalid_format(self, mock_db_session, sample_data): + """Test exporting feedback with unsupported format.""" + + with pytest.raises(ValueError, match="Unsupported format"): + FeedbackService.export_feedbacks( + app_id=sample_data["app"].id, + format_type="xml", # Unsupported format + ) + + def test_export_feedbacks_long_response_truncation(self, mock_db_session, sample_data): + """Test that long AI responses are truncated in export.""" + + # Create message with long response + long_message = Message( + id="long-message-id", + conversation_id="test-conversation-id", + query="What is AI?", + answer="A" * 600, # 600 character response + inputs={"query": "What is AI?"}, + created_at=datetime(2024, 1, 1, 10, 0, 0), + ) + + # Setup mock query result + mock_query = mock.Mock() + mock_query.join.return_value = mock_query + mock_query.outerjoin.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.order_by.return_value = mock_query + mock_query.all.return_value = [ + ( + sample_data["user_feedback"], + long_message, + sample_data["conversation"], + sample_data["app"], + sample_data["user_feedback"].from_account, + ) + ] + + mock_db_session.query.return_value = mock_query + + # Test export + result = FeedbackService.export_feedbacks(app_id=sample_data["app"].id, format_type="json") + + # Check JSON content + json_content = json.loads(result.get_data(as_text=True)) + exported_answer = json_content["feedback_data"][0]["ai_response"] + + # Should be truncated with ellipsis + assert len(exported_answer) <= 503 # 500 + "..." + assert exported_answer.endswith("...") + assert len(exported_answer) > 500 # Should be close to limit + + def test_export_feedbacks_unicode_content(self, mock_db_session, sample_data): + """Test exporting feedback with unicode content (Chinese characters).""" + + # Create feedback with Chinese content (use SimpleNamespace to avoid ORM constructor constraints) + chinese_feedback = SimpleNamespace( + id="chinese-feedback-id", + app_id=sample_data["app"].id, + conversation_id="test-conversation-id", + message_id="test-message-id", + rating="dislike", + from_source="user", + content="回答不够详细，需要更多信息", + from_end_user_id="user-123", + from_account_id=None, + created_at=datetime(2024, 1, 1, 10, 5, 0), + ) + + # Create Chinese message + chinese_message = Message( + id="chinese-message-id", + conversation_id="test-conversation-id", + query="什么是人工智能？", + answer="人工智能是模拟人类智能的技术。", + inputs={"query": "什么是人工智能？"}, + created_at=datetime(2024, 1, 1, 10, 0, 0), + ) + + # Setup mock query result + mock_query = mock.Mock() + mock_query.join.return_value = mock_query + mock_query.outerjoin.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.order_by.return_value = mock_query + mock_query.all.return_value = [ + ( + chinese_feedback, + chinese_message, + sample_data["conversation"], + sample_data["app"], + None, # No account for user feedback + ) + ] + + mock_db_session.query.return_value = mock_query + + # Test export + result = FeedbackService.export_feedbacks(app_id=sample_data["app"].id, format_type="csv") + + # Check that unicode content is preserved + csv_content = result.get_data(as_text=True) + assert "什么是人工智能？" in csv_content + assert "回答不够详细，需要更多信息" in csv_content + assert "人工智能是模拟人类智能的技术" in csv_content + + def test_export_feedbacks_emoji_ratings(self, mock_db_session, sample_data): + """Test that rating emojis are properly formatted in export.""" + + # Setup mock query result with both like and dislike feedback + mock_query = mock.Mock() + mock_query.join.return_value = mock_query + mock_query.outerjoin.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.order_by.return_value = mock_query + mock_query.all.return_value = [ + ( + sample_data["user_feedback"], + sample_data["message"], + sample_data["conversation"], + sample_data["app"], + sample_data["user_feedback"].from_account, + ), + ( + sample_data["admin_feedback"], + sample_data["message"], + sample_data["conversation"], + sample_data["app"], + sample_data["admin_feedback"].from_account, + ), + ] + + mock_db_session.query.return_value = mock_query + + # Test export + result = FeedbackService.export_feedbacks(app_id=sample_data["app"].id, format_type="json") + + # Check JSON content for emoji ratings + json_content = json.loads(result.get_data(as_text=True)) + feedback_data = json_content["feedback_data"] + + # Should have both feedback records + assert len(feedback_data) == 2 + + # Check that emojis are properly set + like_feedback = next(f for f in feedback_data if f["feedback_rating_raw"] == "like") + dislike_feedback = next(f for f in feedback_data if f["feedback_rating_raw"] == "dislike") + + assert like_feedback["feedback_rating"] == "👍" + assert dislike_feedback["feedback_rating"] == "👎" diff --git a/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py b/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py index cb1e79d507..71cedd26c4 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py @@ -257,7 +257,6 @@ class TestWorkflowToolManageService: # Attempt to create second workflow tool with same name second_tool_parameters = self._create_test_workflow_tool_parameters() - with pytest.raises(ValueError) as exc_info: WorkflowToolManageService.create_workflow_tool( user_id=account.id, @@ -309,7 +308,6 @@ class TestWorkflowToolManageService: # Attempt to create workflow tool with non-existent app tool_parameters = self._create_test_workflow_tool_parameters() - with pytest.raises(ValueError) as exc_info: WorkflowToolManageService.create_workflow_tool( user_id=account.id, @@ -365,7 +363,6 @@ class TestWorkflowToolManageService: "required": True, } ] - # Attempt to create workflow tool with invalid parameters with pytest.raises(ValueError) as exc_info: WorkflowToolManageService.create_workflow_tool( @@ -416,7 +413,6 @@ class TestWorkflowToolManageService: # Create first workflow tool first_tool_name = fake.word() first_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, @@ -431,7 +427,6 @@ class TestWorkflowToolManageService: # Attempt to create second workflow tool with same app_id but different name second_tool_name = fake.word() second_tool_parameters = self._create_test_workflow_tool_parameters() - with pytest.raises(ValueError) as exc_info: WorkflowToolManageService.create_workflow_tool( user_id=account.id, @@ -486,7 +481,6 @@ class TestWorkflowToolManageService: # Attempt to create workflow tool for app without workflow tool_parameters = self._create_test_workflow_tool_parameters() - with pytest.raises(ValueError) as exc_info: WorkflowToolManageService.create_workflow_tool( user_id=account.id, @@ -534,7 +528,6 @@ class TestWorkflowToolManageService: # Create initial workflow tool initial_tool_name = fake.word() initial_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, @@ -621,7 +614,6 @@ class TestWorkflowToolManageService: # Attempt to update non-existent workflow tool tool_parameters = self._create_test_workflow_tool_parameters() - with pytest.raises(ValueError) as exc_info: WorkflowToolManageService.update_workflow_tool( user_id=account.id, @@ -671,7 +663,6 @@ class TestWorkflowToolManageService: # Create first workflow tool first_tool_name = fake.word() first_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, diff --git a/api/tests/test_containers_integration_tests/test_workflow_pause_integration.py b/api/tests/test_containers_integration_tests/test_workflow_pause_integration.py index 79da5d4d0e..889e3d1d83 100644 --- a/api/tests/test_containers_integration_tests/test_workflow_pause_integration.py +++ b/api/tests/test_containers_integration_tests/test_workflow_pause_integration.py @@ -334,12 +334,14 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) # Assert - Pause state created assert pause_entity is not None assert pause_entity.id is not None assert pause_entity.workflow_execution_id == workflow_run.id + assert list(pause_entity.get_pause_reasons()) == [] # Convert both to strings for comparison retrieved_state = pause_entity.get_state() if isinstance(retrieved_state, bytes): @@ -366,6 +368,7 @@ class TestWorkflowPauseIntegration: if isinstance(retrieved_state, bytes): retrieved_state = retrieved_state.decode() assert retrieved_state == test_state + assert list(retrieved_entity.get_pause_reasons()) == [] # Act - Resume workflow resumed_entity = repository.resume_workflow_pause( @@ -402,6 +405,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) assert pause_entity is not None @@ -432,6 +436,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) @pytest.mark.parametrize("test_case", resume_workflow_success_cases(), ids=lambda tc: tc.name) @@ -449,6 +454,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) self.session.refresh(workflow_run) @@ -480,6 +486,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) self.session.refresh(workflow_run) @@ -503,6 +510,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) pause_model = self.session.get(WorkflowPauseModel, pause_entity.id) pause_model.resumed_at = naive_utc_now() @@ -530,6 +538,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=nonexistent_id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) def test_resume_nonexistent_workflow_run(self): @@ -543,6 +552,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) nonexistent_id = str(uuid.uuid4()) @@ -570,6 +580,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) # Manually adjust timestamps for testing @@ -648,6 +659,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) pause_entities.append(pause_entity) @@ -750,6 +762,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run1.id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) # Try to access pause from tenant 2 using tenant 1's repository @@ -762,6 +775,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run2.id, state_owner_user_id=account2.id, state=test_state, + pause_reasons=[], ) # Assert - Both pauses should exist and be separate @@ -782,6 +796,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) # Verify pause is properly scoped @@ -802,6 +817,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=test_state, + pause_reasons=[], ) # Assert - Verify file was uploaded to storage @@ -828,9 +844,7 @@ class TestWorkflowPauseIntegration: repository = self._get_workflow_run_repository() pause_entity = repository.create_workflow_pause( - workflow_run_id=workflow_run.id, - state_owner_user_id=self.test_user_id, - state=test_state, + workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=test_state, pause_reasons=[] ) # Get file info before deletion @@ -868,6 +882,7 @@ class TestWorkflowPauseIntegration: workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=large_state_json, + pause_reasons=[], ) # Assert @@ -902,9 +917,7 @@ class TestWorkflowPauseIntegration: # Pause pause_entity = repository.create_workflow_pause( - workflow_run_id=workflow_run.id, - state_owner_user_id=self.test_user_id, - state=state, + workflow_run_id=workflow_run.id, state_owner_user_id=self.test_user_id, state=state, pause_reasons=[] ) assert pause_entity is not None diff --git a/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py b/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py index 807f5e0fa5..534420f21e 100644 --- a/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py +++ b/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py @@ -31,7 +31,7 @@ class TestDataFactory: @staticmethod def create_graph_run_paused_event(outputs: dict[str, object] | None = None) -> GraphRunPausedEvent: - return GraphRunPausedEvent(reason=SchedulingPause(message="test pause"), outputs=outputs or {}) + return GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")], outputs=outputs or {}) @staticmethod def create_graph_run_started_event() -> GraphRunStartedEvent: @@ -255,15 +255,17 @@ class TestPauseStatePersistenceLayer: layer.on_event(event) mock_factory.assert_called_once_with(session_factory) - mock_repo.create_workflow_pause.assert_called_once_with( - workflow_run_id="run-123", - state_owner_user_id="owner-123", - state=mock_repo.create_workflow_pause.call_args.kwargs["state"], - ) - serialized_state = mock_repo.create_workflow_pause.call_args.kwargs["state"] + assert mock_repo.create_workflow_pause.call_count == 1 + call_kwargs = mock_repo.create_workflow_pause.call_args.kwargs + assert call_kwargs["workflow_run_id"] == "run-123" + assert call_kwargs["state_owner_user_id"] == "owner-123" + serialized_state = call_kwargs["state"] resumption_context = WorkflowResumptionContext.loads(serialized_state) assert resumption_context.serialized_graph_runtime_state == expected_state assert resumption_context.get_generate_entity().model_dump() == generate_entity.model_dump() + pause_reasons = call_kwargs["pause_reasons"] + + assert isinstance(pause_reasons, list) def test_on_event_ignores_non_paused_events(self, monkeypatch: pytest.MonkeyPatch): session_factory = Mock(name="session_factory") diff --git a/api/tests/unit_tests/core/rag/retrieval/__init__.py b/api/tests/unit_tests/core/rag/retrieval/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py new file mode 100644 index 0000000000..0163e42992 --- /dev/null +++ b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py @@ -0,0 +1,1696 @@ +""" +Unit tests for dataset retrieval functionality. + +This module provides comprehensive test coverage for the RetrievalService class, +which is responsible for retrieving relevant documents from datasets using various +search strategies. + +Core Retrieval Mechanisms Tested: +================================== +1. **Vector Search (Semantic Search)** + - Uses embedding vectors to find semantically similar documents + - Supports score thresholds and top-k limiting + - Can filter by document IDs and metadata + +2. **Keyword Search** + - Traditional text-based search using keyword matching + - Handles special characters and query escaping + - Supports document filtering + +3. **Full-Text Search** + - BM25-based full-text search for text matching + - Used in hybrid search scenarios + +4. **Hybrid Search** + - Combines vector and full-text search results + - Implements deduplication to avoid duplicate chunks + - Uses DataPostProcessor for score merging with configurable weights + +5. **Score Merging Algorithms** + - Deduplication based on doc_id + - Retains higher-scoring duplicates + - Supports weighted score combination + +6. **Metadata Filtering** + - Filters documents based on metadata conditions + - Supports document ID filtering + +Test Architecture: +================== +- **Fixtures**: Provide reusable mock objects (datasets, documents, Flask app) +- **Mocking Strategy**: Mock at the method level (embedding_search, keyword_search, etc.) + rather than at the class level to properly simulate the ThreadPoolExecutor behavior +- **Pattern**: All tests follow Arrange-Act-Assert (AAA) pattern +- **Isolation**: Each test is independent and doesn't rely on external state + +Running Tests: +============== + # Run all tests in this module + uv run --project api pytest \ + api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py -v + + # Run a specific test class + uv run --project api pytest \ + api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py::TestRetrievalService -v + + # Run a specific test + uv run --project api pytest \ + api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py::\ +TestRetrievalService::test_vector_search_basic -v + +Notes: +====== +- The RetrievalService uses ThreadPoolExecutor for concurrent search operations +- Tests mock the individual search methods to avoid threading complexity +- All mocked search methods modify the all_documents list in-place +- Score thresholds and top-k limits are enforced by the search methods +""" + +from unittest.mock import MagicMock, Mock, patch +from uuid import uuid4 + +import pytest + +from core.rag.datasource.retrieval_service import RetrievalService +from core.rag.models.document import Document +from core.rag.retrieval.retrieval_methods import RetrievalMethod +from models.dataset import Dataset + +# ==================== Helper Functions ==================== + + +def create_mock_document( + content: str, + doc_id: str, + score: float = 0.8, + provider: str = "dify", + additional_metadata: dict | None = None, +) -> Document: + """ + Create a mock Document object for testing. + + This helper function standardizes document creation across tests, + ensuring consistent structure and reducing code duplication. + + Args: + content: The text content of the document + doc_id: Unique identifier for the document chunk + score: Relevance score (0.0 to 1.0) + provider: Document provider ("dify" or "external") + additional_metadata: Optional extra metadata fields + + Returns: + Document: A properly structured Document object + + Example: + >>> doc = create_mock_document("Python is great", "doc1", score=0.95) + >>> assert doc.metadata["score"] == 0.95 + """ + metadata = { + "doc_id": doc_id, + "document_id": str(uuid4()), + "dataset_id": str(uuid4()), + "score": score, + } + + # Merge additional metadata if provided + if additional_metadata: + metadata.update(additional_metadata) + + return Document( + page_content=content, + metadata=metadata, + provider=provider, + ) + + +def create_side_effect_for_search(documents: list[Document]): + """ + Create a side effect function for mocking search methods. + + This helper creates a function that simulates how RetrievalService + search methods work - they modify the all_documents list in-place + rather than returning values directly. + + Args: + documents: List of documents to add to all_documents + + Returns: + Callable: A side effect function compatible with mock.side_effect + + Example: + >>> mock_search.side_effect = create_side_effect_for_search([doc1, doc2]) + + Note: + The RetrievalService uses ThreadPoolExecutor which submits tasks that + modify a shared all_documents list. This pattern simulates that behavior. + """ + + def side_effect(flask_app, dataset_id, query, top_k, *args, all_documents, exceptions, **kwargs): + """ + Side effect function that mimics search method behavior. + + Args: + flask_app: Flask application context (unused in mock) + dataset_id: ID of the dataset being searched + query: Search query string + top_k: Maximum number of results + all_documents: Shared list to append results to + exceptions: Shared list to append errors to + **kwargs: Additional arguments (score_threshold, document_ids_filter, etc.) + """ + all_documents.extend(documents) + + return side_effect + + +def create_side_effect_with_exception(error_message: str): + """ + Create a side effect function that adds an exception to the exceptions list. + + Used for testing error handling in the RetrievalService. + + Args: + error_message: The error message to add to exceptions + + Returns: + Callable: A side effect function that simulates an error + + Example: + >>> mock_search.side_effect = create_side_effect_with_exception("Search failed") + """ + + def side_effect(flask_app, dataset_id, query, top_k, *args, all_documents, exceptions, **kwargs): + """Add error message to exceptions list.""" + exceptions.append(error_message) + + return side_effect + + +class TestRetrievalService: + """ + Comprehensive test suite for RetrievalService class. + + This test class validates all retrieval methods and their interactions, + including edge cases, error handling, and integration scenarios. + + Test Organization: + ================== + 1. Fixtures (lines ~190-240) + - mock_dataset: Standard dataset configuration + - sample_documents: Reusable test documents with varying scores + - mock_flask_app: Flask application context + - mock_thread_pool: Synchronous executor for deterministic testing + + 2. Vector Search Tests (lines ~240-350) + - Basic functionality + - Document filtering + - Empty results + - Metadata filtering + - Score thresholds + + 3. Keyword Search Tests (lines ~350-450) + - Basic keyword matching + - Special character handling + - Document filtering + + 4. Hybrid Search Tests (lines ~450-640) + - Vector + full-text combination + - Deduplication logic + - Weighted score merging + + 5. Full-Text Search Tests (lines ~640-680) + - BM25-based search + + 6. Score Merging Tests (lines ~680-790) + - Deduplication algorithms + - Score comparison + - Provider-specific handling + + 7. Error Handling Tests (lines ~790-920) + - Empty queries + - Non-existent datasets + - Exception propagation + + 8. Additional Tests (lines ~920-1080) + - Query escaping + - Reranking integration + - Top-K limiting + + Mocking Strategy: + ================= + Tests mock at the method level (embedding_search, keyword_search, etc.) + rather than the underlying Vector/Keyword classes. This approach: + - Avoids complexity of mocking ThreadPoolExecutor behavior + - Provides clearer test intent + - Makes tests more maintainable + - Properly simulates the in-place list modification pattern + + Common Patterns: + ================ + 1. **Arrange**: Set up mocks with side_effect functions + 2. **Act**: Call RetrievalService.retrieve() with specific parameters + 3. **Assert**: Verify results, mock calls, and side effects + + Example Test Structure: + ```python + def test_example(self, mock_get_dataset, mock_search, mock_dataset): + # Arrange: Set up test data and mocks + mock_get_dataset.return_value = mock_dataset + mock_search.side_effect = create_side_effect_for_search([doc1, doc2]) + + # Act: Execute the method under test + results = RetrievalService.retrieve(...) + + # Assert: Verify expectations + assert len(results) == 2 + mock_search.assert_called_once() + ``` + """ + + @pytest.fixture + def mock_dataset(self) -> Dataset: + """ + Create a mock Dataset object for testing. + + Returns: + Dataset: Mock dataset with standard configuration + """ + dataset = Mock(spec=Dataset) + dataset.id = str(uuid4()) + dataset.tenant_id = str(uuid4()) + dataset.name = "test_dataset" + dataset.indexing_technique = "high_quality" + dataset.embedding_model = "text-embedding-ada-002" + dataset.embedding_model_provider = "openai" + dataset.retrieval_model = { + "search_method": RetrievalMethod.SEMANTIC_SEARCH, + "reranking_enable": False, + "top_k": 4, + "score_threshold_enabled": False, + } + return dataset + + @pytest.fixture + def sample_documents(self) -> list[Document]: + """ + Create sample documents for testing retrieval results. + + Returns: + list[Document]: List of mock documents with varying scores + """ + return [ + Document( + page_content="Python is a high-level programming language.", + metadata={ + "doc_id": "doc1", + "document_id": str(uuid4()), + "dataset_id": str(uuid4()), + "score": 0.95, + }, + provider="dify", + ), + Document( + page_content="JavaScript is widely used for web development.", + metadata={ + "doc_id": "doc2", + "document_id": str(uuid4()), + "dataset_id": str(uuid4()), + "score": 0.85, + }, + provider="dify", + ), + Document( + page_content="Machine learning is a subset of artificial intelligence.", + metadata={ + "doc_id": "doc3", + "document_id": str(uuid4()), + "dataset_id": str(uuid4()), + "score": 0.75, + }, + provider="dify", + ), + ] + + @pytest.fixture + def mock_flask_app(self): + """ + Create a mock Flask application context. + + Returns: + Mock: Flask app mock with app_context + """ + app = MagicMock() + app.app_context.return_value.__enter__ = Mock() + app.app_context.return_value.__exit__ = Mock() + return app + + @pytest.fixture(autouse=True) + def mock_thread_pool(self): + """ + Mock ThreadPoolExecutor to run tasks synchronously in tests. + + The RetrievalService uses ThreadPoolExecutor to run search operations + concurrently (embedding_search, keyword_search, full_text_index_search). + In tests, we want synchronous execution for: + - Deterministic behavior + - Easier debugging + - Avoiding race conditions + - Simpler assertions + + How it works: + ------------- + 1. Intercepts ThreadPoolExecutor creation + 2. Replaces submit() to execute functions immediately (synchronously) + 3. Functions modify shared all_documents list in-place + 4. Mocks concurrent.futures.wait() since tasks are already done + + Why this approach: + ------------------ + - RetrievalService.retrieve() creates a ThreadPoolExecutor context + - It submits search tasks that modify all_documents list + - concurrent.futures.wait() waits for all tasks to complete + - By executing synchronously, we avoid threading complexity in tests + + Returns: + Mock: Mocked ThreadPoolExecutor that executes tasks synchronously + """ + with patch("core.rag.datasource.retrieval_service.ThreadPoolExecutor") as mock_executor: + # Store futures to track submitted tasks (for debugging if needed) + futures_list = [] + + def sync_submit(fn, *args, **kwargs): + """ + Synchronous replacement for ThreadPoolExecutor.submit(). + + Instead of scheduling the function for async execution, + we execute it immediately in the current thread. + + Args: + fn: The function to execute (e.g., embedding_search) + *args, **kwargs: Arguments to pass to the function + + Returns: + Mock: A mock Future object + """ + future = Mock() + try: + # Execute immediately - this modifies all_documents in place + # The function signature is: fn(flask_app, dataset_id, query, + # top_k, all_documents, exceptions, ...) + fn(*args, **kwargs) + future.result.return_value = None + future.exception.return_value = None + except Exception as e: + # If function raises, store exception in future + future.result.return_value = None + future.exception.return_value = e + + futures_list.append(future) + return future + + # Set up the mock executor instance + mock_executor_instance = Mock() + mock_executor_instance.submit = sync_submit + + # Configure context manager behavior (__enter__ and __exit__) + mock_executor.return_value.__enter__.return_value = mock_executor_instance + mock_executor.return_value.__exit__.return_value = None + + # Mock concurrent.futures.wait to do nothing since tasks are already done + # In real code, this waits for all futures to complete + # In tests, futures complete immediately, so wait is a no-op + with patch("core.rag.datasource.retrieval_service.concurrent.futures.wait"): + yield mock_executor + + # ==================== Vector Search Tests ==================== + + @patch("core.rag.datasource.retrieval_service.RetrievalService.embedding_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_vector_search_basic(self, mock_get_dataset, mock_embedding_search, mock_dataset, sample_documents): + """ + Test basic vector/semantic search functionality. + + This test validates the core vector search flow: + 1. Dataset is retrieved from database + 2. embedding_search is called via ThreadPoolExecutor + 3. Documents are added to shared all_documents list + 4. Results are returned to caller + + Verifies: + - Vector search is called with correct parameters + - Results are returned in expected format + - Score threshold is applied correctly + - Documents maintain their metadata and scores + """ + # ==================== ARRANGE ==================== + # Set up the mock dataset that will be "retrieved" from database + mock_get_dataset.return_value = mock_dataset + + # Create a side effect function that simulates embedding_search behavior + # In the real implementation, embedding_search: + # 1. Gets the dataset + # 2. Creates a Vector instance + # 3. Calls search_by_vector with embeddings + # 4. Extends all_documents with results + def side_effect_embedding_search( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + """Simulate embedding_search adding documents to the shared list.""" + all_documents.extend(sample_documents) + + mock_embedding_search.side_effect = side_effect_embedding_search + + # Define test parameters + query = "What is Python?" # Natural language query + top_k = 3 # Maximum number of results to return + score_threshold = 0.7 # Minimum relevance score (0.0 to 1.0) + + # ==================== ACT ==================== + # Call the retrieve method with SEMANTIC_SEARCH strategy + # This will: + # 1. Check if query is empty (early return if so) + # 2. Get the dataset using _get_dataset + # 3. Create ThreadPoolExecutor + # 4. Submit embedding_search task + # 5. Wait for completion + # 6. Return all_documents list + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.SEMANTIC_SEARCH, + dataset_id=mock_dataset.id, + query=query, + top_k=top_k, + score_threshold=score_threshold, + ) + + # ==================== ASSERT ==================== + # Verify we got the expected number of documents + assert len(results) == 3, "Should return 3 documents from sample_documents" + + # Verify all results are Document objects (type safety) + assert all(isinstance(doc, Document) for doc in results), "All results should be Document instances" + + # Verify documents maintain their scores (highest score first in sample_documents) + assert results[0].metadata["score"] == 0.95, "First document should have highest score from sample_documents" + + # Verify embedding_search was called exactly once + # This confirms the search method was invoked by ThreadPoolExecutor + mock_embedding_search.assert_called_once() + + @patch("core.rag.datasource.retrieval_service.RetrievalService.embedding_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_vector_search_with_document_filter( + self, mock_get_dataset, mock_embedding_search, mock_dataset, sample_documents + ): + """ + Test vector search with document ID filtering. + + Verifies: + - Document ID filter is passed correctly to vector search + - Only specified documents are searched + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + filtered_docs = [sample_documents[0]] + + def side_effect_embedding_search( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + all_documents.extend(filtered_docs) + + mock_embedding_search.side_effect = side_effect_embedding_search + document_ids_filter = [sample_documents[0].metadata["document_id"]] + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.SEMANTIC_SEARCH, + dataset_id=mock_dataset.id, + query="test query", + top_k=5, + document_ids_filter=document_ids_filter, + ) + + # Assert + assert len(results) == 1 + assert results[0].metadata["doc_id"] == "doc1" + # Verify document_ids_filter was passed + call_kwargs = mock_embedding_search.call_args.kwargs + assert call_kwargs["document_ids_filter"] == document_ids_filter + + @patch("core.rag.datasource.retrieval_service.RetrievalService.embedding_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_vector_search_empty_results(self, mock_get_dataset, mock_embedding_search, mock_dataset): + """ + Test vector search when no results match the query. + + Verifies: + - Empty list is returned when no documents match + - No errors are raised + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + # embedding_search doesn't add anything to all_documents + mock_embedding_search.side_effect = lambda *args, **kwargs: None + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.SEMANTIC_SEARCH, + dataset_id=mock_dataset.id, + query="nonexistent query", + top_k=5, + ) + + # Assert + assert results == [] + + # ==================== Keyword Search Tests ==================== + + @patch("core.rag.datasource.retrieval_service.RetrievalService.keyword_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_keyword_search_basic(self, mock_get_dataset, mock_keyword_search, mock_dataset, sample_documents): + """ + Test basic keyword search functionality. + + Verifies: + - Keyword search is invoked correctly + - Query is escaped properly for search + - Results are returned in expected format + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + + def side_effect_keyword_search( + flask_app, dataset_id, query, top_k, all_documents, exceptions, document_ids_filter=None + ): + all_documents.extend(sample_documents) + + mock_keyword_search.side_effect = side_effect_keyword_search + + query = "Python programming" + top_k = 3 + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.KEYWORD_SEARCH, + dataset_id=mock_dataset.id, + query=query, + top_k=top_k, + ) + + # Assert + assert len(results) == 3 + assert all(isinstance(doc, Document) for doc in results) + mock_keyword_search.assert_called_once() + + @patch("core.rag.datasource.retrieval_service.RetrievalService.keyword_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_keyword_search_with_special_characters(self, mock_get_dataset, mock_keyword_search, mock_dataset): + """ + Test keyword search with special characters in query. + + Verifies: + - Special characters are escaped correctly + - Search handles quotes and other special chars + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + mock_keyword_search.side_effect = lambda *args, **kwargs: None + + query = 'Python "programming" language' + + # Act + RetrievalService.retrieve( + retrieval_method=RetrievalMethod.KEYWORD_SEARCH, + dataset_id=mock_dataset.id, + query=query, + top_k=5, + ) + + # Assert + # Verify that keyword_search was called + assert mock_keyword_search.called + # The query escaping happens inside keyword_search method + call_args = mock_keyword_search.call_args + assert call_args is not None + + @patch("core.rag.datasource.retrieval_service.RetrievalService.keyword_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_keyword_search_with_document_filter( + self, mock_get_dataset, mock_keyword_search, mock_dataset, sample_documents + ): + """ + Test keyword search with document ID filtering. + + Verifies: + - Document filter is applied to keyword search + - Only filtered documents are returned + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + filtered_docs = [sample_documents[1]] + + def side_effect_keyword_search( + flask_app, dataset_id, query, top_k, all_documents, exceptions, document_ids_filter=None + ): + all_documents.extend(filtered_docs) + + mock_keyword_search.side_effect = side_effect_keyword_search + document_ids_filter = [sample_documents[1].metadata["document_id"]] + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.KEYWORD_SEARCH, + dataset_id=mock_dataset.id, + query="JavaScript", + top_k=5, + document_ids_filter=document_ids_filter, + ) + + # Assert + assert len(results) == 1 + assert results[0].metadata["doc_id"] == "doc2" + + # ==================== Hybrid Search Tests ==================== + + @patch("core.rag.datasource.retrieval_service.DataPostProcessor") + @patch("core.rag.datasource.retrieval_service.RetrievalService.full_text_index_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService.embedding_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_hybrid_search_basic( + self, + mock_get_dataset, + mock_embedding_search, + mock_fulltext_search, + mock_data_processor_class, + mock_dataset, + sample_documents, + ): + """ + Test basic hybrid search combining vector and full-text search. + + Verifies: + - Both vector and full-text search are executed + - Results are merged and deduplicated + - DataPostProcessor is invoked for score merging + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + + # Vector search returns first 2 docs + def side_effect_embedding( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + all_documents.extend(sample_documents[:2]) + + mock_embedding_search.side_effect = side_effect_embedding + + # Full-text search returns last 2 docs (with overlap) + def side_effect_fulltext( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + all_documents.extend(sample_documents[1:]) + + mock_fulltext_search.side_effect = side_effect_fulltext + + # Mock DataPostProcessor + mock_processor_instance = Mock() + mock_processor_instance.invoke.return_value = sample_documents + mock_data_processor_class.return_value = mock_processor_instance + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.HYBRID_SEARCH, + dataset_id=mock_dataset.id, + query="Python programming", + top_k=3, + score_threshold=0.5, + ) + + # Assert + assert len(results) == 3 + mock_embedding_search.assert_called_once() + mock_fulltext_search.assert_called_once() + mock_processor_instance.invoke.assert_called_once() + + @patch("core.rag.datasource.retrieval_service.DataPostProcessor") + @patch("core.rag.datasource.retrieval_service.RetrievalService.full_text_index_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService.embedding_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_hybrid_search_deduplication( + self, mock_get_dataset, mock_embedding_search, mock_fulltext_search, mock_data_processor_class, mock_dataset + ): + """ + Test that hybrid search properly deduplicates documents. + + Hybrid search combines results from multiple search methods (vector + full-text). + This can lead to duplicate documents when the same chunk is found by both methods. + + Scenario: + --------- + 1. Vector search finds document "duplicate_doc" with score 0.9 + 2. Full-text search also finds "duplicate_doc" but with score 0.6 + 3. Both searches find "unique_doc" + 4. Deduplication should keep only the higher-scoring version (0.9) + + Why deduplication matters: + -------------------------- + - Prevents showing the same content multiple times to users + - Ensures score consistency (keeps best match) + - Improves result quality and user experience + - Happens BEFORE reranking to avoid processing duplicates + + Verifies: + - Duplicate documents (same doc_id) are removed + - Higher scoring duplicate is retained + - Deduplication happens before post-processing + - Final result count is correct + """ + # ==================== ARRANGE ==================== + mock_get_dataset.return_value = mock_dataset + + # Create test documents with intentional duplication + # Same doc_id but different scores to test score comparison logic + doc1_high = Document( + page_content="Content 1", + metadata={ + "doc_id": "duplicate_doc", # Same doc_id as doc1_low + "score": 0.9, # Higher score - should be kept + "document_id": str(uuid4()), + }, + provider="dify", + ) + doc1_low = Document( + page_content="Content 1", + metadata={ + "doc_id": "duplicate_doc", # Same doc_id as doc1_high + "score": 0.6, # Lower score - should be discarded + "document_id": str(uuid4()), + }, + provider="dify", + ) + doc2 = Document( + page_content="Content 2", + metadata={ + "doc_id": "unique_doc", # Unique doc_id + "score": 0.8, + "document_id": str(uuid4()), + }, + provider="dify", + ) + + # Simulate vector search returning high-score duplicate + unique doc + def side_effect_embedding( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + """Vector search finds 2 documents including high-score duplicate.""" + all_documents.extend([doc1_high, doc2]) + + mock_embedding_search.side_effect = side_effect_embedding + + # Simulate full-text search returning low-score duplicate + def side_effect_fulltext( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + """Full-text search finds the same document but with lower score.""" + all_documents.extend([doc1_low]) + + mock_fulltext_search.side_effect = side_effect_fulltext + + # Mock DataPostProcessor to return deduplicated results + # In real implementation, _deduplicate_documents is called before this + mock_processor_instance = Mock() + mock_processor_instance.invoke.return_value = [doc1_high, doc2] + mock_data_processor_class.return_value = mock_processor_instance + + # ==================== ACT ==================== + # Execute hybrid search which should: + # 1. Run both embedding_search and full_text_index_search + # 2. Collect all results in all_documents (3 docs: 2 unique + 1 duplicate) + # 3. Call _deduplicate_documents to remove duplicate (keeps higher score) + # 4. Pass deduplicated results to DataPostProcessor + # 5. Return final results + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.HYBRID_SEARCH, + dataset_id=mock_dataset.id, + query="test", + top_k=5, + ) + + # ==================== ASSERT ==================== + # Verify deduplication worked correctly + assert len(results) == 2, "Should have 2 unique documents after deduplication (not 3)" + + # Verify the correct documents are present + doc_ids = [doc.metadata["doc_id"] for doc in results] + assert "duplicate_doc" in doc_ids, "Duplicate doc should be present (higher score version)" + assert "unique_doc" in doc_ids, "Unique doc should be present" + + # Implicitly verifies that doc1_low (score 0.6) was discarded + # in favor of doc1_high (score 0.9) + + @patch("core.rag.datasource.retrieval_service.DataPostProcessor") + @patch("core.rag.datasource.retrieval_service.RetrievalService.full_text_index_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService.embedding_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_hybrid_search_with_weights( + self, + mock_get_dataset, + mock_embedding_search, + mock_fulltext_search, + mock_data_processor_class, + mock_dataset, + sample_documents, + ): + """ + Test hybrid search with custom weights for score merging. + + Verifies: + - Weights are passed to DataPostProcessor + - Score merging respects weight configuration + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + + def side_effect_embedding( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + all_documents.extend(sample_documents[:2]) + + mock_embedding_search.side_effect = side_effect_embedding + + def side_effect_fulltext( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + all_documents.extend(sample_documents[1:]) + + mock_fulltext_search.side_effect = side_effect_fulltext + + mock_processor_instance = Mock() + mock_processor_instance.invoke.return_value = sample_documents + mock_data_processor_class.return_value = mock_processor_instance + + weights = { + "vector_setting": { + "vector_weight": 0.7, + "embedding_provider_name": "openai", + "embedding_model_name": "text-embedding-ada-002", + }, + "keyword_setting": {"keyword_weight": 0.3}, + } + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.HYBRID_SEARCH, + dataset_id=mock_dataset.id, + query="test query", + top_k=3, + weights=weights, + reranking_mode="weighted_score", + ) + + # Assert + assert len(results) == 3 + # Verify DataPostProcessor was created with weights + mock_data_processor_class.assert_called_once() + # Check that weights were passed (may be in args or kwargs) + call_args = mock_data_processor_class.call_args + if call_args.kwargs: + assert call_args.kwargs.get("weights") == weights + else: + # Weights might be in positional args (position 3) + assert len(call_args.args) >= 4 + + # ==================== Full-Text Search Tests ==================== + + @patch("core.rag.datasource.retrieval_service.RetrievalService.full_text_index_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_fulltext_search_basic(self, mock_get_dataset, mock_fulltext_search, mock_dataset, sample_documents): + """ + Test basic full-text search functionality. + + Verifies: + - Full-text search is invoked correctly + - Results are returned in expected format + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + + def side_effect_fulltext( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + all_documents.extend(sample_documents) + + mock_fulltext_search.side_effect = side_effect_fulltext + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.FULL_TEXT_SEARCH, + dataset_id=mock_dataset.id, + query="programming language", + top_k=3, + ) + + # Assert + assert len(results) == 3 + mock_fulltext_search.assert_called_once() + + # ==================== Score Merging Tests ==================== + + def test_deduplicate_documents_basic(self): + """ + Test basic document deduplication logic. + + Verifies: + - Documents with same doc_id are deduplicated + - First occurrence is kept by default + """ + # Arrange + doc1 = Document( + page_content="Content 1", + metadata={"doc_id": "doc1", "score": 0.8}, + provider="dify", + ) + doc2 = Document( + page_content="Content 2", + metadata={"doc_id": "doc2", "score": 0.7}, + provider="dify", + ) + doc1_duplicate = Document( + page_content="Content 1 duplicate", + metadata={"doc_id": "doc1", "score": 0.6}, + provider="dify", + ) + + documents = [doc1, doc2, doc1_duplicate] + + # Act + result = RetrievalService._deduplicate_documents(documents) + + # Assert + assert len(result) == 2 + doc_ids = [doc.metadata["doc_id"] for doc in result] + assert doc_ids == ["doc1", "doc2"] + + def test_deduplicate_documents_keeps_higher_score(self): + """ + Test that deduplication keeps document with higher score. + + Verifies: + - When duplicates exist, higher scoring version is retained + - Score comparison works correctly + """ + # Arrange + doc_low = Document( + page_content="Content", + metadata={"doc_id": "doc1", "score": 0.5}, + provider="dify", + ) + doc_high = Document( + page_content="Content", + metadata={"doc_id": "doc1", "score": 0.9}, + provider="dify", + ) + + # Low score first + documents = [doc_low, doc_high] + + # Act + result = RetrievalService._deduplicate_documents(documents) + + # Assert + assert len(result) == 1 + assert result[0].metadata["score"] == 0.9 + + def test_deduplicate_documents_empty_list(self): + """ + Test deduplication with empty document list. + + Verifies: + - Empty list returns empty list + - No errors are raised + """ + # Act + result = RetrievalService._deduplicate_documents([]) + + # Assert + assert result == [] + + def test_deduplicate_documents_non_dify_provider(self): + """ + Test deduplication with non-dify provider documents. + + Verifies: + - External provider documents use content-based deduplication + - Different providers are handled correctly + """ + # Arrange + doc1 = Document( + page_content="External content", + metadata={"score": 0.8}, + provider="external", + ) + doc2 = Document( + page_content="External content", + metadata={"score": 0.7}, + provider="external", + ) + + documents = [doc1, doc2] + + # Act + result = RetrievalService._deduplicate_documents(documents) + + # Assert + # External documents without doc_id should use content-based dedup + assert len(result) >= 1 + + # ==================== Metadata Filtering Tests ==================== + + @patch("core.rag.datasource.retrieval_service.RetrievalService.embedding_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_vector_search_with_metadata_filter( + self, mock_get_dataset, mock_embedding_search, mock_dataset, sample_documents + ): + """ + Test vector search with metadata-based document filtering. + + Verifies: + - Metadata filters are applied correctly + - Only documents matching metadata criteria are returned + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + + # Add metadata to documents + filtered_doc = sample_documents[0] + filtered_doc.metadata["category"] = "programming" + + def side_effect_embedding( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + all_documents.append(filtered_doc) + + mock_embedding_search.side_effect = side_effect_embedding + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.SEMANTIC_SEARCH, + dataset_id=mock_dataset.id, + query="Python", + top_k=5, + document_ids_filter=[filtered_doc.metadata["document_id"]], + ) + + # Assert + assert len(results) == 1 + assert results[0].metadata.get("category") == "programming" + + # ==================== Error Handling Tests ==================== + + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_retrieve_with_empty_query(self, mock_get_dataset, mock_dataset): + """ + Test retrieval with empty query string. + + Verifies: + - Empty query returns empty results + - No search operations are performed + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.SEMANTIC_SEARCH, + dataset_id=mock_dataset.id, + query="", + top_k=5, + ) + + # Assert + assert results == [] + + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_retrieve_with_nonexistent_dataset(self, mock_get_dataset): + """ + Test retrieval with non-existent dataset ID. + + Verifies: + - Non-existent dataset returns empty results + - No errors are raised + """ + # Arrange + mock_get_dataset.return_value = None + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.SEMANTIC_SEARCH, + dataset_id="nonexistent_id", + query="test query", + top_k=5, + ) + + # Assert + assert results == [] + + @patch("core.rag.datasource.retrieval_service.RetrievalService.embedding_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_retrieve_with_exception_handling(self, mock_get_dataset, mock_embedding_search, mock_dataset): + """ + Test that exceptions during retrieval are properly handled. + + Verifies: + - Exceptions are caught and added to exceptions list + - ValueError is raised with exception messages + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + + # Make embedding_search add an exception to the exceptions list + def side_effect_with_exception( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + exceptions.append("Search failed") + + mock_embedding_search.side_effect = side_effect_with_exception + + # Act & Assert + with pytest.raises(ValueError) as exc_info: + RetrievalService.retrieve( + retrieval_method=RetrievalMethod.SEMANTIC_SEARCH, + dataset_id=mock_dataset.id, + query="test query", + top_k=5, + ) + + assert "Search failed" in str(exc_info.value) + + # ==================== Score Threshold Tests ==================== + + @patch("core.rag.datasource.retrieval_service.RetrievalService.embedding_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_vector_search_with_score_threshold(self, mock_get_dataset, mock_embedding_search, mock_dataset): + """ + Test vector search with score threshold filtering. + + Verifies: + - Score threshold is passed to search method + - Documents below threshold are filtered out + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + + # Only return documents above threshold + high_score_doc = Document( + page_content="High relevance content", + metadata={"doc_id": "doc1", "score": 0.85}, + provider="dify", + ) + + def side_effect_embedding( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + all_documents.append(high_score_doc) + + mock_embedding_search.side_effect = side_effect_embedding + + score_threshold = 0.8 + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.SEMANTIC_SEARCH, + dataset_id=mock_dataset.id, + query="test query", + top_k=5, + score_threshold=score_threshold, + ) + + # Assert + assert len(results) == 1 + assert results[0].metadata["score"] >= score_threshold + + # ==================== Top-K Limiting Tests ==================== + + @patch("core.rag.datasource.retrieval_service.RetrievalService.embedding_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_retrieve_respects_top_k_limit(self, mock_get_dataset, mock_embedding_search, mock_dataset): + """ + Test that retrieval respects top_k parameter. + + Verifies: + - Only top_k documents are returned + - Limit is applied correctly + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + + # Create more documents than top_k + many_docs = [ + Document( + page_content=f"Content {i}", + metadata={"doc_id": f"doc{i}", "score": 0.9 - i * 0.1}, + provider="dify", + ) + for i in range(10) + ] + + def side_effect_embedding( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + # Return only top_k documents + all_documents.extend(many_docs[:top_k]) + + mock_embedding_search.side_effect = side_effect_embedding + + top_k = 3 + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.SEMANTIC_SEARCH, + dataset_id=mock_dataset.id, + query="test query", + top_k=top_k, + ) + + # Assert + # Verify top_k was passed to embedding_search + assert mock_embedding_search.called + call_kwargs = mock_embedding_search.call_args.kwargs + assert call_kwargs["top_k"] == top_k + # Verify we got the right number of results + assert len(results) == top_k + + # ==================== Query Escaping Tests ==================== + + def test_escape_query_for_search(self): + """ + Test query escaping for special characters. + + Verifies: + - Double quotes are properly escaped + - Other characters remain unchanged + """ + # Test cases with expected outputs + test_cases = [ + ("simple query", "simple query"), + ('query with "quotes"', 'query with \\"quotes\\"'), + ('"quoted phrase"', '\\"quoted phrase\\"'), + ("no special chars", "no special chars"), + ] + + for input_query, expected_output in test_cases: + result = RetrievalService.escape_query_for_search(input_query) + assert result == expected_output + + # ==================== Reranking Tests ==================== + + @patch("core.rag.datasource.retrieval_service.RetrievalService.embedding_search") + @patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset") + def test_semantic_search_with_reranking( + self, mock_get_dataset, mock_embedding_search, mock_dataset, sample_documents + ): + """ + Test semantic search with reranking model. + + Verifies: + - Reranking is applied when configured + - DataPostProcessor is invoked with correct parameters + """ + # Arrange + mock_get_dataset.return_value = mock_dataset + + # Simulate reranking changing order + reranked_docs = list(reversed(sample_documents)) + + def side_effect_embedding( + flask_app, + dataset_id, + query, + top_k, + score_threshold, + reranking_model, + all_documents, + retrieval_method, + exceptions, + document_ids_filter=None, + ): + # embedding_search handles reranking internally + all_documents.extend(reranked_docs) + + mock_embedding_search.side_effect = side_effect_embedding + + reranking_model = { + "reranking_provider_name": "cohere", + "reranking_model_name": "rerank-english-v2.0", + } + + # Act + results = RetrievalService.retrieve( + retrieval_method=RetrievalMethod.SEMANTIC_SEARCH, + dataset_id=mock_dataset.id, + query="test query", + top_k=3, + reranking_model=reranking_model, + ) + + # Assert + # For semantic search with reranking, reranking_model should be passed + assert len(results) == 3 + call_kwargs = mock_embedding_search.call_args.kwargs + assert call_kwargs["reranking_model"] == reranking_model + + +class TestRetrievalMethods: + """ + Test suite for RetrievalMethod enum and utility methods. + + The RetrievalMethod enum defines the available search strategies: + + 1. **SEMANTIC_SEARCH**: Vector-based similarity search using embeddings + - Best for: Natural language queries, conceptual similarity + - Uses: Embedding models (e.g., text-embedding-ada-002) + - Example: "What is machine learning?" matches "AI and ML concepts" + + 2. **FULL_TEXT_SEARCH**: BM25-based text matching + - Best for: Exact phrase matching, keyword presence + - Uses: BM25 algorithm with sparse vectors + - Example: "Python programming" matches documents with those exact terms + + 3. **HYBRID_SEARCH**: Combination of semantic + full-text + - Best for: Comprehensive search with both conceptual and exact matching + - Uses: Both embedding vectors and BM25, with score merging + - Example: Finds both semantically similar and keyword-matching documents + + 4. **KEYWORD_SEARCH**: Traditional keyword-based search (economy mode) + - Best for: Simple, fast searches without embeddings + - Uses: Jieba tokenization and keyword matching + - Example: Basic text search without vector database + + Utility Methods: + ================ + - is_support_semantic_search(): Check if method uses embeddings + - is_support_fulltext_search(): Check if method uses BM25 + + These utilities help determine which search operations to execute + in the RetrievalService.retrieve() method. + """ + + def test_retrieval_method_values(self): + """ + Test that all retrieval method constants are defined correctly. + + This ensures the enum values match the expected string constants + used throughout the codebase for configuration and API calls. + + Verifies: + - All expected retrieval methods exist + - Values are correct strings (not accidentally changed) + - String values match database/config expectations + """ + assert RetrievalMethod.SEMANTIC_SEARCH == "semantic_search" + assert RetrievalMethod.FULL_TEXT_SEARCH == "full_text_search" + assert RetrievalMethod.HYBRID_SEARCH == "hybrid_search" + assert RetrievalMethod.KEYWORD_SEARCH == "keyword_search" + + def test_is_support_semantic_search(self): + """ + Test semantic search support detection. + + Verifies: + - Semantic search method is detected + - Hybrid search method is detected (includes semantic) + - Other methods are not detected + """ + assert RetrievalMethod.is_support_semantic_search(RetrievalMethod.SEMANTIC_SEARCH) is True + assert RetrievalMethod.is_support_semantic_search(RetrievalMethod.HYBRID_SEARCH) is True + assert RetrievalMethod.is_support_semantic_search(RetrievalMethod.FULL_TEXT_SEARCH) is False + assert RetrievalMethod.is_support_semantic_search(RetrievalMethod.KEYWORD_SEARCH) is False + + def test_is_support_fulltext_search(self): + """ + Test full-text search support detection. + + Verifies: + - Full-text search method is detected + - Hybrid search method is detected (includes full-text) + - Other methods are not detected + """ + assert RetrievalMethod.is_support_fulltext_search(RetrievalMethod.FULL_TEXT_SEARCH) is True + assert RetrievalMethod.is_support_fulltext_search(RetrievalMethod.HYBRID_SEARCH) is True + assert RetrievalMethod.is_support_fulltext_search(RetrievalMethod.SEMANTIC_SEARCH) is False + assert RetrievalMethod.is_support_fulltext_search(RetrievalMethod.KEYWORD_SEARCH) is False + + +class TestDocumentModel: + """ + Test suite for Document model used in retrieval. + + The Document class is the core data structure for representing text chunks + in the retrieval system. It's based on Pydantic BaseModel for validation. + + Document Structure: + =================== + - **page_content** (str): The actual text content of the document chunk + - **metadata** (dict): Additional information about the document + - doc_id: Unique identifier for the chunk + - document_id: Parent document ID + - dataset_id: Dataset this document belongs to + - score: Relevance score from search (0.0 to 1.0) + - Custom fields: category, tags, timestamps, etc. + - **provider** (str): Source of the document ("dify" or "external") + - **vector** (list[float] | None): Embedding vector for semantic search + - **children** (list[ChildDocument] | None): Sub-chunks for hierarchical docs + + Document Lifecycle: + =================== + 1. **Creation**: Documents are created when text is indexed + - Content is chunked into manageable pieces + - Embeddings are generated for semantic search + - Metadata is attached for filtering and tracking + + 2. **Storage**: Documents are stored in vector databases + - Vector field stores embeddings + - Metadata enables filtering + - Provider tracks source (internal vs external) + + 3. **Retrieval**: Documents are returned from search operations + - Scores are added during search + - Multiple documents may be combined (hybrid search) + - Deduplication uses doc_id + + 4. **Post-processing**: Documents may be reranked or filtered + - Scores can be recalculated + - Content may be truncated or formatted + - Metadata is used for display + + Why Test the Document Model: + ============================ + - Ensures data structure integrity + - Validates Pydantic model behavior + - Confirms default values work correctly + - Tests equality comparison for deduplication + - Verifies metadata handling + + Related Classes: + ================ + - ChildDocument: For hierarchical document structures + - RetrievalSegments: Combines Document with database segment info + """ + + def test_document_creation_basic(self): + """ + Test basic Document object creation. + + Tests the minimal required fields and default values. + Only page_content is required; all other fields have defaults. + + Verifies: + - Document can be created with minimal fields + - Default values are set correctly + - Pydantic validation works + - No exceptions are raised + """ + doc = Document(page_content="Test content") + + assert doc.page_content == "Test content" + assert doc.metadata == {} # Empty dict by default + assert doc.provider == "dify" # Default provider + assert doc.vector is None # No embedding by default + assert doc.children is None # No child documents by default + + def test_document_creation_with_metadata(self): + """ + Test Document creation with metadata. + + Verifies: + - Metadata is stored correctly + - Metadata can contain various types + """ + metadata = { + "doc_id": "test_doc", + "score": 0.95, + "dataset_id": str(uuid4()), + "category": "test", + } + doc = Document(page_content="Test content", metadata=metadata) + + assert doc.metadata == metadata + assert doc.metadata["score"] == 0.95 + + def test_document_creation_with_vector(self): + """ + Test Document creation with embedding vector. + + Verifies: + - Vector embeddings can be stored + - Vector is optional + """ + vector = [0.1, 0.2, 0.3, 0.4, 0.5] + doc = Document(page_content="Test content", vector=vector) + + assert doc.vector == vector + assert len(doc.vector) == 5 + + def test_document_with_external_provider(self): + """ + Test Document with external provider. + + Verifies: + - Provider can be set to external + - External documents are handled correctly + """ + doc = Document(page_content="External content", provider="external") + + assert doc.provider == "external" + + def test_document_equality(self): + """ + Test Document equality comparison. + + Verifies: + - Documents with same content are considered equal + - Metadata affects equality + """ + doc1 = Document(page_content="Content", metadata={"id": "1"}) + doc2 = Document(page_content="Content", metadata={"id": "1"}) + doc3 = Document(page_content="Different", metadata={"id": "1"}) + + assert doc1 == doc2 + assert doc1 != doc3 diff --git a/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py b/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py index c68aad0b22..02bf8e82f1 100644 --- a/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py +++ b/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py @@ -3,7 +3,7 @@ import pytest from core.app.entities.app_invoke_entities import InvokeFrom from core.tools.__base.tool_runtime import ToolRuntime from core.tools.entities.common_entities import I18nObject -from core.tools.entities.tool_entities import ToolEntity, ToolIdentity +from core.tools.entities.tool_entities import ToolEntity, ToolIdentity, ToolInvokeMessage from core.tools.errors import ToolInvokeError from core.tools.workflow_as_tool.tool import WorkflowTool @@ -51,3 +51,166 @@ def test_workflow_tool_should_raise_tool_invoke_error_when_result_has_error_fiel # actually `run` the tool. list(tool.invoke("test_user", {})) assert exc_info.value.args == ("oops",) + + +def test_workflow_tool_should_generate_variable_messages_for_outputs(monkeypatch: pytest.MonkeyPatch): + """Test that WorkflowTool should generate variable messages when there are outputs""" + entity = ToolEntity( + identity=ToolIdentity(author="test", name="test tool", label=I18nObject(en_US="test tool"), provider="test"), + parameters=[], + description=None, + has_runtime_parameters=False, + ) + runtime = ToolRuntime(tenant_id="test_tool", invoke_from=InvokeFrom.EXPLORE) + tool = WorkflowTool( + workflow_app_id="", + workflow_as_tool_id="", + version="1", + workflow_entities={}, + workflow_call_depth=1, + entity=entity, + runtime=runtime, + ) + + # Mock workflow outputs + mock_outputs = {"result": "success", "count": 42, "data": {"key": "value"}} + + # needs to patch those methods to avoid database access. + monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) + monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) + + # Mock user resolution to avoid database access + from unittest.mock import Mock + + mock_user = Mock() + monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: mock_user) + + # replace `WorkflowAppGenerator.generate` 's return value. + monkeypatch.setattr( + "core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", + lambda *args, **kwargs: {"data": {"outputs": mock_outputs}}, + ) + monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) + + # Execute tool invocation + messages = list(tool.invoke("test_user", {})) + + # Verify generated messages + # Should contain: 3 variable messages + 1 text message + 1 JSON message = 5 messages + assert len(messages) == 5 + + # Verify variable messages + variable_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.VARIABLE] + assert len(variable_messages) == 3 + + # Verify content of each variable message + variable_dict = {msg.message.variable_name: msg.message.variable_value for msg in variable_messages} + assert variable_dict["result"] == "success" + assert variable_dict["count"] == 42 + assert variable_dict["data"] == {"key": "value"} + + # Verify text message + text_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.TEXT] + assert len(text_messages) == 1 + assert '{"result": "success", "count": 42, "data": {"key": "value"}}' in text_messages[0].message.text + + # Verify JSON message + json_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.JSON] + assert len(json_messages) == 1 + assert json_messages[0].message.json_object == mock_outputs + + +def test_workflow_tool_should_handle_empty_outputs(monkeypatch: pytest.MonkeyPatch): + """Test that WorkflowTool should handle empty outputs correctly""" + entity = ToolEntity( + identity=ToolIdentity(author="test", name="test tool", label=I18nObject(en_US="test tool"), provider="test"), + parameters=[], + description=None, + has_runtime_parameters=False, + ) + runtime = ToolRuntime(tenant_id="test_tool", invoke_from=InvokeFrom.EXPLORE) + tool = WorkflowTool( + workflow_app_id="", + workflow_as_tool_id="", + version="1", + workflow_entities={}, + workflow_call_depth=1, + entity=entity, + runtime=runtime, + ) + + # needs to patch those methods to avoid database access. + monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) + monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) + + # Mock user resolution to avoid database access + from unittest.mock import Mock + + mock_user = Mock() + monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: mock_user) + + # replace `WorkflowAppGenerator.generate` 's return value. + monkeypatch.setattr( + "core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", + lambda *args, **kwargs: {"data": {}}, + ) + monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) + + # Execute tool invocation + messages = list(tool.invoke("test_user", {})) + + # Verify generated messages + # Should contain: 0 variable messages + 1 text message + 1 JSON message = 2 messages + assert len(messages) == 2 + + # Verify no variable messages + variable_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.VARIABLE] + assert len(variable_messages) == 0 + + # Verify text message + text_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.TEXT] + assert len(text_messages) == 1 + assert text_messages[0].message.text == "{}" + + # Verify JSON message + json_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.JSON] + assert len(json_messages) == 1 + assert json_messages[0].message.json_object == {} + + +def test_create_variable_message(): + """Test the functionality of creating variable messages""" + entity = ToolEntity( + identity=ToolIdentity(author="test", name="test tool", label=I18nObject(en_US="test tool"), provider="test"), + parameters=[], + description=None, + has_runtime_parameters=False, + ) + runtime = ToolRuntime(tenant_id="test_tool", invoke_from=InvokeFrom.EXPLORE) + tool = WorkflowTool( + workflow_app_id="", + workflow_as_tool_id="", + version="1", + workflow_entities={}, + workflow_call_depth=1, + entity=entity, + runtime=runtime, + ) + + # Test different types of variable values + test_cases = [ + ("string_var", "test string"), + ("int_var", 42), + ("float_var", 3.14), + ("bool_var", True), + ("list_var", [1, 2, 3]), + ("dict_var", {"key": "value"}), + ] + + for var_name, var_value in test_cases: + message = tool.create_variable_message(var_name, var_value) + + assert message.type == ToolInvokeMessage.MessageType.VARIABLE + assert message.message.variable_name == var_name + assert message.message.variable_value == var_value + assert message.message.stream is False diff --git a/api/tests/unit_tests/core/workflow/entities/test_private_workflow_pause.py b/api/tests/unit_tests/core/workflow/entities/test_private_workflow_pause.py index ccb2dff85a..be165bf1c1 100644 --- a/api/tests/unit_tests/core/workflow/entities/test_private_workflow_pause.py +++ b/api/tests/unit_tests/core/workflow/entities/test_private_workflow_pause.py @@ -19,38 +19,18 @@ class TestPrivateWorkflowPauseEntity: mock_pause_model.resumed_at = None # Create entity - entity = _PrivateWorkflowPauseEntity( - pause_model=mock_pause_model, - ) + entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[]) # Verify initialization assert entity._pause_model is mock_pause_model assert entity._cached_state is None - def test_from_models_classmethod(self): - """Test from_models class method.""" - # Create mock models - mock_pause_model = MagicMock(spec=WorkflowPauseModel) - mock_pause_model.id = "pause-123" - mock_pause_model.workflow_run_id = "execution-456" - - # Create entity using from_models - entity = _PrivateWorkflowPauseEntity.from_models( - workflow_pause_model=mock_pause_model, - ) - - # Verify entity creation - assert isinstance(entity, _PrivateWorkflowPauseEntity) - assert entity._pause_model is mock_pause_model - def test_id_property(self): """Test id property returns pause model ID.""" mock_pause_model = MagicMock(spec=WorkflowPauseModel) mock_pause_model.id = "pause-123" - entity = _PrivateWorkflowPauseEntity( - pause_model=mock_pause_model, - ) + entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[]) assert entity.id == "pause-123" @@ -59,9 +39,7 @@ class TestPrivateWorkflowPauseEntity: mock_pause_model = MagicMock(spec=WorkflowPauseModel) mock_pause_model.workflow_run_id = "execution-456" - entity = _PrivateWorkflowPauseEntity( - pause_model=mock_pause_model, - ) + entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[]) assert entity.workflow_execution_id == "execution-456" @@ -72,9 +50,7 @@ class TestPrivateWorkflowPauseEntity: mock_pause_model = MagicMock(spec=WorkflowPauseModel) mock_pause_model.resumed_at = resumed_at - entity = _PrivateWorkflowPauseEntity( - pause_model=mock_pause_model, - ) + entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[]) assert entity.resumed_at == resumed_at @@ -83,9 +59,7 @@ class TestPrivateWorkflowPauseEntity: mock_pause_model = MagicMock(spec=WorkflowPauseModel) mock_pause_model.resumed_at = None - entity = _PrivateWorkflowPauseEntity( - pause_model=mock_pause_model, - ) + entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[]) assert entity.resumed_at is None @@ -98,9 +72,7 @@ class TestPrivateWorkflowPauseEntity: mock_pause_model = MagicMock(spec=WorkflowPauseModel) mock_pause_model.state_object_key = "test-state-key" - entity = _PrivateWorkflowPauseEntity( - pause_model=mock_pause_model, - ) + entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[]) # First call should load from storage result = entity.get_state() @@ -118,9 +90,7 @@ class TestPrivateWorkflowPauseEntity: mock_pause_model = MagicMock(spec=WorkflowPauseModel) mock_pause_model.state_object_key = "test-state-key" - entity = _PrivateWorkflowPauseEntity( - pause_model=mock_pause_model, - ) + entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[]) # First call result1 = entity.get_state() @@ -139,9 +109,7 @@ class TestPrivateWorkflowPauseEntity: mock_pause_model = MagicMock(spec=WorkflowPauseModel) - entity = _PrivateWorkflowPauseEntity( - pause_model=mock_pause_model, - ) + entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[]) # Pre-cache data entity._cached_state = state_data @@ -162,9 +130,7 @@ class TestPrivateWorkflowPauseEntity: mock_pause_model = MagicMock(spec=WorkflowPauseModel) - entity = _PrivateWorkflowPauseEntity( - pause_model=mock_pause_model, - ) + entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[]) result = entity.get_state() diff --git a/api/tests/unit_tests/core/workflow/graph/test_graph_validation.py b/api/tests/unit_tests/core/workflow/graph/test_graph_validation.py index c55c40c5b4..2597a3d65a 100644 --- a/api/tests/unit_tests/core/workflow/graph/test_graph_validation.py +++ b/api/tests/unit_tests/core/workflow/graph/test_graph_validation.py @@ -3,22 +3,27 @@ from __future__ import annotations import time from collections.abc import Mapping from dataclasses import dataclass -from typing import Any import pytest from core.app.entities.app_invoke_entities import InvokeFrom -from core.workflow.entities import GraphInitParams, GraphRuntimeState, VariablePool +from core.workflow.entities import GraphInitParams from core.workflow.enums import ErrorStrategy, NodeExecutionType, NodeType from core.workflow.graph import Graph from core.workflow.graph.validation import GraphValidationError -from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig +from core.workflow.nodes.base.entities import BaseNodeData from core.workflow.nodes.base.node import Node +from core.workflow.runtime import GraphRuntimeState, VariablePool from core.workflow.system_variable import SystemVariable from models.enums import UserFrom -class _TestNode(Node): +class _TestNodeData(BaseNodeData): + type: NodeType | str | None = None + execution_type: NodeExecutionType | str | None = None + + +class _TestNode(Node[_TestNodeData]): node_type = NodeType.ANSWER execution_type = NodeExecutionType.EXECUTABLE @@ -40,31 +45,8 @@ class _TestNode(Node): graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - data = config.get("data", {}) - if isinstance(data, Mapping): - execution_type = data.get("execution_type") - if isinstance(execution_type, str): - self.execution_type = NodeExecutionType(execution_type) - self._base_node_data = BaseNodeData(title=str(data.get("title", self.id))) - self.data: dict[str, object] = {} - def init_node_data(self, data: Mapping[str, object]) -> None: - title = str(data.get("title", self.id)) - desc = data.get("description") - error_strategy_value = data.get("error_strategy") - error_strategy: ErrorStrategy | None = None - if isinstance(error_strategy_value, ErrorStrategy): - error_strategy = error_strategy_value - elif isinstance(error_strategy_value, str): - error_strategy = ErrorStrategy(error_strategy_value) - self._base_node_data = BaseNodeData( - title=title, - desc=str(desc) if desc is not None else None, - error_strategy=error_strategy, - ) - self.data = dict(data) - - node_type_value = data.get("type") + node_type_value = self.data.get("type") if isinstance(node_type_value, NodeType): self.node_type = node_type_value elif isinstance(node_type_value, str): @@ -76,23 +58,19 @@ class _TestNode(Node): def _run(self): raise NotImplementedError - def _get_error_strategy(self) -> ErrorStrategy | None: - return self._base_node_data.error_strategy + def post_init(self) -> None: + super().post_init() + self._maybe_override_execution_type() + self.data = dict(self.node_data.model_dump()) - def _get_retry_config(self) -> RetryConfig: - return self._base_node_data.retry_config - - def _get_title(self) -> str: - return self._base_node_data.title - - def _get_description(self) -> str | None: - return self._base_node_data.desc - - def _get_default_value_dict(self) -> dict[str, Any]: - return self._base_node_data.default_value_dict - - def get_base_node_data(self) -> BaseNodeData: - return self._base_node_data + def _maybe_override_execution_type(self) -> None: + execution_type_value = self.node_data.execution_type + if execution_type_value is None: + return + if isinstance(execution_type_value, NodeExecutionType): + self.execution_type = execution_type_value + else: + self.execution_type = NodeExecutionType(execution_type_value) @dataclass(slots=True) @@ -108,7 +86,6 @@ class _SimpleNodeFactory: graph_init_params=self.graph_init_params, graph_runtime_state=self.graph_runtime_state, ) - node.init_node_data(node_config.get("data", {})) return node diff --git a/api/tests/unit_tests/core/workflow/graph_engine/event_management/test_event_handlers.py b/api/tests/unit_tests/core/workflow/graph_engine/event_management/test_event_handlers.py index 2b8f04979d..5d17b7a243 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/event_management/test_event_handlers.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/event_management/test_event_handlers.py @@ -2,8 +2,6 @@ from __future__ import annotations -from datetime import datetime - from core.workflow.enums import NodeExecutionType, NodeState, NodeType, WorkflowNodeExecutionStatus from core.workflow.graph import Graph from core.workflow.graph_engine.domain.graph_execution import GraphExecution @@ -16,6 +14,7 @@ from core.workflow.graph_events import NodeRunRetryEvent, NodeRunStartedEvent from core.workflow.node_events import NodeRunResult from core.workflow.nodes.base.entities import RetryConfig from core.workflow.runtime import GraphRuntimeState, VariablePool +from libs.datetime_utils import naive_utc_now class _StubEdgeProcessor: @@ -75,7 +74,7 @@ def test_retry_does_not_emit_additional_start_event() -> None: execution_id = "exec-1" node_type = NodeType.CODE - start_time = datetime.utcnow() + start_time = naive_utc_now() start_event = NodeRunStartedEvent( id=execution_id, diff --git a/api/tests/unit_tests/core/workflow/graph_engine/orchestration/test_dispatcher.py b/api/tests/unit_tests/core/workflow/graph_engine/orchestration/test_dispatcher.py index e6d4508fdf..c1fc4acd73 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/orchestration/test_dispatcher.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/orchestration/test_dispatcher.py @@ -3,7 +3,6 @@ from __future__ import annotations import queue -from datetime import datetime from unittest import mock from core.workflow.entities.pause_reason import SchedulingPause @@ -18,6 +17,7 @@ from core.workflow.graph_events import ( NodeRunSucceededEvent, ) from core.workflow.node_events import NodeRunResult +from libs.datetime_utils import naive_utc_now def test_dispatcher_should_consume_remains_events_after_pause(): @@ -109,7 +109,7 @@ def _make_started_event() -> NodeRunStartedEvent: node_id="node-1", node_type=NodeType.CODE, node_title="Test Node", - start_at=datetime.utcnow(), + start_at=naive_utc_now(), ) @@ -119,7 +119,7 @@ def _make_succeeded_event() -> NodeRunSucceededEvent: node_id="node-1", node_type=NodeType.CODE, node_title="Test Node", - start_at=datetime.utcnow(), + start_at=naive_utc_now(), node_run_result=NodeRunResult(status=WorkflowNodeExecutionStatus.SUCCEEDED), ) @@ -153,7 +153,7 @@ def test_dispatcher_drain_event_queue(): node_id="node-1", node_type=NodeType.CODE, node_title="Code", - start_at=datetime.utcnow(), + start_at=naive_utc_now(), ), NodeRunPauseRequestedEvent( id="pause-event", @@ -165,7 +165,7 @@ def test_dispatcher_drain_event_queue(): id="success-event", node_id="node-1", node_type=NodeType.CODE, - start_at=datetime.utcnow(), + start_at=naive_utc_now(), node_run_result=NodeRunResult(status=WorkflowNodeExecutionStatus.SUCCEEDED), ), ] diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_command_system.py b/api/tests/unit_tests/core/workflow/graph_engine/test_command_system.py index 868edf9832..b074a11be9 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/test_command_system.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/test_command_system.py @@ -32,7 +32,7 @@ def test_abort_command(): # Create mock nodes with required attributes - using shared runtime state start_node = StartNode( id="start", - config={"id": "start"}, + config={"id": "start", "data": {"title": "start", "variables": []}}, graph_init_params=GraphInitParams( tenant_id="test_tenant", app_id="test_app", @@ -45,7 +45,6 @@ def test_abort_command(): ), graph_runtime_state=shared_runtime_state, ) - start_node.init_node_data({"title": "start", "variables": []}) mock_graph.nodes["start"] = start_node # Mock graph methods @@ -142,7 +141,7 @@ def test_pause_command(): start_node = StartNode( id="start", - config={"id": "start"}, + config={"id": "start", "data": {"title": "start", "variables": []}}, graph_init_params=GraphInitParams( tenant_id="test_tenant", app_id="test_app", @@ -155,7 +154,6 @@ def test_pause_command(): ), graph_runtime_state=shared_runtime_state, ) - start_node.init_node_data({"title": "start", "variables": []}) mock_graph.nodes["start"] = start_node mock_graph.get_outgoing_edges = MagicMock(return_value=[]) @@ -178,8 +176,7 @@ def test_pause_command(): assert any(isinstance(e, GraphRunStartedEvent) for e in events) pause_events = [e for e in events if isinstance(e, GraphRunPausedEvent)] assert len(pause_events) == 1 - assert pause_events[0].reason == SchedulingPause(message="User requested pause") + assert pause_events[0].reasons == [SchedulingPause(message="User requested pause")] graph_execution = engine.graph_runtime_state.graph_execution - assert graph_execution.paused - assert graph_execution.pause_reason == SchedulingPause(message="User requested pause") + assert graph_execution.pause_reasons == [SchedulingPause(message="User requested pause")] diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_human_input_pause_multi_branch.py b/api/tests/unit_tests/core/workflow/graph_engine/test_human_input_pause_multi_branch.py index c9e7e31e52..c398e4e8c1 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/test_human_input_pause_multi_branch.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/test_human_input_pause_multi_branch.py @@ -14,7 +14,7 @@ from core.workflow.graph_events import ( NodeRunStreamChunkEvent, NodeRunSucceededEvent, ) -from core.workflow.nodes.base.entities import VariableSelector +from core.workflow.nodes.base.entities import OutputVariableEntity, OutputVariableType from core.workflow.nodes.end.end_node import EndNode from core.workflow.nodes.end.entities import EndNodeData from core.workflow.nodes.human_input import HumanInputNode @@ -63,7 +63,6 @@ def _build_branching_graph(mock_config: MockConfig) -> tuple[Graph, GraphRuntime graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - start_node.init_node_data(start_config["data"]) def _create_llm_node(node_id: str, title: str, prompt_text: str) -> MockLLMNode: llm_data = LLMNodeData( @@ -88,7 +87,6 @@ def _build_branching_graph(mock_config: MockConfig) -> tuple[Graph, GraphRuntime graph_runtime_state=graph_runtime_state, mock_config=mock_config, ) - llm_node.init_node_data(llm_config["data"]) return llm_node llm_initial = _create_llm_node("llm_initial", "Initial LLM", "Initial stream") @@ -105,7 +103,6 @@ def _build_branching_graph(mock_config: MockConfig) -> tuple[Graph, GraphRuntime graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - human_node.init_node_data(human_config["data"]) llm_primary = _create_llm_node("llm_primary", "Primary LLM", "Primary stream output") llm_secondary = _create_llm_node("llm_secondary", "Secondary LLM", "Secondary") @@ -113,8 +110,12 @@ def _build_branching_graph(mock_config: MockConfig) -> tuple[Graph, GraphRuntime end_primary_data = EndNodeData( title="End Primary", outputs=[ - VariableSelector(variable="initial_text", value_selector=["llm_initial", "text"]), - VariableSelector(variable="primary_text", value_selector=["llm_primary", "text"]), + OutputVariableEntity( + variable="initial_text", value_type=OutputVariableType.STRING, value_selector=["llm_initial", "text"] + ), + OutputVariableEntity( + variable="primary_text", value_type=OutputVariableType.STRING, value_selector=["llm_primary", "text"] + ), ], desc=None, ) @@ -125,13 +126,18 @@ def _build_branching_graph(mock_config: MockConfig) -> tuple[Graph, GraphRuntime graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - end_primary.init_node_data(end_primary_config["data"]) end_secondary_data = EndNodeData( title="End Secondary", outputs=[ - VariableSelector(variable="initial_text", value_selector=["llm_initial", "text"]), - VariableSelector(variable="secondary_text", value_selector=["llm_secondary", "text"]), + OutputVariableEntity( + variable="initial_text", value_type=OutputVariableType.STRING, value_selector=["llm_initial", "text"] + ), + OutputVariableEntity( + variable="secondary_text", + value_type=OutputVariableType.STRING, + value_selector=["llm_secondary", "text"], + ), ], desc=None, ) @@ -142,7 +148,6 @@ def _build_branching_graph(mock_config: MockConfig) -> tuple[Graph, GraphRuntime graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - end_secondary.init_node_data(end_secondary_config["data"]) graph = ( Graph.new() diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_human_input_pause_single_branch.py b/api/tests/unit_tests/core/workflow/graph_engine/test_human_input_pause_single_branch.py index 27d264365d..ece69b080b 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/test_human_input_pause_single_branch.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/test_human_input_pause_single_branch.py @@ -13,7 +13,7 @@ from core.workflow.graph_events import ( NodeRunStreamChunkEvent, NodeRunSucceededEvent, ) -from core.workflow.nodes.base.entities import VariableSelector +from core.workflow.nodes.base.entities import OutputVariableEntity, OutputVariableType from core.workflow.nodes.end.end_node import EndNode from core.workflow.nodes.end.entities import EndNodeData from core.workflow.nodes.human_input import HumanInputNode @@ -62,7 +62,6 @@ def _build_llm_human_llm_graph(mock_config: MockConfig) -> tuple[Graph, GraphRun graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - start_node.init_node_data(start_config["data"]) def _create_llm_node(node_id: str, title: str, prompt_text: str) -> MockLLMNode: llm_data = LLMNodeData( @@ -87,7 +86,6 @@ def _build_llm_human_llm_graph(mock_config: MockConfig) -> tuple[Graph, GraphRun graph_runtime_state=graph_runtime_state, mock_config=mock_config, ) - llm_node.init_node_data(llm_config["data"]) return llm_node llm_first = _create_llm_node("llm_initial", "Initial LLM", "Initial prompt") @@ -104,15 +102,18 @@ def _build_llm_human_llm_graph(mock_config: MockConfig) -> tuple[Graph, GraphRun graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - human_node.init_node_data(human_config["data"]) llm_second = _create_llm_node("llm_resume", "Follow-up LLM", "Follow-up prompt") end_data = EndNodeData( title="End", outputs=[ - VariableSelector(variable="initial_text", value_selector=["llm_initial", "text"]), - VariableSelector(variable="resume_text", value_selector=["llm_resume", "text"]), + OutputVariableEntity( + variable="initial_text", value_type=OutputVariableType.STRING, value_selector=["llm_initial", "text"] + ), + OutputVariableEntity( + variable="resume_text", value_type=OutputVariableType.STRING, value_selector=["llm_resume", "text"] + ), ], desc=None, ) @@ -123,7 +124,6 @@ def _build_llm_human_llm_graph(mock_config: MockConfig) -> tuple[Graph, GraphRun graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - end_node.init_node_data(end_config["data"]) graph = ( Graph.new() diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_if_else_streaming.py b/api/tests/unit_tests/core/workflow/graph_engine/test_if_else_streaming.py index dfd33f135f..9fa6ee57eb 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/test_if_else_streaming.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/test_if_else_streaming.py @@ -11,7 +11,7 @@ from core.workflow.graph_events import ( NodeRunStreamChunkEvent, NodeRunSucceededEvent, ) -from core.workflow.nodes.base.entities import VariableSelector +from core.workflow.nodes.base.entities import OutputVariableEntity, OutputVariableType from core.workflow.nodes.end.end_node import EndNode from core.workflow.nodes.end.entities import EndNodeData from core.workflow.nodes.if_else.entities import IfElseNodeData @@ -62,7 +62,6 @@ def _build_if_else_graph(branch_value: str, mock_config: MockConfig) -> tuple[Gr graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - start_node.init_node_data(start_config["data"]) def _create_llm_node(node_id: str, title: str, prompt_text: str) -> MockLLMNode: llm_data = LLMNodeData( @@ -87,7 +86,6 @@ def _build_if_else_graph(branch_value: str, mock_config: MockConfig) -> tuple[Gr graph_runtime_state=graph_runtime_state, mock_config=mock_config, ) - llm_node.init_node_data(llm_config["data"]) return llm_node llm_initial = _create_llm_node("llm_initial", "Initial LLM", "Initial stream") @@ -118,7 +116,6 @@ def _build_if_else_graph(branch_value: str, mock_config: MockConfig) -> tuple[Gr graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - if_else_node.init_node_data(if_else_config["data"]) llm_primary = _create_llm_node("llm_primary", "Primary LLM", "Primary stream output") llm_secondary = _create_llm_node("llm_secondary", "Secondary LLM", "Secondary") @@ -126,8 +123,12 @@ def _build_if_else_graph(branch_value: str, mock_config: MockConfig) -> tuple[Gr end_primary_data = EndNodeData( title="End Primary", outputs=[ - VariableSelector(variable="initial_text", value_selector=["llm_initial", "text"]), - VariableSelector(variable="primary_text", value_selector=["llm_primary", "text"]), + OutputVariableEntity( + variable="initial_text", value_type=OutputVariableType.STRING, value_selector=["llm_initial", "text"] + ), + OutputVariableEntity( + variable="primary_text", value_type=OutputVariableType.STRING, value_selector=["llm_primary", "text"] + ), ], desc=None, ) @@ -138,13 +139,18 @@ def _build_if_else_graph(branch_value: str, mock_config: MockConfig) -> tuple[Gr graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - end_primary.init_node_data(end_primary_config["data"]) end_secondary_data = EndNodeData( title="End Secondary", outputs=[ - VariableSelector(variable="initial_text", value_selector=["llm_initial", "text"]), - VariableSelector(variable="secondary_text", value_selector=["llm_secondary", "text"]), + OutputVariableEntity( + variable="initial_text", value_type=OutputVariableType.STRING, value_selector=["llm_initial", "text"] + ), + OutputVariableEntity( + variable="secondary_text", + value_type=OutputVariableType.STRING, + value_selector=["llm_secondary", "text"], + ), ], desc=None, ) @@ -155,7 +161,6 @@ def _build_if_else_graph(branch_value: str, mock_config: MockConfig) -> tuple[Gr graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, ) - end_secondary.init_node_data(end_secondary_config["data"]) graph = ( Graph.new() diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_mock_factory.py b/api/tests/unit_tests/core/workflow/graph_engine/test_mock_factory.py index 03de984bd1..eeffdd27fe 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/test_mock_factory.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/test_mock_factory.py @@ -111,9 +111,6 @@ class MockNodeFactory(DifyNodeFactory): mock_config=self.mock_config, ) - # Initialize node with provided data - mock_instance.init_node_data(node_data) - return mock_instance # For non-mocked node types, use parent implementation diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_mock_iteration_simple.py b/api/tests/unit_tests/core/workflow/graph_engine/test_mock_iteration_simple.py index 48fa00f105..1cda6ced31 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/test_mock_iteration_simple.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/test_mock_iteration_simple.py @@ -142,6 +142,8 @@ def test_mock_loop_node_preserves_config(): "start_node_id": "node1", "loop_variables": [], "outputs": {}, + "break_conditions": [], + "logical_operator": "and", }, } diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_mock_nodes_template_code.py b/api/tests/unit_tests/core/workflow/graph_engine/test_mock_nodes_template_code.py index 23274f5981..4fb693a5c2 100644 --- a/api/tests/unit_tests/core/workflow/graph_engine/test_mock_nodes_template_code.py +++ b/api/tests/unit_tests/core/workflow/graph_engine/test_mock_nodes_template_code.py @@ -63,7 +63,6 @@ class TestMockTemplateTransformNode: graph_runtime_state=graph_runtime_state, mock_config=mock_config, ) - mock_node.init_node_data(node_config["data"]) # Run the node result = mock_node._run() @@ -125,7 +124,6 @@ class TestMockTemplateTransformNode: graph_runtime_state=graph_runtime_state, mock_config=mock_config, ) - mock_node.init_node_data(node_config["data"]) # Run the node result = mock_node._run() @@ -184,7 +182,6 @@ class TestMockTemplateTransformNode: graph_runtime_state=graph_runtime_state, mock_config=mock_config, ) - mock_node.init_node_data(node_config["data"]) # Run the node result = mock_node._run() @@ -246,7 +243,6 @@ class TestMockTemplateTransformNode: graph_runtime_state=graph_runtime_state, mock_config=mock_config, ) - mock_node.init_node_data(node_config["data"]) # Run the node result = mock_node._run() @@ -311,7 +307,6 @@ class TestMockCodeNode: graph_runtime_state=graph_runtime_state, mock_config=mock_config, ) - mock_node.init_node_data(node_config["data"]) # Run the node result = mock_node._run() @@ -376,7 +371,6 @@ class TestMockCodeNode: graph_runtime_state=graph_runtime_state, mock_config=mock_config, ) - mock_node.init_node_data(node_config["data"]) # Run the node result = mock_node._run() @@ -445,7 +439,6 @@ class TestMockCodeNode: graph_runtime_state=graph_runtime_state, mock_config=mock_config, ) - mock_node.init_node_data(node_config["data"]) # Run the node result = mock_node._run() diff --git a/api/tests/unit_tests/core/workflow/nodes/answer/test_answer.py b/api/tests/unit_tests/core/workflow/nodes/answer/test_answer.py index d151bbe015..98d9560e64 100644 --- a/api/tests/unit_tests/core/workflow/nodes/answer/test_answer.py +++ b/api/tests/unit_tests/core/workflow/nodes/answer/test_answer.py @@ -83,9 +83,6 @@ def test_execute_answer(): config=node_config, ) - # Initialize node data - node.init_node_data(node_config["data"]) - # Mock db.session.close() db.session.close = MagicMock() diff --git a/api/tests/unit_tests/core/workflow/nodes/base/test_base_node.py b/api/tests/unit_tests/core/workflow/nodes/base/test_base_node.py index 4b1f224e67..6eead80ac9 100644 --- a/api/tests/unit_tests/core/workflow/nodes/base/test_base_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/base/test_base_node.py @@ -1,4 +1,7 @@ +import pytest + from core.workflow.enums import NodeType +from core.workflow.nodes.base.entities import BaseNodeData from core.workflow.nodes.base.node import Node # Ensures that all node classes are imported. @@ -7,6 +10,12 @@ from core.workflow.nodes.node_mapping import NODE_TYPE_CLASSES_MAPPING _ = NODE_TYPE_CLASSES_MAPPING +class _TestNodeData(BaseNodeData): + """Test node data for unit tests.""" + + pass + + def _get_all_subclasses(root: type[Node]) -> list[type[Node]]: subclasses = [] queue = [root] @@ -34,3 +43,79 @@ def test_ensure_subclasses_of_base_node_has_node_type_and_version_method_defined node_type_and_version = (node_type, node_version) assert node_type_and_version not in type_version_set type_version_set.add(node_type_and_version) + + +def test_extract_node_data_type_from_generic_extracts_type(): + """When a class inherits from Node[T], it should extract T.""" + + class _ConcreteNode(Node[_TestNodeData]): + node_type = NodeType.CODE + + @staticmethod + def version() -> str: + return "1" + + result = _ConcreteNode._extract_node_data_type_from_generic() + + assert result is _TestNodeData + + +def test_extract_node_data_type_from_generic_returns_none_for_base_node(): + """The base Node class itself should return None (no generic parameter).""" + result = Node._extract_node_data_type_from_generic() + + assert result is None + + +def test_extract_node_data_type_from_generic_raises_for_non_base_node_data(): + """When generic parameter is not a BaseNodeData subtype, should raise TypeError.""" + with pytest.raises(TypeError, match="must parameterize Node with a BaseNodeData subtype"): + + class _InvalidNode(Node[str]): # type: ignore[type-arg] + pass + + +def test_extract_node_data_type_from_generic_raises_for_non_type(): + """When generic parameter is not a concrete type, should raise TypeError.""" + from typing import TypeVar + + T = TypeVar("T") + + with pytest.raises(TypeError, match="must parameterize Node with a BaseNodeData subtype"): + + class _InvalidNode(Node[T]): # type: ignore[type-arg] + pass + + +def test_init_subclass_raises_without_generic_or_explicit_type(): + """A subclass must either use Node[T] or explicitly set _node_data_type.""" + with pytest.raises(TypeError, match="must inherit from Node\\[T\\] with a BaseNodeData subtype"): + + class _InvalidNode(Node): + pass + + +def test_init_subclass_rejects_explicit_node_data_type_without_generic(): + """Setting _node_data_type explicitly cannot bypass the Node[T] requirement.""" + with pytest.raises(TypeError, match="must inherit from Node\\[T\\] with a BaseNodeData subtype"): + + class _ExplicitNode(Node): + _node_data_type = _TestNodeData + node_type = NodeType.CODE + + @staticmethod + def version() -> str: + return "1" + + +def test_init_subclass_sets_node_data_type_from_generic(): + """Verify that __init_subclass__ sets _node_data_type from the generic parameter.""" + + class _AutoNode(Node[_TestNodeData]): + node_type = NodeType.CODE + + @staticmethod + def version() -> str: + return "1" + + assert _AutoNode._node_data_type is _TestNodeData diff --git a/api/tests/unit_tests/core/workflow/nodes/code/__init__.py b/api/tests/unit_tests/core/workflow/nodes/code/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/unit_tests/core/workflow/nodes/code/code_node_spec.py b/api/tests/unit_tests/core/workflow/nodes/code/code_node_spec.py new file mode 100644 index 0000000000..f62c714820 --- /dev/null +++ b/api/tests/unit_tests/core/workflow/nodes/code/code_node_spec.py @@ -0,0 +1,488 @@ +from core.helper.code_executor.code_executor import CodeLanguage +from core.variables.types import SegmentType +from core.workflow.nodes.code.code_node import CodeNode +from core.workflow.nodes.code.entities import CodeNodeData +from core.workflow.nodes.code.exc import ( + CodeNodeError, + DepthLimitError, + OutputValidationError, +) + + +class TestCodeNodeExceptions: + """Test suite for code node exceptions.""" + + def test_code_node_error_is_value_error(self): + """Test CodeNodeError inherits from ValueError.""" + error = CodeNodeError("test error") + + assert isinstance(error, ValueError) + assert str(error) == "test error" + + def test_output_validation_error_is_code_node_error(self): + """Test OutputValidationError inherits from CodeNodeError.""" + error = OutputValidationError("validation failed") + + assert isinstance(error, CodeNodeError) + assert isinstance(error, ValueError) + assert str(error) == "validation failed" + + def test_depth_limit_error_is_code_node_error(self): + """Test DepthLimitError inherits from CodeNodeError.""" + error = DepthLimitError("depth exceeded") + + assert isinstance(error, CodeNodeError) + assert isinstance(error, ValueError) + assert str(error) == "depth exceeded" + + def test_code_node_error_with_empty_message(self): + """Test CodeNodeError with empty message.""" + error = CodeNodeError("") + + assert str(error) == "" + + def test_output_validation_error_with_field_info(self): + """Test OutputValidationError with field information.""" + error = OutputValidationError("Output 'result' is not a valid type") + + assert "result" in str(error) + assert "not a valid type" in str(error) + + def test_depth_limit_error_with_limit_info(self): + """Test DepthLimitError with limit information.""" + error = DepthLimitError("Depth limit 5 reached, object too deep") + + assert "5" in str(error) + assert "too deep" in str(error) + + +class TestCodeNodeClassMethods: + """Test suite for CodeNode class methods.""" + + def test_code_node_version(self): + """Test CodeNode version method.""" + version = CodeNode.version() + + assert version == "1" + + def test_get_default_config_python3(self): + """Test get_default_config for Python3.""" + config = CodeNode.get_default_config(filters={"code_language": CodeLanguage.PYTHON3}) + + assert config is not None + assert isinstance(config, dict) + + def test_get_default_config_javascript(self): + """Test get_default_config for JavaScript.""" + config = CodeNode.get_default_config(filters={"code_language": CodeLanguage.JAVASCRIPT}) + + assert config is not None + assert isinstance(config, dict) + + def test_get_default_config_no_filters(self): + """Test get_default_config with no filters defaults to Python3.""" + config = CodeNode.get_default_config() + + assert config is not None + assert isinstance(config, dict) + + def test_get_default_config_empty_filters(self): + """Test get_default_config with empty filters.""" + config = CodeNode.get_default_config(filters={}) + + assert config is not None + + +class TestCodeNodeCheckMethods: + """Test suite for CodeNode check methods.""" + + def test_check_string_none_value(self): + """Test _check_string with None value.""" + node = CodeNode.__new__(CodeNode) + result = node._check_string(None, "test_var") + + assert result is None + + def test_check_string_removes_null_bytes(self): + """Test _check_string removes null bytes.""" + node = CodeNode.__new__(CodeNode) + result = node._check_string("hello\x00world", "test_var") + + assert result == "helloworld" + assert "\x00" not in result + + def test_check_string_valid_string(self): + """Test _check_string with valid string.""" + node = CodeNode.__new__(CodeNode) + result = node._check_string("valid string", "test_var") + + assert result == "valid string" + + def test_check_string_empty_string(self): + """Test _check_string with empty string.""" + node = CodeNode.__new__(CodeNode) + result = node._check_string("", "test_var") + + assert result == "" + + def test_check_string_with_unicode(self): + """Test _check_string with unicode characters.""" + node = CodeNode.__new__(CodeNode) + result = node._check_string("你好世界🌍", "test_var") + + assert result == "你好世界🌍" + + def test_check_boolean_none_value(self): + """Test _check_boolean with None value.""" + node = CodeNode.__new__(CodeNode) + result = node._check_boolean(None, "test_var") + + assert result is None + + def test_check_boolean_true_value(self): + """Test _check_boolean with True value.""" + node = CodeNode.__new__(CodeNode) + result = node._check_boolean(True, "test_var") + + assert result is True + + def test_check_boolean_false_value(self): + """Test _check_boolean with False value.""" + node = CodeNode.__new__(CodeNode) + result = node._check_boolean(False, "test_var") + + assert result is False + + def test_check_number_none_value(self): + """Test _check_number with None value.""" + node = CodeNode.__new__(CodeNode) + result = node._check_number(None, "test_var") + + assert result is None + + def test_check_number_integer_value(self): + """Test _check_number with integer value.""" + node = CodeNode.__new__(CodeNode) + result = node._check_number(42, "test_var") + + assert result == 42 + + def test_check_number_float_value(self): + """Test _check_number with float value.""" + node = CodeNode.__new__(CodeNode) + result = node._check_number(3.14, "test_var") + + assert result == 3.14 + + def test_check_number_zero(self): + """Test _check_number with zero.""" + node = CodeNode.__new__(CodeNode) + result = node._check_number(0, "test_var") + + assert result == 0 + + def test_check_number_negative(self): + """Test _check_number with negative number.""" + node = CodeNode.__new__(CodeNode) + result = node._check_number(-100, "test_var") + + assert result == -100 + + def test_check_number_negative_float(self): + """Test _check_number with negative float.""" + node = CodeNode.__new__(CodeNode) + result = node._check_number(-3.14159, "test_var") + + assert result == -3.14159 + + +class TestCodeNodeConvertBooleanToInt: + """Test suite for _convert_boolean_to_int static method.""" + + def test_convert_none_returns_none(self): + """Test converting None returns None.""" + result = CodeNode._convert_boolean_to_int(None) + + assert result is None + + def test_convert_true_returns_one(self): + """Test converting True returns 1.""" + result = CodeNode._convert_boolean_to_int(True) + + assert result == 1 + assert isinstance(result, int) + + def test_convert_false_returns_zero(self): + """Test converting False returns 0.""" + result = CodeNode._convert_boolean_to_int(False) + + assert result == 0 + assert isinstance(result, int) + + def test_convert_integer_returns_same(self): + """Test converting integer returns same value.""" + result = CodeNode._convert_boolean_to_int(42) + + assert result == 42 + + def test_convert_float_returns_same(self): + """Test converting float returns same value.""" + result = CodeNode._convert_boolean_to_int(3.14) + + assert result == 3.14 + + def test_convert_zero_returns_zero(self): + """Test converting zero returns zero.""" + result = CodeNode._convert_boolean_to_int(0) + + assert result == 0 + + def test_convert_negative_returns_same(self): + """Test converting negative number returns same value.""" + result = CodeNode._convert_boolean_to_int(-100) + + assert result == -100 + + +class TestCodeNodeExtractVariableSelector: + """Test suite for _extract_variable_selector_to_variable_mapping.""" + + def test_extract_empty_variables(self): + """Test extraction with no variables.""" + node_data = { + "title": "Test", + "variables": [], + "code_language": "python3", + "code": "def main(): return {}", + "outputs": {}, + } + + result = CodeNode._extract_variable_selector_to_variable_mapping( + graph_config={}, + node_id="node_1", + node_data=node_data, + ) + + assert result == {} + + def test_extract_single_variable(self): + """Test extraction with single variable.""" + node_data = { + "title": "Test", + "variables": [ + {"variable": "input_text", "value_selector": ["start", "text"]}, + ], + "code_language": "python3", + "code": "def main(): return {}", + "outputs": {}, + } + + result = CodeNode._extract_variable_selector_to_variable_mapping( + graph_config={}, + node_id="node_1", + node_data=node_data, + ) + + assert "node_1.input_text" in result + assert result["node_1.input_text"] == ["start", "text"] + + def test_extract_multiple_variables(self): + """Test extraction with multiple variables.""" + node_data = { + "title": "Test", + "variables": [ + {"variable": "var1", "value_selector": ["node_a", "output1"]}, + {"variable": "var2", "value_selector": ["node_b", "output2"]}, + {"variable": "var3", "value_selector": ["node_c", "output3"]}, + ], + "code_language": "python3", + "code": "def main(): return {}", + "outputs": {}, + } + + result = CodeNode._extract_variable_selector_to_variable_mapping( + graph_config={}, + node_id="code_node", + node_data=node_data, + ) + + assert len(result) == 3 + assert "code_node.var1" in result + assert "code_node.var2" in result + assert "code_node.var3" in result + + def test_extract_with_nested_selector(self): + """Test extraction with nested value selector.""" + node_data = { + "title": "Test", + "variables": [ + {"variable": "deep_var", "value_selector": ["node", "obj", "nested", "value"]}, + ], + "code_language": "python3", + "code": "def main(): return {}", + "outputs": {}, + } + + result = CodeNode._extract_variable_selector_to_variable_mapping( + graph_config={}, + node_id="node_x", + node_data=node_data, + ) + + assert result["node_x.deep_var"] == ["node", "obj", "nested", "value"] + + +class TestCodeNodeDataValidation: + """Test suite for CodeNodeData validation scenarios.""" + + def test_valid_python3_code_node_data(self): + """Test valid Python3 CodeNodeData.""" + data = CodeNodeData( + title="Python Code", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="def main(): return {'result': 1}", + outputs={"result": CodeNodeData.Output(type=SegmentType.NUMBER)}, + ) + + assert data.code_language == CodeLanguage.PYTHON3 + + def test_valid_javascript_code_node_data(self): + """Test valid JavaScript CodeNodeData.""" + data = CodeNodeData( + title="JS Code", + variables=[], + code_language=CodeLanguage.JAVASCRIPT, + code="function main() { return { result: 1 }; }", + outputs={"result": CodeNodeData.Output(type=SegmentType.NUMBER)}, + ) + + assert data.code_language == CodeLanguage.JAVASCRIPT + + def test_code_node_data_with_all_output_types(self): + """Test CodeNodeData with all valid output types.""" + data = CodeNodeData( + title="All Types", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="def main(): return {}", + outputs={ + "str_out": CodeNodeData.Output(type=SegmentType.STRING), + "num_out": CodeNodeData.Output(type=SegmentType.NUMBER), + "bool_out": CodeNodeData.Output(type=SegmentType.BOOLEAN), + "obj_out": CodeNodeData.Output(type=SegmentType.OBJECT), + "arr_str": CodeNodeData.Output(type=SegmentType.ARRAY_STRING), + "arr_num": CodeNodeData.Output(type=SegmentType.ARRAY_NUMBER), + "arr_bool": CodeNodeData.Output(type=SegmentType.ARRAY_BOOLEAN), + "arr_obj": CodeNodeData.Output(type=SegmentType.ARRAY_OBJECT), + }, + ) + + assert len(data.outputs) == 8 + + def test_code_node_data_complex_nested_output(self): + """Test CodeNodeData with complex nested output structure.""" + data = CodeNodeData( + title="Complex Output", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="def main(): return {}", + outputs={ + "response": CodeNodeData.Output( + type=SegmentType.OBJECT, + children={ + "data": CodeNodeData.Output( + type=SegmentType.OBJECT, + children={ + "items": CodeNodeData.Output(type=SegmentType.ARRAY_STRING), + "count": CodeNodeData.Output(type=SegmentType.NUMBER), + }, + ), + "status": CodeNodeData.Output(type=SegmentType.STRING), + "success": CodeNodeData.Output(type=SegmentType.BOOLEAN), + }, + ), + }, + ) + + assert data.outputs["response"].type == SegmentType.OBJECT + assert data.outputs["response"].children is not None + assert "data" in data.outputs["response"].children + assert data.outputs["response"].children["data"].children is not None + + +class TestCodeNodeInitialization: + """Test suite for CodeNode initialization methods.""" + + def test_init_node_data_python3(self): + """Test init_node_data with Python3 configuration.""" + node = CodeNode.__new__(CodeNode) + data = { + "title": "Test Node", + "variables": [], + "code_language": "python3", + "code": "def main(): return {'x': 1}", + "outputs": {"x": {"type": "number"}}, + } + + node.init_node_data(data) + + assert node._node_data.title == "Test Node" + assert node._node_data.code_language == CodeLanguage.PYTHON3 + + def test_init_node_data_javascript(self): + """Test init_node_data with JavaScript configuration.""" + node = CodeNode.__new__(CodeNode) + data = { + "title": "JS Node", + "variables": [], + "code_language": "javascript", + "code": "function main() { return { x: 1 }; }", + "outputs": {"x": {"type": "number"}}, + } + + node.init_node_data(data) + + assert node._node_data.code_language == CodeLanguage.JAVASCRIPT + + def test_get_title(self): + """Test _get_title method.""" + node = CodeNode.__new__(CodeNode) + node._node_data = CodeNodeData( + title="My Code Node", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="", + outputs={}, + ) + + assert node._get_title() == "My Code Node" + + def test_get_description_none(self): + """Test _get_description returns None when not set.""" + node = CodeNode.__new__(CodeNode) + node._node_data = CodeNodeData( + title="Test", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="", + outputs={}, + ) + + assert node._get_description() is None + + def test_get_base_node_data(self): + """Test get_base_node_data returns node data.""" + node = CodeNode.__new__(CodeNode) + node._node_data = CodeNodeData( + title="Base Test", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="", + outputs={}, + ) + + result = node.get_base_node_data() + + assert result == node._node_data + assert result.title == "Base Test" diff --git a/api/tests/unit_tests/core/workflow/nodes/code/entities_spec.py b/api/tests/unit_tests/core/workflow/nodes/code/entities_spec.py new file mode 100644 index 0000000000..d14a6ea69c --- /dev/null +++ b/api/tests/unit_tests/core/workflow/nodes/code/entities_spec.py @@ -0,0 +1,353 @@ +import pytest +from pydantic import ValidationError + +from core.helper.code_executor.code_executor import CodeLanguage +from core.variables.types import SegmentType +from core.workflow.nodes.code.entities import CodeNodeData + + +class TestCodeNodeDataOutput: + """Test suite for CodeNodeData.Output model.""" + + def test_output_with_string_type(self): + """Test Output with STRING type.""" + output = CodeNodeData.Output(type=SegmentType.STRING) + + assert output.type == SegmentType.STRING + assert output.children is None + + def test_output_with_number_type(self): + """Test Output with NUMBER type.""" + output = CodeNodeData.Output(type=SegmentType.NUMBER) + + assert output.type == SegmentType.NUMBER + assert output.children is None + + def test_output_with_boolean_type(self): + """Test Output with BOOLEAN type.""" + output = CodeNodeData.Output(type=SegmentType.BOOLEAN) + + assert output.type == SegmentType.BOOLEAN + + def test_output_with_object_type(self): + """Test Output with OBJECT type.""" + output = CodeNodeData.Output(type=SegmentType.OBJECT) + + assert output.type == SegmentType.OBJECT + + def test_output_with_array_string_type(self): + """Test Output with ARRAY_STRING type.""" + output = CodeNodeData.Output(type=SegmentType.ARRAY_STRING) + + assert output.type == SegmentType.ARRAY_STRING + + def test_output_with_array_number_type(self): + """Test Output with ARRAY_NUMBER type.""" + output = CodeNodeData.Output(type=SegmentType.ARRAY_NUMBER) + + assert output.type == SegmentType.ARRAY_NUMBER + + def test_output_with_array_object_type(self): + """Test Output with ARRAY_OBJECT type.""" + output = CodeNodeData.Output(type=SegmentType.ARRAY_OBJECT) + + assert output.type == SegmentType.ARRAY_OBJECT + + def test_output_with_array_boolean_type(self): + """Test Output with ARRAY_BOOLEAN type.""" + output = CodeNodeData.Output(type=SegmentType.ARRAY_BOOLEAN) + + assert output.type == SegmentType.ARRAY_BOOLEAN + + def test_output_with_nested_children(self): + """Test Output with nested children for OBJECT type.""" + child_output = CodeNodeData.Output(type=SegmentType.STRING) + parent_output = CodeNodeData.Output( + type=SegmentType.OBJECT, + children={"name": child_output}, + ) + + assert parent_output.type == SegmentType.OBJECT + assert parent_output.children is not None + assert "name" in parent_output.children + assert parent_output.children["name"].type == SegmentType.STRING + + def test_output_with_deeply_nested_children(self): + """Test Output with deeply nested children.""" + inner_child = CodeNodeData.Output(type=SegmentType.NUMBER) + middle_child = CodeNodeData.Output( + type=SegmentType.OBJECT, + children={"value": inner_child}, + ) + outer_output = CodeNodeData.Output( + type=SegmentType.OBJECT, + children={"nested": middle_child}, + ) + + assert outer_output.children is not None + assert outer_output.children["nested"].children is not None + assert outer_output.children["nested"].children["value"].type == SegmentType.NUMBER + + def test_output_with_multiple_children(self): + """Test Output with multiple children.""" + output = CodeNodeData.Output( + type=SegmentType.OBJECT, + children={ + "name": CodeNodeData.Output(type=SegmentType.STRING), + "age": CodeNodeData.Output(type=SegmentType.NUMBER), + "active": CodeNodeData.Output(type=SegmentType.BOOLEAN), + }, + ) + + assert output.children is not None + assert len(output.children) == 3 + assert output.children["name"].type == SegmentType.STRING + assert output.children["age"].type == SegmentType.NUMBER + assert output.children["active"].type == SegmentType.BOOLEAN + + def test_output_rejects_invalid_type(self): + """Test Output rejects invalid segment types.""" + with pytest.raises(ValidationError): + CodeNodeData.Output(type=SegmentType.FILE) + + def test_output_rejects_array_file_type(self): + """Test Output rejects ARRAY_FILE type.""" + with pytest.raises(ValidationError): + CodeNodeData.Output(type=SegmentType.ARRAY_FILE) + + +class TestCodeNodeDataDependency: + """Test suite for CodeNodeData.Dependency model.""" + + def test_dependency_basic(self): + """Test Dependency with name and version.""" + dependency = CodeNodeData.Dependency(name="numpy", version="1.24.0") + + assert dependency.name == "numpy" + assert dependency.version == "1.24.0" + + def test_dependency_with_complex_version(self): + """Test Dependency with complex version string.""" + dependency = CodeNodeData.Dependency(name="pandas", version=">=2.0.0,<3.0.0") + + assert dependency.name == "pandas" + assert dependency.version == ">=2.0.0,<3.0.0" + + def test_dependency_with_empty_version(self): + """Test Dependency with empty version.""" + dependency = CodeNodeData.Dependency(name="requests", version="") + + assert dependency.name == "requests" + assert dependency.version == "" + + +class TestCodeNodeData: + """Test suite for CodeNodeData model.""" + + def test_code_node_data_python3(self): + """Test CodeNodeData with Python3 language.""" + data = CodeNodeData( + title="Test Code Node", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="def main(): return {'result': 42}", + outputs={"result": CodeNodeData.Output(type=SegmentType.NUMBER)}, + ) + + assert data.title == "Test Code Node" + assert data.code_language == CodeLanguage.PYTHON3 + assert data.code == "def main(): return {'result': 42}" + assert "result" in data.outputs + assert data.dependencies is None + + def test_code_node_data_javascript(self): + """Test CodeNodeData with JavaScript language.""" + data = CodeNodeData( + title="JS Code Node", + variables=[], + code_language=CodeLanguage.JAVASCRIPT, + code="function main() { return { result: 'hello' }; }", + outputs={"result": CodeNodeData.Output(type=SegmentType.STRING)}, + ) + + assert data.code_language == CodeLanguage.JAVASCRIPT + assert "result" in data.outputs + assert data.outputs["result"].type == SegmentType.STRING + + def test_code_node_data_with_dependencies(self): + """Test CodeNodeData with dependencies.""" + data = CodeNodeData( + title="Code with Deps", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="import numpy as np\ndef main(): return {'sum': 10}", + outputs={"sum": CodeNodeData.Output(type=SegmentType.NUMBER)}, + dependencies=[ + CodeNodeData.Dependency(name="numpy", version="1.24.0"), + CodeNodeData.Dependency(name="pandas", version="2.0.0"), + ], + ) + + assert data.dependencies is not None + assert len(data.dependencies) == 2 + assert data.dependencies[0].name == "numpy" + assert data.dependencies[1].name == "pandas" + + def test_code_node_data_with_multiple_outputs(self): + """Test CodeNodeData with multiple outputs.""" + data = CodeNodeData( + title="Multi Output", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="def main(): return {'name': 'test', 'count': 5, 'items': ['a', 'b']}", + outputs={ + "name": CodeNodeData.Output(type=SegmentType.STRING), + "count": CodeNodeData.Output(type=SegmentType.NUMBER), + "items": CodeNodeData.Output(type=SegmentType.ARRAY_STRING), + }, + ) + + assert len(data.outputs) == 3 + assert data.outputs["name"].type == SegmentType.STRING + assert data.outputs["count"].type == SegmentType.NUMBER + assert data.outputs["items"].type == SegmentType.ARRAY_STRING + + def test_code_node_data_with_object_output(self): + """Test CodeNodeData with nested object output.""" + data = CodeNodeData( + title="Object Output", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="def main(): return {'user': {'name': 'John', 'age': 30}}", + outputs={ + "user": CodeNodeData.Output( + type=SegmentType.OBJECT, + children={ + "name": CodeNodeData.Output(type=SegmentType.STRING), + "age": CodeNodeData.Output(type=SegmentType.NUMBER), + }, + ), + }, + ) + + assert data.outputs["user"].type == SegmentType.OBJECT + assert data.outputs["user"].children is not None + assert len(data.outputs["user"].children) == 2 + + def test_code_node_data_with_array_object_output(self): + """Test CodeNodeData with array of objects output.""" + data = CodeNodeData( + title="Array Object Output", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="def main(): return {'users': [{'name': 'A'}, {'name': 'B'}]}", + outputs={ + "users": CodeNodeData.Output( + type=SegmentType.ARRAY_OBJECT, + children={ + "name": CodeNodeData.Output(type=SegmentType.STRING), + }, + ), + }, + ) + + assert data.outputs["users"].type == SegmentType.ARRAY_OBJECT + assert data.outputs["users"].children is not None + + def test_code_node_data_empty_code(self): + """Test CodeNodeData with empty code.""" + data = CodeNodeData( + title="Empty Code", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="", + outputs={}, + ) + + assert data.code == "" + assert len(data.outputs) == 0 + + def test_code_node_data_multiline_code(self): + """Test CodeNodeData with multiline code.""" + multiline_code = """ +def main(): + result = 0 + for i in range(10): + result += i + return {'sum': result} +""" + data = CodeNodeData( + title="Multiline Code", + variables=[], + code_language=CodeLanguage.PYTHON3, + code=multiline_code, + outputs={"sum": CodeNodeData.Output(type=SegmentType.NUMBER)}, + ) + + assert "for i in range(10)" in data.code + assert "result += i" in data.code + + def test_code_node_data_with_special_characters_in_code(self): + """Test CodeNodeData with special characters in code.""" + code_with_special = "def main(): return {'msg': 'Hello\\nWorld\\t!'}" + data = CodeNodeData( + title="Special Chars", + variables=[], + code_language=CodeLanguage.PYTHON3, + code=code_with_special, + outputs={"msg": CodeNodeData.Output(type=SegmentType.STRING)}, + ) + + assert "\\n" in data.code + assert "\\t" in data.code + + def test_code_node_data_with_unicode_in_code(self): + """Test CodeNodeData with unicode characters in code.""" + unicode_code = "def main(): return {'greeting': '你好世界'}" + data = CodeNodeData( + title="Unicode Code", + variables=[], + code_language=CodeLanguage.PYTHON3, + code=unicode_code, + outputs={"greeting": CodeNodeData.Output(type=SegmentType.STRING)}, + ) + + assert "你好世界" in data.code + + def test_code_node_data_empty_dependencies_list(self): + """Test CodeNodeData with empty dependencies list.""" + data = CodeNodeData( + title="No Deps", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="def main(): return {}", + outputs={}, + dependencies=[], + ) + + assert data.dependencies is not None + assert len(data.dependencies) == 0 + + def test_code_node_data_with_boolean_array_output(self): + """Test CodeNodeData with boolean array output.""" + data = CodeNodeData( + title="Boolean Array", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="def main(): return {'flags': [True, False, True]}", + outputs={"flags": CodeNodeData.Output(type=SegmentType.ARRAY_BOOLEAN)}, + ) + + assert data.outputs["flags"].type == SegmentType.ARRAY_BOOLEAN + + def test_code_node_data_with_number_array_output(self): + """Test CodeNodeData with number array output.""" + data = CodeNodeData( + title="Number Array", + variables=[], + code_language=CodeLanguage.PYTHON3, + code="def main(): return {'values': [1, 2, 3, 4, 5]}", + outputs={"values": CodeNodeData.Output(type=SegmentType.ARRAY_NUMBER)}, + ) + + assert data.outputs["values"].type == SegmentType.ARRAY_NUMBER diff --git a/api/tests/unit_tests/core/workflow/nodes/iteration/__init__.py b/api/tests/unit_tests/core/workflow/nodes/iteration/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/unit_tests/core/workflow/nodes/iteration/entities_spec.py b/api/tests/unit_tests/core/workflow/nodes/iteration/entities_spec.py new file mode 100644 index 0000000000..d669cc7465 --- /dev/null +++ b/api/tests/unit_tests/core/workflow/nodes/iteration/entities_spec.py @@ -0,0 +1,339 @@ +from core.workflow.nodes.iteration.entities import ( + ErrorHandleMode, + IterationNodeData, + IterationStartNodeData, + IterationState, +) + + +class TestErrorHandleMode: + """Test suite for ErrorHandleMode enum.""" + + def test_terminated_value(self): + """Test TERMINATED enum value.""" + assert ErrorHandleMode.TERMINATED == "terminated" + assert ErrorHandleMode.TERMINATED.value == "terminated" + + def test_continue_on_error_value(self): + """Test CONTINUE_ON_ERROR enum value.""" + assert ErrorHandleMode.CONTINUE_ON_ERROR == "continue-on-error" + assert ErrorHandleMode.CONTINUE_ON_ERROR.value == "continue-on-error" + + def test_remove_abnormal_output_value(self): + """Test REMOVE_ABNORMAL_OUTPUT enum value.""" + assert ErrorHandleMode.REMOVE_ABNORMAL_OUTPUT == "remove-abnormal-output" + assert ErrorHandleMode.REMOVE_ABNORMAL_OUTPUT.value == "remove-abnormal-output" + + def test_error_handle_mode_is_str_enum(self): + """Test ErrorHandleMode is a string enum.""" + assert isinstance(ErrorHandleMode.TERMINATED, str) + assert isinstance(ErrorHandleMode.CONTINUE_ON_ERROR, str) + assert isinstance(ErrorHandleMode.REMOVE_ABNORMAL_OUTPUT, str) + + def test_error_handle_mode_comparison(self): + """Test ErrorHandleMode can be compared with strings.""" + assert ErrorHandleMode.TERMINATED == "terminated" + assert ErrorHandleMode.CONTINUE_ON_ERROR == "continue-on-error" + + def test_all_error_handle_modes(self): + """Test all ErrorHandleMode values are accessible.""" + modes = list(ErrorHandleMode) + + assert len(modes) == 3 + assert ErrorHandleMode.TERMINATED in modes + assert ErrorHandleMode.CONTINUE_ON_ERROR in modes + assert ErrorHandleMode.REMOVE_ABNORMAL_OUTPUT in modes + + +class TestIterationNodeData: + """Test suite for IterationNodeData model.""" + + def test_iteration_node_data_basic(self): + """Test IterationNodeData with basic configuration.""" + data = IterationNodeData( + title="Test Iteration", + iterator_selector=["node1", "output"], + output_selector=["iteration", "result"], + ) + + assert data.title == "Test Iteration" + assert data.iterator_selector == ["node1", "output"] + assert data.output_selector == ["iteration", "result"] + + def test_iteration_node_data_default_values(self): + """Test IterationNodeData default values.""" + data = IterationNodeData( + title="Default Test", + iterator_selector=["start", "items"], + output_selector=["iter", "out"], + ) + + assert data.parent_loop_id is None + assert data.is_parallel is False + assert data.parallel_nums == 10 + assert data.error_handle_mode == ErrorHandleMode.TERMINATED + assert data.flatten_output is True + + def test_iteration_node_data_parallel_mode(self): + """Test IterationNodeData with parallel mode enabled.""" + data = IterationNodeData( + title="Parallel Iteration", + iterator_selector=["node", "list"], + output_selector=["iter", "output"], + is_parallel=True, + parallel_nums=5, + ) + + assert data.is_parallel is True + assert data.parallel_nums == 5 + + def test_iteration_node_data_custom_parallel_nums(self): + """Test IterationNodeData with custom parallel numbers.""" + data = IterationNodeData( + title="Custom Parallel", + iterator_selector=["a", "b"], + output_selector=["c", "d"], + parallel_nums=20, + ) + + assert data.parallel_nums == 20 + + def test_iteration_node_data_continue_on_error(self): + """Test IterationNodeData with continue on error mode.""" + data = IterationNodeData( + title="Continue Error", + iterator_selector=["x", "y"], + output_selector=["z", "w"], + error_handle_mode=ErrorHandleMode.CONTINUE_ON_ERROR, + ) + + assert data.error_handle_mode == ErrorHandleMode.CONTINUE_ON_ERROR + + def test_iteration_node_data_remove_abnormal_output(self): + """Test IterationNodeData with remove abnormal output mode.""" + data = IterationNodeData( + title="Remove Abnormal", + iterator_selector=["input", "array"], + output_selector=["output", "result"], + error_handle_mode=ErrorHandleMode.REMOVE_ABNORMAL_OUTPUT, + ) + + assert data.error_handle_mode == ErrorHandleMode.REMOVE_ABNORMAL_OUTPUT + + def test_iteration_node_data_flatten_output_disabled(self): + """Test IterationNodeData with flatten output disabled.""" + data = IterationNodeData( + title="No Flatten", + iterator_selector=["a"], + output_selector=["b"], + flatten_output=False, + ) + + assert data.flatten_output is False + + def test_iteration_node_data_with_parent_loop_id(self): + """Test IterationNodeData with parent loop ID.""" + data = IterationNodeData( + title="Nested Loop", + iterator_selector=["parent", "items"], + output_selector=["child", "output"], + parent_loop_id="parent_loop_123", + ) + + assert data.parent_loop_id == "parent_loop_123" + + def test_iteration_node_data_complex_selectors(self): + """Test IterationNodeData with complex selectors.""" + data = IterationNodeData( + title="Complex Selectors", + iterator_selector=["node1", "output", "data", "items"], + output_selector=["iteration", "result", "value"], + ) + + assert len(data.iterator_selector) == 4 + assert len(data.output_selector) == 3 + + def test_iteration_node_data_all_options(self): + """Test IterationNodeData with all options configured.""" + data = IterationNodeData( + title="Full Config", + iterator_selector=["start", "list"], + output_selector=["end", "result"], + parent_loop_id="outer_loop", + is_parallel=True, + parallel_nums=15, + error_handle_mode=ErrorHandleMode.CONTINUE_ON_ERROR, + flatten_output=False, + ) + + assert data.title == "Full Config" + assert data.parent_loop_id == "outer_loop" + assert data.is_parallel is True + assert data.parallel_nums == 15 + assert data.error_handle_mode == ErrorHandleMode.CONTINUE_ON_ERROR + assert data.flatten_output is False + + +class TestIterationStartNodeData: + """Test suite for IterationStartNodeData model.""" + + def test_iteration_start_node_data_basic(self): + """Test IterationStartNodeData basic creation.""" + data = IterationStartNodeData(title="Iteration Start") + + assert data.title == "Iteration Start" + + def test_iteration_start_node_data_with_description(self): + """Test IterationStartNodeData with description.""" + data = IterationStartNodeData( + title="Start Node", + desc="This is the start of iteration", + ) + + assert data.title == "Start Node" + assert data.desc == "This is the start of iteration" + + +class TestIterationState: + """Test suite for IterationState model.""" + + def test_iteration_state_default_values(self): + """Test IterationState default values.""" + state = IterationState() + + assert state.outputs == [] + assert state.current_output is None + + def test_iteration_state_with_outputs(self): + """Test IterationState with outputs.""" + state = IterationState(outputs=["result1", "result2", "result3"]) + + assert len(state.outputs) == 3 + assert state.outputs[0] == "result1" + assert state.outputs[2] == "result3" + + def test_iteration_state_with_current_output(self): + """Test IterationState with current output.""" + state = IterationState(current_output="current_value") + + assert state.current_output == "current_value" + + def test_iteration_state_get_last_output_with_outputs(self): + """Test get_last_output with outputs present.""" + state = IterationState(outputs=["first", "second", "last"]) + + result = state.get_last_output() + + assert result == "last" + + def test_iteration_state_get_last_output_empty(self): + """Test get_last_output with empty outputs.""" + state = IterationState(outputs=[]) + + result = state.get_last_output() + + assert result is None + + def test_iteration_state_get_last_output_single(self): + """Test get_last_output with single output.""" + state = IterationState(outputs=["only_one"]) + + result = state.get_last_output() + + assert result == "only_one" + + def test_iteration_state_get_current_output(self): + """Test get_current_output method.""" + state = IterationState(current_output={"key": "value"}) + + result = state.get_current_output() + + assert result == {"key": "value"} + + def test_iteration_state_get_current_output_none(self): + """Test get_current_output when None.""" + state = IterationState() + + result = state.get_current_output() + + assert result is None + + def test_iteration_state_with_complex_outputs(self): + """Test IterationState with complex output types.""" + state = IterationState( + outputs=[ + {"id": 1, "name": "first"}, + {"id": 2, "name": "second"}, + [1, 2, 3], + "string_output", + ] + ) + + assert len(state.outputs) == 4 + assert state.outputs[0] == {"id": 1, "name": "first"} + assert state.outputs[2] == [1, 2, 3] + + def test_iteration_state_with_none_outputs(self): + """Test IterationState with None values in outputs.""" + state = IterationState(outputs=["value1", None, "value3"]) + + assert len(state.outputs) == 3 + assert state.outputs[1] is None + + def test_iteration_state_get_last_output_with_none(self): + """Test get_last_output when last output is None.""" + state = IterationState(outputs=["first", None]) + + result = state.get_last_output() + + assert result is None + + def test_iteration_state_metadata_class(self): + """Test IterationState.MetaData class.""" + metadata = IterationState.MetaData(iterator_length=10) + + assert metadata.iterator_length == 10 + + def test_iteration_state_metadata_different_lengths(self): + """Test IterationState.MetaData with different lengths.""" + metadata1 = IterationState.MetaData(iterator_length=0) + metadata2 = IterationState.MetaData(iterator_length=100) + metadata3 = IterationState.MetaData(iterator_length=1000000) + + assert metadata1.iterator_length == 0 + assert metadata2.iterator_length == 100 + assert metadata3.iterator_length == 1000000 + + def test_iteration_state_outputs_modification(self): + """Test modifying IterationState outputs.""" + state = IterationState(outputs=[]) + + state.outputs.append("new_output") + state.outputs.append("another_output") + + assert len(state.outputs) == 2 + assert state.get_last_output() == "another_output" + + def test_iteration_state_current_output_update(self): + """Test updating current_output.""" + state = IterationState() + + state.current_output = "first_value" + assert state.get_current_output() == "first_value" + + state.current_output = "updated_value" + assert state.get_current_output() == "updated_value" + + def test_iteration_state_with_numeric_outputs(self): + """Test IterationState with numeric outputs.""" + state = IterationState(outputs=[1, 2, 3, 4, 5]) + + assert state.get_last_output() == 5 + assert len(state.outputs) == 5 + + def test_iteration_state_with_boolean_outputs(self): + """Test IterationState with boolean outputs.""" + state = IterationState(outputs=[True, False, True]) + + assert state.get_last_output() is True + assert state.outputs[1] is False diff --git a/api/tests/unit_tests/core/workflow/nodes/iteration/iteration_node_spec.py b/api/tests/unit_tests/core/workflow/nodes/iteration/iteration_node_spec.py new file mode 100644 index 0000000000..51af4367f7 --- /dev/null +++ b/api/tests/unit_tests/core/workflow/nodes/iteration/iteration_node_spec.py @@ -0,0 +1,390 @@ +from core.workflow.enums import NodeType +from core.workflow.nodes.iteration.entities import ErrorHandleMode, IterationNodeData +from core.workflow.nodes.iteration.exc import ( + InvalidIteratorValueError, + IterationGraphNotFoundError, + IterationIndexNotFoundError, + IterationNodeError, + IteratorVariableNotFoundError, + StartNodeIdNotFoundError, +) +from core.workflow.nodes.iteration.iteration_node import IterationNode + + +class TestIterationNodeExceptions: + """Test suite for iteration node exceptions.""" + + def test_iteration_node_error_is_value_error(self): + """Test IterationNodeError inherits from ValueError.""" + error = IterationNodeError("test error") + + assert isinstance(error, ValueError) + assert str(error) == "test error" + + def test_iterator_variable_not_found_error(self): + """Test IteratorVariableNotFoundError.""" + error = IteratorVariableNotFoundError("Iterator variable not found") + + assert isinstance(error, IterationNodeError) + assert isinstance(error, ValueError) + assert "Iterator variable not found" in str(error) + + def test_invalid_iterator_value_error(self): + """Test InvalidIteratorValueError.""" + error = InvalidIteratorValueError("Invalid iterator value") + + assert isinstance(error, IterationNodeError) + assert "Invalid iterator value" in str(error) + + def test_start_node_id_not_found_error(self): + """Test StartNodeIdNotFoundError.""" + error = StartNodeIdNotFoundError("Start node ID not found") + + assert isinstance(error, IterationNodeError) + assert "Start node ID not found" in str(error) + + def test_iteration_graph_not_found_error(self): + """Test IterationGraphNotFoundError.""" + error = IterationGraphNotFoundError("Iteration graph not found") + + assert isinstance(error, IterationNodeError) + assert "Iteration graph not found" in str(error) + + def test_iteration_index_not_found_error(self): + """Test IterationIndexNotFoundError.""" + error = IterationIndexNotFoundError("Iteration index not found") + + assert isinstance(error, IterationNodeError) + assert "Iteration index not found" in str(error) + + def test_exception_with_empty_message(self): + """Test exception with empty message.""" + error = IterationNodeError("") + + assert str(error) == "" + + def test_exception_with_detailed_message(self): + """Test exception with detailed message.""" + error = IteratorVariableNotFoundError("Variable 'items' not found in node 'start_node'") + + assert "items" in str(error) + assert "start_node" in str(error) + + def test_all_exceptions_inherit_from_base(self): + """Test all exceptions inherit from IterationNodeError.""" + exceptions = [ + IteratorVariableNotFoundError("test"), + InvalidIteratorValueError("test"), + StartNodeIdNotFoundError("test"), + IterationGraphNotFoundError("test"), + IterationIndexNotFoundError("test"), + ] + + for exc in exceptions: + assert isinstance(exc, IterationNodeError) + assert isinstance(exc, ValueError) + + +class TestIterationNodeClassAttributes: + """Test suite for IterationNode class attributes.""" + + def test_node_type(self): + """Test IterationNode node_type attribute.""" + assert IterationNode.node_type == NodeType.ITERATION + + def test_version(self): + """Test IterationNode version method.""" + version = IterationNode.version() + + assert version == "1" + + +class TestIterationNodeDefaultConfig: + """Test suite for IterationNode get_default_config.""" + + def test_get_default_config_returns_dict(self): + """Test get_default_config returns a dictionary.""" + config = IterationNode.get_default_config() + + assert isinstance(config, dict) + + def test_get_default_config_type(self): + """Test get_default_config includes type.""" + config = IterationNode.get_default_config() + + assert config.get("type") == "iteration" + + def test_get_default_config_has_config_section(self): + """Test get_default_config has config section.""" + config = IterationNode.get_default_config() + + assert "config" in config + assert isinstance(config["config"], dict) + + def test_get_default_config_is_parallel_default(self): + """Test get_default_config is_parallel default value.""" + config = IterationNode.get_default_config() + + assert config["config"]["is_parallel"] is False + + def test_get_default_config_parallel_nums_default(self): + """Test get_default_config parallel_nums default value.""" + config = IterationNode.get_default_config() + + assert config["config"]["parallel_nums"] == 10 + + def test_get_default_config_error_handle_mode_default(self): + """Test get_default_config error_handle_mode default value.""" + config = IterationNode.get_default_config() + + assert config["config"]["error_handle_mode"] == ErrorHandleMode.TERMINATED + + def test_get_default_config_flatten_output_default(self): + """Test get_default_config flatten_output default value.""" + config = IterationNode.get_default_config() + + assert config["config"]["flatten_output"] is True + + def test_get_default_config_with_none_filters(self): + """Test get_default_config with None filters.""" + config = IterationNode.get_default_config(filters=None) + + assert config is not None + assert "type" in config + + def test_get_default_config_with_empty_filters(self): + """Test get_default_config with empty filters.""" + config = IterationNode.get_default_config(filters={}) + + assert config is not None + + +class TestIterationNodeInitialization: + """Test suite for IterationNode initialization.""" + + def test_init_node_data_basic(self): + """Test init_node_data with basic configuration.""" + node = IterationNode.__new__(IterationNode) + data = { + "title": "Test Iteration", + "iterator_selector": ["start", "items"], + "output_selector": ["iteration", "result"], + } + + node.init_node_data(data) + + assert node._node_data.title == "Test Iteration" + assert node._node_data.iterator_selector == ["start", "items"] + + def test_init_node_data_with_parallel(self): + """Test init_node_data with parallel configuration.""" + node = IterationNode.__new__(IterationNode) + data = { + "title": "Parallel Iteration", + "iterator_selector": ["node", "list"], + "output_selector": ["out", "result"], + "is_parallel": True, + "parallel_nums": 5, + } + + node.init_node_data(data) + + assert node._node_data.is_parallel is True + assert node._node_data.parallel_nums == 5 + + def test_init_node_data_with_error_handle_mode(self): + """Test init_node_data with error handle mode.""" + node = IterationNode.__new__(IterationNode) + data = { + "title": "Error Handle Test", + "iterator_selector": ["a", "b"], + "output_selector": ["c", "d"], + "error_handle_mode": "continue-on-error", + } + + node.init_node_data(data) + + assert node._node_data.error_handle_mode == ErrorHandleMode.CONTINUE_ON_ERROR + + def test_get_title(self): + """Test _get_title method.""" + node = IterationNode.__new__(IterationNode) + node._node_data = IterationNodeData( + title="My Iteration", + iterator_selector=["x"], + output_selector=["y"], + ) + + assert node._get_title() == "My Iteration" + + def test_get_description_none(self): + """Test _get_description returns None when not set.""" + node = IterationNode.__new__(IterationNode) + node._node_data = IterationNodeData( + title="Test", + iterator_selector=["a"], + output_selector=["b"], + ) + + assert node._get_description() is None + + def test_get_description_with_value(self): + """Test _get_description with value.""" + node = IterationNode.__new__(IterationNode) + node._node_data = IterationNodeData( + title="Test", + desc="This is a description", + iterator_selector=["a"], + output_selector=["b"], + ) + + assert node._get_description() == "This is a description" + + def test_get_base_node_data(self): + """Test get_base_node_data returns node data.""" + node = IterationNode.__new__(IterationNode) + node._node_data = IterationNodeData( + title="Base Test", + iterator_selector=["x"], + output_selector=["y"], + ) + + result = node.get_base_node_data() + + assert result == node._node_data + + +class TestIterationNodeDataValidation: + """Test suite for IterationNodeData validation scenarios.""" + + def test_valid_iteration_node_data(self): + """Test valid IterationNodeData creation.""" + data = IterationNodeData( + title="Valid Iteration", + iterator_selector=["start", "items"], + output_selector=["end", "result"], + ) + + assert data.title == "Valid Iteration" + + def test_iteration_node_data_with_all_error_modes(self): + """Test IterationNodeData with all error handle modes.""" + modes = [ + ErrorHandleMode.TERMINATED, + ErrorHandleMode.CONTINUE_ON_ERROR, + ErrorHandleMode.REMOVE_ABNORMAL_OUTPUT, + ] + + for mode in modes: + data = IterationNodeData( + title=f"Test {mode}", + iterator_selector=["a"], + output_selector=["b"], + error_handle_mode=mode, + ) + assert data.error_handle_mode == mode + + def test_iteration_node_data_parallel_configuration(self): + """Test IterationNodeData parallel configuration combinations.""" + configs = [ + (False, 10), + (True, 1), + (True, 5), + (True, 20), + (True, 100), + ] + + for is_parallel, parallel_nums in configs: + data = IterationNodeData( + title="Parallel Test", + iterator_selector=["x"], + output_selector=["y"], + is_parallel=is_parallel, + parallel_nums=parallel_nums, + ) + assert data.is_parallel == is_parallel + assert data.parallel_nums == parallel_nums + + def test_iteration_node_data_flatten_output_options(self): + """Test IterationNodeData flatten_output options.""" + data_flatten = IterationNodeData( + title="Flatten True", + iterator_selector=["a"], + output_selector=["b"], + flatten_output=True, + ) + + data_no_flatten = IterationNodeData( + title="Flatten False", + iterator_selector=["a"], + output_selector=["b"], + flatten_output=False, + ) + + assert data_flatten.flatten_output is True + assert data_no_flatten.flatten_output is False + + def test_iteration_node_data_complex_selectors(self): + """Test IterationNodeData with complex selectors.""" + data = IterationNodeData( + title="Complex", + iterator_selector=["node1", "output", "data", "items", "list"], + output_selector=["iteration", "result", "value", "final"], + ) + + assert len(data.iterator_selector) == 5 + assert len(data.output_selector) == 4 + + def test_iteration_node_data_single_element_selectors(self): + """Test IterationNodeData with single element selectors.""" + data = IterationNodeData( + title="Single", + iterator_selector=["items"], + output_selector=["result"], + ) + + assert len(data.iterator_selector) == 1 + assert len(data.output_selector) == 1 + + +class TestIterationNodeErrorStrategies: + """Test suite for IterationNode error strategies.""" + + def test_get_error_strategy_default(self): + """Test _get_error_strategy with default value.""" + node = IterationNode.__new__(IterationNode) + node._node_data = IterationNodeData( + title="Test", + iterator_selector=["a"], + output_selector=["b"], + ) + + result = node._get_error_strategy() + + assert result is None or result == node._node_data.error_strategy + + def test_get_retry_config(self): + """Test _get_retry_config method.""" + node = IterationNode.__new__(IterationNode) + node._node_data = IterationNodeData( + title="Test", + iterator_selector=["a"], + output_selector=["b"], + ) + + result = node._get_retry_config() + + assert result is not None + + def test_get_default_value_dict(self): + """Test _get_default_value_dict method.""" + node = IterationNode.__new__(IterationNode) + node._node_data = IterationNodeData( + title="Test", + iterator_selector=["a"], + output_selector=["b"], + ) + + result = node._get_default_value_dict() + + assert isinstance(result, dict) diff --git a/api/tests/unit_tests/core/workflow/nodes/list_operator/__init__.py b/api/tests/unit_tests/core/workflow/nodes/list_operator/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/api/tests/unit_tests/core/workflow/nodes/list_operator/__init__.py @@ -0,0 +1 @@ + diff --git a/api/tests/unit_tests/core/workflow/nodes/list_operator/node_spec.py b/api/tests/unit_tests/core/workflow/nodes/list_operator/node_spec.py new file mode 100644 index 0000000000..366bec5001 --- /dev/null +++ b/api/tests/unit_tests/core/workflow/nodes/list_operator/node_spec.py @@ -0,0 +1,544 @@ +from unittest.mock import MagicMock + +import pytest +from core.workflow.graph_engine.entities.graph import Graph +from core.workflow.graph_engine.entities.graph_init_params import GraphInitParams +from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState + +from core.variables import ArrayNumberSegment, ArrayStringSegment +from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus +from core.workflow.nodes.list_operator.node import ListOperatorNode +from models.workflow import WorkflowType + + +class TestListOperatorNode: + """Comprehensive tests for ListOperatorNode.""" + + @pytest.fixture + def mock_graph_runtime_state(self): + """Create mock GraphRuntimeState.""" + mock_state = MagicMock(spec=GraphRuntimeState) + mock_variable_pool = MagicMock() + mock_state.variable_pool = mock_variable_pool + return mock_state + + @pytest.fixture + def mock_graph(self): + """Create mock Graph.""" + return MagicMock(spec=Graph) + + @pytest.fixture + def graph_init_params(self): + """Create GraphInitParams fixture.""" + return GraphInitParams( + tenant_id="test", + app_id="test", + workflow_type=WorkflowType.WORKFLOW, + workflow_id="test", + graph_config={}, + user_id="test", + user_from="test", + invoke_from="test", + call_depth=0, + ) + + @pytest.fixture + def list_operator_node_factory(self, graph_init_params, mock_graph, mock_graph_runtime_state): + """Factory fixture for creating ListOperatorNode instances.""" + + def _create_node(config, mock_variable): + mock_graph_runtime_state.variable_pool.get.return_value = mock_variable + return ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + return _create_node + + def test_node_initialization(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test node initializes correctly.""" + config = { + "title": "List Operator", + "variable": ["sys", "list"], + "filter_by": {"enabled": False}, + "order_by": {"enabled": False}, + "limit": {"enabled": False}, + } + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + assert node.node_type == NodeType.LIST_OPERATOR + assert node._node_data.title == "List Operator" + + def test_version(self): + """Test version returns correct value.""" + assert ListOperatorNode.version() == "1" + + def test_run_with_string_array(self, list_operator_node_factory): + """Test with string array.""" + config = { + "title": "Test", + "variable": ["sys", "items"], + "filter_by": {"enabled": False}, + "order_by": {"enabled": False}, + "limit": {"enabled": False}, + } + + mock_var = ArrayStringSegment(value=["apple", "banana", "cherry"]) + node = list_operator_node_factory(config, mock_var) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == ["apple", "banana", "cherry"] + + def test_run_with_empty_array(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test with empty array.""" + config = { + "title": "Test", + "variable": ["sys", "items"], + "filter_by": {"enabled": False}, + "order_by": {"enabled": False}, + "limit": {"enabled": False}, + } + + mock_var = ArrayStringSegment(value=[]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == [] + assert result.outputs["first_record"] is None + assert result.outputs["last_record"] is None + + def test_run_with_filter_contains(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test filter with contains condition.""" + config = { + "title": "Test", + "variable": ["sys", "items"], + "filter_by": { + "enabled": True, + "condition": "contains", + "value": "app", + }, + "order_by": {"enabled": False}, + "limit": {"enabled": False}, + } + + mock_var = ArrayStringSegment(value=["apple", "banana", "pineapple", "cherry"]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == ["apple", "pineapple"] + + def test_run_with_filter_not_contains(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test filter with not contains condition.""" + config = { + "title": "Test", + "variable": ["sys", "items"], + "filter_by": { + "enabled": True, + "condition": "not contains", + "value": "app", + }, + "order_by": {"enabled": False}, + "limit": {"enabled": False}, + } + + mock_var = ArrayStringSegment(value=["apple", "banana", "pineapple", "cherry"]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == ["banana", "cherry"] + + def test_run_with_number_filter_greater_than(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test filter with greater than condition on numbers.""" + config = { + "title": "Test", + "variable": ["sys", "numbers"], + "filter_by": { + "enabled": True, + "condition": ">", + "value": "5", + }, + "order_by": {"enabled": False}, + "limit": {"enabled": False}, + } + + mock_var = ArrayNumberSegment(value=[1, 3, 5, 7, 9, 11]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == [7, 9, 11] + + def test_run_with_order_ascending(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test ordering in ascending order.""" + config = { + "title": "Test", + "variable": ["sys", "items"], + "filter_by": {"enabled": False}, + "order_by": { + "enabled": True, + "value": "asc", + }, + "limit": {"enabled": False}, + } + + mock_var = ArrayStringSegment(value=["cherry", "apple", "banana"]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == ["apple", "banana", "cherry"] + + def test_run_with_order_descending(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test ordering in descending order.""" + config = { + "title": "Test", + "variable": ["sys", "items"], + "filter_by": {"enabled": False}, + "order_by": { + "enabled": True, + "value": "desc", + }, + "limit": {"enabled": False}, + } + + mock_var = ArrayStringSegment(value=["cherry", "apple", "banana"]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == ["cherry", "banana", "apple"] + + def test_run_with_limit(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test with limit enabled.""" + config = { + "title": "Test", + "variable": ["sys", "items"], + "filter_by": {"enabled": False}, + "order_by": {"enabled": False}, + "limit": { + "enabled": True, + "size": 2, + }, + } + + mock_var = ArrayStringSegment(value=["apple", "banana", "cherry", "date"]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == ["apple", "banana"] + + def test_run_with_filter_order_and_limit(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test with filter, order, and limit combined.""" + config = { + "title": "Test", + "variable": ["sys", "numbers"], + "filter_by": { + "enabled": True, + "condition": ">", + "value": "3", + }, + "order_by": { + "enabled": True, + "value": "desc", + }, + "limit": { + "enabled": True, + "size": 3, + }, + } + + mock_var = ArrayNumberSegment(value=[1, 2, 3, 4, 5, 6, 7, 8, 9]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == [9, 8, 7] + + def test_run_with_variable_not_found(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test when variable is not found.""" + config = { + "title": "Test", + "variable": ["sys", "missing"], + "filter_by": {"enabled": False}, + "order_by": {"enabled": False}, + "limit": {"enabled": False}, + } + + mock_graph_runtime_state.variable_pool.get.return_value = None + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.FAILED + assert "Variable not found" in result.error + + def test_run_with_first_and_last_record(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test first_record and last_record outputs.""" + config = { + "title": "Test", + "variable": ["sys", "items"], + "filter_by": {"enabled": False}, + "order_by": {"enabled": False}, + "limit": {"enabled": False}, + } + + mock_var = ArrayStringSegment(value=["first", "middle", "last"]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["first_record"] == "first" + assert result.outputs["last_record"] == "last" + + def test_run_with_filter_startswith(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test filter with startswith condition.""" + config = { + "title": "Test", + "variable": ["sys", "items"], + "filter_by": { + "enabled": True, + "condition": "start with", + "value": "app", + }, + "order_by": {"enabled": False}, + "limit": {"enabled": False}, + } + + mock_var = ArrayStringSegment(value=["apple", "application", "banana", "apricot"]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == ["apple", "application"] + + def test_run_with_filter_endswith(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test filter with endswith condition.""" + config = { + "title": "Test", + "variable": ["sys", "items"], + "filter_by": { + "enabled": True, + "condition": "end with", + "value": "le", + }, + "order_by": {"enabled": False}, + "limit": {"enabled": False}, + } + + mock_var = ArrayStringSegment(value=["apple", "banana", "pineapple", "table"]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == ["apple", "pineapple", "table"] + + def test_run_with_number_filter_equals(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test number filter with equals condition.""" + config = { + "title": "Test", + "variable": ["sys", "numbers"], + "filter_by": { + "enabled": True, + "condition": "=", + "value": "5", + }, + "order_by": {"enabled": False}, + "limit": {"enabled": False}, + } + + mock_var = ArrayNumberSegment(value=[1, 3, 5, 5, 7, 9]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == [5, 5] + + def test_run_with_number_filter_not_equals(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test number filter with not equals condition.""" + config = { + "title": "Test", + "variable": ["sys", "numbers"], + "filter_by": { + "enabled": True, + "condition": "≠", + "value": "5", + }, + "order_by": {"enabled": False}, + "limit": {"enabled": False}, + } + + mock_var = ArrayNumberSegment(value=[1, 3, 5, 7, 9]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == [1, 3, 7, 9] + + def test_run_with_number_order_ascending(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test number ordering in ascending order.""" + config = { + "title": "Test", + "variable": ["sys", "numbers"], + "filter_by": {"enabled": False}, + "order_by": { + "enabled": True, + "value": "asc", + }, + "limit": {"enabled": False}, + } + + mock_var = ArrayNumberSegment(value=[9, 3, 7, 1, 5]) + mock_graph_runtime_state.variable_pool.get.return_value = mock_var + + node = ListOperatorNode( + id="test", + config=config, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["result"].value == [1, 3, 5, 7, 9] diff --git a/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py b/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py index 3ffb5c0fdf..77264022bc 100644 --- a/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py @@ -111,8 +111,6 @@ def llm_node( graph_runtime_state=graph_runtime_state, llm_file_saver=mock_file_saver, ) - # Initialize node data - node.init_node_data(node_config["data"]) return node @@ -498,8 +496,6 @@ def llm_node_for_multimodal(llm_node_data, graph_init_params, graph_runtime_stat graph_runtime_state=graph_runtime_state, llm_file_saver=mock_file_saver, ) - # Initialize node data - node.init_node_data(node_config["data"]) return node, mock_file_saver diff --git a/api/tests/unit_tests/core/workflow/nodes/template_transform/__init__.py b/api/tests/unit_tests/core/workflow/nodes/template_transform/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/api/tests/unit_tests/core/workflow/nodes/template_transform/__init__.py @@ -0,0 +1 @@ + diff --git a/api/tests/unit_tests/core/workflow/nodes/template_transform/entities_spec.py b/api/tests/unit_tests/core/workflow/nodes/template_transform/entities_spec.py new file mode 100644 index 0000000000..5eb302798f --- /dev/null +++ b/api/tests/unit_tests/core/workflow/nodes/template_transform/entities_spec.py @@ -0,0 +1,225 @@ +import pytest +from pydantic import ValidationError + +from core.workflow.enums import ErrorStrategy +from core.workflow.nodes.template_transform.entities import TemplateTransformNodeData + + +class TestTemplateTransformNodeData: + """Test suite for TemplateTransformNodeData entity.""" + + def test_valid_template_transform_node_data(self): + """Test creating valid TemplateTransformNodeData.""" + data = { + "title": "Template Transform", + "desc": "Transform data using Jinja2 template", + "variables": [ + {"variable": "name", "value_selector": ["sys", "user_name"]}, + {"variable": "age", "value_selector": ["sys", "user_age"]}, + ], + "template": "Hello {{ name }}, you are {{ age }} years old!", + } + + node_data = TemplateTransformNodeData.model_validate(data) + + assert node_data.title == "Template Transform" + assert node_data.desc == "Transform data using Jinja2 template" + assert len(node_data.variables) == 2 + assert node_data.variables[0].variable == "name" + assert node_data.variables[0].value_selector == ["sys", "user_name"] + assert node_data.variables[1].variable == "age" + assert node_data.variables[1].value_selector == ["sys", "user_age"] + assert node_data.template == "Hello {{ name }}, you are {{ age }} years old!" + + def test_template_transform_node_data_with_empty_variables(self): + """Test TemplateTransformNodeData with no variables.""" + data = { + "title": "Static Template", + "variables": [], + "template": "This is a static template with no variables.", + } + + node_data = TemplateTransformNodeData.model_validate(data) + + assert node_data.title == "Static Template" + assert len(node_data.variables) == 0 + assert node_data.template == "This is a static template with no variables." + + def test_template_transform_node_data_with_complex_template(self): + """Test TemplateTransformNodeData with complex Jinja2 template.""" + data = { + "title": "Complex Template", + "variables": [ + {"variable": "items", "value_selector": ["sys", "item_list"]}, + {"variable": "total", "value_selector": ["sys", "total_count"]}, + ], + "template": ( + "{% for item in items %}{{ item }}{% if not loop.last %}, {% endif %}{% endfor %}. Total: {{ total }}" + ), + } + + node_data = TemplateTransformNodeData.model_validate(data) + + assert node_data.title == "Complex Template" + assert len(node_data.variables) == 2 + assert "{% for item in items %}" in node_data.template + assert "{{ total }}" in node_data.template + + def test_template_transform_node_data_with_error_strategy(self): + """Test TemplateTransformNodeData with error handling strategy.""" + data = { + "title": "Template with Error Handling", + "variables": [{"variable": "value", "value_selector": ["sys", "input"]}], + "template": "{{ value }}", + "error_strategy": "fail-branch", + } + + node_data = TemplateTransformNodeData.model_validate(data) + + assert node_data.error_strategy == ErrorStrategy.FAIL_BRANCH + + def test_template_transform_node_data_with_retry_config(self): + """Test TemplateTransformNodeData with retry configuration.""" + data = { + "title": "Template with Retry", + "variables": [{"variable": "data", "value_selector": ["sys", "data"]}], + "template": "{{ data }}", + "retry_config": {"enabled": True, "max_retries": 3, "retry_interval": 1000}, + } + + node_data = TemplateTransformNodeData.model_validate(data) + + assert node_data.retry_config.enabled is True + assert node_data.retry_config.max_retries == 3 + assert node_data.retry_config.retry_interval == 1000 + + def test_template_transform_node_data_missing_required_fields(self): + """Test that missing required fields raises ValidationError.""" + data = { + "title": "Incomplete Template", + # Missing 'variables' and 'template' + } + + with pytest.raises(ValidationError) as exc_info: + TemplateTransformNodeData.model_validate(data) + + errors = exc_info.value.errors() + assert len(errors) >= 2 + error_fields = {error["loc"][0] for error in errors} + assert "variables" in error_fields + assert "template" in error_fields + + def test_template_transform_node_data_invalid_variable_selector(self): + """Test that invalid variable selector format raises ValidationError.""" + data = { + "title": "Invalid Variable", + "variables": [ + {"variable": "name", "value_selector": "invalid_format"} # Should be list + ], + "template": "{{ name }}", + } + + with pytest.raises(ValidationError): + TemplateTransformNodeData.model_validate(data) + + def test_template_transform_node_data_with_default_value_dict(self): + """Test TemplateTransformNodeData with default value dictionary.""" + data = { + "title": "Template with Defaults", + "variables": [ + {"variable": "name", "value_selector": ["sys", "user_name"]}, + {"variable": "greeting", "value_selector": ["sys", "greeting"]}, + ], + "template": "{{ greeting }} {{ name }}!", + "default_value_dict": {"greeting": "Hello", "name": "Guest"}, + } + + node_data = TemplateTransformNodeData.model_validate(data) + + assert node_data.default_value_dict == {"greeting": "Hello", "name": "Guest"} + + def test_template_transform_node_data_with_nested_selectors(self): + """Test TemplateTransformNodeData with nested variable selectors.""" + data = { + "title": "Nested Selectors", + "variables": [ + {"variable": "user_info", "value_selector": ["sys", "user", "profile", "name"]}, + {"variable": "settings", "value_selector": ["sys", "config", "app", "theme"]}, + ], + "template": "User: {{ user_info }}, Theme: {{ settings }}", + } + + node_data = TemplateTransformNodeData.model_validate(data) + + assert len(node_data.variables) == 2 + assert node_data.variables[0].value_selector == ["sys", "user", "profile", "name"] + assert node_data.variables[1].value_selector == ["sys", "config", "app", "theme"] + + def test_template_transform_node_data_with_multiline_template(self): + """Test TemplateTransformNodeData with multiline template.""" + data = { + "title": "Multiline Template", + "variables": [ + {"variable": "title", "value_selector": ["sys", "title"]}, + {"variable": "content", "value_selector": ["sys", "content"]}, + ], + "template": """ +# {{ title }} + +{{ content }} + +--- +Generated by Template Transform Node + """, + } + + node_data = TemplateTransformNodeData.model_validate(data) + + assert "# {{ title }}" in node_data.template + assert "{{ content }}" in node_data.template + assert "Generated by Template Transform Node" in node_data.template + + def test_template_transform_node_data_serialization(self): + """Test that TemplateTransformNodeData can be serialized and deserialized.""" + original_data = { + "title": "Serialization Test", + "desc": "Test serialization", + "variables": [{"variable": "test", "value_selector": ["sys", "test"]}], + "template": "{{ test }}", + } + + node_data = TemplateTransformNodeData.model_validate(original_data) + serialized = node_data.model_dump() + deserialized = TemplateTransformNodeData.model_validate(serialized) + + assert deserialized.title == node_data.title + assert deserialized.desc == node_data.desc + assert len(deserialized.variables) == len(node_data.variables) + assert deserialized.template == node_data.template + + def test_template_transform_node_data_with_special_characters(self): + """Test TemplateTransformNodeData with special characters in template.""" + data = { + "title": "Special Characters", + "variables": [{"variable": "text", "value_selector": ["sys", "input"]}], + "template": "Special: {{ text }} | Symbols: @#$%^&*() | Unicode: 你好 🎉", + } + + node_data = TemplateTransformNodeData.model_validate(data) + + assert "@#$%^&*()" in node_data.template + assert "你好" in node_data.template + assert "🎉" in node_data.template + + def test_template_transform_node_data_empty_template(self): + """Test TemplateTransformNodeData with empty template string.""" + data = { + "title": "Empty Template", + "variables": [], + "template": "", + } + + node_data = TemplateTransformNodeData.model_validate(data) + + assert node_data.template == "" + assert len(node_data.variables) == 0 diff --git a/api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.py b/api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.py new file mode 100644 index 0000000000..1a67d5c3e3 --- /dev/null +++ b/api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.py @@ -0,0 +1,414 @@ +from unittest.mock import MagicMock, patch + +import pytest +from core.workflow.graph_engine.entities.graph import Graph +from core.workflow.graph_engine.entities.graph_init_params import GraphInitParams +from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState + +from core.helper.code_executor.code_executor import CodeExecutionError +from core.workflow.enums import ErrorStrategy, NodeType, WorkflowNodeExecutionStatus +from core.workflow.nodes.template_transform.template_transform_node import TemplateTransformNode +from models.workflow import WorkflowType + + +class TestTemplateTransformNode: + """Comprehensive test suite for TemplateTransformNode.""" + + @pytest.fixture + def mock_graph_runtime_state(self): + """Create a mock GraphRuntimeState with variable pool.""" + mock_state = MagicMock(spec=GraphRuntimeState) + mock_variable_pool = MagicMock() + mock_state.variable_pool = mock_variable_pool + return mock_state + + @pytest.fixture + def mock_graph(self): + """Create a mock Graph.""" + return MagicMock(spec=Graph) + + @pytest.fixture + def graph_init_params(self): + """Create a mock GraphInitParams.""" + return GraphInitParams( + tenant_id="test_tenant", + app_id="test_app", + workflow_type=WorkflowType.WORKFLOW, + workflow_id="test_workflow", + graph_config={}, + user_id="test_user", + user_from="test", + invoke_from="test", + call_depth=0, + ) + + @pytest.fixture + def basic_node_data(self): + """Create basic node data for testing.""" + return { + "title": "Template Transform", + "desc": "Transform data using template", + "variables": [ + {"variable": "name", "value_selector": ["sys", "user_name"]}, + {"variable": "age", "value_selector": ["sys", "user_age"]}, + ], + "template": "Hello {{ name }}, you are {{ age }} years old!", + } + + def test_node_initialization(self, basic_node_data, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test that TemplateTransformNode initializes correctly.""" + node = TemplateTransformNode( + id="test_node", + config=basic_node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + assert node.node_type == NodeType.TEMPLATE_TRANSFORM + assert node._node_data.title == "Template Transform" + assert len(node._node_data.variables) == 2 + assert node._node_data.template == "Hello {{ name }}, you are {{ age }} years old!" + + def test_get_title(self, basic_node_data, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test _get_title method.""" + node = TemplateTransformNode( + id="test_node", + config=basic_node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + assert node._get_title() == "Template Transform" + + def test_get_description(self, basic_node_data, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test _get_description method.""" + node = TemplateTransformNode( + id="test_node", + config=basic_node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + assert node._get_description() == "Transform data using template" + + def test_get_error_strategy(self, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test _get_error_strategy method.""" + node_data = { + "title": "Test", + "variables": [], + "template": "test", + "error_strategy": "fail-branch", + } + + node = TemplateTransformNode( + id="test_node", + config=node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + assert node._get_error_strategy() == ErrorStrategy.FAIL_BRANCH + + def test_get_default_config(self): + """Test get_default_config class method.""" + config = TemplateTransformNode.get_default_config() + + assert config["type"] == "template-transform" + assert "config" in config + assert "variables" in config["config"] + assert "template" in config["config"] + assert config["config"]["template"] == "{{ arg1 }}" + + def test_version(self): + """Test version class method.""" + assert TemplateTransformNode.version() == "1" + + @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + def test_run_simple_template( + self, mock_execute, basic_node_data, mock_graph, mock_graph_runtime_state, graph_init_params + ): + """Test _run with simple template transformation.""" + # Setup mock variable pool + mock_name_value = MagicMock() + mock_name_value.to_object.return_value = "Alice" + mock_age_value = MagicMock() + mock_age_value.to_object.return_value = 30 + + variable_map = { + ("sys", "user_name"): mock_name_value, + ("sys", "user_age"): mock_age_value, + } + mock_graph_runtime_state.variable_pool.get.side_effect = lambda selector: variable_map.get(tuple(selector)) + + # Setup mock executor + mock_execute.return_value = {"result": "Hello Alice, you are 30 years old!"} + + node = TemplateTransformNode( + id="test_node", + config=basic_node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["output"] == "Hello Alice, you are 30 years old!" + assert result.inputs["name"] == "Alice" + assert result.inputs["age"] == 30 + + @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + def test_run_with_none_values(self, mock_execute, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test _run with None variable values.""" + node_data = { + "title": "Test", + "variables": [{"variable": "value", "value_selector": ["sys", "missing"]}], + "template": "Value: {{ value }}", + } + + mock_graph_runtime_state.variable_pool.get.return_value = None + mock_execute.return_value = {"result": "Value: "} + + node = TemplateTransformNode( + id="test_node", + config=node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.inputs["value"] is None + + @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + def test_run_with_code_execution_error( + self, mock_execute, basic_node_data, mock_graph, mock_graph_runtime_state, graph_init_params + ): + """Test _run when code execution fails.""" + mock_graph_runtime_state.variable_pool.get.return_value = MagicMock() + mock_execute.side_effect = CodeExecutionError("Template syntax error") + + node = TemplateTransformNode( + id="test_node", + config=basic_node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.FAILED + assert "Template syntax error" in result.error + + @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + @patch("core.workflow.nodes.template_transform.template_transform_node.MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH", 10) + def test_run_output_length_exceeds_limit( + self, mock_execute, basic_node_data, mock_graph, mock_graph_runtime_state, graph_init_params + ): + """Test _run when output exceeds maximum length.""" + mock_graph_runtime_state.variable_pool.get.return_value = MagicMock() + mock_execute.return_value = {"result": "This is a very long output that exceeds the limit"} + + node = TemplateTransformNode( + id="test_node", + config=basic_node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.FAILED + assert "Output length exceeds" in result.error + + @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + def test_run_with_complex_jinja2_template( + self, mock_execute, mock_graph, mock_graph_runtime_state, graph_init_params + ): + """Test _run with complex Jinja2 template including loops and conditions.""" + node_data = { + "title": "Complex Template", + "variables": [ + {"variable": "items", "value_selector": ["sys", "items"]}, + {"variable": "show_total", "value_selector": ["sys", "show_total"]}, + ], + "template": ( + "{% for item in items %}{{ item }}{% if not loop.last %}, {% endif %}{% endfor %}" + "{% if show_total %} (Total: {{ items|length }}){% endif %}" + ), + } + + mock_items = MagicMock() + mock_items.to_object.return_value = ["apple", "banana", "orange"] + mock_show_total = MagicMock() + mock_show_total.to_object.return_value = True + + variable_map = { + ("sys", "items"): mock_items, + ("sys", "show_total"): mock_show_total, + } + mock_graph_runtime_state.variable_pool.get.side_effect = lambda selector: variable_map.get(tuple(selector)) + mock_execute.return_value = {"result": "apple, banana, orange (Total: 3)"} + + node = TemplateTransformNode( + id="test_node", + config=node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["output"] == "apple, banana, orange (Total: 3)" + + def test_extract_variable_selector_to_variable_mapping(self): + """Test _extract_variable_selector_to_variable_mapping class method.""" + node_data = { + "title": "Test", + "variables": [ + {"variable": "var1", "value_selector": ["sys", "input1"]}, + {"variable": "var2", "value_selector": ["sys", "input2"]}, + ], + "template": "{{ var1 }} {{ var2 }}", + } + + mapping = TemplateTransformNode._extract_variable_selector_to_variable_mapping( + graph_config={}, node_id="node_123", node_data=node_data + ) + + assert "node_123.var1" in mapping + assert "node_123.var2" in mapping + assert mapping["node_123.var1"] == ["sys", "input1"] + assert mapping["node_123.var2"] == ["sys", "input2"] + + @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + def test_run_with_empty_variables(self, mock_execute, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test _run with no variables (static template).""" + node_data = { + "title": "Static Template", + "variables": [], + "template": "This is a static message.", + } + + mock_execute.return_value = {"result": "This is a static message."} + + node = TemplateTransformNode( + id="test_node", + config=node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["output"] == "This is a static message." + assert result.inputs == {} + + @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + def test_run_with_numeric_values(self, mock_execute, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test _run with numeric variable values.""" + node_data = { + "title": "Numeric Template", + "variables": [ + {"variable": "price", "value_selector": ["sys", "price"]}, + {"variable": "quantity", "value_selector": ["sys", "quantity"]}, + ], + "template": "Total: ${{ price * quantity }}", + } + + mock_price = MagicMock() + mock_price.to_object.return_value = 10.5 + mock_quantity = MagicMock() + mock_quantity.to_object.return_value = 3 + + variable_map = { + ("sys", "price"): mock_price, + ("sys", "quantity"): mock_quantity, + } + mock_graph_runtime_state.variable_pool.get.side_effect = lambda selector: variable_map.get(tuple(selector)) + mock_execute.return_value = {"result": "Total: $31.5"} + + node = TemplateTransformNode( + id="test_node", + config=node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs["output"] == "Total: $31.5" + + @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + def test_run_with_dict_values(self, mock_execute, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test _run with dictionary variable values.""" + node_data = { + "title": "Dict Template", + "variables": [{"variable": "user", "value_selector": ["sys", "user_data"]}], + "template": "Name: {{ user.name }}, Email: {{ user.email }}", + } + + mock_user = MagicMock() + mock_user.to_object.return_value = {"name": "John Doe", "email": "john@example.com"} + + mock_graph_runtime_state.variable_pool.get.return_value = mock_user + mock_execute.return_value = {"result": "Name: John Doe, Email: john@example.com"} + + node = TemplateTransformNode( + id="test_node", + config=node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert "John Doe" in result.outputs["output"] + assert "john@example.com" in result.outputs["output"] + + @patch("core.workflow.nodes.template_transform.template_transform_node.CodeExecutor.execute_workflow_code_template") + def test_run_with_list_values(self, mock_execute, mock_graph, mock_graph_runtime_state, graph_init_params): + """Test _run with list variable values.""" + node_data = { + "title": "List Template", + "variables": [{"variable": "tags", "value_selector": ["sys", "tags"]}], + "template": "Tags: {% for tag in tags %}#{{ tag }} {% endfor %}", + } + + mock_tags = MagicMock() + mock_tags.to_object.return_value = ["python", "ai", "workflow"] + + mock_graph_runtime_state.variable_pool.get.return_value = mock_tags + mock_execute.return_value = {"result": "Tags: #python #ai #workflow "} + + node = TemplateTransformNode( + id="test_node", + config=node_data, + graph_init_params=graph_init_params, + graph=mock_graph, + graph_runtime_state=mock_graph_runtime_state, + ) + + result = node._run() + + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert "#python" in result.outputs["output"] + assert "#ai" in result.outputs["output"] + assert "#workflow" in result.outputs["output"] diff --git a/api/tests/unit_tests/core/workflow/nodes/test_base_node.py b/api/tests/unit_tests/core/workflow/nodes/test_base_node.py new file mode 100644 index 0000000000..4a57ab2b89 --- /dev/null +++ b/api/tests/unit_tests/core/workflow/nodes/test_base_node.py @@ -0,0 +1,74 @@ +from collections.abc import Mapping + +import pytest + +from core.workflow.entities import GraphInitParams +from core.workflow.enums import NodeType +from core.workflow.nodes.base.entities import BaseNodeData +from core.workflow.nodes.base.node import Node +from core.workflow.runtime import GraphRuntimeState, VariablePool +from core.workflow.system_variable import SystemVariable + + +class _SampleNodeData(BaseNodeData): + foo: str + + +class _SampleNode(Node[_SampleNodeData]): + node_type = NodeType.ANSWER + + @classmethod + def version(cls) -> str: + return "sample-test" + + def _run(self): + raise NotImplementedError + + +def _build_context(graph_config: Mapping[str, object]) -> tuple[GraphInitParams, GraphRuntimeState]: + init_params = GraphInitParams( + tenant_id="tenant", + app_id="app", + workflow_id="workflow", + graph_config=graph_config, + user_id="user", + user_from="account", + invoke_from="debugger", + call_depth=0, + ) + runtime_state = GraphRuntimeState( + variable_pool=VariablePool(system_variables=SystemVariable(user_id="user", files=[]), user_inputs={}), + start_at=0.0, + ) + return init_params, runtime_state + + +def test_node_hydrates_data_during_initialization(): + graph_config: dict[str, object] = {} + init_params, runtime_state = _build_context(graph_config) + + node = _SampleNode( + id="node-1", + config={"id": "node-1", "data": {"title": "Sample", "foo": "bar"}}, + graph_init_params=init_params, + graph_runtime_state=runtime_state, + ) + + assert node.node_data.foo == "bar" + assert node.title == "Sample" + + +def test_missing_generic_argument_raises_type_error(): + graph_config: dict[str, object] = {} + + with pytest.raises(TypeError): + + class _InvalidNode(Node): # type: ignore[type-abstract] + node_type = NodeType.ANSWER + + @classmethod + def version(cls) -> str: + return "1" + + def _run(self): + raise NotImplementedError diff --git a/api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.py b/api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.py index 315c50d946..088c60a337 100644 --- a/api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.py @@ -50,8 +50,6 @@ def document_extractor_node(graph_init_params): graph_init_params=graph_init_params, graph_runtime_state=Mock(), ) - # Initialize node data - node.init_node_data(node_config["data"]) return node diff --git a/api/tests/unit_tests/core/workflow/nodes/test_if_else.py b/api/tests/unit_tests/core/workflow/nodes/test_if_else.py index 962e43a897..dc7175f964 100644 --- a/api/tests/unit_tests/core/workflow/nodes/test_if_else.py +++ b/api/tests/unit_tests/core/workflow/nodes/test_if_else.py @@ -114,9 +114,6 @@ def test_execute_if_else_result_true(): config=node_config, ) - # Initialize node data - node.init_node_data(node_config["data"]) - # Mock db.session.close() db.session.close = MagicMock() @@ -187,9 +184,6 @@ def test_execute_if_else_result_false(): config=node_config, ) - # Initialize node data - node.init_node_data(node_config["data"]) - # Mock db.session.close() db.session.close = MagicMock() @@ -252,9 +246,6 @@ def test_array_file_contains_file_name(): config=node_config, ) - # Initialize node data - node.init_node_data(node_config["data"]) - node.graph_runtime_state.variable_pool.get.return_value = ArrayFileSegment( value=[ File( @@ -347,7 +338,6 @@ def test_execute_if_else_boolean_conditions(condition: Condition): graph_runtime_state=graph_runtime_state, config={"id": "if-else", "data": node_data}, ) - node.init_node_data(node_data) # Mock db.session.close() db.session.close = MagicMock() @@ -417,7 +407,6 @@ def test_execute_if_else_boolean_false_conditions(): "data": node_data, }, ) - node.init_node_data(node_data) # Mock db.session.close() db.session.close = MagicMock() @@ -487,7 +476,6 @@ def test_execute_if_else_boolean_cases_structure(): graph_runtime_state=graph_runtime_state, config={"id": "if-else", "data": node_data}, ) - node.init_node_data(node_data) # Mock db.session.close() db.session.close = MagicMock() diff --git a/api/tests/unit_tests/core/workflow/nodes/test_list_operator.py b/api/tests/unit_tests/core/workflow/nodes/test_list_operator.py index 55fe62ca43..ff3eec0608 100644 --- a/api/tests/unit_tests/core/workflow/nodes/test_list_operator.py +++ b/api/tests/unit_tests/core/workflow/nodes/test_list_operator.py @@ -57,8 +57,6 @@ def list_operator_node(): graph_init_params=graph_init_params, graph_runtime_state=MagicMock(), ) - # Initialize node data - node.init_node_data(node_config["data"]) node.graph_runtime_state = MagicMock() node.graph_runtime_state.variable_pool = MagicMock() return node diff --git a/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node.py b/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node.py new file mode 100644 index 0000000000..09b8191870 --- /dev/null +++ b/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node.py @@ -0,0 +1,159 @@ +import sys +import types +from collections.abc import Generator +from typing import TYPE_CHECKING, Any +from unittest.mock import MagicMock, patch + +import pytest + +from core.file import File, FileTransferMethod, FileType +from core.model_runtime.entities.llm_entities import LLMUsage +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.utils.message_transformer import ToolFileMessageTransformer +from core.variables.segments import ArrayFileSegment +from core.workflow.entities import GraphInitParams +from core.workflow.node_events import StreamChunkEvent, StreamCompletedEvent +from core.workflow.runtime import GraphRuntimeState, VariablePool +from core.workflow.system_variable import SystemVariable + +if TYPE_CHECKING: # pragma: no cover - imported for type checking only + from core.workflow.nodes.tool.tool_node import ToolNode + + +@pytest.fixture +def tool_node(monkeypatch) -> "ToolNode": + module_name = "core.ops.ops_trace_manager" + if module_name not in sys.modules: + ops_stub = types.ModuleType(module_name) + ops_stub.TraceQueueManager = object # pragma: no cover - stub attribute + ops_stub.TraceTask = object # pragma: no cover - stub attribute + monkeypatch.setitem(sys.modules, module_name, ops_stub) + + from core.workflow.nodes.tool.tool_node import ToolNode + + graph_config: dict[str, Any] = { + "nodes": [ + { + "id": "tool-node", + "data": { + "type": "tool", + "title": "Tool", + "desc": "", + "provider_id": "provider", + "provider_type": "builtin", + "provider_name": "provider", + "tool_name": "tool", + "tool_label": "tool", + "tool_configurations": {}, + "tool_parameters": {}, + }, + } + ], + "edges": [], + } + + init_params = GraphInitParams( + tenant_id="tenant-id", + app_id="app-id", + workflow_id="workflow-id", + graph_config=graph_config, + user_id="user-id", + user_from="account", + invoke_from="debugger", + call_depth=0, + ) + + variable_pool = VariablePool(system_variables=SystemVariable(user_id="user-id")) + graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=0.0) + + config = graph_config["nodes"][0] + node = ToolNode( + id="node-instance", + config=config, + graph_init_params=init_params, + graph_runtime_state=graph_runtime_state, + ) + return node + + +def _collect_events(generator: Generator) -> tuple[list[Any], LLMUsage]: + events: list[Any] = [] + try: + while True: + events.append(next(generator)) + except StopIteration as stop: + return events, stop.value + + +def _run_transform(tool_node: "ToolNode", message: ToolInvokeMessage) -> tuple[list[Any], LLMUsage]: + def _identity_transform(messages, *_args, **_kwargs): + return messages + + tool_runtime = MagicMock() + with patch.object(ToolFileMessageTransformer, "transform_tool_invoke_messages", side_effect=_identity_transform): + generator = tool_node._transform_message( + messages=iter([message]), + tool_info={"provider_type": "builtin", "provider_id": "provider"}, + parameters_for_log={}, + user_id="user-id", + tenant_id="tenant-id", + node_id=tool_node._node_id, + tool_runtime=tool_runtime, + ) + return _collect_events(generator) + + +def test_link_messages_with_file_populate_files_output(tool_node: "ToolNode"): + file_obj = File( + tenant_id="tenant-id", + type=FileType.DOCUMENT, + transfer_method=FileTransferMethod.TOOL_FILE, + related_id="file-id", + filename="demo.pdf", + extension=".pdf", + mime_type="application/pdf", + size=123, + storage_key="file-key", + ) + message = ToolInvokeMessage( + type=ToolInvokeMessage.MessageType.LINK, + message=ToolInvokeMessage.TextMessage(text="/files/tools/file-id.pdf"), + meta={"file": file_obj}, + ) + + events, usage = _run_transform(tool_node, message) + + assert isinstance(usage, LLMUsage) + + chunk_events = [event for event in events if isinstance(event, StreamChunkEvent)] + assert chunk_events + assert chunk_events[0].chunk == "File: /files/tools/file-id.pdf\n" + + completed_events = [event for event in events if isinstance(event, StreamCompletedEvent)] + assert len(completed_events) == 1 + outputs = completed_events[0].node_run_result.outputs + assert outputs["text"] == "File: /files/tools/file-id.pdf\n" + + files_segment = outputs["files"] + assert isinstance(files_segment, ArrayFileSegment) + assert files_segment.value == [file_obj] + + +def test_plain_link_messages_remain_links(tool_node: "ToolNode"): + message = ToolInvokeMessage( + type=ToolInvokeMessage.MessageType.LINK, + message=ToolInvokeMessage.TextMessage(text="https://dify.ai"), + meta=None, + ) + + events, _ = _run_transform(tool_node, message) + + chunk_events = [event for event in events if isinstance(event, StreamChunkEvent)] + assert chunk_events + assert chunk_events[0].chunk == "Link: https://dify.ai\n" + + completed_events = [event for event in events if isinstance(event, StreamCompletedEvent)] + assert len(completed_events) == 1 + files_segment = completed_events[0].node_run_result.outputs["files"] + assert isinstance(files_segment, ArrayFileSegment) + assert files_segment.value == [] diff --git a/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v1/test_variable_assigner_v1.py b/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v1/test_variable_assigner_v1.py index 6af4777e0e..ef23a8f565 100644 --- a/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v1/test_variable_assigner_v1.py +++ b/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v1/test_variable_assigner_v1.py @@ -101,9 +101,6 @@ def test_overwrite_string_variable(): conv_var_updater_factory=mock_conv_var_updater_factory, ) - # Initialize node data - node.init_node_data(node_config["data"]) - list(node.run()) expected_var = StringVariable( id=conversation_variable.id, @@ -203,9 +200,6 @@ def test_append_variable_to_array(): conv_var_updater_factory=mock_conv_var_updater_factory, ) - # Initialize node data - node.init_node_data(node_config["data"]) - list(node.run()) expected_value = list(conversation_variable.value) expected_value.append(input_variable.value) @@ -296,9 +290,6 @@ def test_clear_array(): conv_var_updater_factory=mock_conv_var_updater_factory, ) - # Initialize node data - node.init_node_data(node_config["data"]) - list(node.run()) expected_var = ArrayStringVariable( id=conversation_variable.id, diff --git a/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v2/test_variable_assigner_v2.py b/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v2/test_variable_assigner_v2.py index 80071c8616..f793341e73 100644 --- a/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v2/test_variable_assigner_v2.py +++ b/api/tests/unit_tests/core/workflow/nodes/variable_assigner/v2/test_variable_assigner_v2.py @@ -139,11 +139,6 @@ def test_remove_first_from_array(): config=node_config, ) - # Initialize node data - node.init_node_data(node_config["data"]) - - # Skip the mock assertion since we're in a test environment - # Run the node result = list(node.run()) @@ -228,10 +223,6 @@ def test_remove_last_from_array(): config=node_config, ) - # Initialize node data - node.init_node_data(node_config["data"]) - - # Skip the mock assertion since we're in a test environment list(node.run()) got = variable_pool.get(["conversation", conversation_variable.name]) @@ -313,10 +304,6 @@ def test_remove_first_from_empty_array(): config=node_config, ) - # Initialize node data - node.init_node_data(node_config["data"]) - - # Skip the mock assertion since we're in a test environment list(node.run()) got = variable_pool.get(["conversation", conversation_variable.name]) @@ -398,10 +385,6 @@ def test_remove_last_from_empty_array(): config=node_config, ) - # Initialize node data - node.init_node_data(node_config["data"]) - - # Skip the mock assertion since we're in a test environment list(node.run()) got = variable_pool.get(["conversation", conversation_variable.name]) diff --git a/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_node.py b/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_node.py index d7094ae5f2..a599d4f831 100644 --- a/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_node.py @@ -47,7 +47,6 @@ def create_webhook_node(webhook_data: WebhookData, variable_pool: VariablePool) ), ) - node.init_node_data(node_config["data"]) return node diff --git a/api/tests/unit_tests/factories/test_file_factory.py b/api/tests/unit_tests/factories/test_file_factory.py index 777fe5a6e7..e5f45044fa 100644 --- a/api/tests/unit_tests/factories/test_file_factory.py +++ b/api/tests/unit_tests/factories/test_file_factory.py @@ -2,7 +2,7 @@ import re import pytest -from factories.file_factory import _get_remote_file_info +from factories.file_factory import _extract_filename, _get_remote_file_info class _FakeResponse: @@ -113,3 +113,120 @@ class TestGetRemoteFileInfo: # Should generate a random hex filename with .bin extension assert re.match(r"^[0-9a-f]{32}\.bin$", filename) is not None assert mime_type == "application/octet-stream" + + +class TestExtractFilename: + """Tests for _extract_filename function focusing on RFC5987 parsing and security.""" + + def test_no_content_disposition_uses_url_basename(self): + """Test that URL basename is used when no Content-Disposition header.""" + result = _extract_filename("http://example.com/path/file.txt", None) + assert result == "file.txt" + + def test_no_content_disposition_with_percent_encoded_url(self): + """Test that percent-encoded URL basename is decoded.""" + result = _extract_filename("http://example.com/path/file%20name.txt", None) + assert result == "file name.txt" + + def test_no_content_disposition_empty_url_path(self): + """Test that empty URL path returns None.""" + result = _extract_filename("http://example.com/", None) + assert result is None + + def test_simple_filename_header(self): + """Test basic filename extraction from Content-Disposition.""" + result = _extract_filename("http://example.com/", 'attachment; filename="test.txt"') + assert result == "test.txt" + + def test_quoted_filename_with_spaces(self): + """Test filename with spaces in quotes.""" + result = _extract_filename("http://example.com/", 'attachment; filename="my file.txt"') + assert result == "my file.txt" + + def test_unquoted_filename(self): + """Test unquoted filename.""" + result = _extract_filename("http://example.com/", "attachment; filename=test.txt") + assert result == "test.txt" + + def test_percent_encoded_filename(self): + """Test percent-encoded filename.""" + result = _extract_filename("http://example.com/", 'attachment; filename="file%20name.txt"') + assert result == "file name.txt" + + def test_rfc5987_filename_star_utf8(self): + """Test RFC5987 filename* with UTF-8 encoding.""" + result = _extract_filename("http://example.com/", "attachment; filename*=UTF-8''file%20name.txt") + assert result == "file name.txt" + + def test_rfc5987_filename_star_chinese(self): + """Test RFC5987 filename* with Chinese characters.""" + result = _extract_filename( + "http://example.com/", "attachment; filename*=UTF-8''%E6%B5%8B%E8%AF%95%E6%96%87%E4%BB%B6.txt" + ) + assert result == "测试文件.txt" + + def test_rfc5987_filename_star_with_language(self): + """Test RFC5987 filename* with language tag.""" + result = _extract_filename("http://example.com/", "attachment; filename*=UTF-8'en'file%20name.txt") + assert result == "file name.txt" + + def test_rfc5987_filename_star_fallback_charset(self): + """Test RFC5987 filename* with fallback charset.""" + result = _extract_filename("http://example.com/", "attachment; filename*=''file%20name.txt") + assert result == "file name.txt" + + def test_rfc5987_filename_star_malformed_fallback(self): + """Test RFC5987 filename* with malformed format falls back to simple unquote.""" + result = _extract_filename("http://example.com/", "attachment; filename*=malformed%20filename.txt") + assert result == "malformed filename.txt" + + def test_filename_star_takes_precedence_over_filename(self): + """Test that filename* takes precedence over filename.""" + test_string = 'attachment; filename="old.txt"; filename*=UTF-8\'\'new.txt"' + result = _extract_filename("http://example.com/", test_string) + assert result == "new.txt" + + def test_path_injection_protection(self): + """Test that path injection attempts are blocked by os.path.basename.""" + result = _extract_filename("http://example.com/", 'attachment; filename="../../../etc/passwd"') + assert result == "passwd" + + def test_path_injection_protection_rfc5987(self): + """Test that path injection attempts in RFC5987 are blocked.""" + result = _extract_filename("http://example.com/", "attachment; filename*=UTF-8''..%2F..%2F..%2Fetc%2Fpasswd") + assert result == "passwd" + + def test_empty_filename_returns_none(self): + """Test that empty filename returns None.""" + result = _extract_filename("http://example.com/", 'attachment; filename=""') + assert result is None + + def test_whitespace_only_filename_returns_none(self): + """Test that whitespace-only filename returns None.""" + result = _extract_filename("http://example.com/", 'attachment; filename=" "') + assert result is None + + def test_complex_rfc5987_encoding(self): + """Test complex RFC5987 encoding with special characters.""" + result = _extract_filename( + "http://example.com/", + "attachment; filename*=UTF-8''%E4%B8%AD%E6%96%87%E6%96%87%E4%BB%B6%20%28%E5%89%AF%E6%9C%AC%29.pdf", + ) + assert result == "中文文件 (副本).pdf" + + def test_iso8859_1_encoding(self): + """Test ISO-8859-1 encoding in RFC5987.""" + result = _extract_filename("http://example.com/", "attachment; filename*=ISO-8859-1''file%20name.txt") + assert result == "file name.txt" + + def test_encoding_error_fallback(self): + """Test that encoding errors fall back to safe ASCII filename.""" + result = _extract_filename("http://example.com/", "attachment; filename*=INVALID-CHARSET''file%20name.txt") + assert result == "file name.txt" + + def test_mixed_quotes_and_encoding(self): + """Test filename with mixed quotes and percent encoding.""" + result = _extract_filename( + "http://example.com/", 'attachment; filename="file%20with%20quotes%20%26%20encoding.txt"' + ) + assert result == "file with quotes & encoding.txt" diff --git a/api/tests/unit_tests/models/test_tool_models.py b/api/tests/unit_tests/models/test_tool_models.py new file mode 100644 index 0000000000..1a75eb9a01 --- /dev/null +++ b/api/tests/unit_tests/models/test_tool_models.py @@ -0,0 +1,966 @@ +""" +Comprehensive unit tests for Tool models. + +This test suite covers: +- ToolProvider model validation (BuiltinToolProvider, ApiToolProvider) +- BuiltinToolProvider relationships and credential management +- ApiToolProvider credential storage and encryption +- Tool OAuth client models +- ToolLabelBinding relationships +""" + +import json +from uuid import uuid4 + +from core.tools.entities.tool_entities import ApiProviderSchemaType +from models.tools import ( + ApiToolProvider, + BuiltinToolProvider, + ToolLabelBinding, + ToolOAuthSystemClient, + ToolOAuthTenantClient, +) + + +class TestBuiltinToolProviderValidation: + """Test suite for BuiltinToolProvider model validation and operations.""" + + def test_builtin_tool_provider_creation_with_required_fields(self): + """Test creating a builtin tool provider with all required fields.""" + # Arrange + tenant_id = str(uuid4()) + user_id = str(uuid4()) + provider_name = "google" + credentials = {"api_key": "test_key_123"} + + # Act + builtin_provider = BuiltinToolProvider( + tenant_id=tenant_id, + user_id=user_id, + provider=provider_name, + encrypted_credentials=json.dumps(credentials), + name="Google API Key 1", + ) + + # Assert + assert builtin_provider.tenant_id == tenant_id + assert builtin_provider.user_id == user_id + assert builtin_provider.provider == provider_name + assert builtin_provider.name == "Google API Key 1" + assert builtin_provider.encrypted_credentials == json.dumps(credentials) + + def test_builtin_tool_provider_credentials_property(self): + """Test credentials property parses JSON correctly.""" + # Arrange + credentials_data = { + "api_key": "sk-test123", + "auth_type": "api_key", + "endpoint": "https://api.example.com", + } + builtin_provider = BuiltinToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + provider="custom_provider", + name="Custom Provider Key", + encrypted_credentials=json.dumps(credentials_data), + ) + + # Act + result = builtin_provider.credentials + + # Assert + assert result == credentials_data + assert result["api_key"] == "sk-test123" + assert result["auth_type"] == "api_key" + + def test_builtin_tool_provider_credentials_empty_when_none(self): + """Test credentials property returns empty dict when encrypted_credentials is None.""" + # Arrange + builtin_provider = BuiltinToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + provider="test_provider", + name="Test Provider", + encrypted_credentials=None, + ) + + # Act + result = builtin_provider.credentials + + # Assert + assert result == {} + + def test_builtin_tool_provider_credentials_empty_when_empty_string(self): + """Test credentials property returns empty dict when encrypted_credentials is empty.""" + # Arrange + builtin_provider = BuiltinToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + provider="test_provider", + name="Test Provider", + encrypted_credentials="", + ) + + # Act + result = builtin_provider.credentials + + # Assert + assert result == {} + + def test_builtin_tool_provider_default_values(self): + """Test builtin tool provider default values.""" + # Arrange & Act + builtin_provider = BuiltinToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + provider="test_provider", + name="Test Provider", + ) + + # Assert + assert builtin_provider.is_default is False + assert builtin_provider.credential_type == "api-key" + assert builtin_provider.expires_at == -1 + + def test_builtin_tool_provider_with_oauth_credential_type(self): + """Test builtin tool provider with OAuth credential type.""" + # Arrange + credentials = { + "access_token": "oauth_token_123", + "refresh_token": "refresh_token_456", + "token_type": "Bearer", + } + + # Act + builtin_provider = BuiltinToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + provider="google", + name="Google OAuth", + encrypted_credentials=json.dumps(credentials), + credential_type="oauth2", + expires_at=1735689600, + ) + + # Assert + assert builtin_provider.credential_type == "oauth2" + assert builtin_provider.expires_at == 1735689600 + assert builtin_provider.credentials["access_token"] == "oauth_token_123" + + def test_builtin_tool_provider_is_default_flag(self): + """Test is_default flag for builtin tool provider.""" + # Arrange + provider1 = BuiltinToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + provider="google", + name="Google Key 1", + is_default=True, + ) + provider2 = BuiltinToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + provider="google", + name="Google Key 2", + is_default=False, + ) + + # Assert + assert provider1.is_default is True + assert provider2.is_default is False + + def test_builtin_tool_provider_unique_constraint_fields(self): + """Test unique constraint fields (tenant_id, provider, name).""" + # Arrange + tenant_id = str(uuid4()) + provider_name = "google" + credential_name = "My Google Key" + + # Act + builtin_provider = BuiltinToolProvider( + tenant_id=tenant_id, + user_id=str(uuid4()), + provider=provider_name, + name=credential_name, + ) + + # Assert - these fields form unique constraint + assert builtin_provider.tenant_id == tenant_id + assert builtin_provider.provider == provider_name + assert builtin_provider.name == credential_name + + def test_builtin_tool_provider_multiple_credentials_same_provider(self): + """Test multiple credential sets for the same provider.""" + # Arrange + tenant_id = str(uuid4()) + user_id = str(uuid4()) + provider = "openai" + + # Act - create multiple credentials for same provider + provider1 = BuiltinToolProvider( + tenant_id=tenant_id, + user_id=user_id, + provider=provider, + name="OpenAI Key 1", + encrypted_credentials=json.dumps({"api_key": "key1"}), + ) + provider2 = BuiltinToolProvider( + tenant_id=tenant_id, + user_id=user_id, + provider=provider, + name="OpenAI Key 2", + encrypted_credentials=json.dumps({"api_key": "key2"}), + ) + + # Assert - different names allow multiple credentials + assert provider1.provider == provider2.provider + assert provider1.name != provider2.name + assert provider1.credentials != provider2.credentials + + +class TestApiToolProviderValidation: + """Test suite for ApiToolProvider model validation and operations.""" + + def test_api_tool_provider_creation_with_required_fields(self): + """Test creating an API tool provider with all required fields.""" + # Arrange + tenant_id = str(uuid4()) + user_id = str(uuid4()) + provider_name = "Custom API" + schema = '{"openapi": "3.0.0", "info": {"title": "Test API"}}' + tools = [{"name": "test_tool", "description": "A test tool"}] + credentials = {"auth_type": "api_key", "api_key_value": "test123"} + + # Act + api_provider = ApiToolProvider( + tenant_id=tenant_id, + user_id=user_id, + name=provider_name, + icon='{"type": "emoji", "value": "🔧"}', + schema=schema, + schema_type_str="openapi", + description="Custom API for testing", + tools_str=json.dumps(tools), + credentials_str=json.dumps(credentials), + ) + + # Assert + assert api_provider.tenant_id == tenant_id + assert api_provider.user_id == user_id + assert api_provider.name == provider_name + assert api_provider.schema == schema + assert api_provider.schema_type_str == "openapi" + assert api_provider.description == "Custom API for testing" + + def test_api_tool_provider_schema_type_property(self): + """Test schema_type property converts string to enum.""" + # Arrange + api_provider = ApiToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + name="Test API", + icon="{}", + schema="{}", + schema_type_str="openapi", + description="Test", + tools_str="[]", + credentials_str="{}", + ) + + # Act + result = api_provider.schema_type + + # Assert + assert result == ApiProviderSchemaType.OPENAPI + + def test_api_tool_provider_tools_property(self): + """Test tools property parses JSON and returns ApiToolBundle list.""" + # Arrange + tools_data = [ + { + "author": "test", + "server_url": "https://api.weather.com", + "method": "get", + "summary": "Get weather information", + "operation_id": "getWeather", + "parameters": [], + "openapi": { + "operation_id": "getWeather", + "parameters": [], + "method": "get", + "path": "/weather", + "server_url": "https://api.weather.com", + }, + }, + { + "author": "test", + "server_url": "https://api.location.com", + "method": "get", + "summary": "Get location data", + "operation_id": "getLocation", + "parameters": [], + "openapi": { + "operation_id": "getLocation", + "parameters": [], + "method": "get", + "path": "/location", + "server_url": "https://api.location.com", + }, + }, + ] + api_provider = ApiToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + name="Weather API", + icon="{}", + schema="{}", + schema_type_str="openapi", + description="Weather API", + tools_str=json.dumps(tools_data), + credentials_str="{}", + ) + + # Act + result = api_provider.tools + + # Assert + assert len(result) == 2 + assert result[0].operation_id == "getWeather" + assert result[1].operation_id == "getLocation" + + def test_api_tool_provider_credentials_property(self): + """Test credentials property parses JSON correctly.""" + # Arrange + credentials_data = { + "auth_type": "api_key_header", + "api_key_header": "Authorization", + "api_key_value": "Bearer test_token", + "api_key_header_prefix": "bearer", + } + api_provider = ApiToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + name="Secure API", + icon="{}", + schema="{}", + schema_type_str="openapi", + description="Secure API", + tools_str="[]", + credentials_str=json.dumps(credentials_data), + ) + + # Act + result = api_provider.credentials + + # Assert + assert result["auth_type"] == "api_key_header" + assert result["api_key_header"] == "Authorization" + assert result["api_key_value"] == "Bearer test_token" + + def test_api_tool_provider_with_privacy_policy(self): + """Test API tool provider with privacy policy.""" + # Arrange + privacy_policy_url = "https://example.com/privacy" + + # Act + api_provider = ApiToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + name="Privacy API", + icon="{}", + schema="{}", + schema_type_str="openapi", + description="API with privacy policy", + tools_str="[]", + credentials_str="{}", + privacy_policy=privacy_policy_url, + ) + + # Assert + assert api_provider.privacy_policy == privacy_policy_url + + def test_api_tool_provider_with_custom_disclaimer(self): + """Test API tool provider with custom disclaimer.""" + # Arrange + disclaimer = "This API is provided as-is without warranty." + + # Act + api_provider = ApiToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + name="Disclaimer API", + icon="{}", + schema="{}", + schema_type_str="openapi", + description="API with disclaimer", + tools_str="[]", + credentials_str="{}", + custom_disclaimer=disclaimer, + ) + + # Assert + assert api_provider.custom_disclaimer == disclaimer + + def test_api_tool_provider_default_custom_disclaimer(self): + """Test API tool provider default custom_disclaimer is empty string.""" + # Arrange & Act + api_provider = ApiToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + name="Default API", + icon="{}", + schema="{}", + schema_type_str="openapi", + description="API", + tools_str="[]", + credentials_str="{}", + ) + + # Assert + assert api_provider.custom_disclaimer == "" + + def test_api_tool_provider_unique_constraint_fields(self): + """Test unique constraint fields (name, tenant_id).""" + # Arrange + tenant_id = str(uuid4()) + provider_name = "Unique API" + + # Act + api_provider = ApiToolProvider( + tenant_id=tenant_id, + user_id=str(uuid4()), + name=provider_name, + icon="{}", + schema="{}", + schema_type_str="openapi", + description="Unique API", + tools_str="[]", + credentials_str="{}", + ) + + # Assert - these fields form unique constraint + assert api_provider.tenant_id == tenant_id + assert api_provider.name == provider_name + + def test_api_tool_provider_with_no_auth(self): + """Test API tool provider with no authentication.""" + # Arrange + credentials = {"auth_type": "none"} + + # Act + api_provider = ApiToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + name="Public API", + icon="{}", + schema="{}", + schema_type_str="openapi", + description="Public API with no auth", + tools_str="[]", + credentials_str=json.dumps(credentials), + ) + + # Assert + assert api_provider.credentials["auth_type"] == "none" + + def test_api_tool_provider_with_api_key_query_auth(self): + """Test API tool provider with API key in query parameter.""" + # Arrange + credentials = { + "auth_type": "api_key_query", + "api_key_query_param": "apikey", + "api_key_value": "my_secret_key", + } + + # Act + api_provider = ApiToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + name="Query Auth API", + icon="{}", + schema="{}", + schema_type_str="openapi", + description="API with query auth", + tools_str="[]", + credentials_str=json.dumps(credentials), + ) + + # Assert + assert api_provider.credentials["auth_type"] == "api_key_query" + assert api_provider.credentials["api_key_query_param"] == "apikey" + + +class TestToolOAuthModels: + """Test suite for OAuth client models (system and tenant level).""" + + def test_oauth_system_client_creation(self): + """Test creating a system-level OAuth client.""" + # Arrange + plugin_id = "builtin.google" + provider = "google" + oauth_params = json.dumps( + {"client_id": "system_client_id", "client_secret": "system_secret", "scope": "email profile"} + ) + + # Act + oauth_client = ToolOAuthSystemClient( + plugin_id=plugin_id, + provider=provider, + encrypted_oauth_params=oauth_params, + ) + + # Assert + assert oauth_client.plugin_id == plugin_id + assert oauth_client.provider == provider + assert oauth_client.encrypted_oauth_params == oauth_params + + def test_oauth_system_client_unique_constraint(self): + """Test unique constraint on plugin_id and provider.""" + # Arrange + plugin_id = "builtin.github" + provider = "github" + + # Act + oauth_client = ToolOAuthSystemClient( + plugin_id=plugin_id, + provider=provider, + encrypted_oauth_params="{}", + ) + + # Assert - these fields form unique constraint + assert oauth_client.plugin_id == plugin_id + assert oauth_client.provider == provider + + def test_oauth_tenant_client_creation(self): + """Test creating a tenant-level OAuth client.""" + # Arrange + tenant_id = str(uuid4()) + plugin_id = "builtin.google" + provider = "google" + + # Act + oauth_client = ToolOAuthTenantClient( + tenant_id=tenant_id, + plugin_id=plugin_id, + provider=provider, + ) + # Set encrypted_oauth_params after creation (it has init=False) + oauth_params = json.dumps({"client_id": "tenant_client_id", "client_secret": "tenant_secret"}) + oauth_client.encrypted_oauth_params = oauth_params + + # Assert + assert oauth_client.tenant_id == tenant_id + assert oauth_client.plugin_id == plugin_id + assert oauth_client.provider == provider + + def test_oauth_tenant_client_enabled_default(self): + """Test OAuth tenant client enabled flag has init=False and uses server default.""" + # Arrange & Act + oauth_client = ToolOAuthTenantClient( + tenant_id=str(uuid4()), + plugin_id="builtin.slack", + provider="slack", + ) + + # Assert - enabled has init=False, so it won't be set until saved to DB + # We can manually set it to test the field exists + oauth_client.enabled = True + assert oauth_client.enabled is True + + def test_oauth_tenant_client_oauth_params_property(self): + """Test oauth_params property parses JSON correctly.""" + # Arrange + params_data = { + "client_id": "test_client_123", + "client_secret": "secret_456", + "redirect_uri": "https://app.example.com/callback", + } + oauth_client = ToolOAuthTenantClient( + tenant_id=str(uuid4()), + plugin_id="builtin.dropbox", + provider="dropbox", + ) + # Set encrypted_oauth_params after creation (it has init=False) + oauth_client.encrypted_oauth_params = json.dumps(params_data) + + # Act + result = oauth_client.oauth_params + + # Assert + assert result == params_data + assert result["client_id"] == "test_client_123" + assert result["redirect_uri"] == "https://app.example.com/callback" + + def test_oauth_tenant_client_oauth_params_empty_when_none(self): + """Test oauth_params returns empty dict when encrypted_oauth_params is None.""" + # Arrange + oauth_client = ToolOAuthTenantClient( + tenant_id=str(uuid4()), + plugin_id="builtin.test", + provider="test", + ) + # encrypted_oauth_params has init=False, set it to None + oauth_client.encrypted_oauth_params = None + + # Act + result = oauth_client.oauth_params + + # Assert + assert result == {} + + def test_oauth_tenant_client_disabled_state(self): + """Test OAuth tenant client can be disabled.""" + # Arrange + oauth_client = ToolOAuthTenantClient( + tenant_id=str(uuid4()), + plugin_id="builtin.microsoft", + provider="microsoft", + ) + + # Act + oauth_client.enabled = False + + # Assert + assert oauth_client.enabled is False + + +class TestToolLabelBinding: + """Test suite for ToolLabelBinding model.""" + + def test_tool_label_binding_creation(self): + """Test creating a tool label binding.""" + # Arrange + tool_id = "google.search" + tool_type = "builtin" + label_name = "search" + + # Act + label_binding = ToolLabelBinding( + tool_id=tool_id, + tool_type=tool_type, + label_name=label_name, + ) + + # Assert + assert label_binding.tool_id == tool_id + assert label_binding.tool_type == tool_type + assert label_binding.label_name == label_name + + def test_tool_label_binding_unique_constraint(self): + """Test unique constraint on tool_id and label_name.""" + # Arrange + tool_id = "openai.text_generation" + label_name = "text" + + # Act + label_binding = ToolLabelBinding( + tool_id=tool_id, + tool_type="builtin", + label_name=label_name, + ) + + # Assert - these fields form unique constraint + assert label_binding.tool_id == tool_id + assert label_binding.label_name == label_name + + def test_tool_label_binding_multiple_labels_same_tool(self): + """Test multiple labels can be bound to the same tool.""" + # Arrange + tool_id = "google.search" + tool_type = "builtin" + + # Act + binding1 = ToolLabelBinding( + tool_id=tool_id, + tool_type=tool_type, + label_name="search", + ) + binding2 = ToolLabelBinding( + tool_id=tool_id, + tool_type=tool_type, + label_name="productivity", + ) + + # Assert + assert binding1.tool_id == binding2.tool_id + assert binding1.label_name != binding2.label_name + + def test_tool_label_binding_different_tool_types(self): + """Test label bindings for different tool types.""" + # Arrange + tool_types = ["builtin", "api", "workflow"] + + # Act & Assert + for tool_type in tool_types: + binding = ToolLabelBinding( + tool_id=f"test_tool_{tool_type}", + tool_type=tool_type, + label_name="test", + ) + assert binding.tool_type == tool_type + + +class TestCredentialStorage: + """Test suite for credential storage and encryption patterns.""" + + def test_builtin_provider_credential_storage_format(self): + """Test builtin provider stores credentials as JSON string.""" + # Arrange + credentials = { + "api_key": "sk-test123", + "endpoint": "https://api.example.com", + "timeout": 30, + } + + # Act + provider = BuiltinToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + provider="test", + name="Test Provider", + encrypted_credentials=json.dumps(credentials), + ) + + # Assert + assert isinstance(provider.encrypted_credentials, str) + assert provider.credentials == credentials + + def test_api_provider_credential_storage_format(self): + """Test API provider stores credentials as JSON string.""" + # Arrange + credentials = { + "auth_type": "api_key_header", + "api_key_header": "X-API-Key", + "api_key_value": "secret_key_789", + } + + # Act + provider = ApiToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + name="Test API", + icon="{}", + schema="{}", + schema_type_str="openapi", + description="Test", + tools_str="[]", + credentials_str=json.dumps(credentials), + ) + + # Assert + assert isinstance(provider.credentials_str, str) + assert provider.credentials == credentials + + def test_builtin_provider_complex_credential_structure(self): + """Test builtin provider with complex nested credential structure.""" + # Arrange + credentials = { + "auth_type": "oauth2", + "oauth_config": { + "access_token": "token123", + "refresh_token": "refresh456", + "expires_in": 3600, + "token_type": "Bearer", + }, + "additional_headers": {"X-Custom-Header": "value"}, + } + + # Act + provider = BuiltinToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + provider="oauth_provider", + name="OAuth Provider", + encrypted_credentials=json.dumps(credentials), + ) + + # Assert + assert provider.credentials["oauth_config"]["access_token"] == "token123" + assert provider.credentials["additional_headers"]["X-Custom-Header"] == "value" + + def test_api_provider_credential_update_pattern(self): + """Test pattern for updating API provider credentials.""" + # Arrange + original_credentials = {"auth_type": "api_key_header", "api_key_value": "old_key"} + provider = ApiToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + name="Update Test", + icon="{}", + schema="{}", + schema_type_str="openapi", + description="Test", + tools_str="[]", + credentials_str=json.dumps(original_credentials), + ) + + # Act - simulate credential update + new_credentials = {"auth_type": "api_key_header", "api_key_value": "new_key"} + provider.credentials_str = json.dumps(new_credentials) + + # Assert + assert provider.credentials["api_key_value"] == "new_key" + + def test_builtin_provider_credential_expiration(self): + """Test builtin provider credential expiration tracking.""" + # Arrange + future_timestamp = 1735689600 # Future date + past_timestamp = 1609459200 # Past date + + # Act + active_provider = BuiltinToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + provider="active", + name="Active Provider", + expires_at=future_timestamp, + ) + expired_provider = BuiltinToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + provider="expired", + name="Expired Provider", + expires_at=past_timestamp, + ) + never_expires_provider = BuiltinToolProvider( + tenant_id=str(uuid4()), + user_id=str(uuid4()), + provider="permanent", + name="Permanent Provider", + expires_at=-1, + ) + + # Assert + assert active_provider.expires_at == future_timestamp + assert expired_provider.expires_at == past_timestamp + assert never_expires_provider.expires_at == -1 + + def test_oauth_client_credential_storage(self): + """Test OAuth client credential storage pattern.""" + # Arrange + oauth_credentials = { + "client_id": "oauth_client_123", + "client_secret": "oauth_secret_456", + "authorization_url": "https://oauth.example.com/authorize", + "token_url": "https://oauth.example.com/token", + "scope": "read write", + } + + # Act + system_client = ToolOAuthSystemClient( + plugin_id="builtin.oauth_test", + provider="oauth_test", + encrypted_oauth_params=json.dumps(oauth_credentials), + ) + + tenant_client = ToolOAuthTenantClient( + tenant_id=str(uuid4()), + plugin_id="builtin.oauth_test", + provider="oauth_test", + ) + # Set encrypted_oauth_params after creation (it has init=False) + tenant_client.encrypted_oauth_params = json.dumps(oauth_credentials) + + # Assert + assert system_client.encrypted_oauth_params == json.dumps(oauth_credentials) + assert tenant_client.oauth_params == oauth_credentials + + +class TestToolProviderRelationships: + """Test suite for tool provider relationships and associations.""" + + def test_builtin_provider_tenant_relationship(self): + """Test builtin provider belongs to a tenant.""" + # Arrange + tenant_id = str(uuid4()) + + # Act + provider = BuiltinToolProvider( + tenant_id=tenant_id, + user_id=str(uuid4()), + provider="test", + name="Test Provider", + ) + + # Assert + assert provider.tenant_id == tenant_id + + def test_api_provider_user_relationship(self): + """Test API provider belongs to a user.""" + # Arrange + user_id = str(uuid4()) + + # Act + provider = ApiToolProvider( + tenant_id=str(uuid4()), + user_id=user_id, + name="User API", + icon="{}", + schema="{}", + schema_type_str="openapi", + description="Test", + tools_str="[]", + credentials_str="{}", + ) + + # Assert + assert provider.user_id == user_id + + def test_multiple_providers_same_tenant(self): + """Test multiple providers can belong to the same tenant.""" + # Arrange + tenant_id = str(uuid4()) + user_id = str(uuid4()) + + # Act + builtin1 = BuiltinToolProvider( + tenant_id=tenant_id, + user_id=user_id, + provider="google", + name="Google Key 1", + ) + builtin2 = BuiltinToolProvider( + tenant_id=tenant_id, + user_id=user_id, + provider="openai", + name="OpenAI Key 1", + ) + api1 = ApiToolProvider( + tenant_id=tenant_id, + user_id=user_id, + name="Custom API 1", + icon="{}", + schema="{}", + schema_type_str="openapi", + description="Test", + tools_str="[]", + credentials_str="{}", + ) + + # Assert + assert builtin1.tenant_id == tenant_id + assert builtin2.tenant_id == tenant_id + assert api1.tenant_id == tenant_id + + def test_tool_label_bindings_for_provider_tools(self): + """Test tool label bindings can be associated with provider tools.""" + # Arrange + provider_name = "google" + tool_id = f"{provider_name}.search" + + # Act + binding1 = ToolLabelBinding( + tool_id=tool_id, + tool_type="builtin", + label_name="search", + ) + binding2 = ToolLabelBinding( + tool_id=tool_id, + tool_type="builtin", + label_name="web", + ) + + # Assert + assert binding1.tool_id == tool_id + assert binding2.tool_id == tool_id + assert binding1.label_name != binding2.label_name diff --git a/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py b/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py index 73b35b8e63..0c34676252 100644 --- a/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py +++ b/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py @@ -6,10 +6,10 @@ from unittest.mock import Mock, patch import pytest from sqlalchemy.orm import Session, sessionmaker -from core.workflow.entities.workflow_pause import WorkflowPauseEntity from core.workflow.enums import WorkflowExecutionStatus from models.workflow import WorkflowPause as WorkflowPauseModel from models.workflow import WorkflowRun +from repositories.entities.workflow_pause import WorkflowPauseEntity from repositories.sqlalchemy_api_workflow_run_repository import ( DifyAPISQLAlchemyWorkflowRunRepository, _PrivateWorkflowPauseEntity, @@ -129,12 +129,14 @@ class TestCreateWorkflowPause(TestDifyAPISQLAlchemyWorkflowRunRepository): workflow_run_id=workflow_run_id, state_owner_user_id=state_owner_user_id, state=state, + pause_reasons=[], ) # Assert assert isinstance(result, _PrivateWorkflowPauseEntity) assert result.id == "pause-123" assert result.workflow_execution_id == workflow_run_id + assert result.get_pause_reasons() == [] # Verify database interactions mock_session.get.assert_called_once_with(WorkflowRun, workflow_run_id) @@ -156,6 +158,7 @@ class TestCreateWorkflowPause(TestDifyAPISQLAlchemyWorkflowRunRepository): workflow_run_id="workflow-run-123", state_owner_user_id="user-123", state='{"test": "state"}', + pause_reasons=[], ) mock_session.get.assert_called_once_with(WorkflowRun, "workflow-run-123") @@ -174,6 +177,7 @@ class TestCreateWorkflowPause(TestDifyAPISQLAlchemyWorkflowRunRepository): workflow_run_id="workflow-run-123", state_owner_user_id="user-123", state='{"test": "state"}', + pause_reasons=[], ) @@ -316,19 +320,10 @@ class TestDeleteWorkflowPause(TestDifyAPISQLAlchemyWorkflowRunRepository): class TestPrivateWorkflowPauseEntity(TestDifyAPISQLAlchemyWorkflowRunRepository): """Test _PrivateWorkflowPauseEntity class.""" - def test_from_models(self, sample_workflow_pause: Mock): - """Test creating _PrivateWorkflowPauseEntity from models.""" - # Act - entity = _PrivateWorkflowPauseEntity.from_models(sample_workflow_pause) - - # Assert - assert isinstance(entity, _PrivateWorkflowPauseEntity) - assert entity._pause_model == sample_workflow_pause - def test_properties(self, sample_workflow_pause: Mock): """Test entity properties.""" # Arrange - entity = _PrivateWorkflowPauseEntity.from_models(sample_workflow_pause) + entity = _PrivateWorkflowPauseEntity(pause_model=sample_workflow_pause, reason_models=[], human_input_form=[]) # Act & Assert assert entity.id == sample_workflow_pause.id @@ -338,7 +333,7 @@ class TestPrivateWorkflowPauseEntity(TestDifyAPISQLAlchemyWorkflowRunRepository) def test_get_state(self, sample_workflow_pause: Mock): """Test getting state from storage.""" # Arrange - entity = _PrivateWorkflowPauseEntity.from_models(sample_workflow_pause) + entity = _PrivateWorkflowPauseEntity(pause_model=sample_workflow_pause, reason_models=[], human_input_form=[]) expected_state = b'{"test": "state"}' with patch("repositories.sqlalchemy_api_workflow_run_repository.storage") as mock_storage: @@ -354,7 +349,7 @@ class TestPrivateWorkflowPauseEntity(TestDifyAPISQLAlchemyWorkflowRunRepository) def test_get_state_caching(self, sample_workflow_pause: Mock): """Test state caching in get_state method.""" # Arrange - entity = _PrivateWorkflowPauseEntity.from_models(sample_workflow_pause) + entity = _PrivateWorkflowPauseEntity(pause_model=sample_workflow_pause, reason_models=[], human_input_form=[]) expected_state = b'{"test": "state"}' with patch("repositories.sqlalchemy_api_workflow_run_repository.storage") as mock_storage: diff --git a/api/tests/unit_tests/services/controller_api.py b/api/tests/unit_tests/services/controller_api.py new file mode 100644 index 0000000000..762d7b9090 --- /dev/null +++ b/api/tests/unit_tests/services/controller_api.py @@ -0,0 +1,1082 @@ +""" +Comprehensive API/Controller tests for Dataset endpoints. + +This module contains extensive integration tests for the dataset-related +controller endpoints, testing the HTTP API layer that exposes dataset +functionality through REST endpoints. + +The controller endpoints provide HTTP access to: +- Dataset CRUD operations (list, create, update, delete) +- Document management operations +- Segment management operations +- Hit testing (retrieval testing) operations +- External dataset and knowledge API operations + +These tests verify that: +- HTTP requests are properly routed to service methods +- Request validation works correctly +- Response formatting is correct +- Authentication and authorization are enforced +- Error handling returns appropriate HTTP status codes +- Request/response serialization works properly + +================================================================================ +ARCHITECTURE OVERVIEW +================================================================================ + +The controller layer in Dify uses Flask-RESTX to provide RESTful API endpoints. +Controllers act as a thin layer between HTTP requests and service methods, +handling: + +1. Request Parsing: Extracting and validating parameters from HTTP requests +2. Authentication: Verifying user identity and permissions +3. Authorization: Checking if user has permission to perform operations +4. Service Invocation: Calling appropriate service methods +5. Response Formatting: Serializing service results to HTTP responses +6. Error Handling: Converting exceptions to appropriate HTTP status codes + +Key Components: +- Flask-RESTX Resources: Define endpoint classes with HTTP methods +- Decorators: Handle authentication, authorization, and setup requirements +- Request Parsers: Validate and extract request parameters +- Response Models: Define response structure for Swagger documentation +- Error Handlers: Convert exceptions to HTTP error responses + +================================================================================ +TESTING STRATEGY +================================================================================ + +This test suite follows a comprehensive testing strategy that covers: + +1. HTTP Request/Response Testing: + - GET, POST, PATCH, DELETE methods + - Query parameters and request body validation + - Response status codes and body structure + - Headers and content types + +2. Authentication and Authorization: + - Login required checks + - Account initialization checks + - Permission validation + - Role-based access control + +3. Request Validation: + - Required parameter validation + - Parameter type validation + - Parameter range validation + - Custom validation rules + +4. Error Handling: + - 400 Bad Request (validation errors) + - 401 Unauthorized (authentication errors) + - 403 Forbidden (authorization errors) + - 404 Not Found (resource not found) + - 500 Internal Server Error (unexpected errors) + +5. Service Integration: + - Service method invocation + - Service method parameter passing + - Service method return value handling + - Service exception handling + +================================================================================ +""" + +from unittest.mock import Mock, patch +from uuid import uuid4 + +import pytest +from flask import Flask +from flask_restx import Api + +from controllers.console.datasets.datasets import DatasetApi, DatasetListApi +from controllers.console.datasets.external import ( + ExternalApiTemplateListApi, +) +from controllers.console.datasets.hit_testing import HitTestingApi +from models.dataset import Dataset, DatasetPermissionEnum + +# ============================================================================ +# Test Data Factory +# ============================================================================ +# The Test Data Factory pattern is used here to centralize the creation of +# test objects and mock instances. This approach provides several benefits: +# +# 1. Consistency: All test objects are created using the same factory methods, +# ensuring consistent structure across all tests. +# +# 2. Maintainability: If the structure of models or services changes, we only +# need to update the factory methods rather than every individual test. +# +# 3. Reusability: Factory methods can be reused across multiple test classes, +# reducing code duplication. +# +# 4. Readability: Tests become more readable when they use descriptive factory +# method calls instead of complex object construction logic. +# +# ============================================================================ + + +class ControllerApiTestDataFactory: + """ + Factory class for creating test data and mock objects for controller API tests. + + This factory provides static methods to create mock objects for: + - Flask application and test client setup + - Dataset instances and related models + - User and authentication context + - HTTP request/response objects + - Service method return values + + The factory methods help maintain consistency across tests and reduce + code duplication when setting up test scenarios. + """ + + @staticmethod + def create_flask_app(): + """ + Create a Flask test application for API testing. + + Returns: + Flask application instance configured for testing + """ + app = Flask(__name__) + app.config["TESTING"] = True + app.config["SECRET_KEY"] = "test-secret-key" + return app + + @staticmethod + def create_api_instance(app): + """ + Create a Flask-RESTX API instance. + + Args: + app: Flask application instance + + Returns: + Api instance configured for the application + """ + api = Api(app, doc="/docs/") + return api + + @staticmethod + def create_test_client(app, api, resource_class, route): + """ + Create a Flask test client with a resource registered. + + Args: + app: Flask application instance + api: Flask-RESTX API instance + resource_class: Resource class to register + route: URL route for the resource + + Returns: + Flask test client instance + """ + api.add_resource(resource_class, route) + return app.test_client() + + @staticmethod + def create_dataset_mock( + dataset_id: str = "dataset-123", + name: str = "Test Dataset", + tenant_id: str = "tenant-123", + permission: DatasetPermissionEnum = DatasetPermissionEnum.ONLY_ME, + **kwargs, + ) -> Mock: + """ + Create a mock Dataset instance. + + Args: + dataset_id: Unique identifier for the dataset + name: Name of the dataset + tenant_id: Tenant identifier + permission: Dataset permission level + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a Dataset instance + """ + dataset = Mock(spec=Dataset) + dataset.id = dataset_id + dataset.name = name + dataset.tenant_id = tenant_id + dataset.permission = permission + dataset.to_dict.return_value = { + "id": dataset_id, + "name": name, + "tenant_id": tenant_id, + "permission": permission.value, + } + for key, value in kwargs.items(): + setattr(dataset, key, value) + return dataset + + @staticmethod + def create_user_mock( + user_id: str = "user-123", + tenant_id: str = "tenant-123", + is_dataset_editor: bool = True, + **kwargs, + ) -> Mock: + """ + Create a mock user/account instance. + + Args: + user_id: Unique identifier for the user + tenant_id: Tenant identifier + is_dataset_editor: Whether user has dataset editor permissions + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a user/account instance + """ + user = Mock() + user.id = user_id + user.current_tenant_id = tenant_id + user.is_dataset_editor = is_dataset_editor + user.has_edit_permission = True + user.is_dataset_operator = False + for key, value in kwargs.items(): + setattr(user, key, value) + return user + + @staticmethod + def create_paginated_response(items, total, page=1, per_page=20): + """ + Create a mock paginated response. + + Args: + items: List of items in the current page + total: Total number of items + page: Current page number + per_page: Items per page + + Returns: + Mock paginated response object + """ + response = Mock() + response.items = items + response.total = total + response.page = page + response.per_page = per_page + response.pages = (total + per_page - 1) // per_page + return response + + +# ============================================================================ +# Tests for Dataset List Endpoint (GET /datasets) +# ============================================================================ + + +class TestDatasetListApi: + """ + Comprehensive API tests for DatasetListApi (GET /datasets endpoint). + + This test class covers the dataset listing functionality through the + HTTP API, including pagination, search, filtering, and permissions. + + The GET /datasets endpoint: + 1. Requires authentication and account initialization + 2. Supports pagination (page, limit parameters) + 3. Supports search by keyword + 4. Supports filtering by tag IDs + 5. Supports including all datasets (for admins) + 6. Returns paginated list of datasets + + Test scenarios include: + - Successful dataset listing with pagination + - Search functionality + - Tag filtering + - Permission-based filtering + - Error handling (authentication, authorization) + """ + + @pytest.fixture + def app(self): + """ + Create Flask test application. + + Provides a Flask application instance configured for testing. + """ + return ControllerApiTestDataFactory.create_flask_app() + + @pytest.fixture + def api(self, app): + """ + Create Flask-RESTX API instance. + + Provides an API instance for registering resources. + """ + return ControllerApiTestDataFactory.create_api_instance(app) + + @pytest.fixture + def client(self, app, api): + """ + Create test client with DatasetListApi registered. + + Provides a Flask test client that can make HTTP requests to + the dataset list endpoint. + """ + return ControllerApiTestDataFactory.create_test_client(app, api, DatasetListApi, "/datasets") + + @pytest.fixture + def mock_current_user(self): + """ + Mock current user and tenant context. + + Provides mocked current_account_with_tenant function that returns + a user and tenant ID for testing authentication. + """ + with patch("controllers.console.datasets.datasets.current_account_with_tenant") as mock_get_user: + mock_user = ControllerApiTestDataFactory.create_user_mock() + mock_tenant_id = "tenant-123" + mock_get_user.return_value = (mock_user, mock_tenant_id) + yield mock_get_user + + def test_get_datasets_success(self, client, mock_current_user): + """ + Test successful retrieval of dataset list. + + Verifies that when authentication passes, the endpoint returns + a paginated list of datasets. + + This test ensures: + - Authentication is checked + - Service method is called with correct parameters + - Response has correct structure + - Status code is 200 + """ + # Arrange + datasets = [ + ControllerApiTestDataFactory.create_dataset_mock(dataset_id=f"dataset-{i}", name=f"Dataset {i}") + for i in range(3) + ] + + paginated_response = ControllerApiTestDataFactory.create_paginated_response( + items=datasets, total=3, page=1, per_page=20 + ) + + with patch("controllers.console.datasets.datasets.DatasetService.get_datasets") as mock_get_datasets: + mock_get_datasets.return_value = (datasets, 3) + + # Act + response = client.get("/datasets?page=1&limit=20") + + # Assert + assert response.status_code == 200 + data = response.get_json() + assert "data" in data + assert len(data["data"]) == 3 + assert data["total"] == 3 + assert data["page"] == 1 + assert data["limit"] == 20 + + # Verify service was called + mock_get_datasets.assert_called_once() + + def test_get_datasets_with_search(self, client, mock_current_user): + """ + Test dataset listing with search keyword. + + Verifies that search functionality works correctly through the API. + + This test ensures: + - Search keyword is passed to service method + - Filtered results are returned + - Response structure is correct + """ + # Arrange + search_keyword = "test" + datasets = [ControllerApiTestDataFactory.create_dataset_mock(dataset_id="dataset-1", name="Test Dataset")] + + with patch("controllers.console.datasets.datasets.DatasetService.get_datasets") as mock_get_datasets: + mock_get_datasets.return_value = (datasets, 1) + + # Act + response = client.get(f"/datasets?keyword={search_keyword}") + + # Assert + assert response.status_code == 200 + data = response.get_json() + assert len(data["data"]) == 1 + + # Verify search keyword was passed + call_args = mock_get_datasets.call_args + assert call_args[1]["search"] == search_keyword + + def test_get_datasets_with_pagination(self, client, mock_current_user): + """ + Test dataset listing with pagination parameters. + + Verifies that pagination works correctly through the API. + + This test ensures: + - Page and limit parameters are passed correctly + - Pagination metadata is included in response + - Correct datasets are returned for the page + """ + # Arrange + datasets = [ + ControllerApiTestDataFactory.create_dataset_mock(dataset_id=f"dataset-{i}", name=f"Dataset {i}") + for i in range(5) + ] + + with patch("controllers.console.datasets.datasets.DatasetService.get_datasets") as mock_get_datasets: + mock_get_datasets.return_value = (datasets[:3], 5) # First page with 3 items + + # Act + response = client.get("/datasets?page=1&limit=3") + + # Assert + assert response.status_code == 200 + data = response.get_json() + assert len(data["data"]) == 3 + assert data["page"] == 1 + assert data["limit"] == 3 + + # Verify pagination parameters were passed + call_args = mock_get_datasets.call_args + assert call_args[0][0] == 1 # page + assert call_args[0][1] == 3 # per_page + + +# ============================================================================ +# Tests for Dataset Detail Endpoint (GET /datasets/{id}) +# ============================================================================ + + +class TestDatasetApiGet: + """ + Comprehensive API tests for DatasetApi GET method (GET /datasets/{id} endpoint). + + This test class covers the single dataset retrieval functionality through + the HTTP API. + + The GET /datasets/{id} endpoint: + 1. Requires authentication and account initialization + 2. Validates dataset exists + 3. Checks user permissions + 4. Returns dataset details + + Test scenarios include: + - Successful dataset retrieval + - Dataset not found (404) + - Permission denied (403) + - Authentication required + """ + + @pytest.fixture + def app(self): + """Create Flask test application.""" + return ControllerApiTestDataFactory.create_flask_app() + + @pytest.fixture + def api(self, app): + """Create Flask-RESTX API instance.""" + return ControllerApiTestDataFactory.create_api_instance(app) + + @pytest.fixture + def client(self, app, api): + """Create test client with DatasetApi registered.""" + return ControllerApiTestDataFactory.create_test_client(app, api, DatasetApi, "/datasets/") + + @pytest.fixture + def mock_current_user(self): + """Mock current user and tenant context.""" + with patch("controllers.console.datasets.datasets.current_account_with_tenant") as mock_get_user: + mock_user = ControllerApiTestDataFactory.create_user_mock() + mock_tenant_id = "tenant-123" + mock_get_user.return_value = (mock_user, mock_tenant_id) + yield mock_get_user + + def test_get_dataset_success(self, client, mock_current_user): + """ + Test successful retrieval of a single dataset. + + Verifies that when authentication and permissions pass, the endpoint + returns dataset details. + + This test ensures: + - Authentication is checked + - Dataset existence is validated + - Permissions are checked + - Dataset details are returned + - Status code is 200 + """ + # Arrange + dataset_id = str(uuid4()) + dataset = ControllerApiTestDataFactory.create_dataset_mock(dataset_id=dataset_id, name="Test Dataset") + + with ( + patch("controllers.console.datasets.datasets.DatasetService.get_dataset") as mock_get_dataset, + patch("controllers.console.datasets.datasets.DatasetService.check_dataset_permission") as mock_check_perm, + ): + mock_get_dataset.return_value = dataset + mock_check_perm.return_value = None # No exception = permission granted + + # Act + response = client.get(f"/datasets/{dataset_id}") + + # Assert + assert response.status_code == 200 + data = response.get_json() + assert data["id"] == dataset_id + assert data["name"] == "Test Dataset" + + # Verify service methods were called + mock_get_dataset.assert_called_once_with(dataset_id) + mock_check_perm.assert_called_once() + + def test_get_dataset_not_found(self, client, mock_current_user): + """ + Test error handling when dataset is not found. + + Verifies that when dataset doesn't exist, a 404 error is returned. + + This test ensures: + - 404 status code is returned + - Error message is appropriate + - Service method is called + """ + # Arrange + dataset_id = str(uuid4()) + + with ( + patch("controllers.console.datasets.datasets.DatasetService.get_dataset") as mock_get_dataset, + patch("controllers.console.datasets.datasets.DatasetService.check_dataset_permission") as mock_check_perm, + ): + mock_get_dataset.return_value = None # Dataset not found + + # Act + response = client.get(f"/datasets/{dataset_id}") + + # Assert + assert response.status_code == 404 + + # Verify service was called + mock_get_dataset.assert_called_once() + + +# ============================================================================ +# Tests for Dataset Create Endpoint (POST /datasets) +# ============================================================================ + + +class TestDatasetApiCreate: + """ + Comprehensive API tests for DatasetApi POST method (POST /datasets endpoint). + + This test class covers the dataset creation functionality through the HTTP API. + + The POST /datasets endpoint: + 1. Requires authentication and account initialization + 2. Validates request body + 3. Creates dataset via service + 4. Returns created dataset + + Test scenarios include: + - Successful dataset creation + - Request validation errors + - Duplicate name errors + - Authentication required + """ + + @pytest.fixture + def app(self): + """Create Flask test application.""" + return ControllerApiTestDataFactory.create_flask_app() + + @pytest.fixture + def api(self, app): + """Create Flask-RESTX API instance.""" + return ControllerApiTestDataFactory.create_api_instance(app) + + @pytest.fixture + def client(self, app, api): + """Create test client with DatasetApi registered.""" + return ControllerApiTestDataFactory.create_test_client(app, api, DatasetApi, "/datasets") + + @pytest.fixture + def mock_current_user(self): + """Mock current user and tenant context.""" + with patch("controllers.console.datasets.datasets.current_account_with_tenant") as mock_get_user: + mock_user = ControllerApiTestDataFactory.create_user_mock() + mock_tenant_id = "tenant-123" + mock_get_user.return_value = (mock_user, mock_tenant_id) + yield mock_get_user + + def test_create_dataset_success(self, client, mock_current_user): + """ + Test successful creation of a dataset. + + Verifies that when all validation passes, a new dataset is created + and returned. + + This test ensures: + - Request body is validated + - Service method is called with correct parameters + - Created dataset is returned + - Status code is 201 + """ + # Arrange + dataset_id = str(uuid4()) + dataset = ControllerApiTestDataFactory.create_dataset_mock(dataset_id=dataset_id, name="New Dataset") + + request_data = { + "name": "New Dataset", + "description": "Test description", + "permission": "only_me", + } + + with patch("controllers.console.datasets.datasets.DatasetService.create_empty_dataset") as mock_create: + mock_create.return_value = dataset + + # Act + response = client.post( + "/datasets", + json=request_data, + content_type="application/json", + ) + + # Assert + assert response.status_code == 201 + data = response.get_json() + assert data["id"] == dataset_id + assert data["name"] == "New Dataset" + + # Verify service was called + mock_create.assert_called_once() + + +# ============================================================================ +# Tests for Hit Testing Endpoint (POST /datasets/{id}/hit-testing) +# ============================================================================ + + +class TestHitTestingApi: + """ + Comprehensive API tests for HitTestingApi (POST /datasets/{id}/hit-testing endpoint). + + This test class covers the hit testing (retrieval testing) functionality + through the HTTP API. + + The POST /datasets/{id}/hit-testing endpoint: + 1. Requires authentication and account initialization + 2. Validates dataset exists and user has permission + 3. Validates query parameters + 4. Performs retrieval testing + 5. Returns test results + + Test scenarios include: + - Successful hit testing + - Query validation errors + - Dataset not found + - Permission denied + """ + + @pytest.fixture + def app(self): + """Create Flask test application.""" + return ControllerApiTestDataFactory.create_flask_app() + + @pytest.fixture + def api(self, app): + """Create Flask-RESTX API instance.""" + return ControllerApiTestDataFactory.create_api_instance(app) + + @pytest.fixture + def client(self, app, api): + """Create test client with HitTestingApi registered.""" + return ControllerApiTestDataFactory.create_test_client( + app, api, HitTestingApi, "/datasets//hit-testing" + ) + + @pytest.fixture + def mock_current_user(self): + """Mock current user and tenant context.""" + with patch("controllers.console.datasets.hit_testing.current_account_with_tenant") as mock_get_user: + mock_user = ControllerApiTestDataFactory.create_user_mock() + mock_tenant_id = "tenant-123" + mock_get_user.return_value = (mock_user, mock_tenant_id) + yield mock_get_user + + def test_hit_testing_success(self, client, mock_current_user): + """ + Test successful hit testing operation. + + Verifies that when all validation passes, hit testing is performed + and results are returned. + + This test ensures: + - Dataset validation passes + - Query validation passes + - Hit testing service is called + - Results are returned + - Status code is 200 + """ + # Arrange + dataset_id = str(uuid4()) + dataset = ControllerApiTestDataFactory.create_dataset_mock(dataset_id=dataset_id) + + request_data = { + "query": "test query", + "top_k": 10, + } + + expected_result = { + "query": {"content": "test query"}, + "records": [ + {"content": "Result 1", "score": 0.95}, + {"content": "Result 2", "score": 0.85}, + ], + } + + with ( + patch( + "controllers.console.datasets.hit_testing.HitTestingApi.get_and_validate_dataset" + ) as mock_get_dataset, + patch("controllers.console.datasets.hit_testing.HitTestingApi.parse_args") as mock_parse_args, + patch("controllers.console.datasets.hit_testing.HitTestingApi.hit_testing_args_check") as mock_check_args, + patch("controllers.console.datasets.hit_testing.HitTestingApi.perform_hit_testing") as mock_perform, + ): + mock_get_dataset.return_value = dataset + mock_parse_args.return_value = request_data + mock_check_args.return_value = None # No validation error + mock_perform.return_value = expected_result + + # Act + response = client.post( + f"/datasets/{dataset_id}/hit-testing", + json=request_data, + content_type="application/json", + ) + + # Assert + assert response.status_code == 200 + data = response.get_json() + assert "query" in data + assert "records" in data + assert len(data["records"]) == 2 + + # Verify methods were called + mock_get_dataset.assert_called_once() + mock_parse_args.assert_called_once() + mock_check_args.assert_called_once() + mock_perform.assert_called_once() + + +# ============================================================================ +# Tests for External Dataset Endpoints +# ============================================================================ + + +class TestExternalDatasetApi: + """ + Comprehensive API tests for External Dataset endpoints. + + This test class covers the external knowledge API and external dataset + management functionality through the HTTP API. + + Endpoints covered: + - GET /datasets/external-knowledge-api - List external knowledge APIs + - POST /datasets/external-knowledge-api - Create external knowledge API + - GET /datasets/external-knowledge-api/{id} - Get external knowledge API + - PATCH /datasets/external-knowledge-api/{id} - Update external knowledge API + - DELETE /datasets/external-knowledge-api/{id} - Delete external knowledge API + - POST /datasets/external - Create external dataset + + Test scenarios include: + - Successful CRUD operations + - Request validation + - Authentication and authorization + - Error handling + """ + + @pytest.fixture + def app(self): + """Create Flask test application.""" + return ControllerApiTestDataFactory.create_flask_app() + + @pytest.fixture + def api(self, app): + """Create Flask-RESTX API instance.""" + return ControllerApiTestDataFactory.create_api_instance(app) + + @pytest.fixture + def client_list(self, app, api): + """Create test client for external knowledge API list endpoint.""" + return ControllerApiTestDataFactory.create_test_client( + app, api, ExternalApiTemplateListApi, "/datasets/external-knowledge-api" + ) + + @pytest.fixture + def mock_current_user(self): + """Mock current user and tenant context.""" + with patch("controllers.console.datasets.external.current_account_with_tenant") as mock_get_user: + mock_user = ControllerApiTestDataFactory.create_user_mock(is_dataset_editor=True) + mock_tenant_id = "tenant-123" + mock_get_user.return_value = (mock_user, mock_tenant_id) + yield mock_get_user + + def test_get_external_knowledge_apis_success(self, client_list, mock_current_user): + """ + Test successful retrieval of external knowledge API list. + + Verifies that the endpoint returns a paginated list of external + knowledge APIs. + + This test ensures: + - Authentication is checked + - Service method is called + - Paginated response is returned + - Status code is 200 + """ + # Arrange + apis = [{"id": f"api-{i}", "name": f"API {i}", "endpoint": f"https://api{i}.com"} for i in range(3)] + + with patch( + "controllers.console.datasets.external.ExternalDatasetService.get_external_knowledge_apis" + ) as mock_get_apis: + mock_get_apis.return_value = (apis, 3) + + # Act + response = client_list.get("/datasets/external-knowledge-api?page=1&limit=20") + + # Assert + assert response.status_code == 200 + data = response.get_json() + assert "data" in data + assert len(data["data"]) == 3 + assert data["total"] == 3 + + # Verify service was called + mock_get_apis.assert_called_once() + + +# ============================================================================ +# Additional Documentation and Notes +# ============================================================================ +# +# This test suite covers the core API endpoints for dataset operations. +# Additional test scenarios that could be added: +# +# 1. Document Endpoints: +# - POST /datasets/{id}/documents - Upload/create documents +# - GET /datasets/{id}/documents - List documents +# - GET /datasets/{id}/documents/{doc_id} - Get document details +# - PATCH /datasets/{id}/documents/{doc_id} - Update document +# - DELETE /datasets/{id}/documents/{doc_id} - Delete document +# - POST /datasets/{id}/documents/batch - Batch operations +# +# 2. Segment Endpoints: +# - GET /datasets/{id}/segments - List segments +# - GET /datasets/{id}/segments/{segment_id} - Get segment details +# - PATCH /datasets/{id}/segments/{segment_id} - Update segment +# - DELETE /datasets/{id}/segments/{segment_id} - Delete segment +# +# 3. Dataset Update/Delete Endpoints: +# - PATCH /datasets/{id} - Update dataset +# - DELETE /datasets/{id} - Delete dataset +# +# 4. Advanced Scenarios: +# - File upload handling +# - Large payload handling +# - Concurrent request handling +# - Rate limiting +# - CORS headers +# +# These scenarios are not currently implemented but could be added if needed +# based on real-world usage patterns or discovered edge cases. +# +# ============================================================================ + + +# ============================================================================ +# API Testing Best Practices +# ============================================================================ +# +# When writing API tests, consider the following best practices: +# +# 1. Test Structure: +# - Use descriptive test names that explain what is being tested +# - Follow Arrange-Act-Assert pattern +# - Keep tests focused on a single scenario +# - Use fixtures for common setup +# +# 2. Mocking Strategy: +# - Mock external dependencies (database, services, etc.) +# - Mock authentication and authorization +# - Use realistic mock data +# - Verify mock calls to ensure correct integration +# +# 3. Assertions: +# - Verify HTTP status codes +# - Verify response structure +# - Verify response data values +# - Verify service method calls +# - Verify error messages when appropriate +# +# 4. Error Testing: +# - Test all error paths (400, 401, 403, 404, 500) +# - Test validation errors +# - Test authentication failures +# - Test authorization failures +# - Test not found scenarios +# +# 5. Edge Cases: +# - Test with empty data +# - Test with missing required fields +# - Test with invalid data types +# - Test with boundary values +# - Test with special characters +# +# ============================================================================ + + +# ============================================================================ +# Flask-RESTX Resource Testing Patterns +# ============================================================================ +# +# Flask-RESTX resources are tested using Flask's test client. The typical +# pattern involves: +# +# 1. Creating a Flask test application +# 2. Creating a Flask-RESTX API instance +# 3. Registering the resource with a route +# 4. Creating a test client +# 5. Making HTTP requests through the test client +# 6. Asserting on the response +# +# Example pattern: +# +# app = Flask(__name__) +# app.config["TESTING"] = True +# api = Api(app) +# api.add_resource(MyResource, "/my-endpoint") +# client = app.test_client() +# response = client.get("/my-endpoint") +# assert response.status_code == 200 +# +# Decorators on resources (like @login_required) need to be mocked or +# bypassed in tests. This is typically done by mocking the decorator +# functions or the authentication functions they call. +# +# ============================================================================ + + +# ============================================================================ +# Request/Response Validation +# ============================================================================ +# +# API endpoints use Flask-RESTX request parsers to validate incoming requests. +# These parsers: +# +# 1. Extract parameters from query strings, form data, or JSON body +# 2. Validate parameter types (string, integer, float, boolean, etc.) +# 3. Validate parameter ranges and constraints +# 4. Provide default values when parameters are missing +# 5. Raise BadRequest exceptions when validation fails +# +# Response formatting is handled by Flask-RESTX's marshal_with decorator +# or marshal function, which: +# +# 1. Formats response data according to defined models +# 2. Handles nested objects and lists +# 3. Filters out fields not in the model +# 4. Provides consistent response structure +# +# Tests should verify: +# - Request validation works correctly +# - Invalid requests return 400 Bad Request +# - Response structure matches the defined model +# - Response data values are correct +# +# ============================================================================ + + +# ============================================================================ +# Authentication and Authorization Testing +# ============================================================================ +# +# Most API endpoints require authentication and authorization. Testing these +# aspects involves: +# +# 1. Authentication Testing: +# - Test that unauthenticated requests are rejected (401) +# - Test that authenticated requests are accepted +# - Mock the authentication decorators/functions +# - Verify user context is passed correctly +# +# 2. Authorization Testing: +# - Test that unauthorized requests are rejected (403) +# - Test that authorized requests are accepted +# - Test different user roles and permissions +# - Verify permission checks are performed +# +# 3. Common Patterns: +# - Mock current_account_with_tenant() to return test user +# - Mock permission check functions +# - Test with different user roles (admin, editor, operator, etc.) +# - Test with different permission levels (only_me, all_team, etc.) +# +# ============================================================================ + + +# ============================================================================ +# Error Handling in API Tests +# ============================================================================ +# +# API endpoints should handle errors gracefully and return appropriate HTTP +# status codes. Testing error handling involves: +# +# 1. Service Exception Mapping: +# - ValueError -> 400 Bad Request +# - NotFound -> 404 Not Found +# - Forbidden -> 403 Forbidden +# - Unauthorized -> 401 Unauthorized +# - Internal errors -> 500 Internal Server Error +# +# 2. Validation Error Testing: +# - Test missing required parameters +# - Test invalid parameter types +# - Test parameter range violations +# - Test custom validation rules +# +# 3. Error Response Structure: +# - Verify error status code +# - Verify error message is included +# - Verify error structure is consistent +# - Verify error details are helpful +# +# ============================================================================ + + +# ============================================================================ +# Performance and Scalability Considerations +# ============================================================================ +# +# While unit tests focus on correctness, API tests should also consider: +# +# 1. Response Time: +# - Tests should complete quickly +# - Avoid actual database or network calls +# - Use mocks for slow operations +# +# 2. Resource Usage: +# - Tests should not consume excessive memory +# - Tests should clean up after themselves +# - Use fixtures for resource management +# +# 3. Test Isolation: +# - Tests should not depend on each other +# - Tests should not share state +# - Each test should be independently runnable +# +# 4. Maintainability: +# - Tests should be easy to understand +# - Tests should be easy to modify +# - Use descriptive names and comments +# - Follow consistent patterns +# +# ============================================================================ diff --git a/api/tests/unit_tests/services/dataset_collection_binding.py b/api/tests/unit_tests/services/dataset_collection_binding.py new file mode 100644 index 0000000000..2a939a5c1d --- /dev/null +++ b/api/tests/unit_tests/services/dataset_collection_binding.py @@ -0,0 +1,932 @@ +""" +Comprehensive unit tests for DatasetCollectionBindingService. + +This module contains extensive unit tests for the DatasetCollectionBindingService class, +which handles dataset collection binding operations for vector database collections. + +The DatasetCollectionBindingService provides methods for: +- Retrieving or creating dataset collection bindings by provider, model, and type +- Retrieving specific collection bindings by ID and type +- Managing collection bindings for different collection types (dataset, etc.) + +Collection bindings are used to map embedding models (provider + model name) to +specific vector database collections, allowing datasets to share collections when +they use the same embedding model configuration. + +This test suite ensures: +- Correct retrieval of existing bindings +- Proper creation of new bindings when they don't exist +- Accurate filtering by provider, model, and collection type +- Proper error handling for missing bindings +- Database transaction handling (add, commit) +- Collection name generation using Dataset.gen_collection_name_by_id + +================================================================================ +ARCHITECTURE OVERVIEW +================================================================================ + +The DatasetCollectionBindingService is a critical component in the Dify platform's +vector database management system. It serves as an abstraction layer between the +application logic and the underlying vector database collections. + +Key Concepts: +1. Collection Binding: A mapping between an embedding model configuration + (provider + model name) and a vector database collection name. This allows + multiple datasets to share the same collection when they use identical + embedding models, improving resource efficiency. + +2. Collection Type: Different types of collections can exist (e.g., "dataset", + "custom_type"). This allows for separation of collections based on their + intended use case or data structure. + +3. Provider and Model: The combination of provider_name (e.g., "openai", + "cohere", "huggingface") and model_name (e.g., "text-embedding-ada-002") + uniquely identifies an embedding model configuration. + +4. Collection Name Generation: When a new binding is created, a unique collection + name is generated using Dataset.gen_collection_name_by_id() with a UUID. + This ensures each binding has a unique collection identifier. + +================================================================================ +TESTING STRATEGY +================================================================================ + +This test suite follows a comprehensive testing strategy that covers: + +1. Happy Path Scenarios: + - Successful retrieval of existing bindings + - Successful creation of new bindings + - Proper handling of default parameters + +2. Edge Cases: + - Different collection types + - Various provider/model combinations + - Default vs explicit parameter usage + +3. Error Handling: + - Missing bindings (for get_by_id_and_type) + - Database query failures + - Invalid parameter combinations + +4. Database Interaction: + - Query construction and execution + - Transaction management (add, commit) + - Query chaining (where, order_by, first) + +5. Mocking Strategy: + - Database session mocking + - Query builder chain mocking + - UUID generation mocking + - Collection name generation mocking + +================================================================================ +""" + +""" +Import statements for the test module. + +This section imports all necessary dependencies for testing the +DatasetCollectionBindingService, including: +- unittest.mock for creating mock objects +- pytest for test framework functionality +- uuid for UUID generation (used in collection name generation) +- Models and services from the application codebase +""" + +from unittest.mock import Mock, patch + +import pytest + +from models.dataset import Dataset, DatasetCollectionBinding +from services.dataset_service import DatasetCollectionBindingService + +# ============================================================================ +# Test Data Factory +# ============================================================================ +# The Test Data Factory pattern is used here to centralize the creation of +# test objects and mock instances. This approach provides several benefits: +# +# 1. Consistency: All test objects are created using the same factory methods, +# ensuring consistent structure across all tests. +# +# 2. Maintainability: If the structure of DatasetCollectionBinding or Dataset +# changes, we only need to update the factory methods rather than every +# individual test. +# +# 3. Reusability: Factory methods can be reused across multiple test classes, +# reducing code duplication. +# +# 4. Readability: Tests become more readable when they use descriptive factory +# method calls instead of complex object construction logic. +# +# ============================================================================ + + +class DatasetCollectionBindingTestDataFactory: + """ + Factory class for creating test data and mock objects for dataset collection binding tests. + + This factory provides static methods to create mock objects for: + - DatasetCollectionBinding instances + - Database query results + - Collection name generation results + + The factory methods help maintain consistency across tests and reduce + code duplication when setting up test scenarios. + """ + + @staticmethod + def create_collection_binding_mock( + binding_id: str = "binding-123", + provider_name: str = "openai", + model_name: str = "text-embedding-ada-002", + collection_name: str = "collection-abc", + collection_type: str = "dataset", + created_at=None, + **kwargs, + ) -> Mock: + """ + Create a mock DatasetCollectionBinding with specified attributes. + + Args: + binding_id: Unique identifier for the binding + provider_name: Name of the embedding model provider (e.g., "openai", "cohere") + model_name: Name of the embedding model (e.g., "text-embedding-ada-002") + collection_name: Name of the vector database collection + collection_type: Type of collection (default: "dataset") + created_at: Optional datetime for creation timestamp + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a DatasetCollectionBinding instance + """ + binding = Mock(spec=DatasetCollectionBinding) + binding.id = binding_id + binding.provider_name = provider_name + binding.model_name = model_name + binding.collection_name = collection_name + binding.type = collection_type + binding.created_at = created_at + for key, value in kwargs.items(): + setattr(binding, key, value) + return binding + + @staticmethod + def create_dataset_mock( + dataset_id: str = "dataset-123", + **kwargs, + ) -> Mock: + """ + Create a mock Dataset for testing collection name generation. + + Args: + dataset_id: Unique identifier for the dataset + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a Dataset instance + """ + dataset = Mock(spec=Dataset) + dataset.id = dataset_id + for key, value in kwargs.items(): + setattr(dataset, key, value) + return dataset + + +# ============================================================================ +# Tests for get_dataset_collection_binding +# ============================================================================ + + +class TestDatasetCollectionBindingServiceGetBinding: + """ + Comprehensive unit tests for DatasetCollectionBindingService.get_dataset_collection_binding method. + + This test class covers the main collection binding retrieval/creation functionality, + including various provider/model combinations, collection types, and edge cases. + + The get_dataset_collection_binding method: + 1. Queries for existing binding by provider_name, model_name, and collection_type + 2. Orders results by created_at (ascending) and takes the first match + 3. If no binding exists, creates a new one with: + - The provided provider_name and model_name + - A generated collection_name using Dataset.gen_collection_name_by_id + - The provided collection_type + 4. Adds the new binding to the database session and commits + 5. Returns the binding (either existing or newly created) + + Test scenarios include: + - Retrieving existing bindings + - Creating new bindings when none exist + - Different collection types + - Database transaction handling + - Collection name generation + """ + + @pytest.fixture + def mock_db_session(self): + """ + Mock database session for testing database operations. + + Provides a mocked database session that can be used to verify: + - Query construction and execution + - Add operations for new bindings + - Commit operations for transaction completion + + The mock is configured to return a query builder that supports + chaining operations like .where(), .order_by(), and .first(). + """ + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + def test_get_dataset_collection_binding_existing_binding_success(self, mock_db_session): + """ + Test successful retrieval of an existing collection binding. + + Verifies that when a binding already exists in the database for the given + provider, model, and collection type, the method returns the existing binding + without creating a new one. + + This test ensures: + - The query is constructed correctly with all three filters + - Results are ordered by created_at + - The first matching binding is returned + - No new binding is created (db.session.add is not called) + - No commit is performed (db.session.commit is not called) + """ + # Arrange + provider_name = "openai" + model_name = "text-embedding-ada-002" + collection_type = "dataset" + + existing_binding = DatasetCollectionBindingTestDataFactory.create_collection_binding_mock( + binding_id="binding-123", + provider_name=provider_name, + model_name=model_name, + collection_type=collection_type, + ) + + # Mock the query chain: query().where().order_by().first() + mock_query = Mock() + mock_where = Mock() + mock_order_by = Mock() + mock_query.where.return_value = mock_where + mock_where.order_by.return_value = mock_order_by + mock_order_by.first.return_value = existing_binding + mock_db_session.query.return_value = mock_query + + # Act + result = DatasetCollectionBindingService.get_dataset_collection_binding( + provider_name=provider_name, model_name=model_name, collection_type=collection_type + ) + + # Assert + assert result == existing_binding + assert result.id == "binding-123" + assert result.provider_name == provider_name + assert result.model_name == model_name + assert result.type == collection_type + + # Verify query was constructed correctly + # The query should be constructed with DatasetCollectionBinding as the model + mock_db_session.query.assert_called_once_with(DatasetCollectionBinding) + + # Verify the where clause was applied to filter by provider, model, and type + mock_query.where.assert_called_once() + + # Verify the results were ordered by created_at (ascending) + # This ensures we get the oldest binding if multiple exist + mock_where.order_by.assert_called_once() + + # Verify no new binding was created + # Since an existing binding was found, we should not create a new one + mock_db_session.add.assert_not_called() + + # Verify no commit was performed + # Since no new binding was created, no database transaction is needed + mock_db_session.commit.assert_not_called() + + def test_get_dataset_collection_binding_create_new_binding_success(self, mock_db_session): + """ + Test successful creation of a new collection binding when none exists. + + Verifies that when no binding exists in the database for the given + provider, model, and collection type, the method creates a new binding + with a generated collection name and commits it to the database. + + This test ensures: + - The query returns None (no existing binding) + - A new DatasetCollectionBinding is created with correct attributes + - Dataset.gen_collection_name_by_id is called to generate collection name + - The new binding is added to the database session + - The transaction is committed + - The newly created binding is returned + """ + # Arrange + provider_name = "cohere" + model_name = "embed-english-v3.0" + collection_type = "dataset" + generated_collection_name = "collection-generated-xyz" + + # Mock the query chain to return None (no existing binding) + mock_query = Mock() + mock_where = Mock() + mock_order_by = Mock() + mock_query.where.return_value = mock_where + mock_where.order_by.return_value = mock_order_by + mock_order_by.first.return_value = None # No existing binding + mock_db_session.query.return_value = mock_query + + # Mock Dataset.gen_collection_name_by_id to return a generated name + with patch("services.dataset_service.Dataset.gen_collection_name_by_id") as mock_gen_name: + mock_gen_name.return_value = generated_collection_name + + # Mock uuid.uuid4 for the collection name generation + mock_uuid = "test-uuid-123" + with patch("services.dataset_service.uuid.uuid4", return_value=mock_uuid): + # Act + result = DatasetCollectionBindingService.get_dataset_collection_binding( + provider_name=provider_name, model_name=model_name, collection_type=collection_type + ) + + # Assert + assert result is not None + assert result.provider_name == provider_name + assert result.model_name == model_name + assert result.type == collection_type + assert result.collection_name == generated_collection_name + + # Verify Dataset.gen_collection_name_by_id was called with the generated UUID + # This method generates a unique collection name based on the UUID + # The UUID is converted to string before passing to the method + mock_gen_name.assert_called_once_with(str(mock_uuid)) + + # Verify new binding was added to the database session + # The add method should be called exactly once with the new binding instance + mock_db_session.add.assert_called_once() + + # Extract the binding that was added to verify its properties + added_binding = mock_db_session.add.call_args[0][0] + + # Verify the added binding is an instance of DatasetCollectionBinding + # This ensures we're creating the correct type of object + assert isinstance(added_binding, DatasetCollectionBinding) + + # Verify all the binding properties are set correctly + # These should match the input parameters to the method + assert added_binding.provider_name == provider_name + assert added_binding.model_name == model_name + assert added_binding.type == collection_type + + # Verify the collection name was set from the generated name + # This ensures the binding has a valid collection identifier + assert added_binding.collection_name == generated_collection_name + + # Verify the transaction was committed + # This ensures the new binding is persisted to the database + mock_db_session.commit.assert_called_once() + + def test_get_dataset_collection_binding_different_collection_type(self, mock_db_session): + """ + Test retrieval with a different collection type (not "dataset"). + + Verifies that the method correctly filters by collection_type, allowing + different types of collections to coexist with the same provider/model + combination. + + This test ensures: + - Collection type is properly used as a filter in the query + - Different collection types can have separate bindings + - The correct binding is returned based on type + """ + # Arrange + provider_name = "openai" + model_name = "text-embedding-ada-002" + collection_type = "custom_type" + + existing_binding = DatasetCollectionBindingTestDataFactory.create_collection_binding_mock( + binding_id="binding-456", + provider_name=provider_name, + model_name=model_name, + collection_type=collection_type, + ) + + # Mock the query chain + mock_query = Mock() + mock_where = Mock() + mock_order_by = Mock() + mock_query.where.return_value = mock_where + mock_where.order_by.return_value = mock_order_by + mock_order_by.first.return_value = existing_binding + mock_db_session.query.return_value = mock_query + + # Act + result = DatasetCollectionBindingService.get_dataset_collection_binding( + provider_name=provider_name, model_name=model_name, collection_type=collection_type + ) + + # Assert + assert result == existing_binding + assert result.type == collection_type + + # Verify query was constructed with the correct type filter + mock_db_session.query.assert_called_once_with(DatasetCollectionBinding) + mock_query.where.assert_called_once() + + def test_get_dataset_collection_binding_default_collection_type(self, mock_db_session): + """ + Test retrieval with default collection type ("dataset"). + + Verifies that when collection_type is not provided, it defaults to "dataset" + as specified in the method signature. + + This test ensures: + - The default value "dataset" is used when type is not specified + - The query correctly filters by the default type + """ + # Arrange + provider_name = "openai" + model_name = "text-embedding-ada-002" + # collection_type defaults to "dataset" in method signature + + existing_binding = DatasetCollectionBindingTestDataFactory.create_collection_binding_mock( + binding_id="binding-789", + provider_name=provider_name, + model_name=model_name, + collection_type="dataset", # Default type + ) + + # Mock the query chain + mock_query = Mock() + mock_where = Mock() + mock_order_by = Mock() + mock_query.where.return_value = mock_where + mock_where.order_by.return_value = mock_order_by + mock_order_by.first.return_value = existing_binding + mock_db_session.query.return_value = mock_query + + # Act - call without specifying collection_type (uses default) + result = DatasetCollectionBindingService.get_dataset_collection_binding( + provider_name=provider_name, model_name=model_name + ) + + # Assert + assert result == existing_binding + assert result.type == "dataset" + + # Verify query was constructed correctly + mock_db_session.query.assert_called_once_with(DatasetCollectionBinding) + + def test_get_dataset_collection_binding_different_provider_model_combination(self, mock_db_session): + """ + Test retrieval with different provider/model combinations. + + Verifies that bindings are correctly filtered by both provider_name and + model_name, ensuring that different model combinations have separate bindings. + + This test ensures: + - Provider and model are both used as filters + - Different combinations result in different bindings + - The correct binding is returned for each combination + """ + # Arrange + provider_name = "huggingface" + model_name = "sentence-transformers/all-MiniLM-L6-v2" + collection_type = "dataset" + + existing_binding = DatasetCollectionBindingTestDataFactory.create_collection_binding_mock( + binding_id="binding-hf-123", + provider_name=provider_name, + model_name=model_name, + collection_type=collection_type, + ) + + # Mock the query chain + mock_query = Mock() + mock_where = Mock() + mock_order_by = Mock() + mock_query.where.return_value = mock_where + mock_where.order_by.return_value = mock_order_by + mock_order_by.first.return_value = existing_binding + mock_db_session.query.return_value = mock_query + + # Act + result = DatasetCollectionBindingService.get_dataset_collection_binding( + provider_name=provider_name, model_name=model_name, collection_type=collection_type + ) + + # Assert + assert result == existing_binding + assert result.provider_name == provider_name + assert result.model_name == model_name + + # Verify query filters were applied correctly + # The query should filter by both provider_name and model_name + # This ensures different model combinations have separate bindings + mock_db_session.query.assert_called_once_with(DatasetCollectionBinding) + + # Verify the where clause was applied with all three filters: + # - provider_name filter + # - model_name filter + # - collection_type filter + mock_query.where.assert_called_once() + + +# ============================================================================ +# Tests for get_dataset_collection_binding_by_id_and_type +# ============================================================================ +# This section contains tests for the get_dataset_collection_binding_by_id_and_type +# method, which retrieves a specific collection binding by its ID and type. +# +# Key differences from get_dataset_collection_binding: +# 1. This method queries by ID and type, not by provider/model/type +# 2. This method does NOT create a new binding if one doesn't exist +# 3. This method raises ValueError if the binding is not found +# 4. This method is typically used when you already know the binding ID +# +# Use cases: +# - Retrieving a binding that was previously created +# - Validating that a binding exists before using it +# - Accessing binding metadata when you have the ID +# +# ============================================================================ + + +class TestDatasetCollectionBindingServiceGetBindingByIdAndType: + """ + Comprehensive unit tests for DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type method. + + This test class covers collection binding retrieval by ID and type, + including success scenarios and error handling for missing bindings. + + The get_dataset_collection_binding_by_id_and_type method: + 1. Queries for a binding by collection_binding_id and collection_type + 2. Orders results by created_at (ascending) and takes the first match + 3. If no binding exists, raises ValueError("Dataset collection binding not found") + 4. Returns the found binding + + Unlike get_dataset_collection_binding, this method does NOT create a new + binding if one doesn't exist - it only retrieves existing bindings. + + Test scenarios include: + - Successful retrieval of existing bindings + - Error handling for missing bindings + - Different collection types + - Default collection type behavior + """ + + @pytest.fixture + def mock_db_session(self): + """ + Mock database session for testing database operations. + + Provides a mocked database session that can be used to verify: + - Query construction with ID and type filters + - Ordering by created_at + - First result retrieval + + The mock is configured to return a query builder that supports + chaining operations like .where(), .order_by(), and .first(). + """ + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + def test_get_dataset_collection_binding_by_id_and_type_success(self, mock_db_session): + """ + Test successful retrieval of a collection binding by ID and type. + + Verifies that when a binding exists in the database with the given + ID and collection type, the method returns the binding. + + This test ensures: + - The query is constructed correctly with ID and type filters + - Results are ordered by created_at + - The first matching binding is returned + - No error is raised + """ + # Arrange + collection_binding_id = "binding-123" + collection_type = "dataset" + + existing_binding = DatasetCollectionBindingTestDataFactory.create_collection_binding_mock( + binding_id=collection_binding_id, + provider_name="openai", + model_name="text-embedding-ada-002", + collection_type=collection_type, + ) + + # Mock the query chain: query().where().order_by().first() + mock_query = Mock() + mock_where = Mock() + mock_order_by = Mock() + mock_query.where.return_value = mock_where + mock_where.order_by.return_value = mock_order_by + mock_order_by.first.return_value = existing_binding + mock_db_session.query.return_value = mock_query + + # Act + result = DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + collection_binding_id=collection_binding_id, collection_type=collection_type + ) + + # Assert + assert result == existing_binding + assert result.id == collection_binding_id + assert result.type == collection_type + + # Verify query was constructed correctly + mock_db_session.query.assert_called_once_with(DatasetCollectionBinding) + mock_query.where.assert_called_once() + mock_where.order_by.assert_called_once() + + def test_get_dataset_collection_binding_by_id_and_type_not_found_error(self, mock_db_session): + """ + Test error handling when binding is not found. + + Verifies that when no binding exists in the database with the given + ID and collection type, the method raises a ValueError with the + message "Dataset collection binding not found". + + This test ensures: + - The query returns None (no existing binding) + - ValueError is raised with the correct message + - No binding is returned + """ + # Arrange + collection_binding_id = "non-existent-binding" + collection_type = "dataset" + + # Mock the query chain to return None (no existing binding) + mock_query = Mock() + mock_where = Mock() + mock_order_by = Mock() + mock_query.where.return_value = mock_where + mock_where.order_by.return_value = mock_order_by + mock_order_by.first.return_value = None # No existing binding + mock_db_session.query.return_value = mock_query + + # Act & Assert + with pytest.raises(ValueError, match="Dataset collection binding not found"): + DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + collection_binding_id=collection_binding_id, collection_type=collection_type + ) + + # Verify query was attempted + mock_db_session.query.assert_called_once_with(DatasetCollectionBinding) + mock_query.where.assert_called_once() + + def test_get_dataset_collection_binding_by_id_and_type_different_collection_type(self, mock_db_session): + """ + Test retrieval with a different collection type. + + Verifies that the method correctly filters by collection_type, ensuring + that bindings with the same ID but different types are treated as + separate entities. + + This test ensures: + - Collection type is properly used as a filter in the query + - Different collection types can have separate bindings with same ID + - The correct binding is returned based on type + """ + # Arrange + collection_binding_id = "binding-456" + collection_type = "custom_type" + + existing_binding = DatasetCollectionBindingTestDataFactory.create_collection_binding_mock( + binding_id=collection_binding_id, + provider_name="cohere", + model_name="embed-english-v3.0", + collection_type=collection_type, + ) + + # Mock the query chain + mock_query = Mock() + mock_where = Mock() + mock_order_by = Mock() + mock_query.where.return_value = mock_where + mock_where.order_by.return_value = mock_order_by + mock_order_by.first.return_value = existing_binding + mock_db_session.query.return_value = mock_query + + # Act + result = DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + collection_binding_id=collection_binding_id, collection_type=collection_type + ) + + # Assert + assert result == existing_binding + assert result.id == collection_binding_id + assert result.type == collection_type + + # Verify query was constructed with the correct type filter + mock_db_session.query.assert_called_once_with(DatasetCollectionBinding) + mock_query.where.assert_called_once() + + def test_get_dataset_collection_binding_by_id_and_type_default_collection_type(self, mock_db_session): + """ + Test retrieval with default collection type ("dataset"). + + Verifies that when collection_type is not provided, it defaults to "dataset" + as specified in the method signature. + + This test ensures: + - The default value "dataset" is used when type is not specified + - The query correctly filters by the default type + - The correct binding is returned + """ + # Arrange + collection_binding_id = "binding-789" + # collection_type defaults to "dataset" in method signature + + existing_binding = DatasetCollectionBindingTestDataFactory.create_collection_binding_mock( + binding_id=collection_binding_id, + provider_name="openai", + model_name="text-embedding-ada-002", + collection_type="dataset", # Default type + ) + + # Mock the query chain + mock_query = Mock() + mock_where = Mock() + mock_order_by = Mock() + mock_query.where.return_value = mock_where + mock_where.order_by.return_value = mock_order_by + mock_order_by.first.return_value = existing_binding + mock_db_session.query.return_value = mock_query + + # Act - call without specifying collection_type (uses default) + result = DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + collection_binding_id=collection_binding_id + ) + + # Assert + assert result == existing_binding + assert result.id == collection_binding_id + assert result.type == "dataset" + + # Verify query was constructed correctly + mock_db_session.query.assert_called_once_with(DatasetCollectionBinding) + mock_query.where.assert_called_once() + + def test_get_dataset_collection_binding_by_id_and_type_wrong_type_error(self, mock_db_session): + """ + Test error handling when binding exists but with wrong collection type. + + Verifies that when a binding exists with the given ID but a different + collection type, the method raises a ValueError because the binding + doesn't match both the ID and type criteria. + + This test ensures: + - The query correctly filters by both ID and type + - Bindings with matching ID but different type are not returned + - ValueError is raised when no matching binding is found + """ + # Arrange + collection_binding_id = "binding-123" + collection_type = "dataset" + + # Mock the query chain to return None (binding exists but with different type) + mock_query = Mock() + mock_where = Mock() + mock_order_by = Mock() + mock_query.where.return_value = mock_where + mock_where.order_by.return_value = mock_order_by + mock_order_by.first.return_value = None # No matching binding + mock_db_session.query.return_value = mock_query + + # Act & Assert + with pytest.raises(ValueError, match="Dataset collection binding not found"): + DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + collection_binding_id=collection_binding_id, collection_type=collection_type + ) + + # Verify query was attempted with both ID and type filters + # The query should filter by both collection_binding_id and collection_type + # This ensures we only get bindings that match both criteria + mock_db_session.query.assert_called_once_with(DatasetCollectionBinding) + + # Verify the where clause was applied with both filters: + # - collection_binding_id filter (exact match) + # - collection_type filter (exact match) + mock_query.where.assert_called_once() + + # Note: The order_by and first() calls are also part of the query chain, + # but we don't need to verify them separately since they're part of the + # standard query pattern used by both methods in this service. + + +# ============================================================================ +# Additional Test Scenarios and Edge Cases +# ============================================================================ +# The following section could contain additional test scenarios if needed: +# +# Potential additional tests: +# 1. Test with multiple existing bindings (verify ordering by created_at) +# 2. Test with very long provider/model names (boundary testing) +# 3. Test with special characters in provider/model names +# 4. Test concurrent binding creation (thread safety) +# 5. Test database rollback scenarios +# 6. Test with None values for optional parameters +# 7. Test with empty strings for required parameters +# 8. Test collection name generation uniqueness +# 9. Test with different UUID formats +# 10. Test query performance with large datasets +# +# These scenarios are not currently implemented but could be added if needed +# based on real-world usage patterns or discovered edge cases. +# +# ============================================================================ + + +# ============================================================================ +# Integration Notes and Best Practices +# ============================================================================ +# +# When using DatasetCollectionBindingService in production code, consider: +# +# 1. Error Handling: +# - Always handle ValueError exceptions when calling +# get_dataset_collection_binding_by_id_and_type +# - Check return values from get_dataset_collection_binding to ensure +# bindings were created successfully +# +# 2. Performance Considerations: +# - The service queries the database on every call, so consider caching +# bindings if they're accessed frequently +# - Collection bindings are typically long-lived, so caching is safe +# +# 3. Transaction Management: +# - New bindings are automatically committed to the database +# - If you need to rollback, ensure you're within a transaction context +# +# 4. Collection Type Usage: +# - Use "dataset" for standard dataset collections +# - Use custom types only when you need to separate collections by purpose +# - Be consistent with collection type naming across your application +# +# 5. Provider and Model Naming: +# - Use consistent provider names (e.g., "openai", not "OpenAI" or "OPENAI") +# - Use exact model names as provided by the model provider +# - These names are case-sensitive and must match exactly +# +# ============================================================================ + + +# ============================================================================ +# Database Schema Reference +# ============================================================================ +# +# The DatasetCollectionBinding model has the following structure: +# +# - id: StringUUID (primary key, auto-generated) +# - provider_name: String(255) (required, e.g., "openai", "cohere") +# - model_name: String(255) (required, e.g., "text-embedding-ada-002") +# - type: String(40) (required, default: "dataset") +# - collection_name: String(64) (required, unique collection identifier) +# - created_at: DateTime (auto-generated timestamp) +# +# Indexes: +# - Primary key on id +# - Composite index on (provider_name, model_name) for efficient lookups +# +# Relationships: +# - One binding can be referenced by multiple datasets +# - Datasets reference bindings via collection_binding_id +# +# ============================================================================ + + +# ============================================================================ +# Mocking Strategy Documentation +# ============================================================================ +# +# This test suite uses extensive mocking to isolate the unit under test. +# Here's how the mocking strategy works: +# +# 1. Database Session Mocking: +# - db.session is patched to prevent actual database access +# - Query chains are mocked to return predictable results +# - Add and commit operations are tracked for verification +# +# 2. Query Chain Mocking: +# - query() returns a mock query object +# - where() returns a mock where object +# - order_by() returns a mock order_by object +# - first() returns the final result (binding or None) +# +# 3. UUID Generation Mocking: +# - uuid.uuid4() is mocked to return predictable UUIDs +# - This ensures collection names are generated consistently in tests +# +# 4. Collection Name Generation Mocking: +# - Dataset.gen_collection_name_by_id() is mocked +# - This allows us to verify the method is called correctly +# - We can control the generated collection name for testing +# +# Benefits of this approach: +# - Tests run quickly (no database I/O) +# - Tests are deterministic (no random UUIDs) +# - Tests are isolated (no side effects) +# - Tests are maintainable (clear mock setup) +# +# ============================================================================ diff --git a/api/tests/unit_tests/services/dataset_metadata.py b/api/tests/unit_tests/services/dataset_metadata.py new file mode 100644 index 0000000000..5ba18d8dc0 --- /dev/null +++ b/api/tests/unit_tests/services/dataset_metadata.py @@ -0,0 +1,1068 @@ +""" +Comprehensive unit tests for MetadataService. + +This module contains extensive unit tests for the MetadataService class, +which handles dataset metadata CRUD operations and filtering/querying functionality. + +The MetadataService provides methods for: +- Creating, reading, updating, and deleting metadata fields +- Managing built-in metadata fields +- Updating document metadata values +- Metadata filtering and querying operations +- Lock management for concurrent metadata operations + +Metadata in Dify allows users to add custom fields to datasets and documents, +enabling rich filtering and search capabilities. Metadata can be of various +types (string, number, date, boolean, etc.) and can be used to categorize +and filter documents within a dataset. + +This test suite ensures: +- Correct creation of metadata fields with validation +- Proper updating of metadata names and values +- Accurate deletion of metadata fields +- Built-in field management (enable/disable) +- Document metadata updates (partial and full) +- Lock management for concurrent operations +- Metadata querying and filtering functionality + +================================================================================ +ARCHITECTURE OVERVIEW +================================================================================ + +The MetadataService is a critical component in the Dify platform's metadata +management system. It serves as the primary interface for all metadata-related +operations, including field definitions and document-level metadata values. + +Key Concepts: +1. DatasetMetadata: Defines a metadata field for a dataset. Each metadata + field has a name, type, and is associated with a specific dataset. + +2. DatasetMetadataBinding: Links metadata fields to documents. This allows + tracking which documents have which metadata fields assigned. + +3. Document Metadata: The actual metadata values stored on documents. This + is stored as a JSON object in the document's doc_metadata field. + +4. Built-in Fields: System-defined metadata fields that are automatically + available when enabled (document_name, uploader, upload_date, etc.). + +5. Lock Management: Redis-based locking to prevent concurrent metadata + operations that could cause data corruption. + +================================================================================ +TESTING STRATEGY +================================================================================ + +This test suite follows a comprehensive testing strategy that covers: + +1. CRUD Operations: + - Creating metadata fields with validation + - Reading/retrieving metadata fields + - Updating metadata field names + - Deleting metadata fields + +2. Built-in Field Management: + - Enabling built-in fields + - Disabling built-in fields + - Getting built-in field definitions + +3. Document Metadata Operations: + - Updating document metadata (partial and full) + - Managing metadata bindings + - Handling built-in field updates + +4. Lock Management: + - Acquiring locks for dataset operations + - Acquiring locks for document operations + - Handling lock conflicts + +5. Error Handling: + - Validation errors (name length, duplicates) + - Not found errors + - Lock conflict errors + +================================================================================ +""" + +from unittest.mock import Mock, patch + +import pytest + +from core.rag.index_processor.constant.built_in_field import BuiltInField +from models.dataset import Dataset, DatasetMetadata, DatasetMetadataBinding +from services.entities.knowledge_entities.knowledge_entities import ( + MetadataArgs, + MetadataValue, +) +from services.metadata_service import MetadataService + +# ============================================================================ +# Test Data Factory +# ============================================================================ +# The Test Data Factory pattern is used here to centralize the creation of +# test objects and mock instances. This approach provides several benefits: +# +# 1. Consistency: All test objects are created using the same factory methods, +# ensuring consistent structure across all tests. +# +# 2. Maintainability: If the structure of models changes, we only need to +# update the factory methods rather than every individual test. +# +# 3. Reusability: Factory methods can be reused across multiple test classes, +# reducing code duplication. +# +# 4. Readability: Tests become more readable when they use descriptive factory +# method calls instead of complex object construction logic. +# +# ============================================================================ + + +class MetadataTestDataFactory: + """ + Factory class for creating test data and mock objects for metadata service tests. + + This factory provides static methods to create mock objects for: + - DatasetMetadata instances + - DatasetMetadataBinding instances + - Dataset instances + - Document instances + - MetadataArgs and MetadataOperationData entities + - User and tenant context + + The factory methods help maintain consistency across tests and reduce + code duplication when setting up test scenarios. + """ + + @staticmethod + def create_metadata_mock( + metadata_id: str = "metadata-123", + dataset_id: str = "dataset-123", + tenant_id: str = "tenant-123", + name: str = "category", + metadata_type: str = "string", + created_by: str = "user-123", + **kwargs, + ) -> Mock: + """ + Create a mock DatasetMetadata with specified attributes. + + Args: + metadata_id: Unique identifier for the metadata field + dataset_id: ID of the dataset this metadata belongs to + tenant_id: Tenant identifier + name: Name of the metadata field + metadata_type: Type of metadata (string, number, date, etc.) + created_by: ID of the user who created the metadata + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a DatasetMetadata instance + """ + metadata = Mock(spec=DatasetMetadata) + metadata.id = metadata_id + metadata.dataset_id = dataset_id + metadata.tenant_id = tenant_id + metadata.name = name + metadata.type = metadata_type + metadata.created_by = created_by + metadata.updated_by = None + metadata.updated_at = None + for key, value in kwargs.items(): + setattr(metadata, key, value) + return metadata + + @staticmethod + def create_metadata_binding_mock( + binding_id: str = "binding-123", + dataset_id: str = "dataset-123", + tenant_id: str = "tenant-123", + metadata_id: str = "metadata-123", + document_id: str = "document-123", + created_by: str = "user-123", + **kwargs, + ) -> Mock: + """ + Create a mock DatasetMetadataBinding with specified attributes. + + Args: + binding_id: Unique identifier for the binding + dataset_id: ID of the dataset + tenant_id: Tenant identifier + metadata_id: ID of the metadata field + document_id: ID of the document + created_by: ID of the user who created the binding + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a DatasetMetadataBinding instance + """ + binding = Mock(spec=DatasetMetadataBinding) + binding.id = binding_id + binding.dataset_id = dataset_id + binding.tenant_id = tenant_id + binding.metadata_id = metadata_id + binding.document_id = document_id + binding.created_by = created_by + for key, value in kwargs.items(): + setattr(binding, key, value) + return binding + + @staticmethod + def create_dataset_mock( + dataset_id: str = "dataset-123", + tenant_id: str = "tenant-123", + built_in_field_enabled: bool = False, + doc_metadata: list | None = None, + **kwargs, + ) -> Mock: + """ + Create a mock Dataset with specified attributes. + + Args: + dataset_id: Unique identifier for the dataset + tenant_id: Tenant identifier + built_in_field_enabled: Whether built-in fields are enabled + doc_metadata: List of metadata field definitions + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a Dataset instance + """ + dataset = Mock(spec=Dataset) + dataset.id = dataset_id + dataset.tenant_id = tenant_id + dataset.built_in_field_enabled = built_in_field_enabled + dataset.doc_metadata = doc_metadata or [] + for key, value in kwargs.items(): + setattr(dataset, key, value) + return dataset + + @staticmethod + def create_document_mock( + document_id: str = "document-123", + dataset_id: str = "dataset-123", + name: str = "Test Document", + doc_metadata: dict | None = None, + uploader: str = "user-123", + data_source_type: str = "upload_file", + **kwargs, + ) -> Mock: + """ + Create a mock Document with specified attributes. + + Args: + document_id: Unique identifier for the document + dataset_id: ID of the dataset this document belongs to + name: Name of the document + doc_metadata: Dictionary of metadata values + uploader: ID of the user who uploaded the document + data_source_type: Type of data source + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a Document instance + """ + document = Mock() + document.id = document_id + document.dataset_id = dataset_id + document.name = name + document.doc_metadata = doc_metadata or {} + document.uploader = uploader + document.data_source_type = data_source_type + + # Mock datetime objects for upload_date and last_update_date + + document.upload_date = Mock() + document.upload_date.timestamp.return_value = 1234567890.0 + document.last_update_date = Mock() + document.last_update_date.timestamp.return_value = 1234567890.0 + + for key, value in kwargs.items(): + setattr(document, key, value) + return document + + @staticmethod + def create_metadata_args_mock( + name: str = "category", + metadata_type: str = "string", + ) -> Mock: + """ + Create a mock MetadataArgs entity. + + Args: + name: Name of the metadata field + metadata_type: Type of metadata + + Returns: + Mock object configured as a MetadataArgs instance + """ + metadata_args = Mock(spec=MetadataArgs) + metadata_args.name = name + metadata_args.type = metadata_type + return metadata_args + + @staticmethod + def create_metadata_value_mock( + metadata_id: str = "metadata-123", + name: str = "category", + value: str = "test", + ) -> Mock: + """ + Create a mock MetadataValue entity. + + Args: + metadata_id: ID of the metadata field + name: Name of the metadata field + value: Value of the metadata + + Returns: + Mock object configured as a MetadataValue instance + """ + metadata_value = Mock(spec=MetadataValue) + metadata_value.id = metadata_id + metadata_value.name = name + metadata_value.value = value + return metadata_value + + +# ============================================================================ +# Tests for create_metadata +# ============================================================================ + + +class TestMetadataServiceCreateMetadata: + """ + Comprehensive unit tests for MetadataService.create_metadata method. + + This test class covers the metadata field creation functionality, + including validation, duplicate checking, and database operations. + + The create_metadata method: + 1. Validates metadata name length (max 255 characters) + 2. Checks for duplicate metadata names within the dataset + 3. Checks for conflicts with built-in field names + 4. Creates a new DatasetMetadata instance + 5. Adds it to the database session and commits + 6. Returns the created metadata + + Test scenarios include: + - Successful creation with valid data + - Name length validation + - Duplicate name detection + - Built-in field name conflicts + - Database transaction handling + """ + + @pytest.fixture + def mock_db_session(self): + """ + Mock database session for testing database operations. + + Provides a mocked database session that can be used to verify: + - Query construction and execution + - Add operations for new metadata + - Commit operations for transaction completion + """ + with patch("services.metadata_service.db.session") as mock_db: + yield mock_db + + @pytest.fixture + def mock_current_user(self): + """ + Mock current user and tenant context. + + Provides mocked current_account_with_tenant function that returns + a user and tenant ID for testing authentication and authorization. + """ + with patch("services.metadata_service.current_account_with_tenant") as mock_get_user: + mock_user = Mock() + mock_user.id = "user-123" + mock_tenant_id = "tenant-123" + mock_get_user.return_value = (mock_user, mock_tenant_id) + yield mock_get_user + + def test_create_metadata_success(self, mock_db_session, mock_current_user): + """ + Test successful creation of a metadata field. + + Verifies that when all validation passes, a new metadata field + is created and persisted to the database. + + This test ensures: + - Metadata name validation passes + - No duplicate name exists + - No built-in field conflict + - New metadata is added to database + - Transaction is committed + - Created metadata is returned + """ + # Arrange + dataset_id = "dataset-123" + metadata_args = MetadataTestDataFactory.create_metadata_args_mock(name="category", metadata_type="string") + + # Mock query to return None (no existing metadata with same name) + mock_query = Mock() + mock_query.filter_by.return_value = mock_query + mock_query.first.return_value = None + mock_db_session.query.return_value = mock_query + + # Mock BuiltInField enum iteration + with patch("services.metadata_service.BuiltInField") as mock_builtin: + mock_builtin.__iter__ = Mock(return_value=iter([])) + + # Act + result = MetadataService.create_metadata(dataset_id, metadata_args) + + # Assert + assert result is not None + assert isinstance(result, DatasetMetadata) + + # Verify query was made to check for duplicates + mock_db_session.query.assert_called() + mock_query.filter_by.assert_called() + + # Verify metadata was added and committed + mock_db_session.add.assert_called_once() + mock_db_session.commit.assert_called_once() + + def test_create_metadata_name_too_long_error(self, mock_db_session, mock_current_user): + """ + Test error handling when metadata name exceeds 255 characters. + + Verifies that when a metadata name is longer than 255 characters, + a ValueError is raised with an appropriate message. + + This test ensures: + - Name length validation is enforced + - Error message is clear and descriptive + - No database operations are performed + """ + # Arrange + dataset_id = "dataset-123" + long_name = "a" * 256 # 256 characters (exceeds limit) + metadata_args = MetadataTestDataFactory.create_metadata_args_mock(name=long_name, metadata_type="string") + + # Act & Assert + with pytest.raises(ValueError, match="Metadata name cannot exceed 255 characters"): + MetadataService.create_metadata(dataset_id, metadata_args) + + # Verify no database operations were performed + mock_db_session.add.assert_not_called() + mock_db_session.commit.assert_not_called() + + def test_create_metadata_duplicate_name_error(self, mock_db_session, mock_current_user): + """ + Test error handling when metadata name already exists. + + Verifies that when a metadata field with the same name already exists + in the dataset, a ValueError is raised. + + This test ensures: + - Duplicate name detection works correctly + - Error message is clear + - No new metadata is created + """ + # Arrange + dataset_id = "dataset-123" + metadata_args = MetadataTestDataFactory.create_metadata_args_mock(name="category", metadata_type="string") + + # Mock existing metadata with same name + existing_metadata = MetadataTestDataFactory.create_metadata_mock(name="category") + mock_query = Mock() + mock_query.filter_by.return_value = mock_query + mock_query.first.return_value = existing_metadata + mock_db_session.query.return_value = mock_query + + # Act & Assert + with pytest.raises(ValueError, match="Metadata name already exists"): + MetadataService.create_metadata(dataset_id, metadata_args) + + # Verify no new metadata was added + mock_db_session.add.assert_not_called() + mock_db_session.commit.assert_not_called() + + def test_create_metadata_builtin_field_conflict_error(self, mock_db_session, mock_current_user): + """ + Test error handling when metadata name conflicts with built-in field. + + Verifies that when a metadata name matches a built-in field name, + a ValueError is raised. + + This test ensures: + - Built-in field name conflicts are detected + - Error message is clear + - No new metadata is created + """ + # Arrange + dataset_id = "dataset-123" + metadata_args = MetadataTestDataFactory.create_metadata_args_mock( + name=BuiltInField.document_name, metadata_type="string" + ) + + # Mock query to return None (no duplicate in database) + mock_query = Mock() + mock_query.filter_by.return_value = mock_query + mock_query.first.return_value = None + mock_db_session.query.return_value = mock_query + + # Mock BuiltInField to include the conflicting name + with patch("services.metadata_service.BuiltInField") as mock_builtin: + mock_field = Mock() + mock_field.value = BuiltInField.document_name + mock_builtin.__iter__ = Mock(return_value=iter([mock_field])) + + # Act & Assert + with pytest.raises(ValueError, match="Metadata name already exists in Built-in fields"): + MetadataService.create_metadata(dataset_id, metadata_args) + + # Verify no new metadata was added + mock_db_session.add.assert_not_called() + mock_db_session.commit.assert_not_called() + + +# ============================================================================ +# Tests for update_metadata_name +# ============================================================================ + + +class TestMetadataServiceUpdateMetadataName: + """ + Comprehensive unit tests for MetadataService.update_metadata_name method. + + This test class covers the metadata field name update functionality, + including validation, duplicate checking, and document metadata updates. + + The update_metadata_name method: + 1. Validates new name length (max 255 characters) + 2. Checks for duplicate names + 3. Checks for built-in field conflicts + 4. Acquires a lock for the dataset + 5. Updates the metadata name + 6. Updates all related document metadata + 7. Releases the lock + 8. Returns the updated metadata + + Test scenarios include: + - Successful name update + - Name length validation + - Duplicate name detection + - Built-in field conflicts + - Lock management + - Document metadata updates + """ + + @pytest.fixture + def mock_db_session(self): + """Mock database session for testing.""" + with patch("services.metadata_service.db.session") as mock_db: + yield mock_db + + @pytest.fixture + def mock_current_user(self): + """Mock current user and tenant context.""" + with patch("services.metadata_service.current_account_with_tenant") as mock_get_user: + mock_user = Mock() + mock_user.id = "user-123" + mock_tenant_id = "tenant-123" + mock_get_user.return_value = (mock_user, mock_tenant_id) + yield mock_get_user + + @pytest.fixture + def mock_redis_client(self): + """Mock Redis client for lock management.""" + with patch("services.metadata_service.redis_client") as mock_redis: + mock_redis.get.return_value = None # No existing lock + mock_redis.set.return_value = True + mock_redis.delete.return_value = True + yield mock_redis + + def test_update_metadata_name_success(self, mock_db_session, mock_current_user, mock_redis_client): + """ + Test successful update of metadata field name. + + Verifies that when all validation passes, the metadata name is + updated and all related document metadata is updated accordingly. + + This test ensures: + - Name validation passes + - Lock is acquired and released + - Metadata name is updated + - Related document metadata is updated + - Transaction is committed + """ + # Arrange + dataset_id = "dataset-123" + metadata_id = "metadata-123" + new_name = "updated_category" + + existing_metadata = MetadataTestDataFactory.create_metadata_mock(metadata_id=metadata_id, name="category") + + # Mock query for duplicate check (no duplicate) + mock_query = Mock() + mock_query.filter_by.return_value = mock_query + mock_query.first.return_value = None + mock_db_session.query.return_value = mock_query + + # Mock metadata retrieval + def query_side_effect(model): + if model == DatasetMetadata: + mock_meta_query = Mock() + mock_meta_query.filter_by.return_value = mock_meta_query + mock_meta_query.first.return_value = existing_metadata + return mock_meta_query + return mock_query + + mock_db_session.query.side_effect = query_side_effect + + # Mock no metadata bindings (no documents to update) + mock_binding_query = Mock() + mock_binding_query.filter_by.return_value = mock_binding_query + mock_binding_query.all.return_value = [] + + # Mock BuiltInField enum + with patch("services.metadata_service.BuiltInField") as mock_builtin: + mock_builtin.__iter__ = Mock(return_value=iter([])) + + # Act + result = MetadataService.update_metadata_name(dataset_id, metadata_id, new_name) + + # Assert + assert result is not None + assert result.name == new_name + + # Verify lock was acquired and released + mock_redis_client.get.assert_called() + mock_redis_client.set.assert_called() + mock_redis_client.delete.assert_called() + + # Verify metadata was updated and committed + mock_db_session.commit.assert_called() + + def test_update_metadata_name_not_found_error(self, mock_db_session, mock_current_user, mock_redis_client): + """ + Test error handling when metadata is not found. + + Verifies that when the metadata ID doesn't exist, a ValueError + is raised with an appropriate message. + + This test ensures: + - Not found error is handled correctly + - Lock is properly released even on error + - No updates are committed + """ + # Arrange + dataset_id = "dataset-123" + metadata_id = "non-existent-metadata" + new_name = "updated_category" + + # Mock query for duplicate check (no duplicate) + mock_query = Mock() + mock_query.filter_by.return_value = mock_query + mock_query.first.return_value = None + mock_db_session.query.return_value = mock_query + + # Mock metadata retrieval to return None + def query_side_effect(model): + if model == DatasetMetadata: + mock_meta_query = Mock() + mock_meta_query.filter_by.return_value = mock_meta_query + mock_meta_query.first.return_value = None # Not found + return mock_meta_query + return mock_query + + mock_db_session.query.side_effect = query_side_effect + + # Mock BuiltInField enum + with patch("services.metadata_service.BuiltInField") as mock_builtin: + mock_builtin.__iter__ = Mock(return_value=iter([])) + + # Act & Assert + with pytest.raises(ValueError, match="Metadata not found"): + MetadataService.update_metadata_name(dataset_id, metadata_id, new_name) + + # Verify lock was released + mock_redis_client.delete.assert_called() + + +# ============================================================================ +# Tests for delete_metadata +# ============================================================================ + + +class TestMetadataServiceDeleteMetadata: + """ + Comprehensive unit tests for MetadataService.delete_metadata method. + + This test class covers the metadata field deletion functionality, + including document metadata cleanup and lock management. + + The delete_metadata method: + 1. Acquires a lock for the dataset + 2. Retrieves the metadata to delete + 3. Deletes the metadata from the database + 4. Removes metadata from all related documents + 5. Releases the lock + 6. Returns the deleted metadata + + Test scenarios include: + - Successful deletion + - Not found error handling + - Document metadata cleanup + - Lock management + """ + + @pytest.fixture + def mock_db_session(self): + """Mock database session for testing.""" + with patch("services.metadata_service.db.session") as mock_db: + yield mock_db + + @pytest.fixture + def mock_redis_client(self): + """Mock Redis client for lock management.""" + with patch("services.metadata_service.redis_client") as mock_redis: + mock_redis.get.return_value = None + mock_redis.set.return_value = True + mock_redis.delete.return_value = True + yield mock_redis + + def test_delete_metadata_success(self, mock_db_session, mock_redis_client): + """ + Test successful deletion of a metadata field. + + Verifies that when the metadata exists, it is deleted and all + related document metadata is cleaned up. + + This test ensures: + - Lock is acquired and released + - Metadata is deleted from database + - Related document metadata is removed + - Transaction is committed + """ + # Arrange + dataset_id = "dataset-123" + metadata_id = "metadata-123" + + existing_metadata = MetadataTestDataFactory.create_metadata_mock(metadata_id=metadata_id, name="category") + + # Mock metadata retrieval + mock_query = Mock() + mock_query.filter_by.return_value = mock_query + mock_query.first.return_value = existing_metadata + mock_db_session.query.return_value = mock_query + + # Mock no metadata bindings (no documents to update) + mock_binding_query = Mock() + mock_binding_query.filter_by.return_value = mock_binding_query + mock_binding_query.all.return_value = [] + + # Act + result = MetadataService.delete_metadata(dataset_id, metadata_id) + + # Assert + assert result == existing_metadata + + # Verify lock was acquired and released + mock_redis_client.get.assert_called() + mock_redis_client.set.assert_called() + mock_redis_client.delete.assert_called() + + # Verify metadata was deleted and committed + mock_db_session.delete.assert_called_once_with(existing_metadata) + mock_db_session.commit.assert_called() + + def test_delete_metadata_not_found_error(self, mock_db_session, mock_redis_client): + """ + Test error handling when metadata is not found. + + Verifies that when the metadata ID doesn't exist, a ValueError + is raised and the lock is properly released. + + This test ensures: + - Not found error is handled correctly + - Lock is released even on error + - No deletion is performed + """ + # Arrange + dataset_id = "dataset-123" + metadata_id = "non-existent-metadata" + + # Mock metadata retrieval to return None + mock_query = Mock() + mock_query.filter_by.return_value = mock_query + mock_query.first.return_value = None + mock_db_session.query.return_value = mock_query + + # Act & Assert + with pytest.raises(ValueError, match="Metadata not found"): + MetadataService.delete_metadata(dataset_id, metadata_id) + + # Verify lock was released + mock_redis_client.delete.assert_called() + + # Verify no deletion was performed + mock_db_session.delete.assert_not_called() + + +# ============================================================================ +# Tests for get_built_in_fields +# ============================================================================ + + +class TestMetadataServiceGetBuiltInFields: + """ + Comprehensive unit tests for MetadataService.get_built_in_fields method. + + This test class covers the built-in field retrieval functionality. + + The get_built_in_fields method: + 1. Returns a list of built-in field definitions + 2. Each definition includes name and type + + Test scenarios include: + - Successful retrieval of built-in fields + - Correct field definitions + """ + + def test_get_built_in_fields_success(self): + """ + Test successful retrieval of built-in fields. + + Verifies that the method returns the correct list of built-in + field definitions with proper structure. + + This test ensures: + - All built-in fields are returned + - Each field has name and type + - Field definitions are correct + """ + # Act + result = MetadataService.get_built_in_fields() + + # Assert + assert isinstance(result, list) + assert len(result) > 0 + + # Verify each field has required properties + for field in result: + assert "name" in field + assert "type" in field + assert isinstance(field["name"], str) + assert isinstance(field["type"], str) + + # Verify specific built-in fields are present + field_names = [field["name"] for field in result] + assert BuiltInField.document_name in field_names + assert BuiltInField.uploader in field_names + + +# ============================================================================ +# Tests for knowledge_base_metadata_lock_check +# ============================================================================ + + +class TestMetadataServiceLockCheck: + """ + Comprehensive unit tests for MetadataService.knowledge_base_metadata_lock_check method. + + This test class covers the lock management functionality for preventing + concurrent metadata operations. + + The knowledge_base_metadata_lock_check method: + 1. Checks if a lock exists for the dataset or document + 2. Raises ValueError if lock exists (operation in progress) + 3. Sets a lock with expiration time (3600 seconds) + 4. Supports both dataset-level and document-level locks + + Test scenarios include: + - Successful lock acquisition + - Lock conflict detection + - Dataset-level locks + - Document-level locks + """ + + @pytest.fixture + def mock_redis_client(self): + """Mock Redis client for lock management.""" + with patch("services.metadata_service.redis_client") as mock_redis: + yield mock_redis + + def test_lock_check_dataset_success(self, mock_redis_client): + """ + Test successful lock acquisition for dataset operations. + + Verifies that when no lock exists, a new lock is acquired + for the dataset. + + This test ensures: + - Lock check passes when no lock exists + - Lock is set with correct key and expiration + - No error is raised + """ + # Arrange + dataset_id = "dataset-123" + mock_redis_client.get.return_value = None # No existing lock + + # Act (should not raise) + MetadataService.knowledge_base_metadata_lock_check(dataset_id, None) + + # Assert + mock_redis_client.get.assert_called_once_with(f"dataset_metadata_lock_{dataset_id}") + mock_redis_client.set.assert_called_once_with(f"dataset_metadata_lock_{dataset_id}", 1, ex=3600) + + def test_lock_check_dataset_conflict_error(self, mock_redis_client): + """ + Test error handling when dataset lock already exists. + + Verifies that when a lock exists for the dataset, a ValueError + is raised with an appropriate message. + + This test ensures: + - Lock conflict is detected + - Error message is clear + - No new lock is set + """ + # Arrange + dataset_id = "dataset-123" + mock_redis_client.get.return_value = "1" # Lock exists + + # Act & Assert + with pytest.raises(ValueError, match="Another knowledge base metadata operation is running"): + MetadataService.knowledge_base_metadata_lock_check(dataset_id, None) + + # Verify lock was checked but not set + mock_redis_client.get.assert_called_once() + mock_redis_client.set.assert_not_called() + + def test_lock_check_document_success(self, mock_redis_client): + """ + Test successful lock acquisition for document operations. + + Verifies that when no lock exists, a new lock is acquired + for the document. + + This test ensures: + - Lock check passes when no lock exists + - Lock is set with correct key and expiration + - No error is raised + """ + # Arrange + document_id = "document-123" + mock_redis_client.get.return_value = None # No existing lock + + # Act (should not raise) + MetadataService.knowledge_base_metadata_lock_check(None, document_id) + + # Assert + mock_redis_client.get.assert_called_once_with(f"document_metadata_lock_{document_id}") + mock_redis_client.set.assert_called_once_with(f"document_metadata_lock_{document_id}", 1, ex=3600) + + +# ============================================================================ +# Tests for get_dataset_metadatas +# ============================================================================ + + +class TestMetadataServiceGetDatasetMetadatas: + """ + Comprehensive unit tests for MetadataService.get_dataset_metadatas method. + + This test class covers the metadata retrieval functionality for datasets. + + The get_dataset_metadatas method: + 1. Retrieves all metadata fields for a dataset + 2. Excludes built-in fields from the list + 3. Includes usage count for each metadata field + 4. Returns built-in field enabled status + + Test scenarios include: + - Successful retrieval with metadata fields + - Empty metadata list + - Built-in field filtering + - Usage count calculation + """ + + @pytest.fixture + def mock_db_session(self): + """Mock database session for testing.""" + with patch("services.metadata_service.db.session") as mock_db: + yield mock_db + + def test_get_dataset_metadatas_success(self, mock_db_session): + """ + Test successful retrieval of dataset metadata fields. + + Verifies that all metadata fields are returned with correct + structure and usage counts. + + This test ensures: + - All metadata fields are included + - Built-in fields are excluded + - Usage counts are calculated correctly + - Built-in field status is included + """ + # Arrange + dataset = MetadataTestDataFactory.create_dataset_mock( + dataset_id="dataset-123", + built_in_field_enabled=True, + doc_metadata=[ + {"id": "metadata-1", "name": "category", "type": "string"}, + {"id": "metadata-2", "name": "priority", "type": "number"}, + {"id": "built-in", "name": "document_name", "type": "string"}, + ], + ) + + # Mock usage count queries + mock_query = Mock() + mock_query.filter_by.return_value = mock_query + mock_query.count.return_value = 5 # 5 documents use this metadata + mock_db_session.query.return_value = mock_query + + # Act + result = MetadataService.get_dataset_metadatas(dataset) + + # Assert + assert "doc_metadata" in result + assert "built_in_field_enabled" in result + assert result["built_in_field_enabled"] is True + + # Verify built-in fields are excluded + metadata_ids = [meta["id"] for meta in result["doc_metadata"]] + assert "built-in" not in metadata_ids + + # Verify all custom metadata fields are included + assert len(result["doc_metadata"]) == 2 + + # Verify usage counts are included + for meta in result["doc_metadata"]: + assert "count" in meta + assert meta["count"] == 5 + + +# ============================================================================ +# Additional Documentation and Notes +# ============================================================================ +# +# This test suite covers the core metadata CRUD operations and basic +# filtering functionality. Additional test scenarios that could be added: +# +# 1. enable_built_in_field / disable_built_in_field: +# - Testing built-in field enablement +# - Testing built-in field disablement +# - Testing document metadata updates when enabling/disabling +# +# 2. update_documents_metadata: +# - Testing partial updates +# - Testing full updates +# - Testing metadata binding creation +# - Testing built-in field updates +# +# 3. Metadata Filtering and Querying: +# - Testing metadata-based document filtering +# - Testing complex metadata queries +# - Testing metadata value retrieval +# +# These scenarios are not currently implemented but could be added if needed +# based on real-world usage patterns or discovered edge cases. +# +# ============================================================================ diff --git a/api/tests/unit_tests/services/dataset_permission_service.py b/api/tests/unit_tests/services/dataset_permission_service.py new file mode 100644 index 0000000000..b687f472a5 --- /dev/null +++ b/api/tests/unit_tests/services/dataset_permission_service.py @@ -0,0 +1,1412 @@ +""" +Comprehensive unit tests for DatasetPermissionService and DatasetService permission methods. + +This module contains extensive unit tests for dataset permission management, +including partial member list operations, permission validation, and permission +enum handling. + +The DatasetPermissionService provides methods for: +- Retrieving partial member permissions (get_dataset_partial_member_list) +- Updating partial member lists (update_partial_member_list) +- Validating permissions before operations (check_permission) +- Clearing partial member lists (clear_partial_member_list) + +The DatasetService provides permission checking methods: +- check_dataset_permission - validates user access to dataset +- check_dataset_operator_permission - validates operator permissions + +These operations are critical for dataset access control and security, ensuring +that users can only access datasets they have permission to view or modify. + +This test suite ensures: +- Correct retrieval of partial member lists +- Proper update of partial member permissions +- Accurate permission validation logic +- Proper handling of permission enums (only_me, all_team_members, partial_members) +- Security boundaries are maintained +- Error conditions are handled correctly + +================================================================================ +ARCHITECTURE OVERVIEW +================================================================================ + +The Dataset permission system is a multi-layered access control mechanism +that provides fine-grained control over who can access and modify datasets. + +1. Permission Levels: + - only_me: Only the dataset creator can access + - all_team_members: All members of the tenant can access + - partial_members: Only specific users listed in DatasetPermission can access + +2. Permission Storage: + - Dataset.permission: Stores the permission level enum + - DatasetPermission: Stores individual user permissions for partial_members + - Each DatasetPermission record links a dataset to a user account + +3. Permission Validation: + - Tenant-level checks: Users must be in the same tenant + - Role-based checks: OWNER role bypasses some restrictions + - Explicit permission checks: For partial_members, explicit DatasetPermission + records are required + +4. Permission Operations: + - Partial member list management: Add/remove users from partial access + - Permission validation: Check before allowing operations + - Permission clearing: Remove all partial members when changing permission level + +================================================================================ +TESTING STRATEGY +================================================================================ + +This test suite follows a comprehensive testing strategy that covers: + +1. Partial Member List Operations: + - Retrieving member lists + - Adding new members + - Updating existing members + - Removing members + - Empty list handling + +2. Permission Validation: + - Dataset editor permissions + - Dataset operator restrictions + - Permission enum validation + - Partial member list validation + - Tenant isolation + +3. Permission Enum Handling: + - only_me permission behavior + - all_team_members permission behavior + - partial_members permission behavior + - Permission transitions + - Edge cases for each enum value + +4. Security and Access Control: + - Tenant boundary enforcement + - Role-based access control + - Creator privilege validation + - Explicit permission requirement + +5. Error Handling: + - Invalid permission changes + - Missing required data + - Database transaction failures + - Permission denial scenarios + +================================================================================ +""" + +from unittest.mock import Mock, create_autospec, patch + +import pytest + +from models import Account, TenantAccountRole +from models.dataset import ( + Dataset, + DatasetPermission, + DatasetPermissionEnum, +) +from services.dataset_service import DatasetPermissionService, DatasetService +from services.errors.account import NoPermissionError + +# ============================================================================ +# Test Data Factory +# ============================================================================ +# The Test Data Factory pattern is used here to centralize the creation of +# test objects and mock instances. This approach provides several benefits: +# +# 1. Consistency: All test objects are created using the same factory methods, +# ensuring consistent structure across all tests. +# +# 2. Maintainability: If the structure of models or services changes, we only +# need to update the factory methods rather than every individual test. +# +# 3. Reusability: Factory methods can be reused across multiple test classes, +# reducing code duplication. +# +# 4. Readability: Tests become more readable when they use descriptive factory +# method calls instead of complex object construction logic. +# +# ============================================================================ + + +class DatasetPermissionTestDataFactory: + """ + Factory class for creating test data and mock objects for dataset permission tests. + + This factory provides static methods to create mock objects for: + - Dataset instances with various permission configurations + - User/Account instances with different roles and permissions + - DatasetPermission instances + - Permission enum values + - Database query results + + The factory methods help maintain consistency across tests and reduce + code duplication when setting up test scenarios. + """ + + @staticmethod + def create_dataset_mock( + dataset_id: str = "dataset-123", + tenant_id: str = "tenant-123", + permission: DatasetPermissionEnum = DatasetPermissionEnum.ONLY_ME, + created_by: str = "user-123", + name: str = "Test Dataset", + **kwargs, + ) -> Mock: + """ + Create a mock Dataset with specified attributes. + + Args: + dataset_id: Unique identifier for the dataset + tenant_id: Tenant identifier + permission: Permission level enum + created_by: ID of user who created the dataset + name: Dataset name + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a Dataset instance + """ + dataset = Mock(spec=Dataset) + dataset.id = dataset_id + dataset.tenant_id = tenant_id + dataset.permission = permission + dataset.created_by = created_by + dataset.name = name + for key, value in kwargs.items(): + setattr(dataset, key, value) + return dataset + + @staticmethod + def create_user_mock( + user_id: str = "user-123", + tenant_id: str = "tenant-123", + role: TenantAccountRole = TenantAccountRole.NORMAL, + is_dataset_editor: bool = True, + is_dataset_operator: bool = False, + **kwargs, + ) -> Mock: + """ + Create a mock user (Account) with specified attributes. + + Args: + user_id: Unique identifier for the user + tenant_id: Tenant identifier + role: User role (OWNER, ADMIN, NORMAL, DATASET_OPERATOR, etc.) + is_dataset_editor: Whether user has dataset editor permissions + is_dataset_operator: Whether user is a dataset operator + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as an Account instance + """ + user = create_autospec(Account, instance=True) + user.id = user_id + user.current_tenant_id = tenant_id + user.current_role = role + user.is_dataset_editor = is_dataset_editor + user.is_dataset_operator = is_dataset_operator + for key, value in kwargs.items(): + setattr(user, key, value) + return user + + @staticmethod + def create_dataset_permission_mock( + permission_id: str = "permission-123", + dataset_id: str = "dataset-123", + account_id: str = "user-456", + tenant_id: str = "tenant-123", + has_permission: bool = True, + **kwargs, + ) -> Mock: + """ + Create a mock DatasetPermission instance. + + Args: + permission_id: Unique identifier for the permission + dataset_id: Dataset ID + account_id: User account ID + tenant_id: Tenant identifier + has_permission: Whether permission is granted + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a DatasetPermission instance + """ + permission = Mock(spec=DatasetPermission) + permission.id = permission_id + permission.dataset_id = dataset_id + permission.account_id = account_id + permission.tenant_id = tenant_id + permission.has_permission = has_permission + for key, value in kwargs.items(): + setattr(permission, key, value) + return permission + + @staticmethod + def create_user_list_mock(user_ids: list[str]) -> list[dict[str, str]]: + """ + Create a list of user dictionaries for partial member list operations. + + Args: + user_ids: List of user IDs to include + + Returns: + List of user dictionaries with "user_id" keys + """ + return [{"user_id": user_id} for user_id in user_ids] + + +# ============================================================================ +# Tests for get_dataset_partial_member_list +# ============================================================================ + + +class TestDatasetPermissionServiceGetPartialMemberList: + """ + Comprehensive unit tests for DatasetPermissionService.get_dataset_partial_member_list method. + + This test class covers the retrieval of partial member lists for datasets, + which returns a list of account IDs that have explicit permissions for + a given dataset. + + The get_dataset_partial_member_list method: + 1. Queries DatasetPermission table for the dataset ID + 2. Selects account_id values + 3. Returns list of account IDs + + Test scenarios include: + - Retrieving list with multiple members + - Retrieving list with single member + - Retrieving empty list (no partial members) + - Database query validation + """ + + @pytest.fixture + def mock_db_session(self): + """ + Mock database session for testing. + + Provides a mocked database session that can be used to verify + query construction and execution. + """ + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + def test_get_dataset_partial_member_list_with_members(self, mock_db_session): + """ + Test retrieving partial member list with multiple members. + + Verifies that when a dataset has multiple partial members, all + account IDs are returned correctly. + + This test ensures: + - Query is constructed correctly + - All account IDs are returned + - Database query is executed + """ + # Arrange + dataset_id = "dataset-123" + expected_account_ids = ["user-456", "user-789", "user-012"] + + # Mock the scalars query to return account IDs + mock_scalars_result = Mock() + mock_scalars_result.all.return_value = expected_account_ids + mock_db_session.scalars.return_value = mock_scalars_result + + # Act + result = DatasetPermissionService.get_dataset_partial_member_list(dataset_id) + + # Assert + assert result == expected_account_ids + assert len(result) == 3 + + # Verify query was executed + mock_db_session.scalars.assert_called_once() + + def test_get_dataset_partial_member_list_with_single_member(self, mock_db_session): + """ + Test retrieving partial member list with single member. + + Verifies that when a dataset has only one partial member, the + single account ID is returned correctly. + + This test ensures: + - Query works correctly for single member + - Result is a list with one element + - Database query is executed + """ + # Arrange + dataset_id = "dataset-123" + expected_account_ids = ["user-456"] + + # Mock the scalars query to return single account ID + mock_scalars_result = Mock() + mock_scalars_result.all.return_value = expected_account_ids + mock_db_session.scalars.return_value = mock_scalars_result + + # Act + result = DatasetPermissionService.get_dataset_partial_member_list(dataset_id) + + # Assert + assert result == expected_account_ids + assert len(result) == 1 + + # Verify query was executed + mock_db_session.scalars.assert_called_once() + + def test_get_dataset_partial_member_list_empty(self, mock_db_session): + """ + Test retrieving partial member list when no members exist. + + Verifies that when a dataset has no partial members, an empty + list is returned. + + This test ensures: + - Empty list is returned correctly + - Query is executed even when no results + - No errors are raised + """ + # Arrange + dataset_id = "dataset-123" + + # Mock the scalars query to return empty list + mock_scalars_result = Mock() + mock_scalars_result.all.return_value = [] + mock_db_session.scalars.return_value = mock_scalars_result + + # Act + result = DatasetPermissionService.get_dataset_partial_member_list(dataset_id) + + # Assert + assert result == [] + assert len(result) == 0 + + # Verify query was executed + mock_db_session.scalars.assert_called_once() + + +# ============================================================================ +# Tests for update_partial_member_list +# ============================================================================ + + +class TestDatasetPermissionServiceUpdatePartialMemberList: + """ + Comprehensive unit tests for DatasetPermissionService.update_partial_member_list method. + + This test class covers the update of partial member lists for datasets, + which replaces the existing partial member list with a new one. + + The update_partial_member_list method: + 1. Deletes all existing DatasetPermission records for the dataset + 2. Creates new DatasetPermission records for each user in the list + 3. Adds all new permissions to the session + 4. Commits the transaction + 5. Rolls back on error + + Test scenarios include: + - Adding new partial members + - Updating existing partial members + - Replacing entire member list + - Handling empty member list + - Database transaction handling + - Error handling and rollback + """ + + @pytest.fixture + def mock_db_session(self): + """ + Mock database session for testing. + + Provides a mocked database session that can be used to verify + database operations including queries, adds, commits, and rollbacks. + """ + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + def test_update_partial_member_list_add_new_members(self, mock_db_session): + """ + Test adding new partial members to a dataset. + + Verifies that when updating with new members, the old members + are deleted and new members are added correctly. + + This test ensures: + - Old permissions are deleted + - New permissions are created + - All permissions are added to session + - Transaction is committed + """ + # Arrange + tenant_id = "tenant-123" + dataset_id = "dataset-123" + user_list = DatasetPermissionTestDataFactory.create_user_list_mock(["user-456", "user-789"]) + + # Mock the query delete operation + mock_query = Mock() + mock_query.where.return_value = mock_query + mock_query.delete.return_value = None + mock_db_session.query.return_value = mock_query + + # Act + DatasetPermissionService.update_partial_member_list(tenant_id, dataset_id, user_list) + + # Assert + # Verify old permissions were deleted + mock_db_session.query.assert_called() + mock_query.where.assert_called() + + # Verify new permissions were added + mock_db_session.add_all.assert_called_once() + + # Verify transaction was committed + mock_db_session.commit.assert_called_once() + + # Verify no rollback occurred + mock_db_session.rollback.assert_not_called() + + def test_update_partial_member_list_replace_existing(self, mock_db_session): + """ + Test replacing existing partial members with new ones. + + Verifies that when updating with a different member list, the + old members are removed and new members are added. + + This test ensures: + - Old permissions are deleted + - New permissions replace old ones + - Transaction is committed successfully + """ + # Arrange + tenant_id = "tenant-123" + dataset_id = "dataset-123" + user_list = DatasetPermissionTestDataFactory.create_user_list_mock(["user-999", "user-888"]) + + # Mock the query delete operation + mock_query = Mock() + mock_query.where.return_value = mock_query + mock_query.delete.return_value = None + mock_db_session.query.return_value = mock_query + + # Act + DatasetPermissionService.update_partial_member_list(tenant_id, dataset_id, user_list) + + # Assert + # Verify old permissions were deleted + mock_db_session.query.assert_called() + + # Verify new permissions were added + mock_db_session.add_all.assert_called_once() + + # Verify transaction was committed + mock_db_session.commit.assert_called_once() + + def test_update_partial_member_list_empty_list(self, mock_db_session): + """ + Test updating with empty member list (clearing all members). + + Verifies that when updating with an empty list, all existing + permissions are deleted and no new permissions are added. + + This test ensures: + - Old permissions are deleted + - No new permissions are added + - Transaction is committed + """ + # Arrange + tenant_id = "tenant-123" + dataset_id = "dataset-123" + user_list = [] + + # Mock the query delete operation + mock_query = Mock() + mock_query.where.return_value = mock_query + mock_query.delete.return_value = None + mock_db_session.query.return_value = mock_query + + # Act + DatasetPermissionService.update_partial_member_list(tenant_id, dataset_id, user_list) + + # Assert + # Verify old permissions were deleted + mock_db_session.query.assert_called() + + # Verify add_all was called with empty list + mock_db_session.add_all.assert_called_once_with([]) + + # Verify transaction was committed + mock_db_session.commit.assert_called_once() + + def test_update_partial_member_list_database_error_rollback(self, mock_db_session): + """ + Test error handling and rollback on database error. + + Verifies that when a database error occurs during the update, + the transaction is rolled back and the error is re-raised. + + This test ensures: + - Error is caught and handled + - Transaction is rolled back + - Error is re-raised + - No commit occurs after error + """ + # Arrange + tenant_id = "tenant-123" + dataset_id = "dataset-123" + user_list = DatasetPermissionTestDataFactory.create_user_list_mock(["user-456"]) + + # Mock the query delete operation + mock_query = Mock() + mock_query.where.return_value = mock_query + mock_query.delete.return_value = None + mock_db_session.query.return_value = mock_query + + # Mock commit to raise an error + database_error = Exception("Database connection error") + mock_db_session.commit.side_effect = database_error + + # Act & Assert + with pytest.raises(Exception, match="Database connection error"): + DatasetPermissionService.update_partial_member_list(tenant_id, dataset_id, user_list) + + # Verify rollback was called + mock_db_session.rollback.assert_called_once() + + +# ============================================================================ +# Tests for check_permission +# ============================================================================ + + +class TestDatasetPermissionServiceCheckPermission: + """ + Comprehensive unit tests for DatasetPermissionService.check_permission method. + + This test class covers the permission validation logic that ensures + users have the appropriate permissions to modify dataset permissions. + + The check_permission method: + 1. Validates user is a dataset editor + 2. Checks if dataset operator is trying to change permissions + 3. Validates partial member list when setting to partial_members + 4. Ensures dataset operators cannot change permission levels + 5. Ensures dataset operators cannot modify partial member lists + + Test scenarios include: + - Valid permission changes by dataset editors + - Dataset operator restrictions + - Partial member list validation + - Missing dataset editor permissions + - Invalid permission changes + """ + + @pytest.fixture + def mock_get_partial_member_list(self): + """ + Mock get_dataset_partial_member_list method. + + Provides a mocked version of the get_dataset_partial_member_list + method for testing permission validation logic. + """ + with patch.object(DatasetPermissionService, "get_dataset_partial_member_list") as mock_get_list: + yield mock_get_list + + def test_check_permission_dataset_editor_success(self, mock_get_partial_member_list): + """ + Test successful permission check for dataset editor. + + Verifies that when a dataset editor (not operator) tries to + change permissions, the check passes. + + This test ensures: + - Dataset editors can change permissions + - No errors are raised for valid changes + - Partial member list validation is skipped for non-operators + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(is_dataset_editor=True, is_dataset_operator=False) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock(permission=DatasetPermissionEnum.ONLY_ME) + requested_permission = DatasetPermissionEnum.ALL_TEAM + requested_partial_member_list = None + + # Act (should not raise) + DatasetPermissionService.check_permission(user, dataset, requested_permission, requested_partial_member_list) + + # Assert + # Verify get_partial_member_list was not called (not needed for non-operators) + mock_get_partial_member_list.assert_not_called() + + def test_check_permission_not_dataset_editor_error(self): + """ + Test error when user is not a dataset editor. + + Verifies that when a user without dataset editor permissions + tries to change permissions, a NoPermissionError is raised. + + This test ensures: + - Non-editors cannot change permissions + - Error message is clear + - Error type is correct + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(is_dataset_editor=False) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock() + requested_permission = DatasetPermissionEnum.ALL_TEAM + requested_partial_member_list = None + + # Act & Assert + with pytest.raises(NoPermissionError, match="User does not have permission to edit this dataset"): + DatasetPermissionService.check_permission( + user, dataset, requested_permission, requested_partial_member_list + ) + + def test_check_permission_operator_cannot_change_permission_error(self): + """ + Test error when dataset operator tries to change permission level. + + Verifies that when a dataset operator tries to change the permission + level, a NoPermissionError is raised. + + This test ensures: + - Dataset operators cannot change permission levels + - Error message is clear + - Current permission is preserved + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(is_dataset_editor=True, is_dataset_operator=True) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock(permission=DatasetPermissionEnum.ONLY_ME) + requested_permission = DatasetPermissionEnum.ALL_TEAM # Trying to change + requested_partial_member_list = None + + # Act & Assert + with pytest.raises(NoPermissionError, match="Dataset operators cannot change the dataset permissions"): + DatasetPermissionService.check_permission( + user, dataset, requested_permission, requested_partial_member_list + ) + + def test_check_permission_operator_partial_members_missing_list_error(self, mock_get_partial_member_list): + """ + Test error when operator sets partial_members without providing list. + + Verifies that when a dataset operator tries to set permission to + partial_members without providing a member list, a ValueError is raised. + + This test ensures: + - Partial member list is required for partial_members permission + - Error message is clear + - Error type is correct + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(is_dataset_editor=True, is_dataset_operator=True) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock(permission=DatasetPermissionEnum.PARTIAL_TEAM) + requested_permission = "partial_members" + requested_partial_member_list = None # Missing list + + # Act & Assert + with pytest.raises(ValueError, match="Partial member list is required when setting to partial members"): + DatasetPermissionService.check_permission( + user, dataset, requested_permission, requested_partial_member_list + ) + + def test_check_permission_operator_cannot_modify_partial_list_error(self, mock_get_partial_member_list): + """ + Test error when operator tries to modify partial member list. + + Verifies that when a dataset operator tries to change the partial + member list, a ValueError is raised. + + This test ensures: + - Dataset operators cannot modify partial member lists + - Error message is clear + - Current member list is preserved + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(is_dataset_editor=True, is_dataset_operator=True) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock(permission=DatasetPermissionEnum.PARTIAL_TEAM) + requested_permission = "partial_members" + + # Current member list + current_member_list = ["user-456", "user-789"] + mock_get_partial_member_list.return_value = current_member_list + + # Requested member list (different from current) + requested_partial_member_list = DatasetPermissionTestDataFactory.create_user_list_mock( + ["user-456", "user-999"] # Different list + ) + + # Act & Assert + with pytest.raises(ValueError, match="Dataset operators cannot change the dataset permissions"): + DatasetPermissionService.check_permission( + user, dataset, requested_permission, requested_partial_member_list + ) + + def test_check_permission_operator_can_keep_same_partial_list(self, mock_get_partial_member_list): + """ + Test that operator can keep the same partial member list. + + Verifies that when a dataset operator keeps the same partial member + list, the check passes. + + This test ensures: + - Operators can keep existing partial member lists + - No errors are raised for unchanged lists + - Permission validation works correctly + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(is_dataset_editor=True, is_dataset_operator=True) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock(permission=DatasetPermissionEnum.PARTIAL_TEAM) + requested_permission = "partial_members" + + # Current member list + current_member_list = ["user-456", "user-789"] + mock_get_partial_member_list.return_value = current_member_list + + # Requested member list (same as current) + requested_partial_member_list = DatasetPermissionTestDataFactory.create_user_list_mock( + ["user-456", "user-789"] # Same list + ) + + # Act (should not raise) + DatasetPermissionService.check_permission(user, dataset, requested_permission, requested_partial_member_list) + + # Assert + # Verify get_partial_member_list was called to compare lists + mock_get_partial_member_list.assert_called_once_with(dataset.id) + + +# ============================================================================ +# Tests for clear_partial_member_list +# ============================================================================ + + +class TestDatasetPermissionServiceClearPartialMemberList: + """ + Comprehensive unit tests for DatasetPermissionService.clear_partial_member_list method. + + This test class covers the clearing of partial member lists, which removes + all DatasetPermission records for a given dataset. + + The clear_partial_member_list method: + 1. Deletes all DatasetPermission records for the dataset + 2. Commits the transaction + 3. Rolls back on error + + Test scenarios include: + - Clearing list with existing members + - Clearing empty list (no members) + - Database transaction handling + - Error handling and rollback + """ + + @pytest.fixture + def mock_db_session(self): + """ + Mock database session for testing. + + Provides a mocked database session that can be used to verify + database operations including queries, deletes, commits, and rollbacks. + """ + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + def test_clear_partial_member_list_success(self, mock_db_session): + """ + Test successful clearing of partial member list. + + Verifies that when clearing a partial member list, all permissions + are deleted and the transaction is committed. + + This test ensures: + - All permissions are deleted + - Transaction is committed + - No errors are raised + """ + # Arrange + dataset_id = "dataset-123" + + # Mock the query delete operation + mock_query = Mock() + mock_query.where.return_value = mock_query + mock_query.delete.return_value = None + mock_db_session.query.return_value = mock_query + + # Act + DatasetPermissionService.clear_partial_member_list(dataset_id) + + # Assert + # Verify query was executed + mock_db_session.query.assert_called() + + # Verify delete was called + mock_query.where.assert_called() + mock_query.delete.assert_called_once() + + # Verify transaction was committed + mock_db_session.commit.assert_called_once() + + # Verify no rollback occurred + mock_db_session.rollback.assert_not_called() + + def test_clear_partial_member_list_empty_list(self, mock_db_session): + """ + Test clearing partial member list when no members exist. + + Verifies that when clearing an already empty list, the operation + completes successfully without errors. + + This test ensures: + - Operation works correctly for empty lists + - Transaction is committed + - No errors are raised + """ + # Arrange + dataset_id = "dataset-123" + + # Mock the query delete operation + mock_query = Mock() + mock_query.where.return_value = mock_query + mock_query.delete.return_value = None + mock_db_session.query.return_value = mock_query + + # Act + DatasetPermissionService.clear_partial_member_list(dataset_id) + + # Assert + # Verify query was executed + mock_db_session.query.assert_called() + + # Verify transaction was committed + mock_db_session.commit.assert_called_once() + + def test_clear_partial_member_list_database_error_rollback(self, mock_db_session): + """ + Test error handling and rollback on database error. + + Verifies that when a database error occurs during clearing, + the transaction is rolled back and the error is re-raised. + + This test ensures: + - Error is caught and handled + - Transaction is rolled back + - Error is re-raised + - No commit occurs after error + """ + # Arrange + dataset_id = "dataset-123" + + # Mock the query delete operation + mock_query = Mock() + mock_query.where.return_value = mock_query + mock_query.delete.return_value = None + mock_db_session.query.return_value = mock_query + + # Mock commit to raise an error + database_error = Exception("Database connection error") + mock_db_session.commit.side_effect = database_error + + # Act & Assert + with pytest.raises(Exception, match="Database connection error"): + DatasetPermissionService.clear_partial_member_list(dataset_id) + + # Verify rollback was called + mock_db_session.rollback.assert_called_once() + + +# ============================================================================ +# Tests for DatasetService.check_dataset_permission +# ============================================================================ + + +class TestDatasetServiceCheckDatasetPermission: + """ + Comprehensive unit tests for DatasetService.check_dataset_permission method. + + This test class covers the dataset permission checking logic that validates + whether a user has access to a dataset based on permission enums. + + The check_dataset_permission method: + 1. Validates tenant match + 2. Checks OWNER role (bypasses some restrictions) + 3. Validates only_me permission (creator only) + 4. Validates partial_members permission (explicit permission required) + 5. Validates all_team_members permission (all tenant members) + + Test scenarios include: + - Tenant boundary enforcement + - OWNER role bypass + - only_me permission validation + - partial_members permission validation + - all_team_members permission validation + - Permission denial scenarios + """ + + @pytest.fixture + def mock_db_session(self): + """ + Mock database session for testing. + + Provides a mocked database session that can be used to verify + database queries for permission checks. + """ + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + def test_check_dataset_permission_owner_bypass(self, mock_db_session): + """ + Test that OWNER role bypasses permission checks. + + Verifies that when a user has OWNER role, they can access any + dataset in their tenant regardless of permission level. + + This test ensures: + - OWNER role bypasses permission restrictions + - No database queries are needed for OWNER + - Access is granted automatically + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(role=TenantAccountRole.OWNER, tenant_id="tenant-123") + dataset = DatasetPermissionTestDataFactory.create_dataset_mock( + tenant_id="tenant-123", + permission=DatasetPermissionEnum.ONLY_ME, + created_by="other-user-123", # Not the current user + ) + + # Act (should not raise) + DatasetService.check_dataset_permission(dataset, user) + + # Assert + # Verify no permission queries were made (OWNER bypasses) + mock_db_session.query.assert_not_called() + + def test_check_dataset_permission_tenant_mismatch_error(self): + """ + Test error when user and dataset are in different tenants. + + Verifies that when a user tries to access a dataset from a different + tenant, a NoPermissionError is raised. + + This test ensures: + - Tenant boundary is enforced + - Error message is clear + - Error type is correct + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(tenant_id="tenant-123") + dataset = DatasetPermissionTestDataFactory.create_dataset_mock(tenant_id="tenant-456") # Different tenant + + # Act & Assert + with pytest.raises(NoPermissionError, match="You do not have permission to access this dataset"): + DatasetService.check_dataset_permission(dataset, user) + + def test_check_dataset_permission_only_me_creator_success(self): + """ + Test that creator can access only_me dataset. + + Verifies that when a user is the creator of an only_me dataset, + they can access it successfully. + + This test ensures: + - Creators can access their own only_me datasets + - No explicit permission record is needed + - Access is granted correctly + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(user_id="user-123", role=TenantAccountRole.NORMAL) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock( + tenant_id="tenant-123", + permission=DatasetPermissionEnum.ONLY_ME, + created_by="user-123", # User is the creator + ) + + # Act (should not raise) + DatasetService.check_dataset_permission(dataset, user) + + def test_check_dataset_permission_only_me_non_creator_error(self): + """ + Test error when non-creator tries to access only_me dataset. + + Verifies that when a user who is not the creator tries to access + an only_me dataset, a NoPermissionError is raised. + + This test ensures: + - Non-creators cannot access only_me datasets + - Error message is clear + - Error type is correct + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(user_id="user-123", role=TenantAccountRole.NORMAL) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock( + tenant_id="tenant-123", + permission=DatasetPermissionEnum.ONLY_ME, + created_by="other-user-456", # Different creator + ) + + # Act & Assert + with pytest.raises(NoPermissionError, match="You do not have permission to access this dataset"): + DatasetService.check_dataset_permission(dataset, user) + + def test_check_dataset_permission_partial_members_with_permission_success(self, mock_db_session): + """ + Test that user with explicit permission can access partial_members dataset. + + Verifies that when a user has an explicit DatasetPermission record + for a partial_members dataset, they can access it successfully. + + This test ensures: + - Explicit permissions are checked correctly + - Users with permissions can access + - Database query is executed + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(user_id="user-123", role=TenantAccountRole.NORMAL) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock( + tenant_id="tenant-123", + permission=DatasetPermissionEnum.PARTIAL_TEAM, + created_by="other-user-456", # Not the creator + ) + + # Mock permission query to return permission record + mock_permission = DatasetPermissionTestDataFactory.create_dataset_permission_mock( + dataset_id=dataset.id, account_id=user.id + ) + mock_query = Mock() + mock_query.filter_by.return_value = mock_query + mock_query.first.return_value = mock_permission + mock_db_session.query.return_value = mock_query + + # Act (should not raise) + DatasetService.check_dataset_permission(dataset, user) + + # Assert + # Verify permission query was executed + mock_db_session.query.assert_called() + + def test_check_dataset_permission_partial_members_without_permission_error(self, mock_db_session): + """ + Test error when user without permission tries to access partial_members dataset. + + Verifies that when a user does not have an explicit DatasetPermission + record for a partial_members dataset, a NoPermissionError is raised. + + This test ensures: + - Missing permissions are detected + - Error message is clear + - Error type is correct + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(user_id="user-123", role=TenantAccountRole.NORMAL) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock( + tenant_id="tenant-123", + permission=DatasetPermissionEnum.PARTIAL_TEAM, + created_by="other-user-456", # Not the creator + ) + + # Mock permission query to return None (no permission) + mock_query = Mock() + mock_query.filter_by.return_value = mock_query + mock_query.first.return_value = None # No permission found + mock_db_session.query.return_value = mock_query + + # Act & Assert + with pytest.raises(NoPermissionError, match="You do not have permission to access this dataset"): + DatasetService.check_dataset_permission(dataset, user) + + def test_check_dataset_permission_partial_members_creator_success(self, mock_db_session): + """ + Test that creator can access partial_members dataset without explicit permission. + + Verifies that when a user is the creator of a partial_members dataset, + they can access it even without an explicit DatasetPermission record. + + This test ensures: + - Creators can access their own datasets + - No explicit permission record is needed for creators + - Access is granted correctly + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(user_id="user-123", role=TenantAccountRole.NORMAL) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock( + tenant_id="tenant-123", + permission=DatasetPermissionEnum.PARTIAL_TEAM, + created_by="user-123", # User is the creator + ) + + # Act (should not raise) + DatasetService.check_dataset_permission(dataset, user) + + # Assert + # Verify permission query was not executed (creator bypasses) + mock_db_session.query.assert_not_called() + + def test_check_dataset_permission_all_team_members_success(self): + """ + Test that any tenant member can access all_team_members dataset. + + Verifies that when a dataset has all_team_members permission, any + user in the same tenant can access it. + + This test ensures: + - All team members can access + - No explicit permission record is needed + - Access is granted correctly + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(user_id="user-123", role=TenantAccountRole.NORMAL) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock( + tenant_id="tenant-123", + permission=DatasetPermissionEnum.ALL_TEAM, + created_by="other-user-456", # Not the creator + ) + + # Act (should not raise) + DatasetService.check_dataset_permission(dataset, user) + + +# ============================================================================ +# Tests for DatasetService.check_dataset_operator_permission +# ============================================================================ + + +class TestDatasetServiceCheckDatasetOperatorPermission: + """ + Comprehensive unit tests for DatasetService.check_dataset_operator_permission method. + + This test class covers the dataset operator permission checking logic, + which validates whether a dataset operator has access to a dataset. + + The check_dataset_operator_permission method: + 1. Validates dataset exists + 2. Validates user exists + 3. Checks OWNER role (bypasses restrictions) + 4. Validates only_me permission (creator only) + 5. Validates partial_members permission (explicit permission required) + + Test scenarios include: + - Dataset not found error + - User not found error + - OWNER role bypass + - only_me permission validation + - partial_members permission validation + - Permission denial scenarios + """ + + @pytest.fixture + def mock_db_session(self): + """ + Mock database session for testing. + + Provides a mocked database session that can be used to verify + database queries for permission checks. + """ + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + def test_check_dataset_operator_permission_dataset_not_found_error(self): + """ + Test error when dataset is None. + + Verifies that when dataset is None, a ValueError is raised. + + This test ensures: + - Dataset existence is validated + - Error message is clear + - Error type is correct + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock() + dataset = None + + # Act & Assert + with pytest.raises(ValueError, match="Dataset not found"): + DatasetService.check_dataset_operator_permission(user=user, dataset=dataset) + + def test_check_dataset_operator_permission_user_not_found_error(self): + """ + Test error when user is None. + + Verifies that when user is None, a ValueError is raised. + + This test ensures: + - User existence is validated + - Error message is clear + - Error type is correct + """ + # Arrange + user = None + dataset = DatasetPermissionTestDataFactory.create_dataset_mock() + + # Act & Assert + with pytest.raises(ValueError, match="User not found"): + DatasetService.check_dataset_operator_permission(user=user, dataset=dataset) + + def test_check_dataset_operator_permission_owner_bypass(self): + """ + Test that OWNER role bypasses permission checks. + + Verifies that when a user has OWNER role, they can access any + dataset in their tenant regardless of permission level. + + This test ensures: + - OWNER role bypasses permission restrictions + - No database queries are needed for OWNER + - Access is granted automatically + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(role=TenantAccountRole.OWNER, tenant_id="tenant-123") + dataset = DatasetPermissionTestDataFactory.create_dataset_mock( + tenant_id="tenant-123", + permission=DatasetPermissionEnum.ONLY_ME, + created_by="other-user-123", # Not the current user + ) + + # Act (should not raise) + DatasetService.check_dataset_operator_permission(user=user, dataset=dataset) + + def test_check_dataset_operator_permission_only_me_creator_success(self): + """ + Test that creator can access only_me dataset. + + Verifies that when a user is the creator of an only_me dataset, + they can access it successfully. + + This test ensures: + - Creators can access their own only_me datasets + - No explicit permission record is needed + - Access is granted correctly + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(user_id="user-123", role=TenantAccountRole.NORMAL) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock( + tenant_id="tenant-123", + permission=DatasetPermissionEnum.ONLY_ME, + created_by="user-123", # User is the creator + ) + + # Act (should not raise) + DatasetService.check_dataset_operator_permission(user=user, dataset=dataset) + + def test_check_dataset_operator_permission_only_me_non_creator_error(self): + """ + Test error when non-creator tries to access only_me dataset. + + Verifies that when a user who is not the creator tries to access + an only_me dataset, a NoPermissionError is raised. + + This test ensures: + - Non-creators cannot access only_me datasets + - Error message is clear + - Error type is correct + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(user_id="user-123", role=TenantAccountRole.NORMAL) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock( + tenant_id="tenant-123", + permission=DatasetPermissionEnum.ONLY_ME, + created_by="other-user-456", # Different creator + ) + + # Act & Assert + with pytest.raises(NoPermissionError, match="You do not have permission to access this dataset"): + DatasetService.check_dataset_operator_permission(user=user, dataset=dataset) + + def test_check_dataset_operator_permission_partial_members_with_permission_success(self, mock_db_session): + """ + Test that user with explicit permission can access partial_members dataset. + + Verifies that when a user has an explicit DatasetPermission record + for a partial_members dataset, they can access it successfully. + + This test ensures: + - Explicit permissions are checked correctly + - Users with permissions can access + - Database query is executed + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(user_id="user-123", role=TenantAccountRole.NORMAL) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock( + tenant_id="tenant-123", + permission=DatasetPermissionEnum.PARTIAL_TEAM, + created_by="other-user-456", # Not the creator + ) + + # Mock permission query to return permission records + mock_permission = DatasetPermissionTestDataFactory.create_dataset_permission_mock( + dataset_id=dataset.id, account_id=user.id + ) + mock_query = Mock() + mock_query.filter_by.return_value = mock_query + mock_query.all.return_value = [mock_permission] # User has permission + mock_db_session.query.return_value = mock_query + + # Act (should not raise) + DatasetService.check_dataset_operator_permission(user=user, dataset=dataset) + + # Assert + # Verify permission query was executed + mock_db_session.query.assert_called() + + def test_check_dataset_operator_permission_partial_members_without_permission_error(self, mock_db_session): + """ + Test error when user without permission tries to access partial_members dataset. + + Verifies that when a user does not have an explicit DatasetPermission + record for a partial_members dataset, a NoPermissionError is raised. + + This test ensures: + - Missing permissions are detected + - Error message is clear + - Error type is correct + """ + # Arrange + user = DatasetPermissionTestDataFactory.create_user_mock(user_id="user-123", role=TenantAccountRole.NORMAL) + dataset = DatasetPermissionTestDataFactory.create_dataset_mock( + tenant_id="tenant-123", + permission=DatasetPermissionEnum.PARTIAL_TEAM, + created_by="other-user-456", # Not the creator + ) + + # Mock permission query to return empty list (no permission) + mock_query = Mock() + mock_query.filter_by.return_value = mock_query + mock_query.all.return_value = [] # No permissions found + mock_db_session.query.return_value = mock_query + + # Act & Assert + with pytest.raises(NoPermissionError, match="You do not have permission to access this dataset"): + DatasetService.check_dataset_operator_permission(user=user, dataset=dataset) + + +# ============================================================================ +# Additional Documentation and Notes +# ============================================================================ +# +# This test suite covers the core permission management operations for datasets. +# Additional test scenarios that could be added: +# +# 1. Permission Enum Transitions: +# - Testing transitions between permission levels +# - Testing validation during transitions +# - Testing partial member list updates during transitions +# +# 2. Bulk Operations: +# - Testing bulk permission updates +# - Testing bulk partial member list updates +# - Testing performance with large member lists +# +# 3. Edge Cases: +# - Testing with very large partial member lists +# - Testing with special characters in user IDs +# - Testing with deleted users +# - Testing with inactive permissions +# +# 4. Integration Scenarios: +# - Testing permission changes followed by access attempts +# - Testing concurrent permission updates +# - Testing permission inheritance +# +# These scenarios are not currently implemented but could be added if needed +# based on real-world usage patterns or discovered edge cases. +# +# ============================================================================ diff --git a/api/tests/unit_tests/services/dataset_service_update_delete.py b/api/tests/unit_tests/services/dataset_service_update_delete.py new file mode 100644 index 0000000000..3715aadfdc --- /dev/null +++ b/api/tests/unit_tests/services/dataset_service_update_delete.py @@ -0,0 +1,1225 @@ +""" +Comprehensive unit tests for DatasetService update and delete operations. + +This module contains extensive unit tests for the DatasetService class, +specifically focusing on update and delete operations for datasets. + +The DatasetService provides methods for: +- Updating dataset configuration and settings (update_dataset) +- Deleting datasets with proper cleanup (delete_dataset) +- Updating RAG pipeline dataset settings (update_rag_pipeline_dataset_settings) +- Checking if dataset is in use (dataset_use_check) +- Updating dataset API access status (update_dataset_api_status) + +These operations are critical for dataset lifecycle management and require +careful handling of permissions, dependencies, and data integrity. + +This test suite ensures: +- Correct update of dataset properties +- Proper permission validation before updates/deletes +- Cascade deletion handling +- Event signaling for cleanup operations +- RAG pipeline dataset configuration updates +- API status management +- Use check validation + +================================================================================ +ARCHITECTURE OVERVIEW +================================================================================ + +The DatasetService update and delete operations are part of the dataset +lifecycle management system. These operations interact with multiple +components: + +1. Permission System: All update/delete operations require proper + permission validation to ensure users can only modify datasets they + have access to. + +2. Event System: Dataset deletion triggers the dataset_was_deleted event, + which notifies other components to clean up related data (documents, + segments, vector indices, etc.). + +3. Dependency Checking: Before deletion, the system checks if the dataset + is in use by any applications (via AppDatasetJoin). + +4. RAG Pipeline Integration: RAG pipeline datasets have special update + logic that handles chunk structure, indexing techniques, and embedding + model configuration. + +5. API Status Management: Datasets can have their API access enabled or + disabled, which affects whether they can be accessed via the API. + +================================================================================ +TESTING STRATEGY +================================================================================ + +This test suite follows a comprehensive testing strategy that covers: + +1. Update Operations: + - Internal dataset updates + - External dataset updates + - RAG pipeline dataset updates + - Permission validation + - Name duplicate checking + - Configuration validation + +2. Delete Operations: + - Successful deletion + - Permission validation + - Event signaling + - Database cleanup + - Not found handling + +3. Use Check Operations: + - Dataset in use detection + - Dataset not in use detection + - AppDatasetJoin query validation + +4. API Status Operations: + - Enable API access + - Disable API access + - Permission validation + - Current user validation + +5. RAG Pipeline Operations: + - Unpublished dataset updates + - Published dataset updates + - Chunk structure validation + - Indexing technique changes + - Embedding model configuration + +================================================================================ +""" + +import datetime +from unittest.mock import Mock, create_autospec, patch + +import pytest +from sqlalchemy.orm import Session +from werkzeug.exceptions import NotFound + +from models import Account, TenantAccountRole +from models.dataset import ( + AppDatasetJoin, + Dataset, + DatasetPermissionEnum, +) +from services.dataset_service import DatasetService +from services.errors.account import NoPermissionError + +# ============================================================================ +# Test Data Factory +# ============================================================================ +# The Test Data Factory pattern is used here to centralize the creation of +# test objects and mock instances. This approach provides several benefits: +# +# 1. Consistency: All test objects are created using the same factory methods, +# ensuring consistent structure across all tests. +# +# 2. Maintainability: If the structure of models or services changes, we only +# need to update the factory methods rather than every individual test. +# +# 3. Reusability: Factory methods can be reused across multiple test classes, +# reducing code duplication. +# +# 4. Readability: Tests become more readable when they use descriptive factory +# method calls instead of complex object construction logic. +# +# ============================================================================ + + +class DatasetUpdateDeleteTestDataFactory: + """ + Factory class for creating test data and mock objects for dataset update/delete tests. + + This factory provides static methods to create mock objects for: + - Dataset instances with various configurations + - User/Account instances with different roles + - Knowledge configuration objects + - Database session mocks + - Event signal mocks + + The factory methods help maintain consistency across tests and reduce + code duplication when setting up test scenarios. + """ + + @staticmethod + def create_dataset_mock( + dataset_id: str = "dataset-123", + provider: str = "vendor", + name: str = "Test Dataset", + description: str = "Test description", + tenant_id: str = "tenant-123", + indexing_technique: str = "high_quality", + embedding_model_provider: str | None = "openai", + embedding_model: str | None = "text-embedding-ada-002", + collection_binding_id: str | None = "binding-123", + enable_api: bool = True, + permission: DatasetPermissionEnum = DatasetPermissionEnum.ONLY_ME, + created_by: str = "user-123", + chunk_structure: str | None = None, + runtime_mode: str = "general", + **kwargs, + ) -> Mock: + """ + Create a mock Dataset with specified attributes. + + Args: + dataset_id: Unique identifier for the dataset + provider: Dataset provider (vendor, external) + name: Dataset name + description: Dataset description + tenant_id: Tenant identifier + indexing_technique: Indexing technique (high_quality, economy) + embedding_model_provider: Embedding model provider + embedding_model: Embedding model name + collection_binding_id: Collection binding ID + enable_api: Whether API access is enabled + permission: Dataset permission level + created_by: ID of user who created the dataset + chunk_structure: Chunk structure for RAG pipeline datasets + runtime_mode: Runtime mode (general, rag_pipeline) + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a Dataset instance + """ + dataset = Mock(spec=Dataset) + dataset.id = dataset_id + dataset.provider = provider + dataset.name = name + dataset.description = description + dataset.tenant_id = tenant_id + dataset.indexing_technique = indexing_technique + dataset.embedding_model_provider = embedding_model_provider + dataset.embedding_model = embedding_model + dataset.collection_binding_id = collection_binding_id + dataset.enable_api = enable_api + dataset.permission = permission + dataset.created_by = created_by + dataset.chunk_structure = chunk_structure + dataset.runtime_mode = runtime_mode + dataset.retrieval_model = {} + dataset.keyword_number = 10 + for key, value in kwargs.items(): + setattr(dataset, key, value) + return dataset + + @staticmethod + def create_user_mock( + user_id: str = "user-123", + tenant_id: str = "tenant-123", + role: TenantAccountRole = TenantAccountRole.NORMAL, + is_dataset_editor: bool = True, + **kwargs, + ) -> Mock: + """ + Create a mock user (Account) with specified attributes. + + Args: + user_id: Unique identifier for the user + tenant_id: Tenant identifier + role: User role (OWNER, ADMIN, NORMAL, etc.) + is_dataset_editor: Whether user has dataset editor permissions + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as an Account instance + """ + user = create_autospec(Account, instance=True) + user.id = user_id + user.current_tenant_id = tenant_id + user.current_role = role + user.is_dataset_editor = is_dataset_editor + for key, value in kwargs.items(): + setattr(user, key, value) + return user + + @staticmethod + def create_knowledge_configuration_mock( + chunk_structure: str = "tree", + indexing_technique: str = "high_quality", + embedding_model_provider: str = "openai", + embedding_model: str = "text-embedding-ada-002", + keyword_number: int = 10, + retrieval_model: dict | None = None, + **kwargs, + ) -> Mock: + """ + Create a mock KnowledgeConfiguration entity. + + Args: + chunk_structure: Chunk structure type + indexing_technique: Indexing technique + embedding_model_provider: Embedding model provider + embedding_model: Embedding model name + keyword_number: Keyword number for economy indexing + retrieval_model: Retrieval model configuration + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a KnowledgeConfiguration instance + """ + config = Mock() + config.chunk_structure = chunk_structure + config.indexing_technique = indexing_technique + config.embedding_model_provider = embedding_model_provider + config.embedding_model = embedding_model + config.keyword_number = keyword_number + config.retrieval_model = Mock() + config.retrieval_model.model_dump.return_value = retrieval_model or { + "search_method": "semantic_search", + "top_k": 2, + } + for key, value in kwargs.items(): + setattr(config, key, value) + return config + + @staticmethod + def create_app_dataset_join_mock( + app_id: str = "app-123", + dataset_id: str = "dataset-123", + **kwargs, + ) -> Mock: + """ + Create a mock AppDatasetJoin instance. + + Args: + app_id: Application ID + dataset_id: Dataset ID + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as an AppDatasetJoin instance + """ + join = Mock(spec=AppDatasetJoin) + join.app_id = app_id + join.dataset_id = dataset_id + for key, value in kwargs.items(): + setattr(join, key, value) + return join + + +# ============================================================================ +# Tests for update_dataset +# ============================================================================ + + +class TestDatasetServiceUpdateDataset: + """ + Comprehensive unit tests for DatasetService.update_dataset method. + + This test class covers the dataset update functionality, including + internal and external dataset updates, permission validation, and + name duplicate checking. + + The update_dataset method: + 1. Retrieves the dataset by ID + 2. Validates dataset exists + 3. Checks for duplicate names + 4. Validates user permissions + 5. Routes to appropriate update handler (internal or external) + 6. Returns the updated dataset + + Test scenarios include: + - Successful internal dataset updates + - Successful external dataset updates + - Permission validation + - Duplicate name detection + - Dataset not found errors + """ + + @pytest.fixture + def mock_dataset_service_dependencies(self): + """ + Mock dataset service dependencies for testing. + + Provides mocked dependencies including: + - get_dataset method + - check_dataset_permission method + - _has_dataset_same_name method + - Database session + - Current time utilities + """ + with ( + patch("services.dataset_service.DatasetService.get_dataset") as mock_get_dataset, + patch("services.dataset_service.DatasetService.check_dataset_permission") as mock_check_perm, + patch("services.dataset_service.DatasetService._has_dataset_same_name") as mock_has_same_name, + patch("extensions.ext_database.db.session") as mock_db, + patch("services.dataset_service.naive_utc_now") as mock_naive_utc_now, + ): + current_time = datetime.datetime(2023, 1, 1, 12, 0, 0) + mock_naive_utc_now.return_value = current_time + + yield { + "get_dataset": mock_get_dataset, + "check_permission": mock_check_perm, + "has_same_name": mock_has_same_name, + "db_session": mock_db, + "naive_utc_now": mock_naive_utc_now, + "current_time": current_time, + } + + def test_update_dataset_internal_success(self, mock_dataset_service_dependencies): + """ + Test successful update of an internal dataset. + + Verifies that when all validation passes, an internal dataset + is updated correctly through the _update_internal_dataset method. + + This test ensures: + - Dataset is retrieved correctly + - Permission is checked + - Name duplicate check is performed + - Internal update handler is called + - Updated dataset is returned + """ + # Arrange + dataset_id = "dataset-123" + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock( + dataset_id=dataset_id, provider="vendor", name="Old Name" + ) + user = DatasetUpdateDeleteTestDataFactory.create_user_mock() + + update_data = { + "name": "New Name", + "description": "New Description", + } + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + mock_dataset_service_dependencies["has_same_name"].return_value = False + + with patch("services.dataset_service.DatasetService._update_internal_dataset") as mock_update_internal: + mock_update_internal.return_value = dataset + + # Act + result = DatasetService.update_dataset(dataset_id, update_data, user) + + # Assert + assert result == dataset + + # Verify dataset was retrieved + mock_dataset_service_dependencies["get_dataset"].assert_called_once_with(dataset_id) + + # Verify permission was checked + mock_dataset_service_dependencies["check_permission"].assert_called_once_with(dataset, user) + + # Verify name duplicate check was performed + mock_dataset_service_dependencies["has_same_name"].assert_called_once() + + # Verify internal update handler was called + mock_update_internal.assert_called_once() + + def test_update_dataset_external_success(self, mock_dataset_service_dependencies): + """ + Test successful update of an external dataset. + + Verifies that when all validation passes, an external dataset + is updated correctly through the _update_external_dataset method. + + This test ensures: + - Dataset is retrieved correctly + - Permission is checked + - Name duplicate check is performed + - External update handler is called + - Updated dataset is returned + """ + # Arrange + dataset_id = "dataset-123" + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock( + dataset_id=dataset_id, provider="external", name="Old Name" + ) + user = DatasetUpdateDeleteTestDataFactory.create_user_mock() + + update_data = { + "name": "New Name", + "external_knowledge_id": "new-knowledge-id", + } + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + mock_dataset_service_dependencies["has_same_name"].return_value = False + + with patch("services.dataset_service.DatasetService._update_external_dataset") as mock_update_external: + mock_update_external.return_value = dataset + + # Act + result = DatasetService.update_dataset(dataset_id, update_data, user) + + # Assert + assert result == dataset + + # Verify external update handler was called + mock_update_external.assert_called_once() + + def test_update_dataset_not_found_error(self, mock_dataset_service_dependencies): + """ + Test error handling when dataset is not found. + + Verifies that when the dataset ID doesn't exist, a ValueError + is raised with an appropriate message. + + This test ensures: + - Dataset not found error is handled correctly + - No update operations are performed + - Error message is clear + """ + # Arrange + dataset_id = "non-existent-dataset" + user = DatasetUpdateDeleteTestDataFactory.create_user_mock() + + update_data = {"name": "New Name"} + + mock_dataset_service_dependencies["get_dataset"].return_value = None + + # Act & Assert + with pytest.raises(ValueError, match="Dataset not found"): + DatasetService.update_dataset(dataset_id, update_data, user) + + # Verify no update operations were attempted + mock_dataset_service_dependencies["check_permission"].assert_not_called() + mock_dataset_service_dependencies["has_same_name"].assert_not_called() + + def test_update_dataset_duplicate_name_error(self, mock_dataset_service_dependencies): + """ + Test error handling when dataset name already exists. + + Verifies that when a dataset with the same name already exists + in the tenant, a ValueError is raised. + + This test ensures: + - Duplicate name detection works correctly + - Error message is clear + - No update operations are performed + """ + # Arrange + dataset_id = "dataset-123" + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock(dataset_id=dataset_id) + user = DatasetUpdateDeleteTestDataFactory.create_user_mock() + + update_data = {"name": "Existing Name"} + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + mock_dataset_service_dependencies["has_same_name"].return_value = True # Duplicate exists + + # Act & Assert + with pytest.raises(ValueError, match="Dataset name already exists"): + DatasetService.update_dataset(dataset_id, update_data, user) + + # Verify permission check was not called (fails before that) + mock_dataset_service_dependencies["check_permission"].assert_not_called() + + def test_update_dataset_permission_denied_error(self, mock_dataset_service_dependencies): + """ + Test error handling when user lacks permission. + + Verifies that when the user doesn't have permission to update + the dataset, a NoPermissionError is raised. + + This test ensures: + - Permission validation works correctly + - Error is raised before any updates + - Error type is correct + """ + # Arrange + dataset_id = "dataset-123" + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock(dataset_id=dataset_id) + user = DatasetUpdateDeleteTestDataFactory.create_user_mock() + + update_data = {"name": "New Name"} + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + mock_dataset_service_dependencies["has_same_name"].return_value = False + mock_dataset_service_dependencies["check_permission"].side_effect = NoPermissionError("No permission") + + # Act & Assert + with pytest.raises(NoPermissionError): + DatasetService.update_dataset(dataset_id, update_data, user) + + +# ============================================================================ +# Tests for delete_dataset +# ============================================================================ + + +class TestDatasetServiceDeleteDataset: + """ + Comprehensive unit tests for DatasetService.delete_dataset method. + + This test class covers the dataset deletion functionality, including + permission validation, event signaling, and database cleanup. + + The delete_dataset method: + 1. Retrieves the dataset by ID + 2. Returns False if dataset not found + 3. Validates user permissions + 4. Sends dataset_was_deleted event + 5. Deletes dataset from database + 6. Commits transaction + 7. Returns True on success + + Test scenarios include: + - Successful dataset deletion + - Permission validation + - Event signaling + - Database cleanup + - Not found handling + """ + + @pytest.fixture + def mock_dataset_service_dependencies(self): + """ + Mock dataset service dependencies for testing. + + Provides mocked dependencies including: + - get_dataset method + - check_dataset_permission method + - dataset_was_deleted event signal + - Database session + """ + with ( + patch("services.dataset_service.DatasetService.get_dataset") as mock_get_dataset, + patch("services.dataset_service.DatasetService.check_dataset_permission") as mock_check_perm, + patch("services.dataset_service.dataset_was_deleted") as mock_event, + patch("extensions.ext_database.db.session") as mock_db, + ): + yield { + "get_dataset": mock_get_dataset, + "check_permission": mock_check_perm, + "dataset_was_deleted": mock_event, + "db_session": mock_db, + } + + def test_delete_dataset_success(self, mock_dataset_service_dependencies): + """ + Test successful deletion of a dataset. + + Verifies that when all validation passes, a dataset is deleted + correctly with proper event signaling and database cleanup. + + This test ensures: + - Dataset is retrieved correctly + - Permission is checked + - Event is sent for cleanup + - Dataset is deleted from database + - Transaction is committed + - Method returns True + """ + # Arrange + dataset_id = "dataset-123" + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock(dataset_id=dataset_id) + user = DatasetUpdateDeleteTestDataFactory.create_user_mock() + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + + # Act + result = DatasetService.delete_dataset(dataset_id, user) + + # Assert + assert result is True + + # Verify dataset was retrieved + mock_dataset_service_dependencies["get_dataset"].assert_called_once_with(dataset_id) + + # Verify permission was checked + mock_dataset_service_dependencies["check_permission"].assert_called_once_with(dataset, user) + + # Verify event was sent for cleanup + mock_dataset_service_dependencies["dataset_was_deleted"].send.assert_called_once_with(dataset) + + # Verify dataset was deleted and committed + mock_dataset_service_dependencies["db_session"].delete.assert_called_once_with(dataset) + mock_dataset_service_dependencies["db_session"].commit.assert_called_once() + + def test_delete_dataset_not_found(self, mock_dataset_service_dependencies): + """ + Test handling when dataset is not found. + + Verifies that when the dataset ID doesn't exist, the method + returns False without performing any operations. + + This test ensures: + - Method returns False when dataset not found + - No permission checks are performed + - No events are sent + - No database operations are performed + """ + # Arrange + dataset_id = "non-existent-dataset" + user = DatasetUpdateDeleteTestDataFactory.create_user_mock() + + mock_dataset_service_dependencies["get_dataset"].return_value = None + + # Act + result = DatasetService.delete_dataset(dataset_id, user) + + # Assert + assert result is False + + # Verify no operations were performed + mock_dataset_service_dependencies["check_permission"].assert_not_called() + mock_dataset_service_dependencies["dataset_was_deleted"].send.assert_not_called() + mock_dataset_service_dependencies["db_session"].delete.assert_not_called() + + def test_delete_dataset_permission_denied_error(self, mock_dataset_service_dependencies): + """ + Test error handling when user lacks permission. + + Verifies that when the user doesn't have permission to delete + the dataset, a NoPermissionError is raised. + + This test ensures: + - Permission validation works correctly + - Error is raised before deletion + - No database operations are performed + """ + # Arrange + dataset_id = "dataset-123" + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock(dataset_id=dataset_id) + user = DatasetUpdateDeleteTestDataFactory.create_user_mock() + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + mock_dataset_service_dependencies["check_permission"].side_effect = NoPermissionError("No permission") + + # Act & Assert + with pytest.raises(NoPermissionError): + DatasetService.delete_dataset(dataset_id, user) + + # Verify no deletion was attempted + mock_dataset_service_dependencies["db_session"].delete.assert_not_called() + + +# ============================================================================ +# Tests for dataset_use_check +# ============================================================================ + + +class TestDatasetServiceDatasetUseCheck: + """ + Comprehensive unit tests for DatasetService.dataset_use_check method. + + This test class covers the dataset use checking functionality, which + determines if a dataset is currently being used by any applications. + + The dataset_use_check method: + 1. Queries AppDatasetJoin table for the dataset ID + 2. Returns True if dataset is in use + 3. Returns False if dataset is not in use + + Test scenarios include: + - Dataset in use (has AppDatasetJoin records) + - Dataset not in use (no AppDatasetJoin records) + - Database query validation + """ + + @pytest.fixture + def mock_db_session(self): + """ + Mock database session for testing. + + Provides a mocked database session that can be used to verify + query construction and execution. + """ + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + def test_dataset_use_check_in_use(self, mock_db_session): + """ + Test detection when dataset is in use. + + Verifies that when a dataset has associated AppDatasetJoin records, + the method returns True. + + This test ensures: + - Query is constructed correctly + - True is returned when dataset is in use + - Database query is executed + """ + # Arrange + dataset_id = "dataset-123" + + # Mock the exists() query to return True + mock_execute = Mock() + mock_execute.scalar_one.return_value = True + mock_db_session.execute.return_value = mock_execute + + # Act + result = DatasetService.dataset_use_check(dataset_id) + + # Assert + assert result is True + + # Verify query was executed + mock_db_session.execute.assert_called_once() + + def test_dataset_use_check_not_in_use(self, mock_db_session): + """ + Test detection when dataset is not in use. + + Verifies that when a dataset has no associated AppDatasetJoin records, + the method returns False. + + This test ensures: + - Query is constructed correctly + - False is returned when dataset is not in use + - Database query is executed + """ + # Arrange + dataset_id = "dataset-123" + + # Mock the exists() query to return False + mock_execute = Mock() + mock_execute.scalar_one.return_value = False + mock_db_session.execute.return_value = mock_execute + + # Act + result = DatasetService.dataset_use_check(dataset_id) + + # Assert + assert result is False + + # Verify query was executed + mock_db_session.execute.assert_called_once() + + +# ============================================================================ +# Tests for update_dataset_api_status +# ============================================================================ + + +class TestDatasetServiceUpdateDatasetApiStatus: + """ + Comprehensive unit tests for DatasetService.update_dataset_api_status method. + + This test class covers the dataset API status update functionality, + which enables or disables API access for a dataset. + + The update_dataset_api_status method: + 1. Retrieves the dataset by ID + 2. Validates dataset exists + 3. Updates enable_api field + 4. Updates updated_by and updated_at fields + 5. Commits transaction + + Test scenarios include: + - Successful API status enable + - Successful API status disable + - Dataset not found error + - Current user validation + """ + + @pytest.fixture + def mock_dataset_service_dependencies(self): + """ + Mock dataset service dependencies for testing. + + Provides mocked dependencies including: + - get_dataset method + - current_user context + - Database session + - Current time utilities + """ + with ( + patch("services.dataset_service.DatasetService.get_dataset") as mock_get_dataset, + patch( + "services.dataset_service.current_user", create_autospec(Account, instance=True) + ) as mock_current_user, + patch("extensions.ext_database.db.session") as mock_db, + patch("services.dataset_service.naive_utc_now") as mock_naive_utc_now, + ): + current_time = datetime.datetime(2023, 1, 1, 12, 0, 0) + mock_naive_utc_now.return_value = current_time + mock_current_user.id = "user-123" + + yield { + "get_dataset": mock_get_dataset, + "current_user": mock_current_user, + "db_session": mock_db, + "naive_utc_now": mock_naive_utc_now, + "current_time": current_time, + } + + def test_update_dataset_api_status_enable_success(self, mock_dataset_service_dependencies): + """ + Test successful enabling of dataset API access. + + Verifies that when all validation passes, the dataset's API + access is enabled and the update is committed. + + This test ensures: + - Dataset is retrieved correctly + - enable_api is set to True + - updated_by and updated_at are set + - Transaction is committed + """ + # Arrange + dataset_id = "dataset-123" + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock(dataset_id=dataset_id, enable_api=False) + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + + # Act + DatasetService.update_dataset_api_status(dataset_id, True) + + # Assert + assert dataset.enable_api is True + assert dataset.updated_by == "user-123" + assert dataset.updated_at == mock_dataset_service_dependencies["current_time"] + + # Verify dataset was retrieved + mock_dataset_service_dependencies["get_dataset"].assert_called_once_with(dataset_id) + + # Verify transaction was committed + mock_dataset_service_dependencies["db_session"].commit.assert_called_once() + + def test_update_dataset_api_status_disable_success(self, mock_dataset_service_dependencies): + """ + Test successful disabling of dataset API access. + + Verifies that when all validation passes, the dataset's API + access is disabled and the update is committed. + + This test ensures: + - Dataset is retrieved correctly + - enable_api is set to False + - updated_by and updated_at are set + - Transaction is committed + """ + # Arrange + dataset_id = "dataset-123" + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock(dataset_id=dataset_id, enable_api=True) + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + + # Act + DatasetService.update_dataset_api_status(dataset_id, False) + + # Assert + assert dataset.enable_api is False + assert dataset.updated_by == "user-123" + + # Verify transaction was committed + mock_dataset_service_dependencies["db_session"].commit.assert_called_once() + + def test_update_dataset_api_status_not_found_error(self, mock_dataset_service_dependencies): + """ + Test error handling when dataset is not found. + + Verifies that when the dataset ID doesn't exist, a NotFound + exception is raised. + + This test ensures: + - NotFound exception is raised + - No updates are performed + - Error message is appropriate + """ + # Arrange + dataset_id = "non-existent-dataset" + + mock_dataset_service_dependencies["get_dataset"].return_value = None + + # Act & Assert + with pytest.raises(NotFound, match="Dataset not found"): + DatasetService.update_dataset_api_status(dataset_id, True) + + # Verify no commit was attempted + mock_dataset_service_dependencies["db_session"].commit.assert_not_called() + + def test_update_dataset_api_status_missing_current_user_error(self, mock_dataset_service_dependencies): + """ + Test error handling when current_user is missing. + + Verifies that when current_user is None or has no ID, a ValueError + is raised. + + This test ensures: + - ValueError is raised when current_user is None + - Error message is clear + - No updates are committed + """ + # Arrange + dataset_id = "dataset-123" + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock(dataset_id=dataset_id) + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + mock_dataset_service_dependencies["current_user"].id = None # Missing user ID + + # Act & Assert + with pytest.raises(ValueError, match="Current user or current user id not found"): + DatasetService.update_dataset_api_status(dataset_id, True) + + # Verify no commit was attempted + mock_dataset_service_dependencies["db_session"].commit.assert_not_called() + + +# ============================================================================ +# Tests for update_rag_pipeline_dataset_settings +# ============================================================================ + + +class TestDatasetServiceUpdateRagPipelineDatasetSettings: + """ + Comprehensive unit tests for DatasetService.update_rag_pipeline_dataset_settings method. + + This test class covers the RAG pipeline dataset settings update functionality, + including chunk structure, indexing technique, and embedding model configuration. + + The update_rag_pipeline_dataset_settings method: + 1. Validates current_user and tenant + 2. Merges dataset into session + 3. Handles unpublished vs published datasets differently + 4. Updates chunk structure, indexing technique, and retrieval model + 5. Configures embedding model for high_quality indexing + 6. Updates keyword_number for economy indexing + 7. Commits transaction + 8. Triggers index update tasks if needed + + Test scenarios include: + - Unpublished dataset updates + - Published dataset updates + - Chunk structure validation + - Indexing technique changes + - Embedding model configuration + - Error handling + """ + + @pytest.fixture + def mock_session(self): + """ + Mock database session for testing. + + Provides a mocked SQLAlchemy session for testing session operations. + """ + return Mock(spec=Session) + + @pytest.fixture + def mock_dataset_service_dependencies(self): + """ + Mock dataset service dependencies for testing. + + Provides mocked dependencies including: + - current_user context + - ModelManager + - DatasetCollectionBindingService + - Database session operations + - Task scheduling + """ + with ( + patch( + "services.dataset_service.current_user", create_autospec(Account, instance=True) + ) as mock_current_user, + patch("services.dataset_service.ModelManager") as mock_model_manager, + patch( + "services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding" + ) as mock_get_binding, + patch("services.dataset_service.deal_dataset_index_update_task") as mock_task, + ): + mock_current_user.current_tenant_id = "tenant-123" + mock_current_user.id = "user-123" + + yield { + "current_user": mock_current_user, + "model_manager": mock_model_manager, + "get_binding": mock_get_binding, + "task": mock_task, + } + + def test_update_rag_pipeline_dataset_settings_unpublished_success( + self, mock_session, mock_dataset_service_dependencies + ): + """ + Test successful update of unpublished RAG pipeline dataset. + + Verifies that when a dataset is not published, all settings can + be updated including chunk structure and indexing technique. + + This test ensures: + - Current user validation passes + - Dataset is merged into session + - Chunk structure is updated + - Indexing technique is updated + - Embedding model is configured for high_quality + - Retrieval model is updated + - Dataset is added to session + """ + # Arrange + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock( + dataset_id="dataset-123", + runtime_mode="rag_pipeline", + chunk_structure="tree", + indexing_technique="high_quality", + ) + + knowledge_config = DatasetUpdateDeleteTestDataFactory.create_knowledge_configuration_mock( + chunk_structure="list", + indexing_technique="high_quality", + embedding_model_provider="openai", + embedding_model="text-embedding-ada-002", + ) + + # Mock embedding model + mock_embedding_model = Mock() + mock_embedding_model.model = "text-embedding-ada-002" + mock_embedding_model.provider = "openai" + + mock_model_instance = Mock() + mock_model_instance.get_model_instance.return_value = mock_embedding_model + mock_dataset_service_dependencies["model_manager"].return_value = mock_model_instance + + # Mock collection binding + mock_binding = Mock() + mock_binding.id = "binding-123" + mock_dataset_service_dependencies["get_binding"].return_value = mock_binding + + mock_session.merge.return_value = dataset + + # Act + DatasetService.update_rag_pipeline_dataset_settings( + mock_session, dataset, knowledge_config, has_published=False + ) + + # Assert + assert dataset.chunk_structure == "list" + assert dataset.indexing_technique == "high_quality" + assert dataset.embedding_model == "text-embedding-ada-002" + assert dataset.embedding_model_provider == "openai" + assert dataset.collection_binding_id == "binding-123" + + # Verify dataset was added to session + mock_session.add.assert_called_once_with(dataset) + + def test_update_rag_pipeline_dataset_settings_published_chunk_structure_error( + self, mock_session, mock_dataset_service_dependencies + ): + """ + Test error handling when trying to update chunk structure of published dataset. + + Verifies that when a dataset is published and has an existing chunk structure, + attempting to change it raises a ValueError. + + This test ensures: + - Chunk structure change is detected + - ValueError is raised with appropriate message + - No updates are committed + """ + # Arrange + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock( + dataset_id="dataset-123", + runtime_mode="rag_pipeline", + chunk_structure="tree", # Existing structure + indexing_technique="high_quality", + ) + + knowledge_config = DatasetUpdateDeleteTestDataFactory.create_knowledge_configuration_mock( + chunk_structure="list", # Different structure + indexing_technique="high_quality", + ) + + mock_session.merge.return_value = dataset + + # Act & Assert + with pytest.raises(ValueError, match="Chunk structure is not allowed to be updated"): + DatasetService.update_rag_pipeline_dataset_settings( + mock_session, dataset, knowledge_config, has_published=True + ) + + # Verify no commit was attempted + mock_session.commit.assert_not_called() + + def test_update_rag_pipeline_dataset_settings_published_economy_error( + self, mock_session, mock_dataset_service_dependencies + ): + """ + Test error handling when trying to change to economy indexing on published dataset. + + Verifies that when a dataset is published, changing indexing technique to + economy is not allowed and raises a ValueError. + + This test ensures: + - Economy indexing change is detected + - ValueError is raised with appropriate message + - No updates are committed + """ + # Arrange + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock( + dataset_id="dataset-123", + runtime_mode="rag_pipeline", + indexing_technique="high_quality", # Current technique + ) + + knowledge_config = DatasetUpdateDeleteTestDataFactory.create_knowledge_configuration_mock( + indexing_technique="economy", # Trying to change to economy + ) + + mock_session.merge.return_value = dataset + + # Act & Assert + with pytest.raises( + ValueError, match="Knowledge base indexing technique is not allowed to be updated to economy" + ): + DatasetService.update_rag_pipeline_dataset_settings( + mock_session, dataset, knowledge_config, has_published=True + ) + + def test_update_rag_pipeline_dataset_settings_missing_current_user_error( + self, mock_session, mock_dataset_service_dependencies + ): + """ + Test error handling when current_user is missing. + + Verifies that when current_user is None or has no tenant ID, a ValueError + is raised. + + This test ensures: + - Current user validation works correctly + - Error message is clear + - No updates are performed + """ + # Arrange + dataset = DatasetUpdateDeleteTestDataFactory.create_dataset_mock() + knowledge_config = DatasetUpdateDeleteTestDataFactory.create_knowledge_configuration_mock() + + mock_dataset_service_dependencies["current_user"].current_tenant_id = None # Missing tenant + + # Act & Assert + with pytest.raises(ValueError, match="Current user or current tenant not found"): + DatasetService.update_rag_pipeline_dataset_settings( + mock_session, dataset, knowledge_config, has_published=False + ) + + +# ============================================================================ +# Additional Documentation and Notes +# ============================================================================ +# +# This test suite covers the core update and delete operations for datasets. +# Additional test scenarios that could be added: +# +# 1. Update Operations: +# - Testing with different indexing techniques +# - Testing embedding model provider changes +# - Testing retrieval model updates +# - Testing icon_info updates +# - Testing partial_member_list updates +# +# 2. Delete Operations: +# - Testing cascade deletion of related data +# - Testing event handler execution +# - Testing with datasets that have documents +# - Testing with datasets that have segments +# +# 3. RAG Pipeline Operations: +# - Testing economy indexing technique updates +# - Testing embedding model provider errors +# - Testing keyword_number updates +# - Testing index update task triggering +# +# 4. Integration Scenarios: +# - Testing update followed by delete +# - Testing multiple updates in sequence +# - Testing concurrent update attempts +# - Testing with different user roles +# +# These scenarios are not currently implemented but could be added if needed +# based on real-world usage patterns or discovered edge cases. +# +# ============================================================================ diff --git a/api/tests/unit_tests/services/external_dataset_service.py b/api/tests/unit_tests/services/external_dataset_service.py new file mode 100644 index 0000000000..1647eb3e85 --- /dev/null +++ b/api/tests/unit_tests/services/external_dataset_service.py @@ -0,0 +1,920 @@ +""" +Extensive unit tests for ``ExternalDatasetService``. + +This module focuses on the *external dataset service* surface area, which is responsible +for integrating with **external knowledge APIs** and wiring them into Dify datasets. + +The goal of this test suite is twofold: + +- Provide **high‑confidence regression coverage** for all public helpers on + ``ExternalDatasetService``. +- Serve as **executable documentation** for how external API integration is expected + to behave in different scenarios (happy paths, validation failures, and error codes). + +The file intentionally contains **rich comments and generous spacing** in order to make +each scenario easy to scan during reviews. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any, cast +from unittest.mock import MagicMock, Mock, patch + +import httpx +import pytest + +from constants import HIDDEN_VALUE +from models.dataset import Dataset, ExternalKnowledgeApis, ExternalKnowledgeBindings +from services.entities.external_knowledge_entities.external_knowledge_entities import ( + Authorization, + AuthorizationConfig, + ExternalKnowledgeApiSetting, +) +from services.errors.dataset import DatasetNameDuplicateError +from services.external_knowledge_service import ExternalDatasetService + + +class ExternalDatasetTestDataFactory: + """ + Factory helpers for building *lightweight* mocks for external knowledge tests. + + These helpers are intentionally small and explicit: + + - They avoid pulling in unnecessary fixtures. + - They reflect the minimal contract that the service under test cares about. + """ + + @staticmethod + def create_external_api( + api_id: str = "api-123", + tenant_id: str = "tenant-1", + name: str = "Test API", + description: str = "Description", + settings: dict | None = None, + ) -> ExternalKnowledgeApis: + """ + Create a concrete ``ExternalKnowledgeApis`` instance with minimal fields. + + Using the real SQLAlchemy model (instead of a pure Mock) makes it easier to + exercise ``settings_dict`` and other convenience properties if needed. + """ + + instance = ExternalKnowledgeApis( + tenant_id=tenant_id, + name=name, + description=description, + settings=None if settings is None else cast(str, pytest.approx), # type: ignore[assignment] + ) + + # Overwrite generated id for determinism in assertions. + instance.id = api_id + return instance + + @staticmethod + def create_dataset( + dataset_id: str = "ds-1", + tenant_id: str = "tenant-1", + name: str = "External Dataset", + provider: str = "external", + ) -> Dataset: + """ + Build a small ``Dataset`` instance representing an external dataset. + """ + + dataset = Dataset( + tenant_id=tenant_id, + name=name, + description="", + provider=provider, + created_by="user-1", + ) + dataset.id = dataset_id + return dataset + + @staticmethod + def create_external_binding( + tenant_id: str = "tenant-1", + dataset_id: str = "ds-1", + api_id: str = "api-1", + external_knowledge_id: str = "knowledge-1", + ) -> ExternalKnowledgeBindings: + """ + Small helper for a binding between dataset and external knowledge API. + """ + + binding = ExternalKnowledgeBindings( + tenant_id=tenant_id, + dataset_id=dataset_id, + external_knowledge_api_id=api_id, + external_knowledge_id=external_knowledge_id, + created_by="user-1", + ) + return binding + + +# --------------------------------------------------------------------------- +# get_external_knowledge_apis +# --------------------------------------------------------------------------- + + +class TestExternalDatasetServiceGetExternalKnowledgeApis: + """ + Tests for ``ExternalDatasetService.get_external_knowledge_apis``. + + These tests focus on: + + - Basic pagination wiring via ``db.paginate``. + - Optional search keyword behaviour. + """ + + @pytest.fixture + def mock_db_paginate(self): + """ + Patch ``db.paginate`` so we do not touch the real database layer. + """ + + with ( + patch("services.external_knowledge_service.db.paginate") as mock_paginate, + patch("services.external_knowledge_service.select"), + ): + yield mock_paginate + + def test_get_external_knowledge_apis_basic_pagination(self, mock_db_paginate: MagicMock): + """ + It should return ``items`` and ``total`` coming from the paginate object. + """ + + # Arrange + tenant_id = "tenant-1" + page = 1 + per_page = 20 + + mock_items = [Mock(spec=ExternalKnowledgeApis), Mock(spec=ExternalKnowledgeApis)] + mock_pagination = SimpleNamespace(items=mock_items, total=42) + mock_db_paginate.return_value = mock_pagination + + # Act + items, total = ExternalDatasetService.get_external_knowledge_apis(page, per_page, tenant_id) + + # Assert + assert items is mock_items + assert total == 42 + + mock_db_paginate.assert_called_once() + call_kwargs = mock_db_paginate.call_args.kwargs + assert call_kwargs["page"] == page + assert call_kwargs["per_page"] == per_page + assert call_kwargs["max_per_page"] == 100 + assert call_kwargs["error_out"] is False + + def test_get_external_knowledge_apis_with_search_keyword(self, mock_db_paginate: MagicMock): + """ + When a search keyword is provided, the query should be adjusted + (we simply assert that paginate is still called and does not explode). + """ + + # Arrange + tenant_id = "tenant-1" + page = 2 + per_page = 10 + search = "foo" + + mock_pagination = SimpleNamespace(items=[], total=0) + mock_db_paginate.return_value = mock_pagination + + # Act + items, total = ExternalDatasetService.get_external_knowledge_apis(page, per_page, tenant_id, search=search) + + # Assert + assert items == [] + assert total == 0 + mock_db_paginate.assert_called_once() + + +# --------------------------------------------------------------------------- +# validate_api_list +# --------------------------------------------------------------------------- + + +class TestExternalDatasetServiceValidateApiList: + """ + Lightweight validation tests for ``validate_api_list``. + """ + + def test_validate_api_list_success(self): + """ + A minimal valid configuration (endpoint + api_key) should pass. + """ + + config = {"endpoint": "https://example.com", "api_key": "secret"} + + # Act & Assert – no exception expected + ExternalDatasetService.validate_api_list(config) + + @pytest.mark.parametrize( + ("config", "expected_message"), + [ + ({}, "api list is empty"), + ({"api_key": "k"}, "endpoint is required"), + ({"endpoint": "https://example.com"}, "api_key is required"), + ], + ) + def test_validate_api_list_failures(self, config: dict, expected_message: str): + """ + Invalid configs should raise ``ValueError`` with a clear message. + """ + + with pytest.raises(ValueError, match=expected_message): + ExternalDatasetService.validate_api_list(config) + + +# --------------------------------------------------------------------------- +# create_external_knowledge_api & get/update/delete +# --------------------------------------------------------------------------- + + +class TestExternalDatasetServiceCrudExternalKnowledgeApi: + """ + CRUD tests for external knowledge API templates. + """ + + @pytest.fixture + def mock_db_session(self): + """ + Patch ``db.session`` for all CRUD tests in this class. + """ + + with patch("services.external_knowledge_service.db.session") as mock_session: + yield mock_session + + def test_create_external_knowledge_api_success(self, mock_db_session: MagicMock): + """ + ``create_external_knowledge_api`` should persist a new record + when settings are present and valid. + """ + + tenant_id = "tenant-1" + user_id = "user-1" + args = { + "name": "API", + "description": "desc", + "settings": {"endpoint": "https://api.example.com", "api_key": "secret"}, + } + + # We do not want to actually call the remote endpoint here, so we patch the validator. + with patch.object(ExternalDatasetService, "check_endpoint_and_api_key") as mock_check: + result = ExternalDatasetService.create_external_knowledge_api(tenant_id, user_id, args) + + assert isinstance(result, ExternalKnowledgeApis) + mock_check.assert_called_once_with(args["settings"]) + mock_db_session.add.assert_called_once() + mock_db_session.commit.assert_called_once() + + def test_create_external_knowledge_api_missing_settings_raises(self, mock_db_session: MagicMock): + """ + Missing ``settings`` should result in a ``ValueError``. + """ + + tenant_id = "tenant-1" + user_id = "user-1" + args = {"name": "API", "description": "desc"} + + with pytest.raises(ValueError, match="settings is required"): + ExternalDatasetService.create_external_knowledge_api(tenant_id, user_id, args) + + mock_db_session.add.assert_not_called() + mock_db_session.commit.assert_not_called() + + def test_get_external_knowledge_api_found(self, mock_db_session: MagicMock): + """ + ``get_external_knowledge_api`` should return the first matching record. + """ + + api = Mock(spec=ExternalKnowledgeApis) + mock_db_session.query.return_value.filter_by.return_value.first.return_value = api + + result = ExternalDatasetService.get_external_knowledge_api("api-id") + assert result is api + + def test_get_external_knowledge_api_not_found_raises(self, mock_db_session: MagicMock): + """ + When the record is absent, a ``ValueError`` is raised. + """ + + mock_db_session.query.return_value.filter_by.return_value.first.return_value = None + + with pytest.raises(ValueError, match="api template not found"): + ExternalDatasetService.get_external_knowledge_api("missing-id") + + def test_update_external_knowledge_api_success_with_hidden_api_key(self, mock_db_session: MagicMock): + """ + Updating an API should keep the existing API key when the special hidden + value placeholder is sent from the UI. + """ + + tenant_id = "tenant-1" + user_id = "user-1" + api_id = "api-1" + + existing_api = Mock(spec=ExternalKnowledgeApis) + existing_api.settings_dict = {"api_key": "stored-key"} + existing_api.settings = '{"api_key":"stored-key"}' + mock_db_session.query.return_value.filter_by.return_value.first.return_value = existing_api + + args = { + "name": "New Name", + "description": "New Desc", + "settings": {"endpoint": "https://api.example.com", "api_key": HIDDEN_VALUE}, + } + + result = ExternalDatasetService.update_external_knowledge_api(tenant_id, user_id, api_id, args) + + assert result is existing_api + # The placeholder should be replaced with stored key. + assert args["settings"]["api_key"] == "stored-key" + mock_db_session.commit.assert_called_once() + + def test_update_external_knowledge_api_not_found_raises(self, mock_db_session: MagicMock): + """ + Updating a non‑existent API template should raise ``ValueError``. + """ + + mock_db_session.query.return_value.filter_by.return_value.first.return_value = None + + with pytest.raises(ValueError, match="api template not found"): + ExternalDatasetService.update_external_knowledge_api( + tenant_id="tenant-1", + user_id="user-1", + external_knowledge_api_id="missing-id", + args={"name": "n", "description": "d", "settings": {}}, + ) + + def test_delete_external_knowledge_api_success(self, mock_db_session: MagicMock): + """ + ``delete_external_knowledge_api`` should delete and commit when found. + """ + + api = Mock(spec=ExternalKnowledgeApis) + mock_db_session.query.return_value.filter_by.return_value.first.return_value = api + + ExternalDatasetService.delete_external_knowledge_api("tenant-1", "api-1") + + mock_db_session.delete.assert_called_once_with(api) + mock_db_session.commit.assert_called_once() + + def test_delete_external_knowledge_api_not_found_raises(self, mock_db_session: MagicMock): + """ + Deletion of a missing template should raise ``ValueError``. + """ + + mock_db_session.query.return_value.filter_by.return_value.first.return_value = None + + with pytest.raises(ValueError, match="api template not found"): + ExternalDatasetService.delete_external_knowledge_api("tenant-1", "missing") + + +# --------------------------------------------------------------------------- +# external_knowledge_api_use_check & binding lookups +# --------------------------------------------------------------------------- + + +class TestExternalDatasetServiceUsageAndBindings: + """ + Tests for usage checks and dataset binding retrieval. + """ + + @pytest.fixture + def mock_db_session(self): + with patch("services.external_knowledge_service.db.session") as mock_session: + yield mock_session + + def test_external_knowledge_api_use_check_in_use(self, mock_db_session: MagicMock): + """ + When there are bindings, ``external_knowledge_api_use_check`` returns True and count. + """ + + mock_db_session.query.return_value.filter_by.return_value.count.return_value = 3 + + in_use, count = ExternalDatasetService.external_knowledge_api_use_check("api-1") + + assert in_use is True + assert count == 3 + + def test_external_knowledge_api_use_check_not_in_use(self, mock_db_session: MagicMock): + """ + Zero bindings should return ``(False, 0)``. + """ + + mock_db_session.query.return_value.filter_by.return_value.count.return_value = 0 + + in_use, count = ExternalDatasetService.external_knowledge_api_use_check("api-1") + + assert in_use is False + assert count == 0 + + def test_get_external_knowledge_binding_with_dataset_id_found(self, mock_db_session: MagicMock): + """ + Binding lookup should return the first record when present. + """ + + binding = Mock(spec=ExternalKnowledgeBindings) + mock_db_session.query.return_value.filter_by.return_value.first.return_value = binding + + result = ExternalDatasetService.get_external_knowledge_binding_with_dataset_id("tenant-1", "ds-1") + assert result is binding + + def test_get_external_knowledge_binding_with_dataset_id_not_found_raises(self, mock_db_session: MagicMock): + """ + Missing binding should result in a ``ValueError``. + """ + + mock_db_session.query.return_value.filter_by.return_value.first.return_value = None + + with pytest.raises(ValueError, match="external knowledge binding not found"): + ExternalDatasetService.get_external_knowledge_binding_with_dataset_id("tenant-1", "ds-1") + + +# --------------------------------------------------------------------------- +# document_create_args_validate +# --------------------------------------------------------------------------- + + +class TestExternalDatasetServiceDocumentCreateArgsValidate: + """ + Tests for ``document_create_args_validate``. + """ + + @pytest.fixture + def mock_db_session(self): + with patch("services.external_knowledge_service.db.session") as mock_session: + yield mock_session + + def test_document_create_args_validate_success(self, mock_db_session: MagicMock): + """ + All required custom parameters present – validation should pass. + """ + + external_api = Mock(spec=ExternalKnowledgeApis) + external_api.settings = json_settings = ( + '[{"document_process_setting":[{"name":"foo","required":true},{"name":"bar","required":false}]}]' + ) + # Raw string; the service itself calls json.loads on it + mock_db_session.query.return_value.filter_by.return_value.first.return_value = external_api + + process_parameter = {"foo": "value", "bar": "optional"} + + # Act & Assert – no exception + ExternalDatasetService.document_create_args_validate("tenant-1", "api-1", process_parameter) + + assert json_settings in external_api.settings # simple sanity check on our test data + + def test_document_create_args_validate_missing_template_raises(self, mock_db_session: MagicMock): + """ + When the referenced API template is missing, a ``ValueError`` is raised. + """ + + mock_db_session.query.return_value.filter_by.return_value.first.return_value = None + + with pytest.raises(ValueError, match="api template not found"): + ExternalDatasetService.document_create_args_validate("tenant-1", "missing", {}) + + def test_document_create_args_validate_missing_required_parameter_raises(self, mock_db_session: MagicMock): + """ + Required document process parameters must be supplied. + """ + + external_api = Mock(spec=ExternalKnowledgeApis) + external_api.settings = ( + '[{"document_process_setting":[{"name":"foo","required":true},{"name":"bar","required":false}]}]' + ) + mock_db_session.query.return_value.filter_by.return_value.first.return_value = external_api + + process_parameter = {"bar": "present"} # missing "foo" + + with pytest.raises(ValueError, match="foo is required"): + ExternalDatasetService.document_create_args_validate("tenant-1", "api-1", process_parameter) + + +# --------------------------------------------------------------------------- +# process_external_api +# --------------------------------------------------------------------------- + + +class TestExternalDatasetServiceProcessExternalApi: + """ + Tests focused on the HTTP request assembly and method mapping behaviour. + """ + + def test_process_external_api_valid_method_post(self): + """ + For a supported HTTP verb we should delegate to the correct ``ssrf_proxy`` function. + """ + + settings = ExternalKnowledgeApiSetting( + url="https://example.com/path", + request_method="POST", + headers={"X-Test": "1"}, + params={"foo": "bar"}, + ) + + fake_response = httpx.Response(200) + + with patch("services.external_knowledge_service.ssrf_proxy.post") as mock_post: + mock_post.return_value = fake_response + + result = ExternalDatasetService.process_external_api(settings, files=None) + + assert result is fake_response + mock_post.assert_called_once() + kwargs = mock_post.call_args.kwargs + assert kwargs["url"] == settings.url + assert kwargs["headers"] == settings.headers + assert kwargs["follow_redirects"] is True + assert "data" in kwargs + + def test_process_external_api_invalid_method_raises(self): + """ + An unsupported HTTP verb should raise ``InvalidHttpMethodError``. + """ + + settings = ExternalKnowledgeApiSetting( + url="https://example.com", + request_method="INVALID", + headers=None, + params={}, + ) + + from core.workflow.nodes.http_request.exc import InvalidHttpMethodError + + with pytest.raises(InvalidHttpMethodError): + ExternalDatasetService.process_external_api(settings, files=None) + + +# --------------------------------------------------------------------------- +# assembling_headers +# --------------------------------------------------------------------------- + + +class TestExternalDatasetServiceAssemblingHeaders: + """ + Tests for header assembly based on different authentication flavours. + """ + + def test_assembling_headers_bearer_token(self): + """ + For bearer auth we expect ``Authorization: Bearer `` by default. + """ + + auth = Authorization( + type="api-key", + config=AuthorizationConfig(type="bearer", api_key="secret", header=None), + ) + + headers = ExternalDatasetService.assembling_headers(auth) + + assert headers["Authorization"] == "Bearer secret" + + def test_assembling_headers_basic_token_with_custom_header(self): + """ + For basic auth we honour the configured header name. + """ + + auth = Authorization( + type="api-key", + config=AuthorizationConfig(type="basic", api_key="abc123", header="X-Auth"), + ) + + headers = ExternalDatasetService.assembling_headers(auth, headers={"Existing": "1"}) + + assert headers["Existing"] == "1" + assert headers["X-Auth"] == "Basic abc123" + + def test_assembling_headers_custom_type(self): + """ + Custom auth type should inject the raw API key. + """ + + auth = Authorization( + type="api-key", + config=AuthorizationConfig(type="custom", api_key="raw-key", header="X-API-KEY"), + ) + + headers = ExternalDatasetService.assembling_headers(auth, headers=None) + + assert headers["X-API-KEY"] == "raw-key" + + def test_assembling_headers_missing_config_raises(self): + """ + Missing config object should be rejected. + """ + + auth = Authorization(type="api-key", config=None) + + with pytest.raises(ValueError, match="authorization config is required"): + ExternalDatasetService.assembling_headers(auth) + + def test_assembling_headers_missing_api_key_raises(self): + """ + ``api_key`` is required when type is ``api-key``. + """ + + auth = Authorization( + type="api-key", + config=AuthorizationConfig(type="bearer", api_key=None, header="Authorization"), + ) + + with pytest.raises(ValueError, match="api_key is required"): + ExternalDatasetService.assembling_headers(auth) + + def test_assembling_headers_no_auth_type_leaves_headers_unchanged(self): + """ + For ``no-auth`` we should not modify the headers mapping. + """ + + auth = Authorization(type="no-auth", config=None) + + base_headers = {"X": "1"} + result = ExternalDatasetService.assembling_headers(auth, headers=base_headers) + + # A copy is returned, original is not mutated. + assert result == base_headers + assert result is not base_headers + + +# --------------------------------------------------------------------------- +# get_external_knowledge_api_settings +# --------------------------------------------------------------------------- + + +class TestExternalDatasetServiceGetExternalKnowledgeApiSettings: + """ + Simple shape test for ``get_external_knowledge_api_settings``. + """ + + def test_get_external_knowledge_api_settings(self): + settings_dict: dict[str, Any] = { + "url": "https://example.com/retrieval", + "request_method": "post", + "headers": {"Content-Type": "application/json"}, + "params": {"foo": "bar"}, + } + + result = ExternalDatasetService.get_external_knowledge_api_settings(settings_dict) + + assert isinstance(result, ExternalKnowledgeApiSetting) + assert result.url == settings_dict["url"] + assert result.request_method == settings_dict["request_method"] + assert result.headers == settings_dict["headers"] + assert result.params == settings_dict["params"] + + +# --------------------------------------------------------------------------- +# create_external_dataset +# --------------------------------------------------------------------------- + + +class TestExternalDatasetServiceCreateExternalDataset: + """ + Tests around creating the external dataset and its binding row. + """ + + @pytest.fixture + def mock_db_session(self): + with patch("services.external_knowledge_service.db.session") as mock_session: + yield mock_session + + def test_create_external_dataset_success(self, mock_db_session: MagicMock): + """ + A brand new dataset name with valid external knowledge references + should create both the dataset and its binding. + """ + + tenant_id = "tenant-1" + user_id = "user-1" + + args = { + "name": "My Dataset", + "description": "desc", + "external_knowledge_api_id": "api-1", + "external_knowledge_id": "knowledge-1", + "external_retrieval_model": {"top_k": 3}, + } + + # No existing dataset with same name. + mock_db_session.query.return_value.filter_by.return_value.first.side_effect = [ + None, # duplicate‑name check + Mock(spec=ExternalKnowledgeApis), # external knowledge api + ] + + dataset = ExternalDatasetService.create_external_dataset(tenant_id, user_id, args) + + assert isinstance(dataset, Dataset) + assert dataset.provider == "external" + assert dataset.retrieval_model == args["external_retrieval_model"] + + assert mock_db_session.add.call_count >= 2 # dataset + binding + mock_db_session.flush.assert_called_once() + mock_db_session.commit.assert_called_once() + + def test_create_external_dataset_duplicate_name_raises(self, mock_db_session: MagicMock): + """ + When a dataset with the same name already exists, + ``DatasetNameDuplicateError`` is raised. + """ + + existing_dataset = Mock(spec=Dataset) + mock_db_session.query.return_value.filter_by.return_value.first.return_value = existing_dataset + + args = { + "name": "Existing", + "external_knowledge_api_id": "api-1", + "external_knowledge_id": "knowledge-1", + } + + with pytest.raises(DatasetNameDuplicateError): + ExternalDatasetService.create_external_dataset("tenant-1", "user-1", args) + + mock_db_session.add.assert_not_called() + mock_db_session.commit.assert_not_called() + + def test_create_external_dataset_missing_api_template_raises(self, mock_db_session: MagicMock): + """ + If the referenced external knowledge API does not exist, a ``ValueError`` is raised. + """ + + # First call: duplicate name check – not found. + mock_db_session.query.return_value.filter_by.return_value.first.side_effect = [ + None, + None, # external knowledge api lookup + ] + + args = { + "name": "Dataset", + "external_knowledge_api_id": "missing", + "external_knowledge_id": "knowledge-1", + } + + with pytest.raises(ValueError, match="api template not found"): + ExternalDatasetService.create_external_dataset("tenant-1", "user-1", args) + + def test_create_external_dataset_missing_required_ids_raise(self, mock_db_session: MagicMock): + """ + ``external_knowledge_id`` and ``external_knowledge_api_id`` are mandatory. + """ + + # duplicate name check + mock_db_session.query.return_value.filter_by.return_value.first.side_effect = [ + None, + Mock(spec=ExternalKnowledgeApis), + ] + + args_missing_knowledge_id = { + "name": "Dataset", + "external_knowledge_api_id": "api-1", + "external_knowledge_id": None, + } + + with pytest.raises(ValueError, match="external_knowledge_id is required"): + ExternalDatasetService.create_external_dataset("tenant-1", "user-1", args_missing_knowledge_id) + + args_missing_api_id = { + "name": "Dataset", + "external_knowledge_api_id": None, + "external_knowledge_id": "k-1", + } + + with pytest.raises(ValueError, match="external_knowledge_api_id is required"): + ExternalDatasetService.create_external_dataset("tenant-1", "user-1", args_missing_api_id) + + +# --------------------------------------------------------------------------- +# fetch_external_knowledge_retrieval +# --------------------------------------------------------------------------- + + +class TestExternalDatasetServiceFetchExternalKnowledgeRetrieval: + """ + Tests for ``fetch_external_knowledge_retrieval`` which orchestrates + external retrieval requests and normalises the response payload. + """ + + @pytest.fixture + def mock_db_session(self): + with patch("services.external_knowledge_service.db.session") as mock_session: + yield mock_session + + def test_fetch_external_knowledge_retrieval_success(self, mock_db_session: MagicMock): + """ + With a valid binding and API template, records from the external + service should be returned when the HTTP response is 200. + """ + + tenant_id = "tenant-1" + dataset_id = "ds-1" + query = "test query" + external_retrieval_parameters = {"top_k": 3, "score_threshold_enabled": True, "score_threshold": 0.5} + + binding = ExternalDatasetTestDataFactory.create_external_binding( + tenant_id=tenant_id, + dataset_id=dataset_id, + api_id="api-1", + external_knowledge_id="knowledge-1", + ) + + api = Mock(spec=ExternalKnowledgeApis) + api.settings = '{"endpoint":"https://example.com","api_key":"secret"}' + + # First query: binding; second query: api. + mock_db_session.query.return_value.filter_by.return_value.first.side_effect = [ + binding, + api, + ] + + fake_records = [{"content": "doc", "score": 0.9}] + fake_response = Mock(spec=httpx.Response) + fake_response.status_code = 200 + fake_response.json.return_value = {"records": fake_records} + + metadata_condition = SimpleNamespace(model_dump=lambda: {"field": "value"}) + + with patch.object(ExternalDatasetService, "process_external_api", return_value=fake_response) as mock_process: + result = ExternalDatasetService.fetch_external_knowledge_retrieval( + tenant_id=tenant_id, + dataset_id=dataset_id, + query=query, + external_retrieval_parameters=external_retrieval_parameters, + metadata_condition=metadata_condition, + ) + + assert result == fake_records + + mock_process.assert_called_once() + setting_arg = mock_process.call_args.args[0] + assert isinstance(setting_arg, ExternalKnowledgeApiSetting) + assert setting_arg.url.endswith("/retrieval") + + def test_fetch_external_knowledge_retrieval_binding_not_found_raises(self, mock_db_session: MagicMock): + """ + Missing binding should raise ``ValueError``. + """ + + mock_db_session.query.return_value.filter_by.return_value.first.return_value = None + + with pytest.raises(ValueError, match="external knowledge binding not found"): + ExternalDatasetService.fetch_external_knowledge_retrieval( + tenant_id="tenant-1", + dataset_id="missing", + query="q", + external_retrieval_parameters={}, + metadata_condition=None, + ) + + def test_fetch_external_knowledge_retrieval_missing_api_template_raises(self, mock_db_session: MagicMock): + """ + When the API template is missing or has no settings, a ``ValueError`` is raised. + """ + + binding = ExternalDatasetTestDataFactory.create_external_binding() + mock_db_session.query.return_value.filter_by.return_value.first.side_effect = [ + binding, + None, + ] + + with pytest.raises(ValueError, match="external api template not found"): + ExternalDatasetService.fetch_external_knowledge_retrieval( + tenant_id="tenant-1", + dataset_id="ds-1", + query="q", + external_retrieval_parameters={}, + metadata_condition=None, + ) + + def test_fetch_external_knowledge_retrieval_non_200_status_returns_empty_list(self, mock_db_session: MagicMock): + """ + Non‑200 responses should be treated as an empty result set. + """ + + binding = ExternalDatasetTestDataFactory.create_external_binding() + api = Mock(spec=ExternalKnowledgeApis) + api.settings = '{"endpoint":"https://example.com","api_key":"secret"}' + + mock_db_session.query.return_value.filter_by.return_value.first.side_effect = [ + binding, + api, + ] + + fake_response = Mock(spec=httpx.Response) + fake_response.status_code = 500 + fake_response.json.return_value = {} + + with patch.object(ExternalDatasetService, "process_external_api", return_value=fake_response): + result = ExternalDatasetService.fetch_external_knowledge_retrieval( + tenant_id="tenant-1", + dataset_id="ds-1", + query="q", + external_retrieval_parameters={}, + metadata_condition=None, + ) + + assert result == [] diff --git a/api/tests/unit_tests/services/hit_service.py b/api/tests/unit_tests/services/hit_service.py new file mode 100644 index 0000000000..17f3a7e94e --- /dev/null +++ b/api/tests/unit_tests/services/hit_service.py @@ -0,0 +1,802 @@ +""" +Unit tests for HitTestingService. + +This module contains comprehensive unit tests for the HitTestingService class, +which handles retrieval testing operations for datasets, including internal +dataset retrieval and external knowledge base retrieval. +""" + +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from core.rag.models.document import Document +from core.rag.retrieval.retrieval_methods import RetrievalMethod +from models import Account +from models.dataset import Dataset +from services.hit_testing_service import HitTestingService + + +class HitTestingTestDataFactory: + """ + Factory class for creating test data and mock objects for hit testing service tests. + + This factory provides static methods to create mock objects for datasets, users, + documents, and retrieval records used in HitTestingService unit tests. + """ + + @staticmethod + def create_dataset_mock( + dataset_id: str = "dataset-123", + tenant_id: str = "tenant-123", + provider: str = "vendor", + retrieval_model: dict | None = None, + **kwargs, + ) -> Mock: + """ + Create a mock dataset with specified attributes. + + Args: + dataset_id: Unique identifier for the dataset + tenant_id: Tenant identifier + provider: Dataset provider (vendor, external, etc.) + retrieval_model: Optional retrieval model configuration + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a Dataset instance + """ + dataset = Mock(spec=Dataset) + dataset.id = dataset_id + dataset.tenant_id = tenant_id + dataset.provider = provider + dataset.retrieval_model = retrieval_model + for key, value in kwargs.items(): + setattr(dataset, key, value) + return dataset + + @staticmethod + def create_user_mock( + user_id: str = "user-789", + tenant_id: str = "tenant-123", + **kwargs, + ) -> Mock: + """ + Create a mock user (Account) with specified attributes. + + Args: + user_id: Unique identifier for the user + tenant_id: Tenant identifier + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as an Account instance + """ + user = Mock(spec=Account) + user.id = user_id + user.current_tenant_id = tenant_id + user.name = "Test User" + for key, value in kwargs.items(): + setattr(user, key, value) + return user + + @staticmethod + def create_document_mock( + content: str = "Test document content", + metadata: dict | None = None, + **kwargs, + ) -> Mock: + """ + Create a mock Document from core.rag.models.document. + + Args: + content: Document content/text + metadata: Optional metadata dictionary + **kwargs: Additional attributes to set on the mock + + Returns: + Mock object configured as a Document instance + """ + document = Mock(spec=Document) + document.page_content = content + document.metadata = metadata or {} + for key, value in kwargs.items(): + setattr(document, key, value) + return document + + @staticmethod + def create_retrieval_record_mock( + content: str = "Test content", + score: float = 0.95, + **kwargs, + ) -> Mock: + """ + Create a mock retrieval record. + + Args: + content: Record content + score: Retrieval score + **kwargs: Additional fields for the record + + Returns: + Mock object with model_dump method returning record data + """ + record = Mock() + record.model_dump.return_value = { + "content": content, + "score": score, + **kwargs, + } + return record + + +class TestHitTestingServiceRetrieve: + """ + Tests for HitTestingService.retrieve method (hit_testing). + + This test class covers the main retrieval testing functionality, including + various retrieval model configurations, metadata filtering, and query logging. + """ + + @pytest.fixture + def mock_db_session(self): + """ + Mock database session. + + Provides a mocked database session for testing database operations + like adding and committing DatasetQuery records. + """ + with patch("services.hit_testing_service.db.session") as mock_db: + yield mock_db + + def test_retrieve_success_with_default_retrieval_model(self, mock_db_session): + """ + Test successful retrieval with default retrieval model. + + Verifies that the retrieve method works correctly when no custom + retrieval model is provided, using the default retrieval configuration. + """ + # Arrange + dataset = HitTestingTestDataFactory.create_dataset_mock(retrieval_model=None) + account = HitTestingTestDataFactory.create_user_mock() + query = "test query" + retrieval_model = None + external_retrieval_model = {} + + documents = [ + HitTestingTestDataFactory.create_document_mock(content="Doc 1"), + HitTestingTestDataFactory.create_document_mock(content="Doc 2"), + ] + + mock_records = [ + HitTestingTestDataFactory.create_retrieval_record_mock(content="Doc 1"), + HitTestingTestDataFactory.create_retrieval_record_mock(content="Doc 2"), + ] + + with ( + patch("services.hit_testing_service.RetrievalService.retrieve") as mock_retrieve, + patch("services.hit_testing_service.RetrievalService.format_retrieval_documents") as mock_format, + patch("services.hit_testing_service.time.perf_counter") as mock_perf_counter, + ): + mock_perf_counter.side_effect = [0.0, 0.1] # start, end + mock_retrieve.return_value = documents + mock_format.return_value = mock_records + + # Act + result = HitTestingService.retrieve(dataset, query, account, retrieval_model, external_retrieval_model) + + # Assert + assert result["query"]["content"] == query + assert len(result["records"]) == 2 + mock_retrieve.assert_called_once() + mock_db_session.add.assert_called_once() + mock_db_session.commit.assert_called_once() + + def test_retrieve_success_with_custom_retrieval_model(self, mock_db_session): + """ + Test successful retrieval with custom retrieval model. + + Verifies that custom retrieval model parameters (search method, reranking, + score threshold, etc.) are properly passed to RetrievalService. + """ + # Arrange + dataset = HitTestingTestDataFactory.create_dataset_mock() + account = HitTestingTestDataFactory.create_user_mock() + query = "test query" + retrieval_model = { + "search_method": RetrievalMethod.KEYWORD_SEARCH, + "reranking_enable": True, + "reranking_model": {"reranking_provider_name": "cohere", "reranking_model_name": "rerank-1"}, + "top_k": 5, + "score_threshold_enabled": True, + "score_threshold": 0.7, + "weights": {"vector_setting": 0.5, "keyword_setting": 0.5}, + } + external_retrieval_model = {} + + documents = [HitTestingTestDataFactory.create_document_mock()] + mock_records = [HitTestingTestDataFactory.create_retrieval_record_mock()] + + with ( + patch("services.hit_testing_service.RetrievalService.retrieve") as mock_retrieve, + patch("services.hit_testing_service.RetrievalService.format_retrieval_documents") as mock_format, + patch("services.hit_testing_service.time.perf_counter") as mock_perf_counter, + ): + mock_perf_counter.side_effect = [0.0, 0.1] + mock_retrieve.return_value = documents + mock_format.return_value = mock_records + + # Act + result = HitTestingService.retrieve(dataset, query, account, retrieval_model, external_retrieval_model) + + # Assert + assert result["query"]["content"] == query + mock_retrieve.assert_called_once() + call_kwargs = mock_retrieve.call_args[1] + assert call_kwargs["retrieval_method"] == RetrievalMethod.KEYWORD_SEARCH + assert call_kwargs["top_k"] == 5 + assert call_kwargs["score_threshold"] == 0.7 + assert call_kwargs["reranking_model"] == retrieval_model["reranking_model"] + + def test_retrieve_with_metadata_filtering(self, mock_db_session): + """ + Test retrieval with metadata filtering conditions. + + Verifies that metadata filtering conditions are properly processed + and document ID filters are applied to the retrieval query. + """ + # Arrange + dataset = HitTestingTestDataFactory.create_dataset_mock() + account = HitTestingTestDataFactory.create_user_mock() + query = "test query" + retrieval_model = { + "metadata_filtering_conditions": { + "conditions": [ + {"field": "category", "operator": "is", "value": "test"}, + ], + }, + } + external_retrieval_model = {} + + mock_dataset_retrieval = MagicMock() + mock_dataset_retrieval.get_metadata_filter_condition.return_value = ( + {dataset.id: ["doc-1", "doc-2"]}, + None, + ) + + documents = [HitTestingTestDataFactory.create_document_mock()] + mock_records = [HitTestingTestDataFactory.create_retrieval_record_mock()] + + with ( + patch("services.hit_testing_service.RetrievalService.retrieve") as mock_retrieve, + patch("services.hit_testing_service.RetrievalService.format_retrieval_documents") as mock_format, + patch("services.hit_testing_service.DatasetRetrieval") as mock_dataset_retrieval_class, + patch("services.hit_testing_service.time.perf_counter") as mock_perf_counter, + ): + mock_perf_counter.side_effect = [0.0, 0.1] + mock_dataset_retrieval_class.return_value = mock_dataset_retrieval + mock_retrieve.return_value = documents + mock_format.return_value = mock_records + + # Act + result = HitTestingService.retrieve(dataset, query, account, retrieval_model, external_retrieval_model) + + # Assert + assert result["query"]["content"] == query + mock_dataset_retrieval.get_metadata_filter_condition.assert_called_once() + call_kwargs = mock_retrieve.call_args[1] + assert call_kwargs["document_ids_filter"] == ["doc-1", "doc-2"] + + def test_retrieve_with_metadata_filtering_no_documents(self, mock_db_session): + """ + Test retrieval with metadata filtering that returns no documents. + + Verifies that when metadata filtering results in no matching documents, + an empty result is returned without calling RetrievalService. + """ + # Arrange + dataset = HitTestingTestDataFactory.create_dataset_mock() + account = HitTestingTestDataFactory.create_user_mock() + query = "test query" + retrieval_model = { + "metadata_filtering_conditions": { + "conditions": [ + {"field": "category", "operator": "is", "value": "test"}, + ], + }, + } + external_retrieval_model = {} + + mock_dataset_retrieval = MagicMock() + mock_dataset_retrieval.get_metadata_filter_condition.return_value = ({}, True) + + with ( + patch("services.hit_testing_service.DatasetRetrieval") as mock_dataset_retrieval_class, + patch("services.hit_testing_service.RetrievalService.format_retrieval_documents") as mock_format, + ): + mock_dataset_retrieval_class.return_value = mock_dataset_retrieval + mock_format.return_value = [] + + # Act + result = HitTestingService.retrieve(dataset, query, account, retrieval_model, external_retrieval_model) + + # Assert + assert result["query"]["content"] == query + assert result["records"] == [] + + def test_retrieve_with_dataset_retrieval_model(self, mock_db_session): + """ + Test retrieval using dataset's retrieval model when not provided. + + Verifies that when no retrieval model is provided, the dataset's + retrieval model is used as a fallback. + """ + # Arrange + dataset_retrieval_model = { + "search_method": RetrievalMethod.HYBRID_SEARCH, + "top_k": 3, + } + dataset = HitTestingTestDataFactory.create_dataset_mock(retrieval_model=dataset_retrieval_model) + account = HitTestingTestDataFactory.create_user_mock() + query = "test query" + retrieval_model = None + external_retrieval_model = {} + + documents = [HitTestingTestDataFactory.create_document_mock()] + mock_records = [HitTestingTestDataFactory.create_retrieval_record_mock()] + + with ( + patch("services.hit_testing_service.RetrievalService.retrieve") as mock_retrieve, + patch("services.hit_testing_service.RetrievalService.format_retrieval_documents") as mock_format, + patch("services.hit_testing_service.time.perf_counter") as mock_perf_counter, + ): + mock_perf_counter.side_effect = [0.0, 0.1] + mock_retrieve.return_value = documents + mock_format.return_value = mock_records + + # Act + result = HitTestingService.retrieve(dataset, query, account, retrieval_model, external_retrieval_model) + + # Assert + assert result["query"]["content"] == query + call_kwargs = mock_retrieve.call_args[1] + assert call_kwargs["retrieval_method"] == RetrievalMethod.HYBRID_SEARCH + assert call_kwargs["top_k"] == 3 + + +class TestHitTestingServiceExternalRetrieve: + """ + Tests for HitTestingService.external_retrieve method. + + This test class covers external knowledge base retrieval functionality, + including query escaping, response formatting, and provider validation. + """ + + @pytest.fixture + def mock_db_session(self): + """ + Mock database session. + + Provides a mocked database session for testing database operations + like adding and committing DatasetQuery records. + """ + with patch("services.hit_testing_service.db.session") as mock_db: + yield mock_db + + def test_external_retrieve_success(self, mock_db_session): + """ + Test successful external retrieval. + + Verifies that external knowledge base retrieval works correctly, + including query escaping, document formatting, and query logging. + """ + # Arrange + dataset = HitTestingTestDataFactory.create_dataset_mock(provider="external") + account = HitTestingTestDataFactory.create_user_mock() + query = 'test query with "quotes"' + external_retrieval_model = {"top_k": 5, "score_threshold": 0.8} + metadata_filtering_conditions = {} + + external_documents = [ + {"content": "External doc 1", "title": "Title 1", "score": 0.95, "metadata": {"key": "value"}}, + {"content": "External doc 2", "title": "Title 2", "score": 0.85, "metadata": {}}, + ] + + with ( + patch("services.hit_testing_service.RetrievalService.external_retrieve") as mock_external_retrieve, + patch("services.hit_testing_service.time.perf_counter") as mock_perf_counter, + ): + mock_perf_counter.side_effect = [0.0, 0.1] + mock_external_retrieve.return_value = external_documents + + # Act + result = HitTestingService.external_retrieve( + dataset, query, account, external_retrieval_model, metadata_filtering_conditions + ) + + # Assert + assert result["query"]["content"] == query + assert len(result["records"]) == 2 + assert result["records"][0]["content"] == "External doc 1" + assert result["records"][0]["title"] == "Title 1" + assert result["records"][0]["score"] == 0.95 + mock_external_retrieve.assert_called_once() + # Verify query was escaped + assert mock_external_retrieve.call_args[1]["query"] == 'test query with \\"quotes\\"' + mock_db_session.add.assert_called_once() + mock_db_session.commit.assert_called_once() + + def test_external_retrieve_non_external_provider(self, mock_db_session): + """ + Test external retrieval with non-external provider (should return empty). + + Verifies that when the dataset provider is not "external", the method + returns an empty result without performing retrieval or database operations. + """ + # Arrange + dataset = HitTestingTestDataFactory.create_dataset_mock(provider="vendor") + account = HitTestingTestDataFactory.create_user_mock() + query = "test query" + external_retrieval_model = {} + metadata_filtering_conditions = {} + + # Act + result = HitTestingService.external_retrieve( + dataset, query, account, external_retrieval_model, metadata_filtering_conditions + ) + + # Assert + assert result["query"]["content"] == query + assert result["records"] == [] + mock_db_session.add.assert_not_called() + + def test_external_retrieve_with_metadata_filtering(self, mock_db_session): + """ + Test external retrieval with metadata filtering conditions. + + Verifies that metadata filtering conditions are properly passed + to the external retrieval service. + """ + # Arrange + dataset = HitTestingTestDataFactory.create_dataset_mock(provider="external") + account = HitTestingTestDataFactory.create_user_mock() + query = "test query" + external_retrieval_model = {"top_k": 3} + metadata_filtering_conditions = {"category": "test"} + + external_documents = [{"content": "Doc 1", "title": "Title", "score": 0.9, "metadata": {}}] + + with ( + patch("services.hit_testing_service.RetrievalService.external_retrieve") as mock_external_retrieve, + patch("services.hit_testing_service.time.perf_counter") as mock_perf_counter, + ): + mock_perf_counter.side_effect = [0.0, 0.1] + mock_external_retrieve.return_value = external_documents + + # Act + result = HitTestingService.external_retrieve( + dataset, query, account, external_retrieval_model, metadata_filtering_conditions + ) + + # Assert + assert result["query"]["content"] == query + assert len(result["records"]) == 1 + call_kwargs = mock_external_retrieve.call_args[1] + assert call_kwargs["metadata_filtering_conditions"] == metadata_filtering_conditions + + def test_external_retrieve_empty_documents(self, mock_db_session): + """ + Test external retrieval with empty document list. + + Verifies that when external retrieval returns no documents, + an empty result is properly formatted and returned. + """ + # Arrange + dataset = HitTestingTestDataFactory.create_dataset_mock(provider="external") + account = HitTestingTestDataFactory.create_user_mock() + query = "test query" + external_retrieval_model = {} + metadata_filtering_conditions = {} + + with ( + patch("services.hit_testing_service.RetrievalService.external_retrieve") as mock_external_retrieve, + patch("services.hit_testing_service.time.perf_counter") as mock_perf_counter, + ): + mock_perf_counter.side_effect = [0.0, 0.1] + mock_external_retrieve.return_value = [] + + # Act + result = HitTestingService.external_retrieve( + dataset, query, account, external_retrieval_model, metadata_filtering_conditions + ) + + # Assert + assert result["query"]["content"] == query + assert result["records"] == [] + + +class TestHitTestingServiceCompactRetrieveResponse: + """ + Tests for HitTestingService.compact_retrieve_response method. + + This test class covers response formatting for internal dataset retrieval, + ensuring documents are properly formatted into retrieval records. + """ + + def test_compact_retrieve_response_success(self): + """ + Test successful response formatting. + + Verifies that documents are properly formatted into retrieval records + with correct structure and data. + """ + # Arrange + query = "test query" + documents = [ + HitTestingTestDataFactory.create_document_mock(content="Doc 1"), + HitTestingTestDataFactory.create_document_mock(content="Doc 2"), + ] + + mock_records = [ + HitTestingTestDataFactory.create_retrieval_record_mock(content="Doc 1", score=0.95), + HitTestingTestDataFactory.create_retrieval_record_mock(content="Doc 2", score=0.85), + ] + + with patch("services.hit_testing_service.RetrievalService.format_retrieval_documents") as mock_format: + mock_format.return_value = mock_records + + # Act + result = HitTestingService.compact_retrieve_response(query, documents) + + # Assert + assert result["query"]["content"] == query + assert len(result["records"]) == 2 + assert result["records"][0]["content"] == "Doc 1" + assert result["records"][0]["score"] == 0.95 + mock_format.assert_called_once_with(documents) + + def test_compact_retrieve_response_empty_documents(self): + """ + Test response formatting with empty document list. + + Verifies that an empty document list results in an empty records array + while maintaining the correct response structure. + """ + # Arrange + query = "test query" + documents = [] + + with patch("services.hit_testing_service.RetrievalService.format_retrieval_documents") as mock_format: + mock_format.return_value = [] + + # Act + result = HitTestingService.compact_retrieve_response(query, documents) + + # Assert + assert result["query"]["content"] == query + assert result["records"] == [] + + +class TestHitTestingServiceCompactExternalRetrieveResponse: + """ + Tests for HitTestingService.compact_external_retrieve_response method. + + This test class covers response formatting for external knowledge base + retrieval, ensuring proper field extraction and provider validation. + """ + + def test_compact_external_retrieve_response_external_provider(self): + """ + Test external response formatting for external provider. + + Verifies that external documents are properly formatted with all + required fields (content, title, score, metadata). + """ + # Arrange + dataset = HitTestingTestDataFactory.create_dataset_mock(provider="external") + query = "test query" + documents = [ + {"content": "Doc 1", "title": "Title 1", "score": 0.95, "metadata": {"key": "value"}}, + {"content": "Doc 2", "title": "Title 2", "score": 0.85, "metadata": {}}, + ] + + # Act + result = HitTestingService.compact_external_retrieve_response(dataset, query, documents) + + # Assert + assert result["query"]["content"] == query + assert len(result["records"]) == 2 + assert result["records"][0]["content"] == "Doc 1" + assert result["records"][0]["title"] == "Title 1" + assert result["records"][0]["score"] == 0.95 + assert result["records"][0]["metadata"] == {"key": "value"} + + def test_compact_external_retrieve_response_non_external_provider(self): + """ + Test external response formatting for non-external provider. + + Verifies that non-external providers return an empty records array + regardless of input documents. + """ + # Arrange + dataset = HitTestingTestDataFactory.create_dataset_mock(provider="vendor") + query = "test query" + documents = [{"content": "Doc 1"}] + + # Act + result = HitTestingService.compact_external_retrieve_response(dataset, query, documents) + + # Assert + assert result["query"]["content"] == query + assert result["records"] == [] + + def test_compact_external_retrieve_response_missing_fields(self): + """ + Test external response formatting with missing optional fields. + + Verifies that missing optional fields (title, score, metadata) are + handled gracefully by setting them to None. + """ + # Arrange + dataset = HitTestingTestDataFactory.create_dataset_mock(provider="external") + query = "test query" + documents = [ + {"content": "Doc 1"}, # Missing title, score, metadata + {"content": "Doc 2", "title": "Title 2"}, # Missing score, metadata + ] + + # Act + result = HitTestingService.compact_external_retrieve_response(dataset, query, documents) + + # Assert + assert result["query"]["content"] == query + assert len(result["records"]) == 2 + assert result["records"][0]["content"] == "Doc 1" + assert result["records"][0]["title"] is None + assert result["records"][0]["score"] is None + assert result["records"][0]["metadata"] is None + + +class TestHitTestingServiceHitTestingArgsCheck: + """ + Tests for HitTestingService.hit_testing_args_check method. + + This test class covers query argument validation, ensuring queries + meet the required criteria (non-empty, max 250 characters). + """ + + def test_hit_testing_args_check_success(self): + """ + Test successful argument validation. + + Verifies that valid queries pass validation without raising errors. + """ + # Arrange + args = {"query": "valid query"} + + # Act & Assert (should not raise) + HitTestingService.hit_testing_args_check(args) + + def test_hit_testing_args_check_empty_query(self): + """ + Test validation fails with empty query. + + Verifies that empty queries raise a ValueError with appropriate message. + """ + # Arrange + args = {"query": ""} + + # Act & Assert + with pytest.raises(ValueError, match="Query is required and cannot exceed 250 characters"): + HitTestingService.hit_testing_args_check(args) + + def test_hit_testing_args_check_none_query(self): + """ + Test validation fails with None query. + + Verifies that None queries raise a ValueError with appropriate message. + """ + # Arrange + args = {"query": None} + + # Act & Assert + with pytest.raises(ValueError, match="Query is required and cannot exceed 250 characters"): + HitTestingService.hit_testing_args_check(args) + + def test_hit_testing_args_check_too_long_query(self): + """ + Test validation fails with query exceeding 250 characters. + + Verifies that queries longer than 250 characters raise a ValueError. + """ + # Arrange + args = {"query": "a" * 251} + + # Act & Assert + with pytest.raises(ValueError, match="Query is required and cannot exceed 250 characters"): + HitTestingService.hit_testing_args_check(args) + + def test_hit_testing_args_check_exactly_250_characters(self): + """ + Test validation succeeds with exactly 250 characters. + + Verifies that queries with exactly 250 characters (the maximum) + pass validation successfully. + """ + # Arrange + args = {"query": "a" * 250} + + # Act & Assert (should not raise) + HitTestingService.hit_testing_args_check(args) + + +class TestHitTestingServiceEscapeQueryForSearch: + """ + Tests for HitTestingService.escape_query_for_search method. + + This test class covers query escaping functionality for external search, + ensuring special characters are properly escaped. + """ + + def test_escape_query_for_search_with_quotes(self): + """ + Test escaping quotes in query. + + Verifies that double quotes in queries are properly escaped with + backslashes for external search compatibility. + """ + # Arrange + query = 'test query with "quotes"' + + # Act + result = HitTestingService.escape_query_for_search(query) + + # Assert + assert result == 'test query with \\"quotes\\"' + + def test_escape_query_for_search_without_quotes(self): + """ + Test query without quotes (no change). + + Verifies that queries without quotes remain unchanged after escaping. + """ + # Arrange + query = "test query without quotes" + + # Act + result = HitTestingService.escape_query_for_search(query) + + # Assert + assert result == query + + def test_escape_query_for_search_multiple_quotes(self): + """ + Test escaping multiple quotes in query. + + Verifies that all occurrences of double quotes in a query are + properly escaped, not just the first one. + """ + # Arrange + query = 'test "query" with "multiple" quotes' + + # Act + result = HitTestingService.escape_query_for_search(query) + + # Assert + assert result == 'test \\"query\\" with \\"multiple\\" quotes' + + def test_escape_query_for_search_empty_string(self): + """ + Test escaping empty string. + + Verifies that empty strings are handled correctly and remain empty + after the escaping operation. + """ + # Arrange + query = "" + + # Act + result = HitTestingService.escape_query_for_search(query) + + # Assert + assert result == "" diff --git a/api/tests/unit_tests/services/segment_service.py b/api/tests/unit_tests/services/segment_service.py new file mode 100644 index 0000000000..ee05e890b2 --- /dev/null +++ b/api/tests/unit_tests/services/segment_service.py @@ -0,0 +1,1093 @@ +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from models.account import Account +from models.dataset import ChildChunk, Dataset, Document, DocumentSegment +from services.dataset_service import SegmentService +from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgs +from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError + + +class SegmentTestDataFactory: + """Factory class for creating test data and mock objects for segment service tests.""" + + @staticmethod + def create_segment_mock( + segment_id: str = "segment-123", + document_id: str = "doc-123", + dataset_id: str = "dataset-123", + tenant_id: str = "tenant-123", + content: str = "Test segment content", + position: int = 1, + enabled: bool = True, + status: str = "completed", + word_count: int = 3, + tokens: int = 5, + **kwargs, + ) -> Mock: + """Create a mock segment with specified attributes.""" + segment = Mock(spec=DocumentSegment) + segment.id = segment_id + segment.document_id = document_id + segment.dataset_id = dataset_id + segment.tenant_id = tenant_id + segment.content = content + segment.position = position + segment.enabled = enabled + segment.status = status + segment.word_count = word_count + segment.tokens = tokens + segment.index_node_id = f"node-{segment_id}" + segment.index_node_hash = "hash-123" + segment.keywords = [] + segment.answer = None + segment.disabled_at = None + segment.disabled_by = None + segment.updated_by = None + segment.updated_at = None + segment.indexing_at = None + segment.completed_at = None + segment.error = None + for key, value in kwargs.items(): + setattr(segment, key, value) + return segment + + @staticmethod + def create_child_chunk_mock( + chunk_id: str = "chunk-123", + segment_id: str = "segment-123", + document_id: str = "doc-123", + dataset_id: str = "dataset-123", + tenant_id: str = "tenant-123", + content: str = "Test child chunk content", + position: int = 1, + word_count: int = 3, + **kwargs, + ) -> Mock: + """Create a mock child chunk with specified attributes.""" + chunk = Mock(spec=ChildChunk) + chunk.id = chunk_id + chunk.segment_id = segment_id + chunk.document_id = document_id + chunk.dataset_id = dataset_id + chunk.tenant_id = tenant_id + chunk.content = content + chunk.position = position + chunk.word_count = word_count + chunk.index_node_id = f"node-{chunk_id}" + chunk.index_node_hash = "hash-123" + chunk.type = "automatic" + chunk.created_by = "user-123" + chunk.updated_by = None + chunk.updated_at = None + for key, value in kwargs.items(): + setattr(chunk, key, value) + return chunk + + @staticmethod + def create_document_mock( + document_id: str = "doc-123", + dataset_id: str = "dataset-123", + tenant_id: str = "tenant-123", + doc_form: str = "text_model", + word_count: int = 100, + **kwargs, + ) -> Mock: + """Create a mock document with specified attributes.""" + document = Mock(spec=Document) + document.id = document_id + document.dataset_id = dataset_id + document.tenant_id = tenant_id + document.doc_form = doc_form + document.word_count = word_count + for key, value in kwargs.items(): + setattr(document, key, value) + return document + + @staticmethod + def create_dataset_mock( + dataset_id: str = "dataset-123", + tenant_id: str = "tenant-123", + indexing_technique: str = "high_quality", + embedding_model: str = "text-embedding-ada-002", + embedding_model_provider: str = "openai", + **kwargs, + ) -> Mock: + """Create a mock dataset with specified attributes.""" + dataset = Mock(spec=Dataset) + dataset.id = dataset_id + dataset.tenant_id = tenant_id + dataset.indexing_technique = indexing_technique + dataset.embedding_model = embedding_model + dataset.embedding_model_provider = embedding_model_provider + for key, value in kwargs.items(): + setattr(dataset, key, value) + return dataset + + @staticmethod + def create_user_mock( + user_id: str = "user-789", + tenant_id: str = "tenant-123", + **kwargs, + ) -> Mock: + """Create a mock user with specified attributes.""" + user = Mock(spec=Account) + user.id = user_id + user.current_tenant_id = tenant_id + user.name = "Test User" + for key, value in kwargs.items(): + setattr(user, key, value) + return user + + +class TestSegmentServiceCreateSegment: + """Tests for SegmentService.create_segment method.""" + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + @pytest.fixture + def mock_current_user(self): + """Mock current_user.""" + user = SegmentTestDataFactory.create_user_mock() + with patch("services.dataset_service.current_user", user): + yield user + + def test_create_segment_success(self, mock_db_session, mock_current_user): + """Test successful creation of a segment.""" + # Arrange + document = SegmentTestDataFactory.create_document_mock(word_count=100) + dataset = SegmentTestDataFactory.create_dataset_mock(indexing_technique="economy") + args = {"content": "New segment content", "keywords": ["test", "segment"]} + + mock_query = MagicMock() + mock_query.where.return_value.scalar.return_value = None # No existing segments + mock_db_session.query.return_value = mock_query + + mock_segment = SegmentTestDataFactory.create_segment_mock() + mock_db_session.query.return_value.where.return_value.first.return_value = mock_segment + + with ( + patch("services.dataset_service.redis_client.lock") as mock_lock, + patch("services.dataset_service.VectorService.create_segments_vector") as mock_vector_service, + patch("services.dataset_service.helper.generate_text_hash") as mock_hash, + patch("services.dataset_service.naive_utc_now") as mock_now, + ): + mock_lock.return_value.__enter__ = Mock() + mock_lock.return_value.__exit__ = Mock(return_value=None) + mock_hash.return_value = "hash-123" + mock_now.return_value = "2024-01-01T00:00:00" + + # Act + result = SegmentService.create_segment(args, document, dataset) + + # Assert + assert mock_db_session.add.call_count == 2 + + created_segment = mock_db_session.add.call_args_list[0].args[0] + assert isinstance(created_segment, DocumentSegment) + assert created_segment.content == args["content"] + assert created_segment.word_count == len(args["content"]) + + mock_db_session.commit.assert_called_once() + + mock_vector_service.assert_called_once() + vector_call_args = mock_vector_service.call_args[0] + assert vector_call_args[0] == [args["keywords"]] + assert vector_call_args[1][0] == created_segment + assert vector_call_args[2] == dataset + assert vector_call_args[3] == document.doc_form + + assert result == mock_segment + + def test_create_segment_with_qa_model(self, mock_db_session, mock_current_user): + """Test creation of segment with QA model (requires answer).""" + # Arrange + document = SegmentTestDataFactory.create_document_mock(doc_form="qa_model", word_count=100) + dataset = SegmentTestDataFactory.create_dataset_mock(indexing_technique="economy") + args = {"content": "What is AI?", "answer": "AI is Artificial Intelligence", "keywords": ["ai"]} + + mock_query = MagicMock() + mock_query.where.return_value.scalar.return_value = None + mock_db_session.query.return_value = mock_query + + mock_segment = SegmentTestDataFactory.create_segment_mock() + mock_db_session.query.return_value.where.return_value.first.return_value = mock_segment + + with ( + patch("services.dataset_service.redis_client.lock") as mock_lock, + patch("services.dataset_service.VectorService.create_segments_vector") as mock_vector_service, + patch("services.dataset_service.helper.generate_text_hash") as mock_hash, + patch("services.dataset_service.naive_utc_now") as mock_now, + ): + mock_lock.return_value.__enter__ = Mock() + mock_lock.return_value.__exit__ = Mock(return_value=None) + mock_hash.return_value = "hash-123" + mock_now.return_value = "2024-01-01T00:00:00" + + # Act + result = SegmentService.create_segment(args, document, dataset) + + # Assert + assert result == mock_segment + mock_db_session.add.assert_called() + mock_db_session.commit.assert_called() + + def test_create_segment_with_high_quality_indexing(self, mock_db_session, mock_current_user): + """Test creation of segment with high quality indexing technique.""" + # Arrange + document = SegmentTestDataFactory.create_document_mock(word_count=100) + dataset = SegmentTestDataFactory.create_dataset_mock(indexing_technique="high_quality") + args = {"content": "New segment content", "keywords": ["test"]} + + mock_query = MagicMock() + mock_query.where.return_value.scalar.return_value = None + mock_db_session.query.return_value = mock_query + + mock_embedding_model = MagicMock() + mock_embedding_model.get_text_embedding_num_tokens.return_value = [10] + mock_model_manager = MagicMock() + mock_model_manager.get_model_instance.return_value = mock_embedding_model + + mock_segment = SegmentTestDataFactory.create_segment_mock() + mock_db_session.query.return_value.where.return_value.first.return_value = mock_segment + + with ( + patch("services.dataset_service.redis_client.lock") as mock_lock, + patch("services.dataset_service.VectorService.create_segments_vector") as mock_vector_service, + patch("services.dataset_service.ModelManager") as mock_model_manager_class, + patch("services.dataset_service.helper.generate_text_hash") as mock_hash, + patch("services.dataset_service.naive_utc_now") as mock_now, + ): + mock_lock.return_value.__enter__ = Mock() + mock_lock.return_value.__exit__ = Mock(return_value=None) + mock_model_manager_class.return_value = mock_model_manager + mock_hash.return_value = "hash-123" + mock_now.return_value = "2024-01-01T00:00:00" + + # Act + result = SegmentService.create_segment(args, document, dataset) + + # Assert + assert result == mock_segment + mock_model_manager.get_model_instance.assert_called_once() + mock_embedding_model.get_text_embedding_num_tokens.assert_called_once() + + def test_create_segment_vector_index_failure(self, mock_db_session, mock_current_user): + """Test segment creation when vector indexing fails.""" + # Arrange + document = SegmentTestDataFactory.create_document_mock(word_count=100) + dataset = SegmentTestDataFactory.create_dataset_mock(indexing_technique="economy") + args = {"content": "New segment content", "keywords": ["test"]} + + mock_query = MagicMock() + mock_query.where.return_value.scalar.return_value = None + mock_db_session.query.return_value = mock_query + + mock_segment = SegmentTestDataFactory.create_segment_mock(enabled=False, status="error") + mock_db_session.query.return_value.where.return_value.first.return_value = mock_segment + + with ( + patch("services.dataset_service.redis_client.lock") as mock_lock, + patch("services.dataset_service.VectorService.create_segments_vector") as mock_vector_service, + patch("services.dataset_service.helper.generate_text_hash") as mock_hash, + patch("services.dataset_service.naive_utc_now") as mock_now, + ): + mock_lock.return_value.__enter__ = Mock() + mock_lock.return_value.__exit__ = Mock(return_value=None) + mock_vector_service.side_effect = Exception("Vector indexing failed") + mock_hash.return_value = "hash-123" + mock_now.return_value = "2024-01-01T00:00:00" + + # Act + result = SegmentService.create_segment(args, document, dataset) + + # Assert + assert result == mock_segment + assert mock_db_session.commit.call_count == 2 # Once for creation, once for error update + + +class TestSegmentServiceUpdateSegment: + """Tests for SegmentService.update_segment method.""" + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + @pytest.fixture + def mock_current_user(self): + """Mock current_user.""" + user = SegmentTestDataFactory.create_user_mock() + with patch("services.dataset_service.current_user", user): + yield user + + def test_update_segment_content_success(self, mock_db_session, mock_current_user): + """Test successful update of segment content.""" + # Arrange + segment = SegmentTestDataFactory.create_segment_mock(enabled=True, word_count=10) + document = SegmentTestDataFactory.create_document_mock(word_count=100) + dataset = SegmentTestDataFactory.create_dataset_mock(indexing_technique="economy") + args = SegmentUpdateArgs(content="Updated content", keywords=["updated"]) + + mock_db_session.query.return_value.where.return_value.first.return_value = segment + + with ( + patch("services.dataset_service.redis_client.get") as mock_redis_get, + patch("services.dataset_service.VectorService.update_segment_vector") as mock_vector_service, + patch("services.dataset_service.helper.generate_text_hash") as mock_hash, + patch("services.dataset_service.naive_utc_now") as mock_now, + ): + mock_redis_get.return_value = None # Not indexing + mock_hash.return_value = "new-hash" + mock_now.return_value = "2024-01-01T00:00:00" + + # Act + result = SegmentService.update_segment(args, segment, document, dataset) + + # Assert + assert result == segment + assert segment.content == "Updated content" + assert segment.keywords == ["updated"] + assert segment.word_count == len("Updated content") + assert document.word_count == 100 + (len("Updated content") - 10) + mock_db_session.add.assert_called() + mock_db_session.commit.assert_called() + + def test_update_segment_disable(self, mock_db_session, mock_current_user): + """Test disabling a segment.""" + # Arrange + segment = SegmentTestDataFactory.create_segment_mock(enabled=True) + document = SegmentTestDataFactory.create_document_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + args = SegmentUpdateArgs(enabled=False) + + with ( + patch("services.dataset_service.redis_client.get") as mock_redis_get, + patch("services.dataset_service.redis_client.setex") as mock_redis_setex, + patch("services.dataset_service.disable_segment_from_index_task") as mock_task, + patch("services.dataset_service.naive_utc_now") as mock_now, + ): + mock_redis_get.return_value = None + mock_now.return_value = "2024-01-01T00:00:00" + + # Act + result = SegmentService.update_segment(args, segment, document, dataset) + + # Assert + assert result == segment + assert segment.enabled is False + mock_db_session.add.assert_called() + mock_db_session.commit.assert_called() + mock_task.delay.assert_called_once() + + def test_update_segment_indexing_in_progress(self, mock_db_session, mock_current_user): + """Test update fails when segment is currently indexing.""" + # Arrange + segment = SegmentTestDataFactory.create_segment_mock(enabled=True) + document = SegmentTestDataFactory.create_document_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + args = SegmentUpdateArgs(content="Updated content") + + with patch("services.dataset_service.redis_client.get") as mock_redis_get: + mock_redis_get.return_value = "1" # Indexing in progress + + # Act & Assert + with pytest.raises(ValueError, match="Segment is indexing"): + SegmentService.update_segment(args, segment, document, dataset) + + def test_update_segment_disabled_segment(self, mock_db_session, mock_current_user): + """Test update fails when segment is disabled.""" + # Arrange + segment = SegmentTestDataFactory.create_segment_mock(enabled=False) + document = SegmentTestDataFactory.create_document_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + args = SegmentUpdateArgs(content="Updated content") + + with patch("services.dataset_service.redis_client.get") as mock_redis_get: + mock_redis_get.return_value = None + + # Act & Assert + with pytest.raises(ValueError, match="Can't update disabled segment"): + SegmentService.update_segment(args, segment, document, dataset) + + def test_update_segment_with_qa_model(self, mock_db_session, mock_current_user): + """Test update segment with QA model (includes answer).""" + # Arrange + segment = SegmentTestDataFactory.create_segment_mock(enabled=True, word_count=10) + document = SegmentTestDataFactory.create_document_mock(doc_form="qa_model", word_count=100) + dataset = SegmentTestDataFactory.create_dataset_mock(indexing_technique="economy") + args = SegmentUpdateArgs(content="Updated question", answer="Updated answer", keywords=["qa"]) + + mock_db_session.query.return_value.where.return_value.first.return_value = segment + + with ( + patch("services.dataset_service.redis_client.get") as mock_redis_get, + patch("services.dataset_service.VectorService.update_segment_vector") as mock_vector_service, + patch("services.dataset_service.helper.generate_text_hash") as mock_hash, + patch("services.dataset_service.naive_utc_now") as mock_now, + ): + mock_redis_get.return_value = None + mock_hash.return_value = "new-hash" + mock_now.return_value = "2024-01-01T00:00:00" + + # Act + result = SegmentService.update_segment(args, segment, document, dataset) + + # Assert + assert result == segment + assert segment.content == "Updated question" + assert segment.answer == "Updated answer" + assert segment.keywords == ["qa"] + new_word_count = len("Updated question") + len("Updated answer") + assert segment.word_count == new_word_count + assert document.word_count == 100 + (new_word_count - 10) + mock_db_session.commit.assert_called() + + +class TestSegmentServiceDeleteSegment: + """Tests for SegmentService.delete_segment method.""" + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + def test_delete_segment_success(self, mock_db_session): + """Test successful deletion of a segment.""" + # Arrange + segment = SegmentTestDataFactory.create_segment_mock(enabled=True, word_count=50) + document = SegmentTestDataFactory.create_document_mock(word_count=100) + dataset = SegmentTestDataFactory.create_dataset_mock() + + mock_scalars = MagicMock() + mock_scalars.all.return_value = [] + mock_db_session.scalars.return_value = mock_scalars + + with ( + patch("services.dataset_service.redis_client.get") as mock_redis_get, + patch("services.dataset_service.redis_client.setex") as mock_redis_setex, + patch("services.dataset_service.delete_segment_from_index_task") as mock_task, + patch("services.dataset_service.select") as mock_select, + ): + mock_redis_get.return_value = None + mock_select.return_value.where.return_value = mock_select + + # Act + SegmentService.delete_segment(segment, document, dataset) + + # Assert + mock_db_session.delete.assert_called_once_with(segment) + mock_db_session.commit.assert_called_once() + mock_task.delay.assert_called_once() + + def test_delete_segment_disabled(self, mock_db_session): + """Test deletion of disabled segment (no index deletion).""" + # Arrange + segment = SegmentTestDataFactory.create_segment_mock(enabled=False, word_count=50) + document = SegmentTestDataFactory.create_document_mock(word_count=100) + dataset = SegmentTestDataFactory.create_dataset_mock() + + with ( + patch("services.dataset_service.redis_client.get") as mock_redis_get, + patch("services.dataset_service.delete_segment_from_index_task") as mock_task, + ): + mock_redis_get.return_value = None + + # Act + SegmentService.delete_segment(segment, document, dataset) + + # Assert + mock_db_session.delete.assert_called_once_with(segment) + mock_db_session.commit.assert_called_once() + mock_task.delay.assert_not_called() + + def test_delete_segment_indexing_in_progress(self, mock_db_session): + """Test deletion fails when segment is currently being deleted.""" + # Arrange + segment = SegmentTestDataFactory.create_segment_mock(enabled=True) + document = SegmentTestDataFactory.create_document_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + + with patch("services.dataset_service.redis_client.get") as mock_redis_get: + mock_redis_get.return_value = "1" # Deletion in progress + + # Act & Assert + with pytest.raises(ValueError, match="Segment is deleting"): + SegmentService.delete_segment(segment, document, dataset) + + +class TestSegmentServiceDeleteSegments: + """Tests for SegmentService.delete_segments method.""" + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + @pytest.fixture + def mock_current_user(self): + """Mock current_user.""" + user = SegmentTestDataFactory.create_user_mock() + with patch("services.dataset_service.current_user", user): + yield user + + def test_delete_segments_success(self, mock_db_session, mock_current_user): + """Test successful deletion of multiple segments.""" + # Arrange + segment_ids = ["segment-1", "segment-2"] + document = SegmentTestDataFactory.create_document_mock(word_count=200) + dataset = SegmentTestDataFactory.create_dataset_mock() + + segments_info = [ + ("node-1", "segment-1", 50), + ("node-2", "segment-2", 30), + ] + + mock_query = MagicMock() + mock_query.with_entities.return_value.where.return_value.all.return_value = segments_info + mock_db_session.query.return_value = mock_query + + mock_scalars = MagicMock() + mock_scalars.all.return_value = [] + mock_select = MagicMock() + mock_select.where.return_value = mock_select + mock_db_session.scalars.return_value = mock_scalars + + with ( + patch("services.dataset_service.delete_segment_from_index_task") as mock_task, + patch("services.dataset_service.select") as mock_select_func, + ): + mock_select_func.return_value = mock_select + + # Act + SegmentService.delete_segments(segment_ids, document, dataset) + + # Assert + mock_db_session.query.return_value.where.return_value.delete.assert_called_once() + mock_db_session.commit.assert_called_once() + mock_task.delay.assert_called_once() + + def test_delete_segments_empty_list(self, mock_db_session, mock_current_user): + """Test deletion with empty list (should return early).""" + # Arrange + document = SegmentTestDataFactory.create_document_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + + # Act + SegmentService.delete_segments([], document, dataset) + + # Assert + mock_db_session.query.assert_not_called() + + +class TestSegmentServiceUpdateSegmentsStatus: + """Tests for SegmentService.update_segments_status method.""" + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + @pytest.fixture + def mock_current_user(self): + """Mock current_user.""" + user = SegmentTestDataFactory.create_user_mock() + with patch("services.dataset_service.current_user", user): + yield user + + def test_update_segments_status_enable(self, mock_db_session, mock_current_user): + """Test enabling multiple segments.""" + # Arrange + segment_ids = ["segment-1", "segment-2"] + document = SegmentTestDataFactory.create_document_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + + segments = [ + SegmentTestDataFactory.create_segment_mock(segment_id="segment-1", enabled=False), + SegmentTestDataFactory.create_segment_mock(segment_id="segment-2", enabled=False), + ] + + mock_scalars = MagicMock() + mock_scalars.all.return_value = segments + mock_select = MagicMock() + mock_select.where.return_value = mock_select + mock_db_session.scalars.return_value = mock_scalars + + with ( + patch("services.dataset_service.redis_client.get") as mock_redis_get, + patch("services.dataset_service.enable_segments_to_index_task") as mock_task, + patch("services.dataset_service.select") as mock_select_func, + ): + mock_redis_get.return_value = None + mock_select_func.return_value = mock_select + + # Act + SegmentService.update_segments_status(segment_ids, "enable", dataset, document) + + # Assert + assert all(seg.enabled is True for seg in segments) + mock_db_session.commit.assert_called_once() + mock_task.delay.assert_called_once() + + def test_update_segments_status_disable(self, mock_db_session, mock_current_user): + """Test disabling multiple segments.""" + # Arrange + segment_ids = ["segment-1", "segment-2"] + document = SegmentTestDataFactory.create_document_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + + segments = [ + SegmentTestDataFactory.create_segment_mock(segment_id="segment-1", enabled=True), + SegmentTestDataFactory.create_segment_mock(segment_id="segment-2", enabled=True), + ] + + mock_scalars = MagicMock() + mock_scalars.all.return_value = segments + mock_select = MagicMock() + mock_select.where.return_value = mock_select + mock_db_session.scalars.return_value = mock_scalars + + with ( + patch("services.dataset_service.redis_client.get") as mock_redis_get, + patch("services.dataset_service.disable_segments_from_index_task") as mock_task, + patch("services.dataset_service.naive_utc_now") as mock_now, + patch("services.dataset_service.select") as mock_select_func, + ): + mock_redis_get.return_value = None + mock_now.return_value = "2024-01-01T00:00:00" + mock_select_func.return_value = mock_select + + # Act + SegmentService.update_segments_status(segment_ids, "disable", dataset, document) + + # Assert + assert all(seg.enabled is False for seg in segments) + mock_db_session.commit.assert_called_once() + mock_task.delay.assert_called_once() + + def test_update_segments_status_empty_list(self, mock_db_session, mock_current_user): + """Test update with empty list (should return early).""" + # Arrange + document = SegmentTestDataFactory.create_document_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + + # Act + SegmentService.update_segments_status([], "enable", dataset, document) + + # Assert + mock_db_session.scalars.assert_not_called() + + +class TestSegmentServiceGetSegments: + """Tests for SegmentService.get_segments method.""" + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + @pytest.fixture + def mock_current_user(self): + """Mock current_user.""" + user = SegmentTestDataFactory.create_user_mock() + with patch("services.dataset_service.current_user", user): + yield user + + def test_get_segments_success(self, mock_db_session, mock_current_user): + """Test successful retrieval of segments.""" + # Arrange + document_id = "doc-123" + tenant_id = "tenant-123" + segments = [ + SegmentTestDataFactory.create_segment_mock(segment_id="segment-1"), + SegmentTestDataFactory.create_segment_mock(segment_id="segment-2"), + ] + + mock_paginate = MagicMock() + mock_paginate.items = segments + mock_paginate.total = 2 + mock_db_session.paginate.return_value = mock_paginate + + # Act + items, total = SegmentService.get_segments(document_id, tenant_id) + + # Assert + assert len(items) == 2 + assert total == 2 + mock_db_session.paginate.assert_called_once() + + def test_get_segments_with_status_filter(self, mock_db_session, mock_current_user): + """Test retrieval with status filter.""" + # Arrange + document_id = "doc-123" + tenant_id = "tenant-123" + status_list = ["completed", "error"] + + mock_paginate = MagicMock() + mock_paginate.items = [] + mock_paginate.total = 0 + mock_db_session.paginate.return_value = mock_paginate + + # Act + items, total = SegmentService.get_segments(document_id, tenant_id, status_list=status_list) + + # Assert + assert len(items) == 0 + assert total == 0 + + def test_get_segments_with_keyword(self, mock_db_session, mock_current_user): + """Test retrieval with keyword search.""" + # Arrange + document_id = "doc-123" + tenant_id = "tenant-123" + keyword = "test" + + mock_paginate = MagicMock() + mock_paginate.items = [SegmentTestDataFactory.create_segment_mock()] + mock_paginate.total = 1 + mock_db_session.paginate.return_value = mock_paginate + + # Act + items, total = SegmentService.get_segments(document_id, tenant_id, keyword=keyword) + + # Assert + assert len(items) == 1 + assert total == 1 + + +class TestSegmentServiceGetSegmentById: + """Tests for SegmentService.get_segment_by_id method.""" + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + def test_get_segment_by_id_success(self, mock_db_session): + """Test successful retrieval of segment by ID.""" + # Arrange + segment_id = "segment-123" + tenant_id = "tenant-123" + segment = SegmentTestDataFactory.create_segment_mock(segment_id=segment_id) + + mock_query = MagicMock() + mock_query.where.return_value.first.return_value = segment + mock_db_session.query.return_value = mock_query + + # Act + result = SegmentService.get_segment_by_id(segment_id, tenant_id) + + # Assert + assert result == segment + + def test_get_segment_by_id_not_found(self, mock_db_session): + """Test retrieval when segment is not found.""" + # Arrange + segment_id = "non-existent" + tenant_id = "tenant-123" + + mock_query = MagicMock() + mock_query.where.return_value.first.return_value = None + mock_db_session.query.return_value = mock_query + + # Act + result = SegmentService.get_segment_by_id(segment_id, tenant_id) + + # Assert + assert result is None + + +class TestSegmentServiceGetChildChunks: + """Tests for SegmentService.get_child_chunks method.""" + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + @pytest.fixture + def mock_current_user(self): + """Mock current_user.""" + user = SegmentTestDataFactory.create_user_mock() + with patch("services.dataset_service.current_user", user): + yield user + + def test_get_child_chunks_success(self, mock_db_session, mock_current_user): + """Test successful retrieval of child chunks.""" + # Arrange + segment_id = "segment-123" + document_id = "doc-123" + dataset_id = "dataset-123" + page = 1 + limit = 20 + + mock_paginate = MagicMock() + mock_paginate.items = [ + SegmentTestDataFactory.create_child_chunk_mock(chunk_id="chunk-1"), + SegmentTestDataFactory.create_child_chunk_mock(chunk_id="chunk-2"), + ] + mock_paginate.total = 2 + mock_db_session.paginate.return_value = mock_paginate + + # Act + result = SegmentService.get_child_chunks(segment_id, document_id, dataset_id, page, limit) + + # Assert + assert result == mock_paginate + mock_db_session.paginate.assert_called_once() + + def test_get_child_chunks_with_keyword(self, mock_db_session, mock_current_user): + """Test retrieval with keyword search.""" + # Arrange + segment_id = "segment-123" + document_id = "doc-123" + dataset_id = "dataset-123" + page = 1 + limit = 20 + keyword = "test" + + mock_paginate = MagicMock() + mock_paginate.items = [] + mock_paginate.total = 0 + mock_db_session.paginate.return_value = mock_paginate + + # Act + result = SegmentService.get_child_chunks(segment_id, document_id, dataset_id, page, limit, keyword=keyword) + + # Assert + assert result == mock_paginate + + +class TestSegmentServiceGetChildChunkById: + """Tests for SegmentService.get_child_chunk_by_id method.""" + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + def test_get_child_chunk_by_id_success(self, mock_db_session): + """Test successful retrieval of child chunk by ID.""" + # Arrange + chunk_id = "chunk-123" + tenant_id = "tenant-123" + chunk = SegmentTestDataFactory.create_child_chunk_mock(chunk_id=chunk_id) + + mock_query = MagicMock() + mock_query.where.return_value.first.return_value = chunk + mock_db_session.query.return_value = mock_query + + # Act + result = SegmentService.get_child_chunk_by_id(chunk_id, tenant_id) + + # Assert + assert result == chunk + + def test_get_child_chunk_by_id_not_found(self, mock_db_session): + """Test retrieval when child chunk is not found.""" + # Arrange + chunk_id = "non-existent" + tenant_id = "tenant-123" + + mock_query = MagicMock() + mock_query.where.return_value.first.return_value = None + mock_db_session.query.return_value = mock_query + + # Act + result = SegmentService.get_child_chunk_by_id(chunk_id, tenant_id) + + # Assert + assert result is None + + +class TestSegmentServiceCreateChildChunk: + """Tests for SegmentService.create_child_chunk method.""" + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + @pytest.fixture + def mock_current_user(self): + """Mock current_user.""" + user = SegmentTestDataFactory.create_user_mock() + with patch("services.dataset_service.current_user", user): + yield user + + def test_create_child_chunk_success(self, mock_db_session, mock_current_user): + """Test successful creation of a child chunk.""" + # Arrange + content = "New child chunk content" + segment = SegmentTestDataFactory.create_segment_mock() + document = SegmentTestDataFactory.create_document_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + + mock_query = MagicMock() + mock_query.where.return_value.scalar.return_value = None + mock_db_session.query.return_value = mock_query + + with ( + patch("services.dataset_service.redis_client.lock") as mock_lock, + patch("services.dataset_service.VectorService.create_child_chunk_vector") as mock_vector_service, + patch("services.dataset_service.helper.generate_text_hash") as mock_hash, + ): + mock_lock.return_value.__enter__ = Mock() + mock_lock.return_value.__exit__ = Mock(return_value=None) + mock_hash.return_value = "hash-123" + + # Act + result = SegmentService.create_child_chunk(content, segment, document, dataset) + + # Assert + assert result is not None + mock_db_session.add.assert_called_once() + mock_db_session.commit.assert_called_once() + mock_vector_service.assert_called_once() + + def test_create_child_chunk_vector_index_failure(self, mock_db_session, mock_current_user): + """Test child chunk creation when vector indexing fails.""" + # Arrange + content = "New child chunk content" + segment = SegmentTestDataFactory.create_segment_mock() + document = SegmentTestDataFactory.create_document_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + + mock_query = MagicMock() + mock_query.where.return_value.scalar.return_value = None + mock_db_session.query.return_value = mock_query + + with ( + patch("services.dataset_service.redis_client.lock") as mock_lock, + patch("services.dataset_service.VectorService.create_child_chunk_vector") as mock_vector_service, + patch("services.dataset_service.helper.generate_text_hash") as mock_hash, + ): + mock_lock.return_value.__enter__ = Mock() + mock_lock.return_value.__exit__ = Mock(return_value=None) + mock_vector_service.side_effect = Exception("Vector indexing failed") + mock_hash.return_value = "hash-123" + + # Act & Assert + with pytest.raises(ChildChunkIndexingError): + SegmentService.create_child_chunk(content, segment, document, dataset) + + mock_db_session.rollback.assert_called_once() + + +class TestSegmentServiceUpdateChildChunk: + """Tests for SegmentService.update_child_chunk method.""" + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + @pytest.fixture + def mock_current_user(self): + """Mock current_user.""" + user = SegmentTestDataFactory.create_user_mock() + with patch("services.dataset_service.current_user", user): + yield user + + def test_update_child_chunk_success(self, mock_db_session, mock_current_user): + """Test successful update of a child chunk.""" + # Arrange + content = "Updated child chunk content" + chunk = SegmentTestDataFactory.create_child_chunk_mock() + segment = SegmentTestDataFactory.create_segment_mock() + document = SegmentTestDataFactory.create_document_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + + with ( + patch("services.dataset_service.VectorService.update_child_chunk_vector") as mock_vector_service, + patch("services.dataset_service.naive_utc_now") as mock_now, + ): + mock_now.return_value = "2024-01-01T00:00:00" + + # Act + result = SegmentService.update_child_chunk(content, chunk, segment, document, dataset) + + # Assert + assert result == chunk + assert chunk.content == content + assert chunk.word_count == len(content) + mock_db_session.add.assert_called_once_with(chunk) + mock_db_session.commit.assert_called_once() + mock_vector_service.assert_called_once() + + def test_update_child_chunk_vector_index_failure(self, mock_db_session, mock_current_user): + """Test child chunk update when vector indexing fails.""" + # Arrange + content = "Updated content" + chunk = SegmentTestDataFactory.create_child_chunk_mock() + segment = SegmentTestDataFactory.create_segment_mock() + document = SegmentTestDataFactory.create_document_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + + with ( + patch("services.dataset_service.VectorService.update_child_chunk_vector") as mock_vector_service, + patch("services.dataset_service.naive_utc_now") as mock_now, + ): + mock_vector_service.side_effect = Exception("Vector indexing failed") + mock_now.return_value = "2024-01-01T00:00:00" + + # Act & Assert + with pytest.raises(ChildChunkIndexingError): + SegmentService.update_child_chunk(content, chunk, segment, document, dataset) + + mock_db_session.rollback.assert_called_once() + + +class TestSegmentServiceDeleteChildChunk: + """Tests for SegmentService.delete_child_chunk method.""" + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.dataset_service.db.session") as mock_db: + yield mock_db + + def test_delete_child_chunk_success(self, mock_db_session): + """Test successful deletion of a child chunk.""" + # Arrange + chunk = SegmentTestDataFactory.create_child_chunk_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + + with patch("services.dataset_service.VectorService.delete_child_chunk_vector") as mock_vector_service: + # Act + SegmentService.delete_child_chunk(chunk, dataset) + + # Assert + mock_db_session.delete.assert_called_once_with(chunk) + mock_db_session.commit.assert_called_once() + mock_vector_service.assert_called_once_with(chunk, dataset) + + def test_delete_child_chunk_vector_index_failure(self, mock_db_session): + """Test child chunk deletion when vector indexing fails.""" + # Arrange + chunk = SegmentTestDataFactory.create_child_chunk_mock() + dataset = SegmentTestDataFactory.create_dataset_mock() + + with patch("services.dataset_service.VectorService.delete_child_chunk_vector") as mock_vector_service: + mock_vector_service.side_effect = Exception("Vector deletion failed") + + # Act & Assert + with pytest.raises(ChildChunkDeleteIndexError): + SegmentService.delete_child_chunk(chunk, dataset) + + mock_db_session.rollback.assert_called_once() diff --git a/api/tests/unit_tests/services/test_app_task_service.py b/api/tests/unit_tests/services/test_app_task_service.py new file mode 100644 index 0000000000..e00486f77c --- /dev/null +++ b/api/tests/unit_tests/services/test_app_task_service.py @@ -0,0 +1,106 @@ +from unittest.mock import patch + +import pytest + +from core.app.entities.app_invoke_entities import InvokeFrom +from models.model import AppMode +from services.app_task_service import AppTaskService + + +class TestAppTaskService: + """Test suite for AppTaskService.stop_task method.""" + + @pytest.mark.parametrize( + ("app_mode", "should_call_graph_engine"), + [ + (AppMode.CHAT, False), + (AppMode.COMPLETION, False), + (AppMode.AGENT_CHAT, False), + (AppMode.CHANNEL, False), + (AppMode.RAG_PIPELINE, False), + (AppMode.ADVANCED_CHAT, True), + (AppMode.WORKFLOW, True), + ], + ) + @patch("services.app_task_service.AppQueueManager") + @patch("services.app_task_service.GraphEngineManager") + def test_stop_task_with_different_app_modes( + self, mock_graph_engine_manager, mock_app_queue_manager, app_mode, should_call_graph_engine + ): + """Test stop_task behavior with different app modes. + + Verifies that: + - Legacy Redis flag is always set via AppQueueManager + - GraphEngine stop command is only sent for ADVANCED_CHAT and WORKFLOW modes + """ + # Arrange + task_id = "task-123" + invoke_from = InvokeFrom.WEB_APP + user_id = "user-456" + + # Act + AppTaskService.stop_task(task_id, invoke_from, user_id, app_mode) + + # Assert + mock_app_queue_manager.set_stop_flag.assert_called_once_with(task_id, invoke_from, user_id) + if should_call_graph_engine: + mock_graph_engine_manager.send_stop_command.assert_called_once_with(task_id) + else: + mock_graph_engine_manager.send_stop_command.assert_not_called() + + @pytest.mark.parametrize( + "invoke_from", + [ + InvokeFrom.WEB_APP, + InvokeFrom.SERVICE_API, + InvokeFrom.DEBUGGER, + InvokeFrom.EXPLORE, + ], + ) + @patch("services.app_task_service.AppQueueManager") + @patch("services.app_task_service.GraphEngineManager") + def test_stop_task_with_different_invoke_sources( + self, mock_graph_engine_manager, mock_app_queue_manager, invoke_from + ): + """Test stop_task behavior with different invoke sources. + + Verifies that the method works correctly regardless of the invoke source. + """ + # Arrange + task_id = "task-789" + user_id = "user-999" + app_mode = AppMode.ADVANCED_CHAT + + # Act + AppTaskService.stop_task(task_id, invoke_from, user_id, app_mode) + + # Assert + mock_app_queue_manager.set_stop_flag.assert_called_once_with(task_id, invoke_from, user_id) + mock_graph_engine_manager.send_stop_command.assert_called_once_with(task_id) + + @patch("services.app_task_service.GraphEngineManager") + @patch("services.app_task_service.AppQueueManager") + def test_stop_task_legacy_mechanism_called_even_if_graph_engine_fails( + self, mock_app_queue_manager, mock_graph_engine_manager + ): + """Test that legacy Redis flag is set even if GraphEngine fails. + + This ensures backward compatibility: the legacy mechanism should complete + before attempting the GraphEngine command, so the stop flag is set + regardless of GraphEngine success. + """ + # Arrange + task_id = "task-123" + invoke_from = InvokeFrom.WEB_APP + user_id = "user-456" + app_mode = AppMode.ADVANCED_CHAT + + # Simulate GraphEngine failure + mock_graph_engine_manager.send_stop_command.side_effect = Exception("GraphEngine error") + + # Act & Assert - should raise the exception since it's not caught + with pytest.raises(Exception, match="GraphEngine error"): + AppTaskService.stop_task(task_id, invoke_from, user_id, app_mode) + + # Verify legacy mechanism was still called before the exception + mock_app_queue_manager.set_stop_flag.assert_called_once_with(task_id, invoke_from, user_id) diff --git a/api/tests/unit_tests/services/test_billing_service.py b/api/tests/unit_tests/services/test_billing_service.py index dc13143417..915aee3fa7 100644 --- a/api/tests/unit_tests/services/test_billing_service.py +++ b/api/tests/unit_tests/services/test_billing_service.py @@ -1,3 +1,18 @@ +"""Comprehensive unit tests for BillingService. + +This test module covers all aspects of the billing service including: +- HTTP request handling with retry logic +- Subscription tier management and billing information retrieval +- Usage calculation and credit management (positive/negative deltas) +- Rate limit enforcement for compliance downloads and education features +- Account management and permission checks +- Cache management for billing data +- Partner integration features + +All tests use mocking to avoid external dependencies and ensure fast, reliable execution. +Tests follow the Arrange-Act-Assert pattern for clarity. +""" + import json from unittest.mock import MagicMock, patch @@ -5,11 +20,20 @@ import httpx import pytest from werkzeug.exceptions import InternalServerError +from enums.cloud_plan import CloudPlan +from models import Account, TenantAccountJoin, TenantAccountRole from services.billing_service import BillingService class TestBillingServiceSendRequest: - """Unit tests for BillingService._send_request method.""" + """Unit tests for BillingService._send_request method. + + Tests cover: + - Successful GET/PUT/POST/DELETE requests + - Error handling for various HTTP status codes + - Retry logic on network failures + - Request header and parameter validation + """ @pytest.fixture def mock_httpx_request(self): @@ -234,3 +258,1042 @@ class TestBillingServiceSendRequest: # Should retry multiple times (wait=2, stop_before_delay=10 means ~5 attempts) assert mock_httpx_request.call_count > 1 + + +class TestBillingServiceSubscriptionInfo: + """Unit tests for subscription tier and billing info retrieval. + + Tests cover: + - Billing information retrieval + - Knowledge base rate limits with default and custom values + - Payment link generation for subscriptions and model providers + - Invoice retrieval + """ + + @pytest.fixture + def mock_send_request(self): + """Mock _send_request method.""" + with patch.object(BillingService, "_send_request") as mock: + yield mock + + def test_get_info_success(self, mock_send_request): + """Test successful retrieval of billing information.""" + # Arrange + tenant_id = "tenant-123" + expected_response = { + "subscription_plan": "professional", + "billing_cycle": "monthly", + "status": "active", + } + mock_send_request.return_value = expected_response + + # Act + result = BillingService.get_info(tenant_id) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with("GET", "/subscription/info", params={"tenant_id": tenant_id}) + + def test_get_knowledge_rate_limit_with_defaults(self, mock_send_request): + """Test knowledge rate limit retrieval with default values.""" + # Arrange + tenant_id = "tenant-456" + mock_send_request.return_value = {} + + # Act + result = BillingService.get_knowledge_rate_limit(tenant_id) + + # Assert + assert result["limit"] == 10 # Default limit + assert result["subscription_plan"] == CloudPlan.SANDBOX # Default plan + mock_send_request.assert_called_once_with( + "GET", "/subscription/knowledge-rate-limit", params={"tenant_id": tenant_id} + ) + + def test_get_knowledge_rate_limit_with_custom_values(self, mock_send_request): + """Test knowledge rate limit retrieval with custom values.""" + # Arrange + tenant_id = "tenant-789" + mock_send_request.return_value = {"limit": 100, "subscription_plan": CloudPlan.PROFESSIONAL} + + # Act + result = BillingService.get_knowledge_rate_limit(tenant_id) + + # Assert + assert result["limit"] == 100 + assert result["subscription_plan"] == CloudPlan.PROFESSIONAL + + def test_get_subscription_payment_link(self, mock_send_request): + """Test subscription payment link generation.""" + # Arrange + plan = "professional" + interval = "monthly" + email = "user@example.com" + tenant_id = "tenant-123" + expected_response = {"payment_link": "https://payment.example.com/checkout"} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.get_subscription(plan, interval, email, tenant_id) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with( + "GET", + "/subscription/payment-link", + params={"plan": plan, "interval": interval, "prefilled_email": email, "tenant_id": tenant_id}, + ) + + def test_get_model_provider_payment_link(self, mock_send_request): + """Test model provider payment link generation.""" + # Arrange + provider_name = "openai" + tenant_id = "tenant-123" + account_id = "account-456" + email = "user@example.com" + expected_response = {"payment_link": "https://payment.example.com/provider"} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.get_model_provider_payment_link(provider_name, tenant_id, account_id, email) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with( + "GET", + "/model-provider/payment-link", + params={ + "provider_name": provider_name, + "tenant_id": tenant_id, + "account_id": account_id, + "prefilled_email": email, + }, + ) + + def test_get_invoices(self, mock_send_request): + """Test invoice retrieval.""" + # Arrange + email = "user@example.com" + tenant_id = "tenant-123" + expected_response = {"invoices": [{"id": "inv-1", "amount": 100}]} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.get_invoices(email, tenant_id) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with( + "GET", "/invoices", params={"prefilled_email": email, "tenant_id": tenant_id} + ) + + +class TestBillingServiceUsageCalculation: + """Unit tests for usage calculation and credit management. + + Tests cover: + - Feature plan usage information retrieval + - Credit addition (positive delta) + - Credit consumption (negative delta) + - Usage refunds + - Specific feature usage queries + """ + + @pytest.fixture + def mock_send_request(self): + """Mock _send_request method.""" + with patch.object(BillingService, "_send_request") as mock: + yield mock + + def test_get_tenant_feature_plan_usage_info(self, mock_send_request): + """Test retrieval of tenant feature plan usage information.""" + # Arrange + tenant_id = "tenant-123" + expected_response = {"features": {"trigger": {"used": 50, "limit": 100}, "workflow": {"used": 20, "limit": 50}}} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.get_tenant_feature_plan_usage_info(tenant_id) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with("GET", "/tenant-feature-usage/info", params={"tenant_id": tenant_id}) + + def test_update_tenant_feature_plan_usage_positive_delta(self, mock_send_request): + """Test updating tenant feature usage with positive delta (adding credits).""" + # Arrange + tenant_id = "tenant-123" + feature_key = "trigger" + delta = 10 + expected_response = {"result": "success", "history_id": "hist-uuid-123"} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.update_tenant_feature_plan_usage(tenant_id, feature_key, delta) + + # Assert + assert result == expected_response + assert result["result"] == "success" + assert "history_id" in result + mock_send_request.assert_called_once_with( + "POST", + "/tenant-feature-usage/usage", + params={"tenant_id": tenant_id, "feature_key": feature_key, "delta": delta}, + ) + + def test_update_tenant_feature_plan_usage_negative_delta(self, mock_send_request): + """Test updating tenant feature usage with negative delta (consuming credits).""" + # Arrange + tenant_id = "tenant-456" + feature_key = "workflow" + delta = -5 + expected_response = {"result": "success", "history_id": "hist-uuid-456"} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.update_tenant_feature_plan_usage(tenant_id, feature_key, delta) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with( + "POST", + "/tenant-feature-usage/usage", + params={"tenant_id": tenant_id, "feature_key": feature_key, "delta": delta}, + ) + + def test_refund_tenant_feature_plan_usage(self, mock_send_request): + """Test refunding a previous usage charge.""" + # Arrange + history_id = "hist-uuid-789" + expected_response = {"result": "success", "history_id": history_id} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.refund_tenant_feature_plan_usage(history_id) + + # Assert + assert result == expected_response + assert result["result"] == "success" + mock_send_request.assert_called_once_with( + "POST", "/tenant-feature-usage/refund", params={"quota_usage_history_id": history_id} + ) + + def test_get_tenant_feature_plan_usage(self, mock_send_request): + """Test getting specific feature usage for a tenant.""" + # Arrange + tenant_id = "tenant-123" + feature_key = "trigger" + expected_response = {"used": 75, "limit": 100, "remaining": 25} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.get_tenant_feature_plan_usage(tenant_id, feature_key) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with( + "GET", "/billing/tenant_feature_plan/usage", params={"tenant_id": tenant_id, "feature_key": feature_key} + ) + + +class TestBillingServiceRateLimitEnforcement: + """Unit tests for rate limit enforcement mechanisms. + + Tests cover: + - Compliance download rate limiting (4 requests per 60 seconds) + - Education verification rate limiting (10 requests per 60 seconds) + - Education activation rate limiting (10 requests per 60 seconds) + - Rate limit increment after successful operations + - Proper exception raising when limits are exceeded + """ + + @pytest.fixture + def mock_send_request(self): + """Mock _send_request method.""" + with patch.object(BillingService, "_send_request") as mock: + yield mock + + def test_compliance_download_rate_limiter_not_limited(self, mock_send_request): + """Test compliance download when rate limit is not exceeded.""" + # Arrange + doc_name = "compliance_report.pdf" + account_id = "account-123" + tenant_id = "tenant-456" + ip = "192.168.1.1" + device_info = "Mozilla/5.0" + expected_response = {"download_link": "https://example.com/download"} + + # Mock the rate limiter to return False (not limited) + with ( + patch.object( + BillingService.compliance_download_rate_limiter, "is_rate_limited", return_value=False + ) as mock_is_limited, + patch.object(BillingService.compliance_download_rate_limiter, "increment_rate_limit") as mock_increment, + ): + mock_send_request.return_value = expected_response + + # Act + result = BillingService.get_compliance_download_link(doc_name, account_id, tenant_id, ip, device_info) + + # Assert + assert result == expected_response + mock_is_limited.assert_called_once_with(f"{account_id}:{tenant_id}") + mock_send_request.assert_called_once_with( + "POST", + "/compliance/download", + json={ + "doc_name": doc_name, + "account_id": account_id, + "tenant_id": tenant_id, + "ip_address": ip, + "device_info": device_info, + }, + ) + # Verify rate limit was incremented after successful download + mock_increment.assert_called_once_with(f"{account_id}:{tenant_id}") + + def test_compliance_download_rate_limiter_exceeded(self, mock_send_request): + """Test compliance download when rate limit is exceeded.""" + # Arrange + doc_name = "compliance_report.pdf" + account_id = "account-123" + tenant_id = "tenant-456" + ip = "192.168.1.1" + device_info = "Mozilla/5.0" + + # Import the error class to properly catch it + from controllers.console.error import ComplianceRateLimitError + + # Mock the rate limiter to return True (rate limited) + with patch.object( + BillingService.compliance_download_rate_limiter, "is_rate_limited", return_value=True + ) as mock_is_limited: + # Act & Assert + with pytest.raises(ComplianceRateLimitError): + BillingService.get_compliance_download_link(doc_name, account_id, tenant_id, ip, device_info) + + mock_is_limited.assert_called_once_with(f"{account_id}:{tenant_id}") + mock_send_request.assert_not_called() + + def test_education_verify_rate_limit_not_exceeded(self, mock_send_request): + """Test education verification when rate limit is not exceeded.""" + # Arrange + account_id = "account-123" + account_email = "student@university.edu" + expected_response = {"verified": True, "institution": "University"} + + # Mock the rate limiter to return False (not limited) + with ( + patch.object( + BillingService.EducationIdentity.verification_rate_limit, "is_rate_limited", return_value=False + ) as mock_is_limited, + patch.object( + BillingService.EducationIdentity.verification_rate_limit, "increment_rate_limit" + ) as mock_increment, + ): + mock_send_request.return_value = expected_response + + # Act + result = BillingService.EducationIdentity.verify(account_id, account_email) + + # Assert + assert result == expected_response + mock_is_limited.assert_called_once_with(account_email) + mock_send_request.assert_called_once_with("GET", "/education/verify", params={"account_id": account_id}) + mock_increment.assert_called_once_with(account_email) + + def test_education_verify_rate_limit_exceeded(self, mock_send_request): + """Test education verification when rate limit is exceeded.""" + # Arrange + account_id = "account-123" + account_email = "student@university.edu" + + # Import the error class to properly catch it + from controllers.console.error import EducationVerifyLimitError + + # Mock the rate limiter to return True (rate limited) + with patch.object( + BillingService.EducationIdentity.verification_rate_limit, "is_rate_limited", return_value=True + ) as mock_is_limited: + # Act & Assert + with pytest.raises(EducationVerifyLimitError): + BillingService.EducationIdentity.verify(account_id, account_email) + + mock_is_limited.assert_called_once_with(account_email) + mock_send_request.assert_not_called() + + def test_education_activate_rate_limit_not_exceeded(self, mock_send_request): + """Test education activation when rate limit is not exceeded.""" + # Arrange + account = MagicMock(spec=Account) + account.id = "account-123" + account.email = "student@university.edu" + account.current_tenant_id = "tenant-456" + token = "verification-token" + institution = "MIT" + role = "student" + expected_response = {"result": "success", "activated": True} + + # Mock the rate limiter to return False (not limited) + with ( + patch.object( + BillingService.EducationIdentity.activation_rate_limit, "is_rate_limited", return_value=False + ) as mock_is_limited, + patch.object( + BillingService.EducationIdentity.activation_rate_limit, "increment_rate_limit" + ) as mock_increment, + ): + mock_send_request.return_value = expected_response + + # Act + result = BillingService.EducationIdentity.activate(account, token, institution, role) + + # Assert + assert result == expected_response + mock_is_limited.assert_called_once_with(account.email) + mock_send_request.assert_called_once_with( + "POST", + "/education/", + json={"institution": institution, "token": token, "role": role}, + params={"account_id": account.id, "curr_tenant_id": account.current_tenant_id}, + ) + mock_increment.assert_called_once_with(account.email) + + def test_education_activate_rate_limit_exceeded(self, mock_send_request): + """Test education activation when rate limit is exceeded.""" + # Arrange + account = MagicMock(spec=Account) + account.id = "account-123" + account.email = "student@university.edu" + account.current_tenant_id = "tenant-456" + token = "verification-token" + institution = "MIT" + role = "student" + + # Import the error class to properly catch it + from controllers.console.error import EducationActivateLimitError + + # Mock the rate limiter to return True (rate limited) + with patch.object( + BillingService.EducationIdentity.activation_rate_limit, "is_rate_limited", return_value=True + ) as mock_is_limited: + # Act & Assert + with pytest.raises(EducationActivateLimitError): + BillingService.EducationIdentity.activate(account, token, institution, role) + + mock_is_limited.assert_called_once_with(account.email) + mock_send_request.assert_not_called() + + +class TestBillingServiceEducationIdentity: + """Unit tests for education identity verification and management. + + Tests cover: + - Education verification status checking + - Institution autocomplete with pagination + - Default parameter handling + """ + + @pytest.fixture + def mock_send_request(self): + """Mock _send_request method.""" + with patch.object(BillingService, "_send_request") as mock: + yield mock + + def test_education_status(self, mock_send_request): + """Test checking education verification status.""" + # Arrange + account_id = "account-123" + expected_response = {"verified": True, "institution": "MIT", "role": "student"} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.EducationIdentity.status(account_id) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with("GET", "/education/status", params={"account_id": account_id}) + + def test_education_autocomplete(self, mock_send_request): + """Test education institution autocomplete.""" + # Arrange + keywords = "Massachusetts" + page = 0 + limit = 20 + expected_response = { + "institutions": [ + {"name": "Massachusetts Institute of Technology", "domain": "mit.edu"}, + {"name": "University of Massachusetts", "domain": "umass.edu"}, + ] + } + mock_send_request.return_value = expected_response + + # Act + result = BillingService.EducationIdentity.autocomplete(keywords, page, limit) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with( + "GET", "/education/autocomplete", params={"keywords": keywords, "page": page, "limit": limit} + ) + + def test_education_autocomplete_with_defaults(self, mock_send_request): + """Test education institution autocomplete with default parameters.""" + # Arrange + keywords = "Stanford" + expected_response = {"institutions": [{"name": "Stanford University", "domain": "stanford.edu"}]} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.EducationIdentity.autocomplete(keywords) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with( + "GET", "/education/autocomplete", params={"keywords": keywords, "page": 0, "limit": 20} + ) + + +class TestBillingServiceAccountManagement: + """Unit tests for account-related billing operations. + + Tests cover: + - Account deletion + - Email freeze status checking + - Account deletion feedback submission + - Tenant owner/admin permission validation + - Error handling for missing tenant joins + """ + + @pytest.fixture + def mock_send_request(self): + """Mock _send_request method.""" + with patch.object(BillingService, "_send_request") as mock: + yield mock + + @pytest.fixture + def mock_db_session(self): + """Mock database session.""" + with patch("services.billing_service.db.session") as mock_session: + yield mock_session + + def test_delete_account(self, mock_send_request): + """Test account deletion.""" + # Arrange + account_id = "account-123" + expected_response = {"result": "success", "deleted": True} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.delete_account(account_id) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with("DELETE", "/account/", params={"account_id": account_id}) + + def test_is_email_in_freeze_true(self, mock_send_request): + """Test checking if email is frozen (returns True).""" + # Arrange + email = "frozen@example.com" + mock_send_request.return_value = {"data": True} + + # Act + result = BillingService.is_email_in_freeze(email) + + # Assert + assert result is True + mock_send_request.assert_called_once_with("GET", "/account/in-freeze", params={"email": email}) + + def test_is_email_in_freeze_false(self, mock_send_request): + """Test checking if email is frozen (returns False).""" + # Arrange + email = "active@example.com" + mock_send_request.return_value = {"data": False} + + # Act + result = BillingService.is_email_in_freeze(email) + + # Assert + assert result is False + mock_send_request.assert_called_once_with("GET", "/account/in-freeze", params={"email": email}) + + def test_is_email_in_freeze_exception_returns_false(self, mock_send_request): + """Test that is_email_in_freeze returns False on exception.""" + # Arrange + email = "error@example.com" + mock_send_request.side_effect = Exception("Network error") + + # Act + result = BillingService.is_email_in_freeze(email) + + # Assert + assert result is False + + def test_update_account_deletion_feedback(self, mock_send_request): + """Test updating account deletion feedback.""" + # Arrange + email = "user@example.com" + feedback = "Service was too expensive" + expected_response = {"result": "success"} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.update_account_deletion_feedback(email, feedback) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with( + "POST", "/account/delete-feedback", json={"email": email, "feedback": feedback} + ) + + def test_is_tenant_owner_or_admin_owner(self, mock_db_session): + """Test tenant owner/admin check for owner role.""" + # Arrange + current_user = MagicMock(spec=Account) + current_user.id = "account-123" + current_user.current_tenant_id = "tenant-456" + + mock_join = MagicMock(spec=TenantAccountJoin) + mock_join.role = TenantAccountRole.OWNER + + mock_query = MagicMock() + mock_query.where.return_value.first.return_value = mock_join + mock_db_session.query.return_value = mock_query + + # Act - should not raise exception + BillingService.is_tenant_owner_or_admin(current_user) + + # Assert + mock_db_session.query.assert_called_once() + + def test_is_tenant_owner_or_admin_admin(self, mock_db_session): + """Test tenant owner/admin check for admin role.""" + # Arrange + current_user = MagicMock(spec=Account) + current_user.id = "account-123" + current_user.current_tenant_id = "tenant-456" + + mock_join = MagicMock(spec=TenantAccountJoin) + mock_join.role = TenantAccountRole.ADMIN + + mock_query = MagicMock() + mock_query.where.return_value.first.return_value = mock_join + mock_db_session.query.return_value = mock_query + + # Act - should not raise exception + BillingService.is_tenant_owner_or_admin(current_user) + + # Assert + mock_db_session.query.assert_called_once() + + def test_is_tenant_owner_or_admin_normal_user_raises_error(self, mock_db_session): + """Test tenant owner/admin check raises error for normal user.""" + # Arrange + current_user = MagicMock(spec=Account) + current_user.id = "account-123" + current_user.current_tenant_id = "tenant-456" + + mock_join = MagicMock(spec=TenantAccountJoin) + mock_join.role = TenantAccountRole.NORMAL + + mock_query = MagicMock() + mock_query.where.return_value.first.return_value = mock_join + mock_db_session.query.return_value = mock_query + + # Act & Assert + with pytest.raises(ValueError) as exc_info: + BillingService.is_tenant_owner_or_admin(current_user) + assert "Only team owner or team admin can perform this action" in str(exc_info.value) + + def test_is_tenant_owner_or_admin_no_join_raises_error(self, mock_db_session): + """Test tenant owner/admin check raises error when join not found.""" + # Arrange + current_user = MagicMock(spec=Account) + current_user.id = "account-123" + current_user.current_tenant_id = "tenant-456" + + mock_query = MagicMock() + mock_query.where.return_value.first.return_value = None + mock_db_session.query.return_value = mock_query + + # Act & Assert + with pytest.raises(ValueError) as exc_info: + BillingService.is_tenant_owner_or_admin(current_user) + assert "Tenant account join not found" in str(exc_info.value) + + +class TestBillingServiceCacheManagement: + """Unit tests for billing cache management. + + Tests cover: + - Billing info cache invalidation + - Proper Redis key formatting + """ + + @pytest.fixture + def mock_redis_client(self): + """Mock Redis client.""" + with patch("services.billing_service.redis_client") as mock_redis: + yield mock_redis + + def test_clean_billing_info_cache(self, mock_redis_client): + """Test cleaning billing info cache.""" + # Arrange + tenant_id = "tenant-123" + expected_key = f"tenant:{tenant_id}:billing_info" + + # Act + BillingService.clean_billing_info_cache(tenant_id) + + # Assert + mock_redis_client.delete.assert_called_once_with(expected_key) + + +class TestBillingServicePartnerIntegration: + """Unit tests for partner integration features. + + Tests cover: + - Partner tenant binding synchronization + - Click ID tracking + """ + + @pytest.fixture + def mock_send_request(self): + """Mock _send_request method.""" + with patch.object(BillingService, "_send_request") as mock: + yield mock + + def test_sync_partner_tenants_bindings(self, mock_send_request): + """Test syncing partner tenant bindings.""" + # Arrange + account_id = "account-123" + partner_key = "partner-xyz" + click_id = "click-789" + expected_response = {"result": "success", "synced": True} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.sync_partner_tenants_bindings(account_id, partner_key, click_id) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with( + "PUT", f"/partners/{partner_key}/tenants", json={"account_id": account_id, "click_id": click_id} + ) + + +class TestBillingServiceEdgeCases: + """Unit tests for edge cases and error scenarios. + + Tests cover: + - Empty responses from billing API + - Malformed JSON responses + - Boundary conditions for rate limits + - Multiple subscription tiers + - Zero and negative usage deltas + """ + + @pytest.fixture + def mock_send_request(self): + """Mock _send_request method.""" + with patch.object(BillingService, "_send_request") as mock: + yield mock + + def test_get_info_empty_response(self, mock_send_request): + """Test handling of empty billing info response.""" + # Arrange + tenant_id = "tenant-empty" + mock_send_request.return_value = {} + + # Act + result = BillingService.get_info(tenant_id) + + # Assert + assert result == {} + mock_send_request.assert_called_once() + + def test_update_tenant_feature_plan_usage_zero_delta(self, mock_send_request): + """Test updating tenant feature usage with zero delta (no change).""" + # Arrange + tenant_id = "tenant-123" + feature_key = "trigger" + delta = 0 # No change + expected_response = {"result": "success", "history_id": "hist-uuid-zero"} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.update_tenant_feature_plan_usage(tenant_id, feature_key, delta) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with( + "POST", + "/tenant-feature-usage/usage", + params={"tenant_id": tenant_id, "feature_key": feature_key, "delta": delta}, + ) + + def test_update_tenant_feature_plan_usage_large_negative_delta(self, mock_send_request): + """Test updating tenant feature usage with large negative delta.""" + # Arrange + tenant_id = "tenant-456" + feature_key = "workflow" + delta = -1000 # Large consumption + expected_response = {"result": "success", "history_id": "hist-uuid-large"} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.update_tenant_feature_plan_usage(tenant_id, feature_key, delta) + + # Assert + assert result == expected_response + mock_send_request.assert_called_once() + + def test_get_knowledge_rate_limit_all_subscription_tiers(self, mock_send_request): + """Test knowledge rate limit for all subscription tiers.""" + # Test SANDBOX tier + mock_send_request.return_value = {"limit": 10, "subscription_plan": CloudPlan.SANDBOX} + result = BillingService.get_knowledge_rate_limit("tenant-sandbox") + assert result["subscription_plan"] == CloudPlan.SANDBOX + assert result["limit"] == 10 + + # Test PROFESSIONAL tier + mock_send_request.return_value = {"limit": 100, "subscription_plan": CloudPlan.PROFESSIONAL} + result = BillingService.get_knowledge_rate_limit("tenant-pro") + assert result["subscription_plan"] == CloudPlan.PROFESSIONAL + assert result["limit"] == 100 + + # Test TEAM tier + mock_send_request.return_value = {"limit": 500, "subscription_plan": CloudPlan.TEAM} + result = BillingService.get_knowledge_rate_limit("tenant-team") + assert result["subscription_plan"] == CloudPlan.TEAM + assert result["limit"] == 500 + + def test_get_subscription_with_empty_optional_params(self, mock_send_request): + """Test subscription payment link with empty optional parameters.""" + # Arrange + plan = "professional" + interval = "yearly" + expected_response = {"payment_link": "https://payment.example.com/checkout"} + mock_send_request.return_value = expected_response + + # Act - empty email and tenant_id + result = BillingService.get_subscription(plan, interval, "", "") + + # Assert + assert result == expected_response + mock_send_request.assert_called_once_with( + "GET", + "/subscription/payment-link", + params={"plan": plan, "interval": interval, "prefilled_email": "", "tenant_id": ""}, + ) + + def test_get_invoices_with_empty_params(self, mock_send_request): + """Test invoice retrieval with empty parameters.""" + # Arrange + expected_response = {"invoices": []} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.get_invoices("", "") + + # Assert + assert result == expected_response + assert result["invoices"] == [] + + def test_refund_with_invalid_history_id_format(self, mock_send_request): + """Test refund with various history ID formats.""" + # Arrange - test with different ID formats + test_ids = ["hist-123", "uuid-abc-def", "12345", ""] + + for history_id in test_ids: + expected_response = {"result": "success", "history_id": history_id} + mock_send_request.return_value = expected_response + + # Act + result = BillingService.refund_tenant_feature_plan_usage(history_id) + + # Assert + assert result["history_id"] == history_id + + def test_is_tenant_owner_or_admin_editor_role_raises_error(self): + """Test tenant owner/admin check raises error for editor role.""" + # Arrange + current_user = MagicMock(spec=Account) + current_user.id = "account-123" + current_user.current_tenant_id = "tenant-456" + + mock_join = MagicMock(spec=TenantAccountJoin) + mock_join.role = TenantAccountRole.EDITOR # Editor is not privileged + + with patch("services.billing_service.db.session") as mock_session: + mock_query = MagicMock() + mock_query.where.return_value.first.return_value = mock_join + mock_session.query.return_value = mock_query + + # Act & Assert + with pytest.raises(ValueError) as exc_info: + BillingService.is_tenant_owner_or_admin(current_user) + assert "Only team owner or team admin can perform this action" in str(exc_info.value) + + def test_is_tenant_owner_or_admin_dataset_operator_raises_error(self): + """Test tenant owner/admin check raises error for dataset operator role.""" + # Arrange + current_user = MagicMock(spec=Account) + current_user.id = "account-123" + current_user.current_tenant_id = "tenant-456" + + mock_join = MagicMock(spec=TenantAccountJoin) + mock_join.role = TenantAccountRole.DATASET_OPERATOR # Dataset operator is not privileged + + with patch("services.billing_service.db.session") as mock_session: + mock_query = MagicMock() + mock_query.where.return_value.first.return_value = mock_join + mock_session.query.return_value = mock_query + + # Act & Assert + with pytest.raises(ValueError) as exc_info: + BillingService.is_tenant_owner_or_admin(current_user) + assert "Only team owner or team admin can perform this action" in str(exc_info.value) + + +class TestBillingServiceIntegrationScenarios: + """Integration-style tests simulating real-world usage scenarios. + + These tests combine multiple service methods to test common workflows: + - Complete subscription upgrade flow + - Usage tracking and refund workflow + - Rate limit boundary testing + """ + + @pytest.fixture + def mock_send_request(self): + """Mock _send_request method.""" + with patch.object(BillingService, "_send_request") as mock: + yield mock + + def test_subscription_upgrade_workflow(self, mock_send_request): + """Test complete subscription upgrade workflow.""" + # Arrange + tenant_id = "tenant-upgrade" + + # Step 1: Get current billing info + mock_send_request.return_value = { + "subscription_plan": "sandbox", + "billing_cycle": "monthly", + "status": "active", + } + current_info = BillingService.get_info(tenant_id) + assert current_info["subscription_plan"] == "sandbox" + + # Step 2: Get payment link for upgrade + mock_send_request.return_value = {"payment_link": "https://payment.example.com/upgrade"} + payment_link = BillingService.get_subscription("professional", "monthly", "user@example.com", tenant_id) + assert "payment_link" in payment_link + + # Step 3: Verify new rate limits after upgrade + mock_send_request.return_value = {"limit": 100, "subscription_plan": CloudPlan.PROFESSIONAL} + rate_limit = BillingService.get_knowledge_rate_limit(tenant_id) + assert rate_limit["subscription_plan"] == CloudPlan.PROFESSIONAL + assert rate_limit["limit"] == 100 + + def test_usage_tracking_and_refund_workflow(self, mock_send_request): + """Test usage tracking with subsequent refund.""" + # Arrange + tenant_id = "tenant-usage" + feature_key = "workflow" + + # Step 1: Consume credits + mock_send_request.return_value = {"result": "success", "history_id": "hist-consume-123"} + consume_result = BillingService.update_tenant_feature_plan_usage(tenant_id, feature_key, -10) + history_id = consume_result["history_id"] + assert history_id == "hist-consume-123" + + # Step 2: Check current usage + mock_send_request.return_value = {"used": 10, "limit": 100, "remaining": 90} + usage = BillingService.get_tenant_feature_plan_usage(tenant_id, feature_key) + assert usage["used"] == 10 + assert usage["remaining"] == 90 + + # Step 3: Refund the usage + mock_send_request.return_value = {"result": "success", "history_id": history_id} + refund_result = BillingService.refund_tenant_feature_plan_usage(history_id) + assert refund_result["result"] == "success" + + # Step 4: Verify usage after refund + mock_send_request.return_value = {"used": 0, "limit": 100, "remaining": 100} + updated_usage = BillingService.get_tenant_feature_plan_usage(tenant_id, feature_key) + assert updated_usage["used"] == 0 + assert updated_usage["remaining"] == 100 + + def test_compliance_download_multiple_requests_within_limit(self, mock_send_request): + """Test multiple compliance downloads within rate limit.""" + # Arrange + account_id = "account-compliance" + tenant_id = "tenant-compliance" + doc_name = "compliance_report.pdf" + ip = "192.168.1.1" + device_info = "Mozilla/5.0" + + # Mock rate limiter to allow 3 requests (under limit of 4) + with ( + patch.object( + BillingService.compliance_download_rate_limiter, "is_rate_limited", side_effect=[False, False, False] + ) as mock_is_limited, + patch.object(BillingService.compliance_download_rate_limiter, "increment_rate_limit") as mock_increment, + ): + mock_send_request.return_value = {"download_link": "https://example.com/download"} + + # Act - Make 3 requests + for i in range(3): + result = BillingService.get_compliance_download_link(doc_name, account_id, tenant_id, ip, device_info) + assert "download_link" in result + + # Assert - All 3 requests succeeded + assert mock_is_limited.call_count == 3 + assert mock_increment.call_count == 3 + + def test_education_verification_and_activation_flow(self, mock_send_request): + """Test complete education verification and activation flow.""" + # Arrange + account = MagicMock(spec=Account) + account.id = "account-edu" + account.email = "student@mit.edu" + account.current_tenant_id = "tenant-edu" + + # Step 1: Search for institution + with ( + patch.object( + BillingService.EducationIdentity.verification_rate_limit, "is_rate_limited", return_value=False + ), + patch.object(BillingService.EducationIdentity.verification_rate_limit, "increment_rate_limit"), + ): + mock_send_request.return_value = { + "institutions": [{"name": "Massachusetts Institute of Technology", "domain": "mit.edu"}] + } + institutions = BillingService.EducationIdentity.autocomplete("MIT") + assert len(institutions["institutions"]) > 0 + + # Step 2: Verify email + with ( + patch.object( + BillingService.EducationIdentity.verification_rate_limit, "is_rate_limited", return_value=False + ), + patch.object(BillingService.EducationIdentity.verification_rate_limit, "increment_rate_limit"), + ): + mock_send_request.return_value = {"verified": True, "institution": "MIT"} + verify_result = BillingService.EducationIdentity.verify(account.id, account.email) + assert verify_result["verified"] is True + + # Step 3: Check status + mock_send_request.return_value = {"verified": True, "institution": "MIT", "role": "student"} + status = BillingService.EducationIdentity.status(account.id) + assert status["verified"] is True + + # Step 4: Activate education benefits + with ( + patch.object(BillingService.EducationIdentity.activation_rate_limit, "is_rate_limited", return_value=False), + patch.object(BillingService.EducationIdentity.activation_rate_limit, "increment_rate_limit"), + ): + mock_send_request.return_value = {"result": "success", "activated": True} + activate_result = BillingService.EducationIdentity.activate(account, "token-123", "MIT", "student") + assert activate_result["activated"] is True diff --git a/api/tests/unit_tests/services/test_conversation_service.py b/api/tests/unit_tests/services/test_conversation_service.py index 9c1c044f03..81135dbbdf 100644 --- a/api/tests/unit_tests/services/test_conversation_service.py +++ b/api/tests/unit_tests/services/test_conversation_service.py @@ -1,17 +1,293 @@ +""" +Comprehensive unit tests for ConversationService. + +This test suite provides complete coverage of conversation management operations in Dify, +following TDD principles with the Arrange-Act-Assert pattern. + +## Test Coverage + +### 1. Conversation Pagination (TestConversationServicePagination) +Tests conversation listing and filtering: +- Empty include_ids returns empty results +- Non-empty include_ids filters conversations properly +- Empty exclude_ids doesn't filter results +- Non-empty exclude_ids excludes specified conversations +- Null user handling +- Sorting and pagination edge cases + +### 2. Message Creation (TestConversationServiceMessageCreation) +Tests message operations within conversations: +- Message pagination without first_id +- Message pagination with first_id specified +- Error handling for non-existent messages +- Empty result handling for null user/conversation +- Message ordering (ascending/descending) +- Has_more flag calculation + +### 3. Conversation Summarization (TestConversationServiceSummarization) +Tests auto-generated conversation names: +- Successful LLM-based name generation +- Error handling when conversation has no messages +- Graceful handling of LLM service failures +- Manual vs auto-generated naming +- Name update timestamp tracking + +### 4. Message Annotation (TestConversationServiceMessageAnnotation) +Tests annotation creation and management: +- Creating annotations from existing messages +- Creating standalone annotations +- Updating existing annotations +- Paginated annotation retrieval +- Annotation search with keywords +- Annotation export functionality + +### 5. Conversation Export (TestConversationServiceExport) +Tests data retrieval for export: +- Successful conversation retrieval +- Error handling for non-existent conversations +- Message retrieval +- Annotation export +- Batch data export operations + +## Testing Approach + +- **Mocking Strategy**: All external dependencies (database, LLM, Redis) are mocked + for fast, isolated unit tests +- **Factory Pattern**: ConversationServiceTestDataFactory provides consistent test data +- **Fixtures**: Mock objects are configured per test method +- **Assertions**: Each test verifies return values and side effects + (database operations, method calls) + +## Key Concepts + +**Conversation Sources:** +- console: Created by workspace members +- api: Created by end users via API + +**Message Pagination:** +- first_id: Paginate from a specific message forward +- last_id: Paginate from a specific message backward +- Supports ascending/descending order + +**Annotations:** +- Can be attached to messages or standalone +- Support full-text search +- Indexed for semantic retrieval +""" + import uuid -from unittest.mock import MagicMock, patch +from datetime import UTC, datetime +from decimal import Decimal +from unittest.mock import MagicMock, Mock, create_autospec, patch + +import pytest from core.app.entities.app_invoke_entities import InvokeFrom +from models import Account +from models.model import App, Conversation, EndUser, Message, MessageAnnotation +from services.annotation_service import AppAnnotationService from services.conversation_service import ConversationService +from services.errors.conversation import ConversationNotExistsError +from services.errors.message import FirstMessageNotExistsError, MessageNotExistsError +from services.message_service import MessageService -class TestConversationService: +class ConversationServiceTestDataFactory: + """ + Factory for creating test data and mock objects. + + Provides reusable methods to create consistent mock objects for testing + conversation-related operations. + """ + + @staticmethod + def create_account_mock(account_id: str = "account-123", **kwargs) -> Mock: + """ + Create a mock Account object. + + Args: + account_id: Unique identifier for the account + **kwargs: Additional attributes to set on the mock + + Returns: + Mock Account object with specified attributes + """ + account = create_autospec(Account, instance=True) + account.id = account_id + for key, value in kwargs.items(): + setattr(account, key, value) + return account + + @staticmethod + def create_end_user_mock(user_id: str = "user-123", **kwargs) -> Mock: + """ + Create a mock EndUser object. + + Args: + user_id: Unique identifier for the end user + **kwargs: Additional attributes to set on the mock + + Returns: + Mock EndUser object with specified attributes + """ + user = create_autospec(EndUser, instance=True) + user.id = user_id + for key, value in kwargs.items(): + setattr(user, key, value) + return user + + @staticmethod + def create_app_mock(app_id: str = "app-123", tenant_id: str = "tenant-123", **kwargs) -> Mock: + """ + Create a mock App object. + + Args: + app_id: Unique identifier for the app + tenant_id: Tenant/workspace identifier + **kwargs: Additional attributes to set on the mock + + Returns: + Mock App object with specified attributes + """ + app = create_autospec(App, instance=True) + app.id = app_id + app.tenant_id = tenant_id + app.name = kwargs.get("name", "Test App") + app.mode = kwargs.get("mode", "chat") + app.status = kwargs.get("status", "normal") + for key, value in kwargs.items(): + setattr(app, key, value) + return app + + @staticmethod + def create_conversation_mock( + conversation_id: str = "conv-123", + app_id: str = "app-123", + from_source: str = "console", + **kwargs, + ) -> Mock: + """ + Create a mock Conversation object. + + Args: + conversation_id: Unique identifier for the conversation + app_id: Associated app identifier + from_source: Source of conversation ('console' or 'api') + **kwargs: Additional attributes to set on the mock + + Returns: + Mock Conversation object with specified attributes + """ + conversation = create_autospec(Conversation, instance=True) + conversation.id = conversation_id + conversation.app_id = app_id + conversation.from_source = from_source + conversation.from_end_user_id = kwargs.get("from_end_user_id") + conversation.from_account_id = kwargs.get("from_account_id") + conversation.is_deleted = kwargs.get("is_deleted", False) + conversation.name = kwargs.get("name", "Test Conversation") + conversation.status = kwargs.get("status", "normal") + conversation.created_at = kwargs.get("created_at", datetime.now(UTC)) + conversation.updated_at = kwargs.get("updated_at", datetime.now(UTC)) + for key, value in kwargs.items(): + setattr(conversation, key, value) + return conversation + + @staticmethod + def create_message_mock( + message_id: str = "msg-123", + conversation_id: str = "conv-123", + app_id: str = "app-123", + **kwargs, + ) -> Mock: + """ + Create a mock Message object. + + Args: + message_id: Unique identifier for the message + conversation_id: Associated conversation identifier + app_id: Associated app identifier + **kwargs: Additional attributes to set on the mock + + Returns: + Mock Message object with specified attributes including + query, answer, tokens, and pricing information + """ + message = create_autospec(Message, instance=True) + message.id = message_id + message.conversation_id = conversation_id + message.app_id = app_id + message.query = kwargs.get("query", "Test query") + message.answer = kwargs.get("answer", "Test answer") + message.from_source = kwargs.get("from_source", "console") + message.from_end_user_id = kwargs.get("from_end_user_id") + message.from_account_id = kwargs.get("from_account_id") + message.created_at = kwargs.get("created_at", datetime.now(UTC)) + message.message = kwargs.get("message", {}) + message.message_tokens = kwargs.get("message_tokens", 0) + message.answer_tokens = kwargs.get("answer_tokens", 0) + message.message_unit_price = kwargs.get("message_unit_price", Decimal(0)) + message.answer_unit_price = kwargs.get("answer_unit_price", Decimal(0)) + message.message_price_unit = kwargs.get("message_price_unit", Decimal("0.001")) + message.answer_price_unit = kwargs.get("answer_price_unit", Decimal("0.001")) + message.currency = kwargs.get("currency", "USD") + message.status = kwargs.get("status", "normal") + for key, value in kwargs.items(): + setattr(message, key, value) + return message + + @staticmethod + def create_annotation_mock( + annotation_id: str = "anno-123", + app_id: str = "app-123", + message_id: str = "msg-123", + **kwargs, + ) -> Mock: + """ + Create a mock MessageAnnotation object. + + Args: + annotation_id: Unique identifier for the annotation + app_id: Associated app identifier + message_id: Associated message identifier (optional for standalone annotations) + **kwargs: Additional attributes to set on the mock + + Returns: + Mock MessageAnnotation object with specified attributes including + question, content, and hit tracking + """ + annotation = create_autospec(MessageAnnotation, instance=True) + annotation.id = annotation_id + annotation.app_id = app_id + annotation.message_id = message_id + annotation.conversation_id = kwargs.get("conversation_id") + annotation.question = kwargs.get("question", "Test question") + annotation.content = kwargs.get("content", "Test annotation") + annotation.account_id = kwargs.get("account_id", "account-123") + annotation.hit_count = kwargs.get("hit_count", 0) + annotation.created_at = kwargs.get("created_at", datetime.now(UTC)) + annotation.updated_at = kwargs.get("updated_at", datetime.now(UTC)) + for key, value in kwargs.items(): + setattr(annotation, key, value) + return annotation + + +class TestConversationServicePagination: + """Test conversation pagination operations.""" + def test_pagination_with_empty_include_ids(self): - """Test that empty include_ids returns empty result""" - mock_session = MagicMock() - mock_app_model = MagicMock(id=str(uuid.uuid4())) - mock_user = MagicMock(id=str(uuid.uuid4())) + """ + Test that empty include_ids returns empty result. + When include_ids is an empty list, the service should short-circuit + and return empty results without querying the database. + """ + # Arrange - Set up test data + mock_session = MagicMock() # Mock database session + mock_app_model = ConversationServiceTestDataFactory.create_app_mock() + mock_user = ConversationServiceTestDataFactory.create_account_mock() + + # Act - Call the service method with empty include_ids result = ConversationService.pagination_by_last_id( session=mock_session, app_model=mock_app_model, @@ -19,25 +295,188 @@ class TestConversationService: last_id=None, limit=20, invoke_from=InvokeFrom.WEB_APP, - include_ids=[], # Empty include_ids should return empty result + include_ids=[], # Empty list should trigger early return exclude_ids=None, ) + # Assert - Verify empty result without database query + assert result.data == [] # No conversations returned + assert result.has_more is False # No more pages available + assert result.limit == 20 # Limit preserved in response + + def test_pagination_with_non_empty_include_ids(self): + """ + Test that non-empty include_ids filters properly. + + When include_ids contains conversation IDs, the query should filter + to only return conversations matching those IDs. + """ + # Arrange - Set up test data and mocks + mock_session = MagicMock() # Mock database session + mock_app_model = ConversationServiceTestDataFactory.create_app_mock() + mock_user = ConversationServiceTestDataFactory.create_account_mock() + + # Create 3 mock conversations that would match the filter + mock_conversations = [ + ConversationServiceTestDataFactory.create_conversation_mock(conversation_id=str(uuid.uuid4())) + for _ in range(3) + ] + # Mock the database query results + mock_session.scalars.return_value.all.return_value = mock_conversations + mock_session.scalar.return_value = 0 # No additional conversations beyond current page + + # Act + with patch("services.conversation_service.select") as mock_select: + mock_stmt = MagicMock() + mock_select.return_value = mock_stmt + mock_stmt.where.return_value = mock_stmt + mock_stmt.order_by.return_value = mock_stmt + mock_stmt.limit.return_value = mock_stmt + mock_stmt.subquery.return_value = MagicMock() + + result = ConversationService.pagination_by_last_id( + session=mock_session, + app_model=mock_app_model, + user=mock_user, + last_id=None, + limit=20, + invoke_from=InvokeFrom.WEB_APP, + include_ids=["conv1", "conv2"], + exclude_ids=None, + ) + + # Assert + assert mock_stmt.where.called + + def test_pagination_with_empty_exclude_ids(self): + """ + Test that empty exclude_ids doesn't filter. + + When exclude_ids is an empty list, the query should not filter out + any conversations. + """ + # Arrange + mock_session = MagicMock() + mock_app_model = ConversationServiceTestDataFactory.create_app_mock() + mock_user = ConversationServiceTestDataFactory.create_account_mock() + mock_conversations = [ + ConversationServiceTestDataFactory.create_conversation_mock(conversation_id=str(uuid.uuid4())) + for _ in range(5) + ] + mock_session.scalars.return_value.all.return_value = mock_conversations + mock_session.scalar.return_value = 0 + + # Act + with patch("services.conversation_service.select") as mock_select: + mock_stmt = MagicMock() + mock_select.return_value = mock_stmt + mock_stmt.where.return_value = mock_stmt + mock_stmt.order_by.return_value = mock_stmt + mock_stmt.limit.return_value = mock_stmt + mock_stmt.subquery.return_value = MagicMock() + + result = ConversationService.pagination_by_last_id( + session=mock_session, + app_model=mock_app_model, + user=mock_user, + last_id=None, + limit=20, + invoke_from=InvokeFrom.WEB_APP, + include_ids=None, + exclude_ids=[], + ) + + # Assert + assert len(result.data) == 5 + + def test_pagination_with_non_empty_exclude_ids(self): + """ + Test that non-empty exclude_ids filters properly. + + When exclude_ids contains conversation IDs, the query should filter + out conversations matching those IDs. + """ + # Arrange + mock_session = MagicMock() + mock_app_model = ConversationServiceTestDataFactory.create_app_mock() + mock_user = ConversationServiceTestDataFactory.create_account_mock() + mock_conversations = [ + ConversationServiceTestDataFactory.create_conversation_mock(conversation_id=str(uuid.uuid4())) + for _ in range(3) + ] + mock_session.scalars.return_value.all.return_value = mock_conversations + mock_session.scalar.return_value = 0 + + # Act + with patch("services.conversation_service.select") as mock_select: + mock_stmt = MagicMock() + mock_select.return_value = mock_stmt + mock_stmt.where.return_value = mock_stmt + mock_stmt.order_by.return_value = mock_stmt + mock_stmt.limit.return_value = mock_stmt + mock_stmt.subquery.return_value = MagicMock() + + result = ConversationService.pagination_by_last_id( + session=mock_session, + app_model=mock_app_model, + user=mock_user, + last_id=None, + limit=20, + invoke_from=InvokeFrom.WEB_APP, + include_ids=None, + exclude_ids=["conv1", "conv2"], + ) + + # Assert + assert mock_stmt.where.called + + def test_pagination_returns_empty_when_user_is_none(self): + """ + Test that pagination returns empty result when user is None. + + This ensures proper handling of unauthenticated requests. + """ + # Arrange + mock_session = MagicMock() + mock_app_model = ConversationServiceTestDataFactory.create_app_mock() + + # Act + result = ConversationService.pagination_by_last_id( + session=mock_session, + app_model=mock_app_model, + user=None, # No user provided + last_id=None, + limit=20, + invoke_from=InvokeFrom.WEB_APP, + ) + + # Assert - should return empty result without querying database assert result.data == [] assert result.has_more is False assert result.limit == 20 - def test_pagination_with_non_empty_include_ids(self): - """Test that non-empty include_ids filters properly""" - mock_session = MagicMock() - mock_app_model = MagicMock(id=str(uuid.uuid4())) - mock_user = MagicMock(id=str(uuid.uuid4())) + def test_pagination_with_sorting_descending(self): + """ + Test pagination with descending sort order. - # Mock the query results - mock_conversations = [MagicMock(id=str(uuid.uuid4())) for _ in range(3)] - mock_session.scalars.return_value.all.return_value = mock_conversations + Verifies that conversations are sorted by updated_at in descending order (newest first). + """ + # Arrange + mock_session = MagicMock() + mock_app_model = ConversationServiceTestDataFactory.create_app_mock() + mock_user = ConversationServiceTestDataFactory.create_account_mock() + + # Create conversations with different timestamps + conversations = [ + ConversationServiceTestDataFactory.create_conversation_mock( + conversation_id=f"conv-{i}", updated_at=datetime(2024, 1, i + 1, tzinfo=UTC) + ) + for i in range(3) + ] + mock_session.scalars.return_value.all.return_value = conversations mock_session.scalar.return_value = 0 + # Act with patch("services.conversation_service.select") as mock_select: mock_stmt = MagicMock() mock_select.return_value = mock_stmt @@ -53,75 +492,902 @@ class TestConversationService: last_id=None, limit=20, invoke_from=InvokeFrom.WEB_APP, - include_ids=["conv1", "conv2"], # Non-empty include_ids - exclude_ids=None, + sort_by="-updated_at", # Descending sort ) - # Verify the where clause was called with id.in_ - assert mock_stmt.where.called + # Assert + assert len(result.data) == 3 + mock_stmt.order_by.assert_called() - def test_pagination_with_empty_exclude_ids(self): - """Test that empty exclude_ids doesn't filter""" - mock_session = MagicMock() - mock_app_model = MagicMock(id=str(uuid.uuid4())) - mock_user = MagicMock(id=str(uuid.uuid4())) - # Mock the query results - mock_conversations = [MagicMock(id=str(uuid.uuid4())) for _ in range(5)] - mock_session.scalars.return_value.all.return_value = mock_conversations - mock_session.scalar.return_value = 0 +class TestConversationServiceMessageCreation: + """ + Test message creation and pagination. - with patch("services.conversation_service.select") as mock_select: - mock_stmt = MagicMock() - mock_select.return_value = mock_stmt - mock_stmt.where.return_value = mock_stmt - mock_stmt.order_by.return_value = mock_stmt - mock_stmt.limit.return_value = mock_stmt - mock_stmt.subquery.return_value = MagicMock() + Tests MessageService operations for creating and retrieving messages + within conversations. + """ - result = ConversationService.pagination_by_last_id( - session=mock_session, - app_model=mock_app_model, - user=mock_user, - last_id=None, - limit=20, - invoke_from=InvokeFrom.WEB_APP, - include_ids=None, - exclude_ids=[], # Empty exclude_ids should not filter + @patch("services.message_service.db.session") + @patch("services.message_service.ConversationService.get_conversation") + def test_pagination_by_first_id_without_first_id(self, mock_get_conversation, mock_db_session): + """ + Test message pagination without specifying first_id. + + When first_id is None, the service should return the most recent messages + up to the specified limit. + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + conversation = ConversationServiceTestDataFactory.create_conversation_mock() + + # Create 3 test messages in the conversation + messages = [ + ConversationServiceTestDataFactory.create_message_mock( + message_id=f"msg-{i}", conversation_id=conversation.id + ) + for i in range(3) + ] + + # Mock the conversation lookup to return our test conversation + mock_get_conversation.return_value = conversation + + # Set up the database query mock chain + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query # WHERE clause returns self for chaining + mock_query.order_by.return_value = mock_query # ORDER BY returns self for chaining + mock_query.limit.return_value = mock_query # LIMIT returns self for chaining + mock_query.all.return_value = messages # Final .all() returns the messages + + # Act - Call the pagination method without first_id + result = MessageService.pagination_by_first_id( + app_model=app_model, + user=user, + conversation_id=conversation.id, + first_id=None, # No starting point specified + limit=10, + ) + + # Assert - Verify the results + assert len(result.data) == 3 # All 3 messages returned + assert result.has_more is False # No more messages available (3 < limit of 10) + # Verify conversation was looked up with correct parameters + mock_get_conversation.assert_called_once_with(app_model=app_model, user=user, conversation_id=conversation.id) + + @patch("services.message_service.db.session") + @patch("services.message_service.ConversationService.get_conversation") + def test_pagination_by_first_id_with_first_id(self, mock_get_conversation, mock_db_session): + """ + Test message pagination with first_id specified. + + When first_id is provided, the service should return messages starting + from the specified message up to the limit. + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + conversation = ConversationServiceTestDataFactory.create_conversation_mock() + first_message = ConversationServiceTestDataFactory.create_message_mock( + message_id="msg-first", conversation_id=conversation.id + ) + messages = [ + ConversationServiceTestDataFactory.create_message_mock( + message_id=f"msg-{i}", conversation_id=conversation.id + ) + for i in range(2) + ] + + # Mock the conversation lookup to return our test conversation + mock_get_conversation.return_value = conversation + + # Set up the database query mock chain + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query # WHERE clause returns self for chaining + mock_query.order_by.return_value = mock_query # ORDER BY returns self for chaining + mock_query.limit.return_value = mock_query # LIMIT returns self for chaining + mock_query.first.return_value = first_message # First message returned + mock_query.all.return_value = messages # Remaining messages returned + + # Act - Call the pagination method with first_id + result = MessageService.pagination_by_first_id( + app_model=app_model, + user=user, + conversation_id=conversation.id, + first_id="msg-first", + limit=10, + ) + + # Assert - Verify the results + assert len(result.data) == 2 # Only 2 messages returned after first_id + assert result.has_more is False # No more messages available (2 < limit of 10) + + @patch("services.message_service.db.session") + @patch("services.message_service.ConversationService.get_conversation") + def test_pagination_by_first_id_raises_error_when_first_message_not_found( + self, mock_get_conversation, mock_db_session + ): + """ + Test that FirstMessageNotExistsError is raised when first_id doesn't exist. + + When the specified first_id does not exist in the conversation, + the service should raise an error. + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + conversation = ConversationServiceTestDataFactory.create_conversation_mock() + + # Mock the conversation lookup to return our test conversation + mock_get_conversation.return_value = conversation + + # Set up the database query mock chain + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query # WHERE clause returns self for chaining + mock_query.first.return_value = None # No message found for first_id + + # Act & Assert + with pytest.raises(FirstMessageNotExistsError): + MessageService.pagination_by_first_id( + app_model=app_model, + user=user, + conversation_id=conversation.id, + first_id="non-existent-msg", + limit=10, ) - # Result should contain the mocked conversations - assert len(result.data) == 5 + def test_pagination_returns_empty_when_no_user(self): + """ + Test that pagination returns empty result when user is None. - def test_pagination_with_non_empty_exclude_ids(self): - """Test that non-empty exclude_ids filters properly""" - mock_session = MagicMock() - mock_app_model = MagicMock(id=str(uuid.uuid4())) - mock_user = MagicMock(id=str(uuid.uuid4())) + This ensures proper handling of unauthenticated requests. + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() - # Mock the query results - mock_conversations = [MagicMock(id=str(uuid.uuid4())) for _ in range(3)] - mock_session.scalars.return_value.all.return_value = mock_conversations - mock_session.scalar.return_value = 0 + # Act + result = MessageService.pagination_by_first_id( + app_model=app_model, + user=None, + conversation_id="conv-123", + first_id=None, + limit=10, + ) - with patch("services.conversation_service.select") as mock_select: - mock_stmt = MagicMock() - mock_select.return_value = mock_stmt - mock_stmt.where.return_value = mock_stmt - mock_stmt.order_by.return_value = mock_stmt - mock_stmt.limit.return_value = mock_stmt - mock_stmt.subquery.return_value = MagicMock() + # Assert + assert result.data == [] + assert result.has_more is False - result = ConversationService.pagination_by_last_id( - session=mock_session, - app_model=mock_app_model, - user=mock_user, - last_id=None, - limit=20, - invoke_from=InvokeFrom.WEB_APP, - include_ids=None, - exclude_ids=["conv1", "conv2"], # Non-empty exclude_ids + def test_pagination_returns_empty_when_no_conversation_id(self): + """ + Test that pagination returns empty result when conversation_id is None. + + This ensures proper handling of invalid requests. + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + + # Act + result = MessageService.pagination_by_first_id( + app_model=app_model, + user=user, + conversation_id="", + first_id=None, + limit=10, + ) + + # Assert + assert result.data == [] + assert result.has_more is False + + @patch("services.message_service.db.session") + @patch("services.message_service.ConversationService.get_conversation") + def test_pagination_with_has_more_flag(self, mock_get_conversation, mock_db_session): + """ + Test that has_more flag is correctly set when there are more messages. + + The service fetches limit+1 messages to determine if more exist. + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + conversation = ConversationServiceTestDataFactory.create_conversation_mock() + + # Create limit+1 messages to trigger has_more + limit = 5 + messages = [ + ConversationServiceTestDataFactory.create_message_mock( + message_id=f"msg-{i}", conversation_id=conversation.id ) + for i in range(limit + 1) # One extra message + ] - # Verify the where clause was called for exclusion - assert mock_stmt.where.called + # Mock the conversation lookup to return our test conversation + mock_get_conversation.return_value = conversation + + # Set up the database query mock chain + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query # WHERE clause returns self for chaining + mock_query.order_by.return_value = mock_query # ORDER BY returns self for chaining + mock_query.limit.return_value = mock_query # LIMIT returns self for chaining + mock_query.all.return_value = messages # Final .all() returns the messages + + # Act + result = MessageService.pagination_by_first_id( + app_model=app_model, + user=user, + conversation_id=conversation.id, + first_id=None, + limit=limit, + ) + + # Assert + assert len(result.data) == limit # Extra message should be removed + assert result.has_more is True # Flag should be set + + @patch("services.message_service.db.session") + @patch("services.message_service.ConversationService.get_conversation") + def test_pagination_with_ascending_order(self, mock_get_conversation, mock_db_session): + """ + Test message pagination with ascending order. + + Messages should be returned in chronological order (oldest first). + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + conversation = ConversationServiceTestDataFactory.create_conversation_mock() + + # Create messages with different timestamps + messages = [ + ConversationServiceTestDataFactory.create_message_mock( + message_id=f"msg-{i}", conversation_id=conversation.id, created_at=datetime(2024, 1, i + 1, tzinfo=UTC) + ) + for i in range(3) + ] + + # Mock the conversation lookup to return our test conversation + mock_get_conversation.return_value = conversation + + # Set up the database query mock chain + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query # WHERE clause returns self for chaining + mock_query.order_by.return_value = mock_query # ORDER BY returns self for chaining + mock_query.limit.return_value = mock_query # LIMIT returns self for chaining + mock_query.all.return_value = messages # Final .all() returns the messages + + # Act + result = MessageService.pagination_by_first_id( + app_model=app_model, + user=user, + conversation_id=conversation.id, + first_id=None, + limit=10, + order="asc", # Ascending order + ) + + # Assert + assert len(result.data) == 3 + # Messages should be in ascending order after reversal + + +class TestConversationServiceSummarization: + """ + Test conversation summarization (auto-generated names). + + Tests the auto_generate_name functionality that creates conversation + titles based on the first message. + """ + + @patch("services.conversation_service.LLMGenerator.generate_conversation_name") + @patch("services.conversation_service.db.session") + def test_auto_generate_name_success(self, mock_db_session, mock_llm_generator): + """ + Test successful auto-generation of conversation name. + + The service uses an LLM to generate a descriptive name based on + the first message in the conversation. + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + conversation = ConversationServiceTestDataFactory.create_conversation_mock() + + # Create the first message that will be used to generate the name + first_message = ConversationServiceTestDataFactory.create_message_mock( + conversation_id=conversation.id, query="What is machine learning?" + ) + # Expected name from LLM + generated_name = "Machine Learning Discussion" + + # Set up database query mock to return the first message + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query # Filter by app_id and conversation_id + mock_query.order_by.return_value = mock_query # Order by created_at ascending + mock_query.first.return_value = first_message # Return the first message + + # Mock the LLM to return our expected name + mock_llm_generator.return_value = generated_name + + # Act + result = ConversationService.auto_generate_name(app_model, conversation) + + # Assert + assert conversation.name == generated_name # Name updated on conversation object + # Verify LLM was called with correct parameters + mock_llm_generator.assert_called_once_with( + app_model.tenant_id, first_message.query, conversation.id, app_model.id + ) + mock_db_session.commit.assert_called_once() # Changes committed to database + + @patch("services.conversation_service.db.session") + def test_auto_generate_name_raises_error_when_no_message(self, mock_db_session): + """ + Test that MessageNotExistsError is raised when conversation has no messages. + + When the conversation has no messages, the service should raise an error. + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + conversation = ConversationServiceTestDataFactory.create_conversation_mock() + + # Set up database query mock to return no messages + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query # Filter by app_id and conversation_id + mock_query.order_by.return_value = mock_query # Order by created_at ascending + mock_query.first.return_value = None # No messages found + + # Act & Assert + with pytest.raises(MessageNotExistsError): + ConversationService.auto_generate_name(app_model, conversation) + + @patch("services.conversation_service.LLMGenerator.generate_conversation_name") + @patch("services.conversation_service.db.session") + def test_auto_generate_name_handles_llm_failure_gracefully(self, mock_db_session, mock_llm_generator): + """ + Test that LLM generation failures are suppressed and don't crash. + + When the LLM fails to generate a name, the service should not crash + and should return the original conversation name. + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + conversation = ConversationServiceTestDataFactory.create_conversation_mock() + first_message = ConversationServiceTestDataFactory.create_message_mock(conversation_id=conversation.id) + original_name = conversation.name + + # Set up database query mock to return the first message + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query # Filter by app_id and conversation_id + mock_query.order_by.return_value = mock_query # Order by created_at ascending + mock_query.first.return_value = first_message # Return the first message + + # Mock the LLM to raise an exception + mock_llm_generator.side_effect = Exception("LLM service unavailable") + + # Act + result = ConversationService.auto_generate_name(app_model, conversation) + + # Assert + assert conversation.name == original_name # Name remains unchanged + mock_db_session.commit.assert_called_once() # Changes committed to database + + @patch("services.conversation_service.db.session") + @patch("services.conversation_service.ConversationService.get_conversation") + @patch("services.conversation_service.ConversationService.auto_generate_name") + def test_rename_with_auto_generate(self, mock_auto_generate, mock_get_conversation, mock_db_session): + """ + Test renaming conversation with auto-generation enabled. + + When auto_generate is True, the service should call the auto_generate_name + method to generate a new name for the conversation. + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + conversation = ConversationServiceTestDataFactory.create_conversation_mock() + conversation.name = "Auto-generated Name" + + # Mock the conversation lookup to return our test conversation + mock_get_conversation.return_value = conversation + + # Mock the auto_generate_name method to return the conversation + mock_auto_generate.return_value = conversation + + # Act + result = ConversationService.rename( + app_model=app_model, + conversation_id=conversation.id, + user=user, + name="", + auto_generate=True, + ) + + # Assert + mock_auto_generate.assert_called_once_with(app_model, conversation) + assert result == conversation + + @patch("services.conversation_service.db.session") + @patch("services.conversation_service.ConversationService.get_conversation") + @patch("services.conversation_service.naive_utc_now") + def test_rename_with_manual_name(self, mock_naive_utc_now, mock_get_conversation, mock_db_session): + """ + Test renaming conversation with manual name. + + When auto_generate is False, the service should update the conversation + name with the provided manual name. + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + conversation = ConversationServiceTestDataFactory.create_conversation_mock() + new_name = "My Custom Conversation Name" + mock_time = datetime(2024, 1, 1, 12, 0, 0) + + # Mock the conversation lookup to return our test conversation + mock_get_conversation.return_value = conversation + + # Mock the current time to return our mock time + mock_naive_utc_now.return_value = mock_time + + # Act + result = ConversationService.rename( + app_model=app_model, + conversation_id=conversation.id, + user=user, + name=new_name, + auto_generate=False, + ) + + # Assert + assert conversation.name == new_name + assert conversation.updated_at == mock_time + mock_db_session.commit.assert_called_once() + + +class TestConversationServiceMessageAnnotation: + """ + Test message annotation operations. + + Tests AppAnnotationService operations for creating and managing + message annotations. + """ + + @patch("services.annotation_service.db.session") + @patch("services.annotation_service.current_account_with_tenant") + def test_create_annotation_from_message(self, mock_current_account, mock_db_session): + """ + Test creating annotation from existing message. + + Annotations can be attached to messages to provide curated responses + that override the AI-generated answers. + """ + # Arrange + app_id = "app-123" + message_id = "msg-123" + account = ConversationServiceTestDataFactory.create_account_mock() + tenant_id = "tenant-123" + app = ConversationServiceTestDataFactory.create_app_mock(app_id=app_id, tenant_id=tenant_id) + + # Create a message that doesn't have an annotation yet + message = ConversationServiceTestDataFactory.create_message_mock( + message_id=message_id, app_id=app_id, query="What is AI?" + ) + message.annotation = None # No existing annotation + + # Mock the authentication context to return current user and tenant + mock_current_account.return_value = (account, tenant_id) + + # Set up database query mock + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + # First call returns app, second returns message, third returns None (no annotation setting) + mock_query.first.side_effect = [app, message, None] + + # Annotation data to create + args = {"message_id": message_id, "answer": "AI is artificial intelligence"} + + # Act + with patch("services.annotation_service.add_annotation_to_index_task"): + result = AppAnnotationService.up_insert_app_annotation_from_message(args, app_id) + + # Assert + mock_db_session.add.assert_called_once() # Annotation added to session + mock_db_session.commit.assert_called_once() # Changes committed + + @patch("services.annotation_service.db.session") + @patch("services.annotation_service.current_account_with_tenant") + def test_create_annotation_without_message(self, mock_current_account, mock_db_session): + """ + Test creating standalone annotation without message. + + Annotations can be created without a message reference for bulk imports + or manual annotation creation. + """ + # Arrange + app_id = "app-123" + account = ConversationServiceTestDataFactory.create_account_mock() + tenant_id = "tenant-123" + app = ConversationServiceTestDataFactory.create_app_mock(app_id=app_id, tenant_id=tenant_id) + + # Mock the authentication context to return current user and tenant + mock_current_account.return_value = (account, tenant_id) + + # Set up database query mock + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + # First call returns app, second returns None (no message) + mock_query.first.side_effect = [app, None] + + # Annotation data to create + args = { + "question": "What is natural language processing?", + "answer": "NLP is a field of AI focused on language understanding", + } + + # Act + with patch("services.annotation_service.add_annotation_to_index_task"): + result = AppAnnotationService.up_insert_app_annotation_from_message(args, app_id) + + # Assert + mock_db_session.add.assert_called_once() # Annotation added to session + mock_db_session.commit.assert_called_once() # Changes committed + + @patch("services.annotation_service.db.session") + @patch("services.annotation_service.current_account_with_tenant") + def test_update_existing_annotation(self, mock_current_account, mock_db_session): + """ + Test updating an existing annotation. + + When a message already has an annotation, calling the service again + should update the existing annotation rather than creating a new one. + """ + # Arrange + app_id = "app-123" + message_id = "msg-123" + account = ConversationServiceTestDataFactory.create_account_mock() + tenant_id = "tenant-123" + app = ConversationServiceTestDataFactory.create_app_mock(app_id=app_id, tenant_id=tenant_id) + message = ConversationServiceTestDataFactory.create_message_mock(message_id=message_id, app_id=app_id) + + # Create an existing annotation with old content + existing_annotation = ConversationServiceTestDataFactory.create_annotation_mock( + app_id=app_id, message_id=message_id, content="Old annotation" + ) + message.annotation = existing_annotation # Message already has annotation + + # Mock the authentication context to return current user and tenant + mock_current_account.return_value = (account, tenant_id) + + # Set up database query mock + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + # First call returns app, second returns message, third returns None (no annotation setting) + mock_query.first.side_effect = [app, message, None] + + # New content to update the annotation with + args = {"message_id": message_id, "answer": "Updated annotation content"} + + # Act + with patch("services.annotation_service.add_annotation_to_index_task"): + result = AppAnnotationService.up_insert_app_annotation_from_message(args, app_id) + + # Assert + assert existing_annotation.content == "Updated annotation content" # Content updated + mock_db_session.add.assert_called_once() # Annotation re-added to session + mock_db_session.commit.assert_called_once() # Changes committed + + @patch("services.annotation_service.db.paginate") + @patch("services.annotation_service.db.session") + @patch("services.annotation_service.current_account_with_tenant") + def test_get_annotation_list(self, mock_current_account, mock_db_session, mock_db_paginate): + """ + Test retrieving paginated annotation list. + + Annotations can be retrieved in a paginated list for display in the UI. + """ + """Test retrieving paginated annotation list.""" + # Arrange + app_id = "app-123" + account = ConversationServiceTestDataFactory.create_account_mock() + tenant_id = "tenant-123" + app = ConversationServiceTestDataFactory.create_app_mock(app_id=app_id, tenant_id=tenant_id) + annotations = [ + ConversationServiceTestDataFactory.create_annotation_mock(annotation_id=f"anno-{i}", app_id=app_id) + for i in range(5) + ] + + mock_current_account.return_value = (account, tenant_id) + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.first.return_value = app + + mock_paginate = MagicMock() + mock_paginate.items = annotations + mock_paginate.total = 5 + mock_db_paginate.return_value = mock_paginate + + # Act + result_items, result_total = AppAnnotationService.get_annotation_list_by_app_id( + app_id=app_id, page=1, limit=10, keyword="" + ) + + # Assert + assert len(result_items) == 5 + assert result_total == 5 + + @patch("services.annotation_service.db.paginate") + @patch("services.annotation_service.db.session") + @patch("services.annotation_service.current_account_with_tenant") + def test_get_annotation_list_with_keyword_search(self, mock_current_account, mock_db_session, mock_db_paginate): + """ + Test retrieving annotations with keyword filtering. + + Annotations can be searched by question or content using case-insensitive matching. + """ + # Arrange + app_id = "app-123" + account = ConversationServiceTestDataFactory.create_account_mock() + tenant_id = "tenant-123" + app = ConversationServiceTestDataFactory.create_app_mock(app_id=app_id, tenant_id=tenant_id) + + # Create annotations with searchable content + annotations = [ + ConversationServiceTestDataFactory.create_annotation_mock( + annotation_id="anno-1", + app_id=app_id, + question="What is machine learning?", + content="ML is a subset of AI", + ), + ConversationServiceTestDataFactory.create_annotation_mock( + annotation_id="anno-2", + app_id=app_id, + question="What is deep learning?", + content="Deep learning uses neural networks", + ), + ] + + mock_current_account.return_value = (account, tenant_id) + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.first.return_value = app + + mock_paginate = MagicMock() + mock_paginate.items = [annotations[0]] # Only first annotation matches + mock_paginate.total = 1 + mock_db_paginate.return_value = mock_paginate + + # Act + result_items, result_total = AppAnnotationService.get_annotation_list_by_app_id( + app_id=app_id, + page=1, + limit=10, + keyword="machine", # Search keyword + ) + + # Assert + assert len(result_items) == 1 + assert result_total == 1 + + @patch("services.annotation_service.db.session") + @patch("services.annotation_service.current_account_with_tenant") + def test_insert_annotation_directly(self, mock_current_account, mock_db_session): + """ + Test direct annotation insertion without message reference. + + This is used for bulk imports or manual annotation creation. + """ + # Arrange + app_id = "app-123" + account = ConversationServiceTestDataFactory.create_account_mock() + tenant_id = "tenant-123" + app = ConversationServiceTestDataFactory.create_app_mock(app_id=app_id, tenant_id=tenant_id) + + mock_current_account.return_value = (account, tenant_id) + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.first.side_effect = [app, None] + + args = { + "question": "What is natural language processing?", + "answer": "NLP is a field of AI focused on language understanding", + } + + # Act + with patch("services.annotation_service.add_annotation_to_index_task"): + result = AppAnnotationService.insert_app_annotation_directly(args, app_id) + + # Assert + mock_db_session.add.assert_called_once() + mock_db_session.commit.assert_called_once() + + +class TestConversationServiceExport: + """ + Test conversation export/retrieval operations. + + Tests retrieving conversation data for export purposes. + """ + + @patch("services.conversation_service.db.session") + def test_get_conversation_success(self, mock_db_session): + """Test successful retrieval of conversation.""" + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + conversation = ConversationServiceTestDataFactory.create_conversation_mock( + app_id=app_model.id, from_account_id=user.id, from_source="console" + ) + + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.first.return_value = conversation + + # Act + result = ConversationService.get_conversation(app_model=app_model, conversation_id=conversation.id, user=user) + + # Assert + assert result == conversation + + @patch("services.conversation_service.db.session") + def test_get_conversation_not_found(self, mock_db_session): + """Test ConversationNotExistsError when conversation doesn't exist.""" + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.first.return_value = None + + # Act & Assert + with pytest.raises(ConversationNotExistsError): + ConversationService.get_conversation(app_model=app_model, conversation_id="non-existent", user=user) + + @patch("services.annotation_service.db.session") + @patch("services.annotation_service.current_account_with_tenant") + def test_export_annotation_list(self, mock_current_account, mock_db_session): + """Test exporting all annotations for an app.""" + # Arrange + app_id = "app-123" + account = ConversationServiceTestDataFactory.create_account_mock() + tenant_id = "tenant-123" + app = ConversationServiceTestDataFactory.create_app_mock(app_id=app_id, tenant_id=tenant_id) + annotations = [ + ConversationServiceTestDataFactory.create_annotation_mock(annotation_id=f"anno-{i}", app_id=app_id) + for i in range(10) + ] + + mock_current_account.return_value = (account, tenant_id) + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.order_by.return_value = mock_query + mock_query.first.return_value = app + mock_query.all.return_value = annotations + + # Act + result = AppAnnotationService.export_annotation_list_by_app_id(app_id) + + # Assert + assert len(result) == 10 + assert result == annotations + + @patch("services.message_service.db.session") + def test_get_message_success(self, mock_db_session): + """Test successful retrieval of a message.""" + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + message = ConversationServiceTestDataFactory.create_message_mock( + app_id=app_model.id, from_account_id=user.id, from_source="console" + ) + + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.first.return_value = message + + # Act + result = MessageService.get_message(app_model=app_model, user=user, message_id=message.id) + + # Assert + assert result == message + + @patch("services.message_service.db.session") + def test_get_message_not_found(self, mock_db_session): + """Test MessageNotExistsError when message doesn't exist.""" + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.first.return_value = None + + # Act & Assert + with pytest.raises(MessageNotExistsError): + MessageService.get_message(app_model=app_model, user=user, message_id="non-existent") + + @patch("services.conversation_service.db.session") + def test_get_conversation_for_end_user(self, mock_db_session): + """ + Test retrieving conversation created by end user via API. + + End users (API) and accounts (console) have different access patterns. + """ + # Arrange + app_model = ConversationServiceTestDataFactory.create_app_mock() + end_user = ConversationServiceTestDataFactory.create_end_user_mock() + + # Conversation created by end user via API + conversation = ConversationServiceTestDataFactory.create_conversation_mock( + app_id=app_model.id, + from_end_user_id=end_user.id, + from_source="api", # API source for end users + ) + + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query + mock_query.first.return_value = conversation + + # Act + result = ConversationService.get_conversation( + app_model=app_model, conversation_id=conversation.id, user=end_user + ) + + # Assert + assert result == conversation + # Verify query filters for API source + mock_query.where.assert_called() + + @patch("services.conversation_service.delete_conversation_related_data") # Mock Celery task + @patch("services.conversation_service.db.session") # Mock database session + def test_delete_conversation(self, mock_db_session, mock_delete_task): + """ + Test conversation deletion with async cleanup. + + Deletion is a two-step process: + 1. Immediately delete the conversation record from database + 2. Trigger async background task to clean up related data + (messages, annotations, vector embeddings, file uploads) + """ + # Arrange - Set up test data + app_model = ConversationServiceTestDataFactory.create_app_mock() + user = ConversationServiceTestDataFactory.create_account_mock() + conversation_id = "conv-to-delete" + + # Set up database query mock + mock_query = MagicMock() + mock_db_session.query.return_value = mock_query + mock_query.where.return_value = mock_query # Filter by conversation_id + + # Act - Delete the conversation + ConversationService.delete(app_model=app_model, conversation_id=conversation_id, user=user) + + # Assert - Verify two-step deletion process + # Step 1: Immediate database deletion + mock_query.delete.assert_called_once() # DELETE query executed + mock_db_session.commit.assert_called_once() # Transaction committed + + # Step 2: Async cleanup task triggered + # The Celery task will handle cleanup of messages, annotations, etc. + mock_delete_task.delay.assert_called_once_with(conversation_id) diff --git a/api/tests/unit_tests/services/test_dataset_service.py b/api/tests/unit_tests/services/test_dataset_service.py new file mode 100644 index 0000000000..87fd29bbc0 --- /dev/null +++ b/api/tests/unit_tests/services/test_dataset_service.py @@ -0,0 +1,1200 @@ +""" +Comprehensive unit tests for DatasetService. + +This test suite provides complete coverage of dataset management operations in Dify, +following TDD principles with the Arrange-Act-Assert pattern. + +## Test Coverage + +### 1. Dataset Creation (TestDatasetServiceCreateDataset) +Tests the creation of knowledge base datasets with various configurations: +- Internal datasets (provider='vendor') with economy or high-quality indexing +- External datasets (provider='external') connected to third-party APIs +- Embedding model configuration for semantic search +- Duplicate name validation +- Permission and access control setup + +### 2. Dataset Updates (TestDatasetServiceUpdateDataset) +Tests modification of existing dataset settings: +- Basic field updates (name, description, permission) +- Indexing technique switching (economy ↔ high_quality) +- Embedding model changes with vector index rebuilding +- Retrieval configuration updates +- External knowledge binding updates + +### 3. Dataset Deletion (TestDatasetServiceDeleteDataset) +Tests safe deletion with cascade cleanup: +- Normal deletion with documents and embeddings +- Empty dataset deletion (regression test for #27073) +- Permission verification +- Event-driven cleanup (vector DB, file storage) + +### 4. Document Indexing (TestDatasetServiceDocumentIndexing) +Tests async document processing operations: +- Pause/resume indexing for resource management +- Retry failed documents +- Status transitions through indexing pipeline +- Redis-based concurrency control + +### 5. Retrieval Configuration (TestDatasetServiceRetrievalConfiguration) +Tests search and ranking settings: +- Search method configuration (semantic, full-text, hybrid) +- Top-k and score threshold tuning +- Reranking model integration for improved relevance + +## Testing Approach + +- **Mocking Strategy**: All external dependencies (database, Redis, model providers) + are mocked to ensure fast, isolated unit tests +- **Factory Pattern**: DatasetServiceTestDataFactory provides consistent test data +- **Fixtures**: Pytest fixtures set up common mock configurations per test class +- **Assertions**: Each test verifies both the return value and all side effects + (database operations, event signals, async task triggers) + +## Key Concepts + +**Indexing Techniques:** +- economy: Keyword-based search (fast, less accurate) +- high_quality: Vector embeddings for semantic search (slower, more accurate) + +**Dataset Providers:** +- vendor: Internal storage and indexing +- external: Third-party knowledge sources via API + +**Document Lifecycle:** +waiting → parsing → cleaning → splitting → indexing → completed (or error) +""" + +from unittest.mock import Mock, create_autospec, patch +from uuid import uuid4 + +import pytest + +from core.model_runtime.entities.model_entities import ModelType +from models.account import Account, TenantAccountRole +from models.dataset import Dataset, DatasetPermissionEnum, Document, ExternalKnowledgeBindings +from services.dataset_service import DatasetService +from services.entities.knowledge_entities.knowledge_entities import RetrievalModel +from services.errors.dataset import DatasetNameDuplicateError + + +class DatasetServiceTestDataFactory: + """ + Factory class for creating test data and mock objects. + + This factory provides reusable methods to create mock objects for testing. + Using a factory pattern ensures consistency across tests and reduces code duplication. + All methods return properly configured Mock objects that simulate real model instances. + """ + + @staticmethod + def create_account_mock( + account_id: str = "account-123", + tenant_id: str = "tenant-123", + role: TenantAccountRole = TenantAccountRole.NORMAL, + **kwargs, + ) -> Mock: + """ + Create a mock account with specified attributes. + + Args: + account_id: Unique identifier for the account + tenant_id: Tenant ID the account belongs to + role: User role (NORMAL, ADMIN, etc.) + **kwargs: Additional attributes to set on the mock + + Returns: + Mock: A properly configured Account mock object + """ + account = create_autospec(Account, instance=True) + account.id = account_id + account.current_tenant_id = tenant_id + account.current_role = role + for key, value in kwargs.items(): + setattr(account, key, value) + return account + + @staticmethod + def create_dataset_mock( + dataset_id: str = "dataset-123", + name: str = "Test Dataset", + tenant_id: str = "tenant-123", + created_by: str = "user-123", + provider: str = "vendor", + indexing_technique: str | None = "high_quality", + **kwargs, + ) -> Mock: + """ + Create a mock dataset with specified attributes. + + Args: + dataset_id: Unique identifier for the dataset + name: Display name of the dataset + tenant_id: Tenant ID the dataset belongs to + created_by: User ID who created the dataset + provider: Dataset provider type ('vendor' for internal, 'external' for external) + indexing_technique: Indexing method ('high_quality', 'economy', or None) + **kwargs: Additional attributes (embedding_model, retrieval_model, etc.) + + Returns: + Mock: A properly configured Dataset mock object + """ + dataset = create_autospec(Dataset, instance=True) + dataset.id = dataset_id + dataset.name = name + dataset.tenant_id = tenant_id + dataset.created_by = created_by + dataset.provider = provider + dataset.indexing_technique = indexing_technique + dataset.permission = kwargs.get("permission", DatasetPermissionEnum.ONLY_ME) + dataset.embedding_model_provider = kwargs.get("embedding_model_provider") + dataset.embedding_model = kwargs.get("embedding_model") + dataset.collection_binding_id = kwargs.get("collection_binding_id") + dataset.retrieval_model = kwargs.get("retrieval_model") + dataset.description = kwargs.get("description") + dataset.doc_form = kwargs.get("doc_form") + for key, value in kwargs.items(): + if not hasattr(dataset, key): + setattr(dataset, key, value) + return dataset + + @staticmethod + def create_embedding_model_mock(model: str = "text-embedding-ada-002", provider: str = "openai") -> Mock: + """ + Create a mock embedding model for high-quality indexing. + + Embedding models are used to convert text into vector representations + for semantic search capabilities. + + Args: + model: Model name (e.g., 'text-embedding-ada-002') + provider: Model provider (e.g., 'openai', 'cohere') + + Returns: + Mock: Embedding model mock with model and provider attributes + """ + embedding_model = Mock() + embedding_model.model = model + embedding_model.provider = provider + return embedding_model + + @staticmethod + def create_retrieval_model_mock() -> Mock: + """ + Create a mock retrieval model configuration. + + Retrieval models define how documents are searched and ranked, + including search method, top-k results, and score thresholds. + + Returns: + Mock: RetrievalModel mock with model_dump() method + """ + retrieval_model = Mock(spec=RetrievalModel) + retrieval_model.model_dump.return_value = { + "search_method": "semantic_search", + "top_k": 2, + "score_threshold": 0.0, + } + retrieval_model.reranking_model = None + return retrieval_model + + @staticmethod + def create_collection_binding_mock(binding_id: str = "binding-456") -> Mock: + """ + Create a mock collection binding for vector database. + + Collection bindings link datasets to their vector storage locations + in the vector database (e.g., Qdrant, Weaviate). + + Args: + binding_id: Unique identifier for the collection binding + + Returns: + Mock: Collection binding mock object + """ + binding = Mock() + binding.id = binding_id + return binding + + @staticmethod + def create_external_binding_mock( + dataset_id: str = "dataset-123", + external_knowledge_id: str = "knowledge-123", + external_knowledge_api_id: str = "api-123", + ) -> Mock: + """ + Create a mock external knowledge binding. + + External knowledge bindings connect datasets to external knowledge sources + (e.g., third-party APIs, external databases) for retrieval. + + Args: + dataset_id: Dataset ID this binding belongs to + external_knowledge_id: External knowledge source identifier + external_knowledge_api_id: External API configuration identifier + + Returns: + Mock: ExternalKnowledgeBindings mock object + """ + binding = Mock(spec=ExternalKnowledgeBindings) + binding.dataset_id = dataset_id + binding.external_knowledge_id = external_knowledge_id + binding.external_knowledge_api_id = external_knowledge_api_id + return binding + + @staticmethod + def create_document_mock( + document_id: str = "doc-123", + dataset_id: str = "dataset-123", + indexing_status: str = "completed", + **kwargs, + ) -> Mock: + """ + Create a mock document for testing document operations. + + Documents are the individual files/content items within a dataset + that go through indexing, parsing, and chunking processes. + + Args: + document_id: Unique identifier for the document + dataset_id: Parent dataset ID + indexing_status: Current status ('waiting', 'indexing', 'completed', 'error') + **kwargs: Additional attributes (is_paused, enabled, archived, etc.) + + Returns: + Mock: Document mock object + """ + document = Mock(spec=Document) + document.id = document_id + document.dataset_id = dataset_id + document.indexing_status = indexing_status + for key, value in kwargs.items(): + setattr(document, key, value) + return document + + +# ==================== Dataset Creation Tests ==================== + + +class TestDatasetServiceCreateDataset: + """ + Comprehensive unit tests for dataset creation logic. + + Covers: + - Internal dataset creation with various indexing techniques + - External dataset creation with external knowledge bindings + - RAG pipeline dataset creation + - Error handling for duplicate names and missing configurations + """ + + @pytest.fixture + def mock_dataset_service_dependencies(self): + """ + Common mock setup for dataset service dependencies. + + This fixture patches all external dependencies that DatasetService.create_empty_dataset + interacts with, including: + - db.session: Database operations (query, add, commit) + - ModelManager: Embedding model management + - check_embedding_model_setting: Validates embedding model configuration + - check_reranking_model_setting: Validates reranking model configuration + - ExternalDatasetService: Handles external knowledge API operations + + Yields: + dict: Dictionary of mocked dependencies for use in tests + """ + with ( + patch("services.dataset_service.db.session") as mock_db, + patch("services.dataset_service.ModelManager") as mock_model_manager, + patch("services.dataset_service.DatasetService.check_embedding_model_setting") as mock_check_embedding, + patch("services.dataset_service.DatasetService.check_reranking_model_setting") as mock_check_reranking, + patch("services.dataset_service.ExternalDatasetService") as mock_external_service, + ): + yield { + "db_session": mock_db, + "model_manager": mock_model_manager, + "check_embedding": mock_check_embedding, + "check_reranking": mock_check_reranking, + "external_service": mock_external_service, + } + + def test_create_internal_dataset_basic_success(self, mock_dataset_service_dependencies): + """ + Test successful creation of basic internal dataset. + + Verifies that a dataset can be created with minimal configuration: + - No indexing technique specified (None) + - Default permission (only_me) + - Vendor provider (internal dataset) + + This is the simplest dataset creation scenario. + """ + # Arrange: Set up test data and mocks + tenant_id = str(uuid4()) + account = DatasetServiceTestDataFactory.create_account_mock(tenant_id=tenant_id) + name = "Test Dataset" + description = "Test description" + + # Mock database query to return None (no duplicate name exists) + mock_query = Mock() + mock_query.filter_by.return_value.first.return_value = None + mock_dataset_service_dependencies["db_session"].query.return_value = mock_query + + # Mock database session operations for dataset creation + mock_db = mock_dataset_service_dependencies["db_session"] + mock_db.add = Mock() # Tracks dataset being added to session + mock_db.flush = Mock() # Flushes to get dataset ID + mock_db.commit = Mock() # Commits transaction + + # Act + result = DatasetService.create_empty_dataset( + tenant_id=tenant_id, + name=name, + description=description, + indexing_technique=None, + account=account, + ) + + # Assert + assert result is not None + assert result.name == name + assert result.description == description + assert result.tenant_id == tenant_id + assert result.created_by == account.id + assert result.updated_by == account.id + assert result.provider == "vendor" + assert result.permission == "only_me" + mock_db.add.assert_called_once() + mock_db.commit.assert_called_once() + + def test_create_internal_dataset_with_economy_indexing(self, mock_dataset_service_dependencies): + """Test successful creation of internal dataset with economy indexing.""" + # Arrange + tenant_id = str(uuid4()) + account = DatasetServiceTestDataFactory.create_account_mock(tenant_id=tenant_id) + name = "Economy Dataset" + + # Mock database query + mock_query = Mock() + mock_query.filter_by.return_value.first.return_value = None + mock_dataset_service_dependencies["db_session"].query.return_value = mock_query + + mock_db = mock_dataset_service_dependencies["db_session"] + mock_db.add = Mock() + mock_db.flush = Mock() + mock_db.commit = Mock() + + # Act + result = DatasetService.create_empty_dataset( + tenant_id=tenant_id, + name=name, + description=None, + indexing_technique="economy", + account=account, + ) + + # Assert + assert result.indexing_technique == "economy" + assert result.embedding_model_provider is None + assert result.embedding_model is None + mock_db.commit.assert_called_once() + + def test_create_internal_dataset_with_high_quality_indexing(self, mock_dataset_service_dependencies): + """Test creation with high_quality indexing using default embedding model.""" + # Arrange + tenant_id = str(uuid4()) + account = DatasetServiceTestDataFactory.create_account_mock(tenant_id=tenant_id) + name = "High Quality Dataset" + + # Mock database query + mock_query = Mock() + mock_query.filter_by.return_value.first.return_value = None + mock_dataset_service_dependencies["db_session"].query.return_value = mock_query + + # Mock model manager + embedding_model = DatasetServiceTestDataFactory.create_embedding_model_mock() + mock_model_manager_instance = Mock() + mock_model_manager_instance.get_default_model_instance.return_value = embedding_model + mock_dataset_service_dependencies["model_manager"].return_value = mock_model_manager_instance + + mock_db = mock_dataset_service_dependencies["db_session"] + mock_db.add = Mock() + mock_db.flush = Mock() + mock_db.commit = Mock() + + # Act + result = DatasetService.create_empty_dataset( + tenant_id=tenant_id, + name=name, + description=None, + indexing_technique="high_quality", + account=account, + ) + + # Assert + assert result.indexing_technique == "high_quality" + assert result.embedding_model_provider == embedding_model.provider + assert result.embedding_model == embedding_model.model + mock_model_manager_instance.get_default_model_instance.assert_called_once_with( + tenant_id=tenant_id, model_type=ModelType.TEXT_EMBEDDING + ) + mock_db.commit.assert_called_once() + + def test_create_dataset_duplicate_name_error(self, mock_dataset_service_dependencies): + """Test error when creating dataset with duplicate name.""" + # Arrange + tenant_id = str(uuid4()) + account = DatasetServiceTestDataFactory.create_account_mock(tenant_id=tenant_id) + name = "Duplicate Dataset" + + # Mock database query to return existing dataset + existing_dataset = DatasetServiceTestDataFactory.create_dataset_mock(name=name, tenant_id=tenant_id) + mock_query = Mock() + mock_query.filter_by.return_value.first.return_value = existing_dataset + mock_dataset_service_dependencies["db_session"].query.return_value = mock_query + + # Act & Assert + with pytest.raises(DatasetNameDuplicateError) as context: + DatasetService.create_empty_dataset( + tenant_id=tenant_id, + name=name, + description=None, + indexing_technique=None, + account=account, + ) + + assert f"Dataset with name {name} already exists" in str(context.value) + + def test_create_external_dataset_success(self, mock_dataset_service_dependencies): + """Test successful creation of external dataset with external knowledge binding.""" + # Arrange + tenant_id = str(uuid4()) + account = DatasetServiceTestDataFactory.create_account_mock(tenant_id=tenant_id) + name = "External Dataset" + external_knowledge_api_id = "api-123" + external_knowledge_id = "knowledge-123" + + # Mock database query + mock_query = Mock() + mock_query.filter_by.return_value.first.return_value = None + mock_dataset_service_dependencies["db_session"].query.return_value = mock_query + + # Mock external knowledge API + external_api = Mock() + external_api.id = external_knowledge_api_id + mock_dataset_service_dependencies["external_service"].get_external_knowledge_api.return_value = external_api + + mock_db = mock_dataset_service_dependencies["db_session"] + mock_db.add = Mock() + mock_db.flush = Mock() + mock_db.commit = Mock() + + # Act + result = DatasetService.create_empty_dataset( + tenant_id=tenant_id, + name=name, + description=None, + indexing_technique=None, + account=account, + provider="external", + external_knowledge_api_id=external_knowledge_api_id, + external_knowledge_id=external_knowledge_id, + ) + + # Assert + assert result.provider == "external" + assert mock_db.add.call_count == 2 # Dataset + ExternalKnowledgeBinding + mock_db.commit.assert_called_once() + + +# ==================== Dataset Update Tests ==================== + + +class TestDatasetServiceUpdateDataset: + """ + Comprehensive unit tests for dataset update settings. + + Covers: + - Basic field updates (name, description, permission) + - Indexing technique changes (economy <-> high_quality) + - Embedding model updates + - Retrieval configuration updates + - External dataset updates + """ + + @pytest.fixture + def mock_dataset_service_dependencies(self): + """Common mock setup for dataset service dependencies.""" + with ( + patch("services.dataset_service.DatasetService.get_dataset") as mock_get_dataset, + patch("services.dataset_service.DatasetService._has_dataset_same_name") as mock_has_same_name, + patch("services.dataset_service.DatasetService.check_dataset_permission") as mock_check_perm, + patch("services.dataset_service.db.session") as mock_db, + patch("services.dataset_service.naive_utc_now") as mock_time, + patch( + "services.dataset_service.DatasetService._update_pipeline_knowledge_base_node_data" + ) as mock_update_pipeline, + ): + mock_time.return_value = "2024-01-01T00:00:00" + yield { + "get_dataset": mock_get_dataset, + "has_dataset_same_name": mock_has_same_name, + "check_permission": mock_check_perm, + "db_session": mock_db, + "current_time": "2024-01-01T00:00:00", + "update_pipeline": mock_update_pipeline, + } + + @pytest.fixture + def mock_internal_provider_dependencies(self): + """Mock dependencies for internal dataset provider operations.""" + with ( + patch("services.dataset_service.ModelManager") as mock_model_manager, + patch("services.dataset_service.DatasetCollectionBindingService") as mock_binding_service, + patch("services.dataset_service.deal_dataset_vector_index_task") as mock_task, + patch("services.dataset_service.current_user") as mock_current_user, + ): + # Mock current_user as Account instance + mock_current_user_account = DatasetServiceTestDataFactory.create_account_mock( + account_id="user-123", tenant_id="tenant-123" + ) + mock_current_user.return_value = mock_current_user_account + mock_current_user.current_tenant_id = "tenant-123" + mock_current_user.id = "user-123" + # Make isinstance check pass + mock_current_user.__class__ = Account + + yield { + "model_manager": mock_model_manager, + "get_binding": mock_binding_service.get_dataset_collection_binding, + "task": mock_task, + "current_user": mock_current_user, + } + + @pytest.fixture + def mock_external_provider_dependencies(self): + """Mock dependencies for external dataset provider operations.""" + with ( + patch("services.dataset_service.Session") as mock_session, + patch("services.dataset_service.db.engine") as mock_engine, + ): + yield mock_session + + def test_update_internal_dataset_basic_success(self, mock_dataset_service_dependencies): + """Test successful update of internal dataset with basic fields.""" + # Arrange + dataset = DatasetServiceTestDataFactory.create_dataset_mock( + provider="vendor", + indexing_technique="high_quality", + embedding_model_provider="openai", + embedding_model="text-embedding-ada-002", + collection_binding_id="binding-123", + ) + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + + user = DatasetServiceTestDataFactory.create_account_mock() + + update_data = { + "name": "new_name", + "description": "new_description", + "indexing_technique": "high_quality", + "retrieval_model": "new_model", + "embedding_model_provider": "openai", + "embedding_model": "text-embedding-ada-002", + } + + mock_dataset_service_dependencies["has_dataset_same_name"].return_value = False + + # Act + result = DatasetService.update_dataset("dataset-123", update_data, user) + + # Assert + mock_dataset_service_dependencies["check_permission"].assert_called_once_with(dataset, user) + mock_dataset_service_dependencies[ + "db_session" + ].query.return_value.filter_by.return_value.update.assert_called_once() + mock_dataset_service_dependencies["db_session"].commit.assert_called_once() + assert result == dataset + + def test_update_dataset_not_found_error(self, mock_dataset_service_dependencies): + """Test error when updating non-existent dataset.""" + # Arrange + mock_dataset_service_dependencies["get_dataset"].return_value = None + user = DatasetServiceTestDataFactory.create_account_mock() + + # Act & Assert + with pytest.raises(ValueError) as context: + DatasetService.update_dataset("non-existent", {}, user) + + assert "Dataset not found" in str(context.value) + + def test_update_dataset_duplicate_name_error(self, mock_dataset_service_dependencies): + """Test error when updating dataset to duplicate name.""" + # Arrange + dataset = DatasetServiceTestDataFactory.create_dataset_mock() + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + mock_dataset_service_dependencies["has_dataset_same_name"].return_value = True + + user = DatasetServiceTestDataFactory.create_account_mock() + update_data = {"name": "duplicate_name"} + + # Act & Assert + with pytest.raises(ValueError) as context: + DatasetService.update_dataset("dataset-123", update_data, user) + + assert "Dataset name already exists" in str(context.value) + + def test_update_indexing_technique_to_economy( + self, mock_dataset_service_dependencies, mock_internal_provider_dependencies + ): + """Test updating indexing technique from high_quality to economy.""" + # Arrange + dataset = DatasetServiceTestDataFactory.create_dataset_mock( + provider="vendor", indexing_technique="high_quality" + ) + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + + user = DatasetServiceTestDataFactory.create_account_mock() + + update_data = {"indexing_technique": "economy", "retrieval_model": "new_model"} + mock_dataset_service_dependencies["has_dataset_same_name"].return_value = False + + # Act + result = DatasetService.update_dataset("dataset-123", update_data, user) + + # Assert + mock_dataset_service_dependencies[ + "db_session" + ].query.return_value.filter_by.return_value.update.assert_called_once() + # Verify embedding model fields are cleared + call_args = mock_dataset_service_dependencies[ + "db_session" + ].query.return_value.filter_by.return_value.update.call_args[0][0] + assert call_args["embedding_model"] is None + assert call_args["embedding_model_provider"] is None + assert call_args["collection_binding_id"] is None + assert result == dataset + + def test_update_indexing_technique_to_high_quality( + self, mock_dataset_service_dependencies, mock_internal_provider_dependencies + ): + """Test updating indexing technique from economy to high_quality.""" + # Arrange + dataset = DatasetServiceTestDataFactory.create_dataset_mock(provider="vendor", indexing_technique="economy") + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + + user = DatasetServiceTestDataFactory.create_account_mock() + + # Mock embedding model + embedding_model = DatasetServiceTestDataFactory.create_embedding_model_mock() + mock_internal_provider_dependencies[ + "model_manager" + ].return_value.get_model_instance.return_value = embedding_model + + # Mock collection binding + binding = DatasetServiceTestDataFactory.create_collection_binding_mock() + mock_internal_provider_dependencies["get_binding"].return_value = binding + + update_data = { + "indexing_technique": "high_quality", + "embedding_model_provider": "openai", + "embedding_model": "text-embedding-ada-002", + "retrieval_model": "new_model", + } + mock_dataset_service_dependencies["has_dataset_same_name"].return_value = False + + # Act + result = DatasetService.update_dataset("dataset-123", update_data, user) + + # Assert + mock_internal_provider_dependencies["model_manager"].return_value.get_model_instance.assert_called_once() + mock_internal_provider_dependencies["get_binding"].assert_called_once() + mock_internal_provider_dependencies["task"].delay.assert_called_once() + call_args = mock_internal_provider_dependencies["task"].delay.call_args[0] + assert call_args[0] == "dataset-123" + assert call_args[1] == "add" + + # Verify return value + assert result == dataset + + # Note: External dataset update test removed due to Flask app context complexity in unit tests + # External dataset functionality is covered by integration tests + + def test_update_external_dataset_missing_knowledge_id_error(self, mock_dataset_service_dependencies): + """Test error when external knowledge id is missing.""" + # Arrange + dataset = DatasetServiceTestDataFactory.create_dataset_mock(provider="external") + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + + user = DatasetServiceTestDataFactory.create_account_mock() + update_data = {"name": "new_name", "external_knowledge_api_id": "api_id"} + mock_dataset_service_dependencies["has_dataset_same_name"].return_value = False + + # Act & Assert + with pytest.raises(ValueError) as context: + DatasetService.update_dataset("dataset-123", update_data, user) + + assert "External knowledge id is required" in str(context.value) + + +# ==================== Dataset Deletion Tests ==================== + + +class TestDatasetServiceDeleteDataset: + """ + Comprehensive unit tests for dataset deletion with cascade operations. + + Covers: + - Normal dataset deletion with documents + - Empty dataset deletion (no documents) + - Dataset deletion with partial None values + - Permission checks + - Event handling for cascade operations + + Dataset deletion is a critical operation that triggers cascade cleanup: + - Documents and segments are removed from vector database + - File storage is cleaned up + - Related bindings and metadata are deleted + - The dataset_was_deleted event notifies listeners for cleanup + """ + + @pytest.fixture + def mock_dataset_service_dependencies(self): + """ + Common mock setup for dataset deletion dependencies. + + Patches: + - get_dataset: Retrieves the dataset to delete + - check_dataset_permission: Verifies user has delete permission + - db.session: Database operations (delete, commit) + - dataset_was_deleted: Signal/event for cascade cleanup operations + + The dataset_was_deleted signal is crucial - it triggers cleanup handlers + that remove vector embeddings, files, and related data. + """ + with ( + patch("services.dataset_service.DatasetService.get_dataset") as mock_get_dataset, + patch("services.dataset_service.DatasetService.check_dataset_permission") as mock_check_perm, + patch("services.dataset_service.db.session") as mock_db, + patch("services.dataset_service.dataset_was_deleted") as mock_dataset_was_deleted, + ): + yield { + "get_dataset": mock_get_dataset, + "check_permission": mock_check_perm, + "db_session": mock_db, + "dataset_was_deleted": mock_dataset_was_deleted, + } + + def test_delete_dataset_with_documents_success(self, mock_dataset_service_dependencies): + """Test successful deletion of a dataset with documents.""" + # Arrange + dataset = DatasetServiceTestDataFactory.create_dataset_mock( + doc_form="text_model", indexing_technique="high_quality" + ) + user = DatasetServiceTestDataFactory.create_account_mock() + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + + # Act + result = DatasetService.delete_dataset(dataset.id, user) + + # Assert + assert result is True + mock_dataset_service_dependencies["get_dataset"].assert_called_once_with(dataset.id) + mock_dataset_service_dependencies["check_permission"].assert_called_once_with(dataset, user) + mock_dataset_service_dependencies["dataset_was_deleted"].send.assert_called_once_with(dataset) + mock_dataset_service_dependencies["db_session"].delete.assert_called_once_with(dataset) + mock_dataset_service_dependencies["db_session"].commit.assert_called_once() + + def test_delete_empty_dataset_success(self, mock_dataset_service_dependencies): + """ + Test successful deletion of an empty dataset (no documents, doc_form is None). + + Empty datasets are created but never had documents uploaded. They have: + - doc_form = None (no document format configured) + - indexing_technique = None (no indexing method set) + + This test ensures empty datasets can be deleted without errors. + The event handler should gracefully skip cleanup operations when + there's no actual data to clean up. + + This test provides regression protection for issue #27073 where + deleting empty datasets caused internal server errors. + """ + # Arrange + dataset = DatasetServiceTestDataFactory.create_dataset_mock(doc_form=None, indexing_technique=None) + user = DatasetServiceTestDataFactory.create_account_mock() + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + + # Act + result = DatasetService.delete_dataset(dataset.id, user) + + # Assert - Verify complete deletion flow + assert result is True + mock_dataset_service_dependencies["get_dataset"].assert_called_once_with(dataset.id) + mock_dataset_service_dependencies["check_permission"].assert_called_once_with(dataset, user) + # Event is sent even for empty datasets - handlers check for None values + mock_dataset_service_dependencies["dataset_was_deleted"].send.assert_called_once_with(dataset) + mock_dataset_service_dependencies["db_session"].delete.assert_called_once_with(dataset) + mock_dataset_service_dependencies["db_session"].commit.assert_called_once() + + def test_delete_dataset_not_found(self, mock_dataset_service_dependencies): + """Test deletion attempt when dataset doesn't exist.""" + # Arrange + dataset_id = "non-existent-dataset" + user = DatasetServiceTestDataFactory.create_account_mock() + + mock_dataset_service_dependencies["get_dataset"].return_value = None + + # Act + result = DatasetService.delete_dataset(dataset_id, user) + + # Assert + assert result is False + mock_dataset_service_dependencies["get_dataset"].assert_called_once_with(dataset_id) + mock_dataset_service_dependencies["check_permission"].assert_not_called() + mock_dataset_service_dependencies["dataset_was_deleted"].send.assert_not_called() + mock_dataset_service_dependencies["db_session"].delete.assert_not_called() + mock_dataset_service_dependencies["db_session"].commit.assert_not_called() + + def test_delete_dataset_with_partial_none_values(self, mock_dataset_service_dependencies): + """Test deletion of dataset with partial None values (doc_form exists but indexing_technique is None).""" + # Arrange + dataset = DatasetServiceTestDataFactory.create_dataset_mock(doc_form="text_model", indexing_technique=None) + user = DatasetServiceTestDataFactory.create_account_mock() + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + + # Act + result = DatasetService.delete_dataset(dataset.id, user) + + # Assert + assert result is True + mock_dataset_service_dependencies["dataset_was_deleted"].send.assert_called_once_with(dataset) + mock_dataset_service_dependencies["db_session"].delete.assert_called_once_with(dataset) + mock_dataset_service_dependencies["db_session"].commit.assert_called_once() + + +# ==================== Document Indexing Logic Tests ==================== + + +class TestDatasetServiceDocumentIndexing: + """ + Comprehensive unit tests for document indexing logic. + + Covers: + - Document indexing status transitions + - Pause/resume document indexing + - Retry document indexing + - Sync website document indexing + - Document indexing task triggering + + Document indexing is an async process with multiple stages: + 1. waiting: Document queued for processing + 2. parsing: Extracting text from file + 3. cleaning: Removing unwanted content + 4. splitting: Breaking into chunks + 5. indexing: Creating embeddings and storing in vector DB + 6. completed: Successfully indexed + 7. error: Failed at some stage + + Users can pause/resume indexing or retry failed documents. + """ + + @pytest.fixture + def mock_document_service_dependencies(self): + """ + Common mock setup for document service dependencies. + + Patches: + - redis_client: Caches indexing state and prevents concurrent operations + - db.session: Database operations for document status updates + - current_user: User context for tracking who paused/resumed + + Redis is used to: + - Store pause flags (document_{id}_is_paused) + - Prevent duplicate retry operations (document_{id}_is_retried) + - Track active indexing operations (document_{id}_indexing) + """ + with ( + patch("services.dataset_service.redis_client") as mock_redis, + patch("services.dataset_service.db.session") as mock_db, + patch("services.dataset_service.current_user") as mock_current_user, + ): + mock_current_user.id = "user-123" + yield { + "redis_client": mock_redis, + "db_session": mock_db, + "current_user": mock_current_user, + } + + def test_pause_document_success(self, mock_document_service_dependencies): + """ + Test successful pause of document indexing. + + Pausing allows users to temporarily stop indexing without canceling it. + This is useful when: + - System resources are needed elsewhere + - User wants to modify document settings before continuing + - Indexing is taking too long and needs to be deferred + + When paused: + - is_paused flag is set to True + - paused_by and paused_at are recorded + - Redis flag prevents indexing worker from processing + - Document remains in current indexing stage + """ + # Arrange + document = DatasetServiceTestDataFactory.create_document_mock(indexing_status="indexing") + mock_db = mock_document_service_dependencies["db_session"] + mock_redis = mock_document_service_dependencies["redis_client"] + + # Act + from services.dataset_service import DocumentService + + DocumentService.pause_document(document) + + # Assert - Verify pause state is persisted + assert document.is_paused is True + mock_db.add.assert_called_once_with(document) + mock_db.commit.assert_called_once() + # setnx (set if not exists) prevents race conditions + mock_redis.setnx.assert_called_once() + + def test_pause_document_invalid_status_error(self, mock_document_service_dependencies): + """Test error when pausing document with invalid status.""" + # Arrange + document = DatasetServiceTestDataFactory.create_document_mock(indexing_status="completed") + + # Act & Assert + from services.dataset_service import DocumentService + from services.errors.document import DocumentIndexingError + + with pytest.raises(DocumentIndexingError): + DocumentService.pause_document(document) + + def test_recover_document_success(self, mock_document_service_dependencies): + """Test successful recovery of paused document indexing.""" + # Arrange + document = DatasetServiceTestDataFactory.create_document_mock(indexing_status="indexing", is_paused=True) + mock_db = mock_document_service_dependencies["db_session"] + mock_redis = mock_document_service_dependencies["redis_client"] + + # Act + with patch("services.dataset_service.recover_document_indexing_task") as mock_task: + from services.dataset_service import DocumentService + + DocumentService.recover_document(document) + + # Assert + assert document.is_paused is False + mock_db.add.assert_called_once_with(document) + mock_db.commit.assert_called_once() + mock_redis.delete.assert_called_once() + mock_task.delay.assert_called_once_with(document.dataset_id, document.id) + + def test_retry_document_indexing_success(self, mock_document_service_dependencies): + """Test successful retry of document indexing.""" + # Arrange + dataset_id = "dataset-123" + documents = [ + DatasetServiceTestDataFactory.create_document_mock(document_id="doc-1", indexing_status="error"), + DatasetServiceTestDataFactory.create_document_mock(document_id="doc-2", indexing_status="error"), + ] + mock_db = mock_document_service_dependencies["db_session"] + mock_redis = mock_document_service_dependencies["redis_client"] + mock_redis.get.return_value = None + + # Act + with patch("services.dataset_service.retry_document_indexing_task") as mock_task: + from services.dataset_service import DocumentService + + DocumentService.retry_document(dataset_id, documents) + + # Assert + for doc in documents: + assert doc.indexing_status == "waiting" + assert mock_db.add.call_count == len(documents) + # Commit is called once per document + assert mock_db.commit.call_count == len(documents) + mock_task.delay.assert_called_once() + + +# ==================== Retrieval Configuration Tests ==================== + + +class TestDatasetServiceRetrievalConfiguration: + """ + Comprehensive unit tests for retrieval configuration. + + Covers: + - Retrieval model configuration + - Search method configuration + - Top-k and score threshold settings + - Reranking model configuration + + Retrieval configuration controls how documents are searched and ranked: + + Search Methods: + - semantic_search: Uses vector similarity (cosine distance) + - full_text_search: Uses keyword matching (BM25) + - hybrid_search: Combines both methods with weighted scores + + Parameters: + - top_k: Number of results to return (default: 2-10) + - score_threshold: Minimum similarity score (0.0-1.0) + - reranking_enable: Whether to use reranking model for better results + + Reranking: + After initial retrieval, a reranking model (e.g., Cohere rerank) can + reorder results for better relevance. This is more accurate but slower. + """ + + @pytest.fixture + def mock_dataset_service_dependencies(self): + """ + Common mock setup for retrieval configuration tests. + + Patches: + - get_dataset: Retrieves dataset with retrieval configuration + - db.session: Database operations for configuration updates + """ + with ( + patch("services.dataset_service.DatasetService.get_dataset") as mock_get_dataset, + patch("services.dataset_service.db.session") as mock_db, + ): + yield { + "get_dataset": mock_get_dataset, + "db_session": mock_db, + } + + def test_get_dataset_retrieval_configuration(self, mock_dataset_service_dependencies): + """Test retrieving dataset with retrieval configuration.""" + # Arrange + dataset_id = "dataset-123" + retrieval_model_config = { + "search_method": "semantic_search", + "top_k": 5, + "score_threshold": 0.5, + "reranking_enable": True, + } + dataset = DatasetServiceTestDataFactory.create_dataset_mock( + dataset_id=dataset_id, retrieval_model=retrieval_model_config + ) + + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + + # Act + result = DatasetService.get_dataset(dataset_id) + + # Assert + assert result is not None + assert result.retrieval_model == retrieval_model_config + assert result.retrieval_model["search_method"] == "semantic_search" + assert result.retrieval_model["top_k"] == 5 + assert result.retrieval_model["score_threshold"] == 0.5 + + def test_update_dataset_retrieval_configuration(self, mock_dataset_service_dependencies): + """Test updating dataset retrieval configuration.""" + # Arrange + dataset = DatasetServiceTestDataFactory.create_dataset_mock( + provider="vendor", + indexing_technique="high_quality", + retrieval_model={"search_method": "semantic_search", "top_k": 2}, + ) + + with ( + patch("services.dataset_service.DatasetService._has_dataset_same_name") as mock_has_same_name, + patch("services.dataset_service.DatasetService.check_dataset_permission") as mock_check_perm, + patch("services.dataset_service.naive_utc_now") as mock_time, + patch( + "services.dataset_service.DatasetService._update_pipeline_knowledge_base_node_data" + ) as mock_update_pipeline, + ): + mock_dataset_service_dependencies["get_dataset"].return_value = dataset + mock_has_same_name.return_value = False + mock_time.return_value = "2024-01-01T00:00:00" + + user = DatasetServiceTestDataFactory.create_account_mock() + + new_retrieval_config = { + "search_method": "full_text_search", + "top_k": 10, + "score_threshold": 0.7, + } + + update_data = { + "indexing_technique": "high_quality", + "retrieval_model": new_retrieval_config, + } + + # Act + result = DatasetService.update_dataset("dataset-123", update_data, user) + + # Assert + mock_dataset_service_dependencies[ + "db_session" + ].query.return_value.filter_by.return_value.update.assert_called_once() + call_args = mock_dataset_service_dependencies[ + "db_session" + ].query.return_value.filter_by.return_value.update.call_args[0][0] + assert call_args["retrieval_model"] == new_retrieval_config + assert result == dataset + + def test_create_dataset_with_retrieval_model_and_reranking(self, mock_dataset_service_dependencies): + """Test creating dataset with retrieval model and reranking configuration.""" + # Arrange + tenant_id = str(uuid4()) + account = DatasetServiceTestDataFactory.create_account_mock(tenant_id=tenant_id) + name = "Dataset with Reranking" + + # Mock database query + mock_query = Mock() + mock_query.filter_by.return_value.first.return_value = None + mock_dataset_service_dependencies["db_session"].query.return_value = mock_query + + # Mock retrieval model with reranking + retrieval_model = Mock(spec=RetrievalModel) + retrieval_model.model_dump.return_value = { + "search_method": "semantic_search", + "top_k": 3, + "score_threshold": 0.6, + "reranking_enable": True, + } + reranking_model = Mock() + reranking_model.reranking_provider_name = "cohere" + reranking_model.reranking_model_name = "rerank-english-v2.0" + retrieval_model.reranking_model = reranking_model + + # Mock model manager + embedding_model = DatasetServiceTestDataFactory.create_embedding_model_mock() + mock_model_manager_instance = Mock() + mock_model_manager_instance.get_default_model_instance.return_value = embedding_model + + with ( + patch("services.dataset_service.ModelManager") as mock_model_manager, + patch("services.dataset_service.DatasetService.check_embedding_model_setting") as mock_check_embedding, + patch("services.dataset_service.DatasetService.check_reranking_model_setting") as mock_check_reranking, + ): + mock_model_manager.return_value = mock_model_manager_instance + + mock_db = mock_dataset_service_dependencies["db_session"] + mock_db.add = Mock() + mock_db.flush = Mock() + mock_db.commit = Mock() + + # Act + result = DatasetService.create_empty_dataset( + tenant_id=tenant_id, + name=name, + description=None, + indexing_technique="high_quality", + account=account, + retrieval_model=retrieval_model, + ) + + # Assert + assert result.retrieval_model == retrieval_model.model_dump() + mock_check_reranking.assert_called_once_with(tenant_id, "cohere", "rerank-english-v2.0") + mock_db.commit.assert_called_once() diff --git a/api/tests/unit_tests/services/test_webhook_service.py b/api/tests/unit_tests/services/test_webhook_service.py index 010295bcd6..6afe52d97b 100644 --- a/api/tests/unit_tests/services/test_webhook_service.py +++ b/api/tests/unit_tests/services/test_webhook_service.py @@ -118,10 +118,8 @@ class TestWebhookServiceUnit: "/webhook", method="POST", headers={"Content-Type": "application/json"}, data="invalid json" ): webhook_trigger = MagicMock() - webhook_data = WebhookService.extract_webhook_data(webhook_trigger) - - assert webhook_data["method"] == "POST" - assert webhook_data["body"] == {} # Should default to empty dict + with pytest.raises(ValueError, match="Invalid JSON body"): + WebhookService.extract_webhook_data(webhook_trigger) def test_generate_webhook_response_default(self): """Test webhook response generation with default values.""" @@ -435,6 +433,27 @@ class TestWebhookServiceUnit: assert result["body"]["message"] == "hello" # Already string assert result["body"]["age"] == 25 # Already number + def test_extract_and_validate_webhook_data_invalid_json_error(self): + """Invalid JSON should bubble up as a ValueError with details.""" + app = Flask(__name__) + + with app.test_request_context( + "/webhook", + method="POST", + headers={"Content-Type": "application/json"}, + data='{"invalid": }', + ): + webhook_trigger = MagicMock() + node_config = { + "data": { + "method": "post", + "content_type": "application/json", + } + } + + with pytest.raises(ValueError, match="Invalid JSON body"): + WebhookService.extract_and_validate_webhook_data(webhook_trigger, node_config) + def test_extract_and_validate_webhook_data_validation_error(self): """Test unified data extraction with validation error.""" app = Flask(__name__) diff --git a/api/tests/unit_tests/services/test_workflow_run_service_pause.py b/api/tests/unit_tests/services/test_workflow_run_service_pause.py index a062d9444e..f45a72927e 100644 --- a/api/tests/unit_tests/services/test_workflow_run_service_pause.py +++ b/api/tests/unit_tests/services/test_workflow_run_service_pause.py @@ -17,6 +17,7 @@ from sqlalchemy import Engine from sqlalchemy.orm import Session, sessionmaker from core.workflow.enums import WorkflowExecutionStatus +from models.workflow import WorkflowPause from repositories.api_workflow_run_repository import APIWorkflowRunRepository from repositories.sqlalchemy_api_workflow_run_repository import _PrivateWorkflowPauseEntity from services.workflow_run_service import ( @@ -63,7 +64,7 @@ class TestDataFactory: **kwargs, ) -> MagicMock: """Create a mock WorkflowPauseModel object.""" - mock_pause = MagicMock() + mock_pause = MagicMock(spec=WorkflowPause) mock_pause.id = id mock_pause.tenant_id = tenant_id mock_pause.app_id = app_id @@ -77,38 +78,15 @@ class TestDataFactory: return mock_pause - @staticmethod - def create_upload_file_mock( - id: str = "file-456", - key: str = "upload_files/test/state.json", - name: str = "state.json", - tenant_id: str = "tenant-456", - **kwargs, - ) -> MagicMock: - """Create a mock UploadFile object.""" - mock_file = MagicMock() - mock_file.id = id - mock_file.key = key - mock_file.name = name - mock_file.tenant_id = tenant_id - - for key, value in kwargs.items(): - setattr(mock_file, key, value) - - return mock_file - @staticmethod def create_pause_entity_mock( pause_model: MagicMock | None = None, - upload_file: MagicMock | None = None, ) -> _PrivateWorkflowPauseEntity: """Create a mock _PrivateWorkflowPauseEntity object.""" if pause_model is None: pause_model = TestDataFactory.create_workflow_pause_mock() - if upload_file is None: - upload_file = TestDataFactory.create_upload_file_mock() - return _PrivateWorkflowPauseEntity.from_models(pause_model, upload_file) + return _PrivateWorkflowPauseEntity(pause_model=pause_model, reason_models=[], human_input_form=[]) class TestWorkflowRunService: diff --git a/api/tests/unit_tests/services/test_workflow_service.py b/api/tests/unit_tests/services/test_workflow_service.py new file mode 100644 index 0000000000..ae5b194afb --- /dev/null +++ b/api/tests/unit_tests/services/test_workflow_service.py @@ -0,0 +1,1114 @@ +""" +Unit tests for WorkflowService. + +This test suite covers: +- Workflow creation from template +- Workflow validation (graph and features structure) +- Draft/publish transitions +- Version management +- Execution triggering +""" + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from core.workflow.enums import NodeType +from libs.datetime_utils import naive_utc_now +from models.model import App, AppMode +from models.workflow import Workflow, WorkflowType +from services.errors.app import IsDraftWorkflowError, TriggerNodeLimitExceededError, WorkflowHashNotEqualError +from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError +from services.workflow_service import WorkflowService + + +class TestWorkflowAssociatedDataFactory: + """ + Factory class for creating test data and mock objects for workflow service tests. + + This factory provides reusable methods to create mock objects for: + - App models with configurable attributes + - Workflow models with graph and feature configurations + - Account models for user authentication + - Valid workflow graph structures for testing + + All factory methods return MagicMock objects that simulate database models + without requiring actual database connections. + """ + + @staticmethod + def create_app_mock( + app_id: str = "app-123", + tenant_id: str = "tenant-456", + mode: str = AppMode.WORKFLOW.value, + workflow_id: str | None = None, + **kwargs, + ) -> MagicMock: + """ + Create a mock App with specified attributes. + + Args: + app_id: Unique identifier for the app + tenant_id: Workspace/tenant identifier + mode: App mode (workflow, chat, completion, etc.) + workflow_id: Optional ID of the published workflow + **kwargs: Additional attributes to set on the mock + + Returns: + MagicMock object configured as an App model + """ + app = MagicMock(spec=App) + app.id = app_id + app.tenant_id = tenant_id + app.mode = mode + app.workflow_id = workflow_id + for key, value in kwargs.items(): + setattr(app, key, value) + return app + + @staticmethod + def create_workflow_mock( + workflow_id: str = "workflow-789", + tenant_id: str = "tenant-456", + app_id: str = "app-123", + version: str = Workflow.VERSION_DRAFT, + workflow_type: str = WorkflowType.WORKFLOW.value, + graph: dict | None = None, + features: dict | None = None, + unique_hash: str | None = None, + **kwargs, + ) -> MagicMock: + """ + Create a mock Workflow with specified attributes. + + Args: + workflow_id: Unique identifier for the workflow + tenant_id: Workspace/tenant identifier + app_id: Associated app identifier + version: Workflow version ("draft" or timestamp-based version) + workflow_type: Type of workflow (workflow, chat, rag-pipeline) + graph: Workflow graph structure containing nodes and edges + features: Feature configuration (file upload, text-to-speech, etc.) + unique_hash: Hash for optimistic locking during updates + **kwargs: Additional attributes to set on the mock + + Returns: + MagicMock object configured as a Workflow model with graph/features + """ + workflow = MagicMock(spec=Workflow) + workflow.id = workflow_id + workflow.tenant_id = tenant_id + workflow.app_id = app_id + workflow.version = version + workflow.type = workflow_type + + # Set up graph and features with defaults if not provided + # Graph contains the workflow structure (nodes and their connections) + if graph is None: + graph = {"nodes": [], "edges": []} + # Features contain app-level configurations like file upload settings + if features is None: + features = {} + + workflow.graph = json.dumps(graph) + workflow.features = json.dumps(features) + workflow.graph_dict = graph + workflow.features_dict = features + workflow.unique_hash = unique_hash or "test-hash-123" + workflow.environment_variables = [] + workflow.conversation_variables = [] + workflow.rag_pipeline_variables = [] + workflow.created_by = "user-123" + workflow.updated_by = None + workflow.created_at = naive_utc_now() + workflow.updated_at = naive_utc_now() + + # Mock walk_nodes method to iterate through workflow nodes + # This is used by the service to traverse and validate workflow structure + def walk_nodes_side_effect(specific_node_type=None): + nodes = graph.get("nodes", []) + # Filter by node type if specified (e.g., only LLM nodes) + if specific_node_type: + return ( + (node["id"], node["data"]) + for node in nodes + if node.get("data", {}).get("type") == specific_node_type.value + ) + # Return all nodes if no filter specified + return ((node["id"], node["data"]) for node in nodes) + + workflow.walk_nodes = walk_nodes_side_effect + + for key, value in kwargs.items(): + setattr(workflow, key, value) + return workflow + + @staticmethod + def create_account_mock(account_id: str = "user-123", **kwargs) -> MagicMock: + """Create a mock Account with specified attributes.""" + account = MagicMock() + account.id = account_id + for key, value in kwargs.items(): + setattr(account, key, value) + return account + + @staticmethod + def create_valid_workflow_graph(include_start: bool = True, include_trigger: bool = False) -> dict: + """ + Create a valid workflow graph structure for testing. + + Args: + include_start: Whether to include a START node (for regular workflows) + include_trigger: Whether to include trigger nodes (webhook, schedule, etc.) + + Returns: + Dictionary containing nodes and edges arrays representing workflow graph + + Note: + Start nodes and trigger nodes cannot coexist in the same workflow. + This is validated by the workflow service. + """ + nodes = [] + edges = [] + + # Add START node for regular workflows (user-initiated) + if include_start: + nodes.append( + { + "id": "start", + "data": { + "type": NodeType.START.value, + "title": "START", + "variables": [], + }, + } + ) + + # Add trigger node for event-driven workflows (webhook, schedule, etc.) + if include_trigger: + nodes.append( + { + "id": "trigger-1", + "data": { + "type": "http-request", + "title": "HTTP Request Trigger", + }, + } + ) + + # Add an LLM node as a sample processing node + # This represents an AI model interaction in the workflow + nodes.append( + { + "id": "llm-1", + "data": { + "type": NodeType.LLM.value, + "title": "LLM", + "model": { + "provider": "openai", + "name": "gpt-4", + }, + }, + } + ) + + return {"nodes": nodes, "edges": edges} + + +class TestWorkflowService: + """ + Comprehensive unit tests for WorkflowService methods. + + This test suite covers: + - Workflow creation from template + - Workflow validation (graph and features) + - Draft/publish transitions + - Version management + - Workflow deletion and error handling + """ + + @pytest.fixture + def workflow_service(self): + """ + Create a WorkflowService instance with mocked dependencies. + + This fixture patches the database to avoid real database connections + during testing. Each test gets a fresh service instance. + """ + with patch("services.workflow_service.db"): + service = WorkflowService() + return service + + @pytest.fixture + def mock_db_session(self): + """ + Mock database session for testing database operations. + + Provides mock implementations of: + - session.add(): Adding new records + - session.commit(): Committing transactions + - session.query(): Querying database + - session.execute(): Executing SQL statements + """ + with patch("services.workflow_service.db") as mock_db: + mock_session = MagicMock() + mock_db.session = mock_session + mock_session.add = MagicMock() + mock_session.commit = MagicMock() + mock_session.query = MagicMock() + mock_session.execute = MagicMock() + yield mock_db + + @pytest.fixture + def mock_sqlalchemy_session(self): + """ + Mock SQLAlchemy Session for publish_workflow tests. + + This is a separate fixture because publish_workflow uses + SQLAlchemy's Session class directly rather than the Flask-SQLAlchemy + db.session object. + """ + mock_session = MagicMock() + mock_session.add = MagicMock() + mock_session.commit = MagicMock() + mock_session.scalar = MagicMock() + return mock_session + + # ==================== Workflow Existence Tests ==================== + # These tests verify the service can check if a draft workflow exists + + def test_is_workflow_exist_returns_true(self, workflow_service, mock_db_session): + """ + Test is_workflow_exist returns True when draft workflow exists. + + Verifies that the service correctly identifies when an app has a draft workflow. + This is used to determine whether to create or update a workflow. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock() + + # Mock the database query to return True + mock_db_session.session.execute.return_value.scalar_one.return_value = True + + result = workflow_service.is_workflow_exist(app) + + assert result is True + + def test_is_workflow_exist_returns_false(self, workflow_service, mock_db_session): + """Test is_workflow_exist returns False when no draft workflow exists.""" + app = TestWorkflowAssociatedDataFactory.create_app_mock() + + # Mock the database query to return False + mock_db_session.session.execute.return_value.scalar_one.return_value = False + + result = workflow_service.is_workflow_exist(app) + + assert result is False + + # ==================== Get Draft Workflow Tests ==================== + # These tests verify retrieval of draft workflows (version="draft") + + def test_get_draft_workflow_success(self, workflow_service, mock_db_session): + """ + Test get_draft_workflow returns draft workflow successfully. + + Draft workflows are the working copy that users edit before publishing. + Each app can have only one draft workflow at a time. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock() + mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock() + + # Mock database query + mock_query = MagicMock() + mock_db_session.session.query.return_value = mock_query + mock_query.where.return_value.first.return_value = mock_workflow + + result = workflow_service.get_draft_workflow(app) + + assert result == mock_workflow + + def test_get_draft_workflow_returns_none(self, workflow_service, mock_db_session): + """Test get_draft_workflow returns None when no draft exists.""" + app = TestWorkflowAssociatedDataFactory.create_app_mock() + + # Mock database query to return None + mock_query = MagicMock() + mock_db_session.session.query.return_value = mock_query + mock_query.where.return_value.first.return_value = None + + result = workflow_service.get_draft_workflow(app) + + assert result is None + + def test_get_draft_workflow_with_workflow_id(self, workflow_service, mock_db_session): + """Test get_draft_workflow with workflow_id calls get_published_workflow_by_id.""" + app = TestWorkflowAssociatedDataFactory.create_app_mock() + workflow_id = "workflow-123" + mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock(version="v1") + + # Mock database query + mock_query = MagicMock() + mock_db_session.session.query.return_value = mock_query + mock_query.where.return_value.first.return_value = mock_workflow + + result = workflow_service.get_draft_workflow(app, workflow_id=workflow_id) + + assert result == mock_workflow + + # ==================== Get Published Workflow Tests ==================== + # These tests verify retrieval of published workflows (versioned snapshots) + + def test_get_published_workflow_by_id_success(self, workflow_service, mock_db_session): + """Test get_published_workflow_by_id returns published workflow.""" + app = TestWorkflowAssociatedDataFactory.create_app_mock() + workflow_id = "workflow-123" + mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock(workflow_id=workflow_id, version="v1") + + # Mock database query + mock_query = MagicMock() + mock_db_session.session.query.return_value = mock_query + mock_query.where.return_value.first.return_value = mock_workflow + + result = workflow_service.get_published_workflow_by_id(app, workflow_id) + + assert result == mock_workflow + + def test_get_published_workflow_by_id_raises_error_for_draft(self, workflow_service, mock_db_session): + """ + Test get_published_workflow_by_id raises error when workflow is draft. + + This prevents using draft workflows in production contexts where only + published, stable versions should be used (e.g., API execution). + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock() + workflow_id = "workflow-123" + mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock( + workflow_id=workflow_id, version=Workflow.VERSION_DRAFT + ) + + # Mock database query + mock_query = MagicMock() + mock_db_session.session.query.return_value = mock_query + mock_query.where.return_value.first.return_value = mock_workflow + + with pytest.raises(IsDraftWorkflowError): + workflow_service.get_published_workflow_by_id(app, workflow_id) + + def test_get_published_workflow_by_id_returns_none(self, workflow_service, mock_db_session): + """Test get_published_workflow_by_id returns None when workflow not found.""" + app = TestWorkflowAssociatedDataFactory.create_app_mock() + workflow_id = "nonexistent-workflow" + + # Mock database query to return None + mock_query = MagicMock() + mock_db_session.session.query.return_value = mock_query + mock_query.where.return_value.first.return_value = None + + result = workflow_service.get_published_workflow_by_id(app, workflow_id) + + assert result is None + + def test_get_published_workflow_success(self, workflow_service, mock_db_session): + """Test get_published_workflow returns published workflow.""" + workflow_id = "workflow-123" + app = TestWorkflowAssociatedDataFactory.create_app_mock(workflow_id=workflow_id) + mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock(workflow_id=workflow_id, version="v1") + + # Mock database query + mock_query = MagicMock() + mock_db_session.session.query.return_value = mock_query + mock_query.where.return_value.first.return_value = mock_workflow + + result = workflow_service.get_published_workflow(app) + + assert result == mock_workflow + + def test_get_published_workflow_returns_none_when_no_workflow_id(self, workflow_service): + """Test get_published_workflow returns None when app has no workflow_id.""" + app = TestWorkflowAssociatedDataFactory.create_app_mock(workflow_id=None) + + result = workflow_service.get_published_workflow(app) + + assert result is None + + # ==================== Sync Draft Workflow Tests ==================== + # These tests verify creating and updating draft workflows with validation + + def test_sync_draft_workflow_creates_new_draft(self, workflow_service, mock_db_session): + """ + Test sync_draft_workflow creates new draft workflow when none exists. + + When a user first creates a workflow app, this creates the initial draft. + The draft is validated before creation to ensure graph and features are valid. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock() + account = TestWorkflowAssociatedDataFactory.create_account_mock() + graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() + features = {"file_upload": {"enabled": False}} + + # Mock get_draft_workflow to return None (no existing draft) + # This simulates the first time a workflow is created for an app + mock_query = MagicMock() + mock_db_session.session.query.return_value = mock_query + mock_query.where.return_value.first.return_value = None + + with ( + patch.object(workflow_service, "validate_features_structure"), + patch.object(workflow_service, "validate_graph_structure"), + patch("services.workflow_service.app_draft_workflow_was_synced"), + ): + result = workflow_service.sync_draft_workflow( + app_model=app, + graph=graph, + features=features, + unique_hash=None, + account=account, + environment_variables=[], + conversation_variables=[], + ) + + # Verify workflow was added to session + mock_db_session.session.add.assert_called_once() + mock_db_session.session.commit.assert_called_once() + + def test_sync_draft_workflow_updates_existing_draft(self, workflow_service, mock_db_session): + """ + Test sync_draft_workflow updates existing draft workflow. + + When users edit their workflow, this updates the existing draft. + The unique_hash is used for optimistic locking to prevent conflicts. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock() + account = TestWorkflowAssociatedDataFactory.create_account_mock() + graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() + features = {"file_upload": {"enabled": False}} + unique_hash = "test-hash-123" + + # Mock existing draft workflow + mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock(unique_hash=unique_hash) + + mock_query = MagicMock() + mock_db_session.session.query.return_value = mock_query + mock_query.where.return_value.first.return_value = mock_workflow + + with ( + patch.object(workflow_service, "validate_features_structure"), + patch.object(workflow_service, "validate_graph_structure"), + patch("services.workflow_service.app_draft_workflow_was_synced"), + ): + result = workflow_service.sync_draft_workflow( + app_model=app, + graph=graph, + features=features, + unique_hash=unique_hash, + account=account, + environment_variables=[], + conversation_variables=[], + ) + + # Verify workflow was updated + assert mock_workflow.graph == json.dumps(graph) + assert mock_workflow.features == json.dumps(features) + assert mock_workflow.updated_by == account.id + mock_db_session.session.commit.assert_called_once() + + def test_sync_draft_workflow_raises_hash_not_equal_error(self, workflow_service, mock_db_session): + """ + Test sync_draft_workflow raises error when hash doesn't match. + + This implements optimistic locking: if the workflow was modified by another + user/session since it was loaded, the hash won't match and the update fails. + This prevents overwriting concurrent changes. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock() + account = TestWorkflowAssociatedDataFactory.create_account_mock() + graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() + features = {} + + # Mock existing draft workflow with different hash + mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock(unique_hash="old-hash") + + mock_query = MagicMock() + mock_db_session.session.query.return_value = mock_query + mock_query.where.return_value.first.return_value = mock_workflow + + with pytest.raises(WorkflowHashNotEqualError): + workflow_service.sync_draft_workflow( + app_model=app, + graph=graph, + features=features, + unique_hash="new-hash", + account=account, + environment_variables=[], + conversation_variables=[], + ) + + # ==================== Workflow Validation Tests ==================== + # These tests verify graph structure and feature configuration validation + + def test_validate_graph_structure_empty_graph(self, workflow_service): + """Test validate_graph_structure accepts empty graph.""" + graph = {"nodes": []} + + # Should not raise any exception + workflow_service.validate_graph_structure(graph) + + def test_validate_graph_structure_valid_graph(self, workflow_service): + """Test validate_graph_structure accepts valid graph.""" + graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() + + # Should not raise any exception + workflow_service.validate_graph_structure(graph) + + def test_validate_graph_structure_start_and_trigger_coexist_raises_error(self, workflow_service): + """ + Test validate_graph_structure raises error when start and trigger nodes coexist. + + Workflows can be either: + - User-initiated (with START node): User provides input to start execution + - Event-driven (with trigger nodes): External events trigger execution + + These two patterns cannot be mixed in a single workflow. + """ + # Create a graph with both start and trigger nodes + # Use actual trigger node types: trigger-webhook, trigger-schedule, trigger-plugin + graph = { + "nodes": [ + { + "id": "start", + "data": { + "type": "start", + "title": "START", + }, + }, + { + "id": "trigger-1", + "data": { + "type": "trigger-webhook", + "title": "Webhook Trigger", + }, + }, + ], + "edges": [], + } + + with pytest.raises(ValueError, match="Start node and trigger nodes cannot coexist"): + workflow_service.validate_graph_structure(graph) + + def test_validate_features_structure_workflow_mode(self, workflow_service): + """ + Test validate_features_structure for workflow mode. + + Different app modes have different feature configurations. + This ensures the features match the expected schema for workflow apps. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock(mode=AppMode.WORKFLOW.value) + features = {"file_upload": {"enabled": False}} + + with patch("services.workflow_service.WorkflowAppConfigManager.config_validate") as mock_validate: + workflow_service.validate_features_structure(app, features) + mock_validate.assert_called_once_with( + tenant_id=app.tenant_id, config=features, only_structure_validate=True + ) + + def test_validate_features_structure_advanced_chat_mode(self, workflow_service): + """Test validate_features_structure for advanced chat mode.""" + app = TestWorkflowAssociatedDataFactory.create_app_mock(mode=AppMode.ADVANCED_CHAT.value) + features = {"opening_statement": "Hello"} + + with patch("services.workflow_service.AdvancedChatAppConfigManager.config_validate") as mock_validate: + workflow_service.validate_features_structure(app, features) + mock_validate.assert_called_once_with( + tenant_id=app.tenant_id, config=features, only_structure_validate=True + ) + + def test_validate_features_structure_invalid_mode_raises_error(self, workflow_service): + """Test validate_features_structure raises error for invalid mode.""" + app = TestWorkflowAssociatedDataFactory.create_app_mock(mode=AppMode.COMPLETION.value) + features = {} + + with pytest.raises(ValueError, match="Invalid app mode"): + workflow_service.validate_features_structure(app, features) + + # ==================== Publish Workflow Tests ==================== + # These tests verify creating published versions from draft workflows + + def test_publish_workflow_success(self, workflow_service, mock_sqlalchemy_session): + """ + Test publish_workflow creates new published version. + + Publishing creates a timestamped snapshot of the draft workflow. + This allows users to: + - Roll back to previous versions + - Use stable versions in production + - Continue editing draft without affecting published version + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock() + account = TestWorkflowAssociatedDataFactory.create_account_mock() + graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() + + # Mock draft workflow + mock_draft = TestWorkflowAssociatedDataFactory.create_workflow_mock(version=Workflow.VERSION_DRAFT, graph=graph) + mock_sqlalchemy_session.scalar.return_value = mock_draft + + with ( + patch.object(workflow_service, "validate_graph_structure"), + patch("services.workflow_service.app_published_workflow_was_updated"), + patch("services.workflow_service.dify_config") as mock_config, + patch("services.workflow_service.Workflow.new") as mock_workflow_new, + ): + # Disable billing + mock_config.BILLING_ENABLED = False + + # Mock Workflow.new to return a new workflow + mock_new_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock(version="v1") + mock_workflow_new.return_value = mock_new_workflow + + result = workflow_service.publish_workflow( + session=mock_sqlalchemy_session, + app_model=app, + account=account, + marked_name="Version 1", + marked_comment="Initial release", + ) + + # Verify workflow was added to session + mock_sqlalchemy_session.add.assert_called_once_with(mock_new_workflow) + assert result == mock_new_workflow + + def test_publish_workflow_no_draft_raises_error(self, workflow_service, mock_sqlalchemy_session): + """ + Test publish_workflow raises error when no draft exists. + + Cannot publish if there's no draft to publish from. + Users must create and save a draft before publishing. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock() + account = TestWorkflowAssociatedDataFactory.create_account_mock() + + # Mock no draft workflow + mock_sqlalchemy_session.scalar.return_value = None + + with pytest.raises(ValueError, match="No valid workflow found"): + workflow_service.publish_workflow(session=mock_sqlalchemy_session, app_model=app, account=account) + + def test_publish_workflow_trigger_limit_exceeded(self, workflow_service, mock_sqlalchemy_session): + """ + Test publish_workflow raises error when trigger node limit exceeded in SANDBOX plan. + + Free/sandbox tier users have limits on the number of trigger nodes. + This prevents resource abuse while allowing users to test the feature. + The limit is enforced at publish time, not during draft editing. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock() + account = TestWorkflowAssociatedDataFactory.create_account_mock() + + # Create graph with 3 trigger nodes (exceeds SANDBOX limit of 2) + # Trigger nodes enable event-driven automation which consumes resources + graph = { + "nodes": [ + {"id": "trigger-1", "data": {"type": "trigger-webhook"}}, + {"id": "trigger-2", "data": {"type": "trigger-schedule"}}, + {"id": "trigger-3", "data": {"type": "trigger-plugin"}}, + ], + "edges": [], + } + mock_draft = TestWorkflowAssociatedDataFactory.create_workflow_mock(version=Workflow.VERSION_DRAFT, graph=graph) + mock_sqlalchemy_session.scalar.return_value = mock_draft + + with ( + patch.object(workflow_service, "validate_graph_structure"), + patch("services.workflow_service.dify_config") as mock_config, + patch("services.workflow_service.BillingService") as MockBillingService, + patch("services.workflow_service.app_published_workflow_was_updated"), + ): + # Enable billing and set SANDBOX plan + mock_config.BILLING_ENABLED = True + MockBillingService.get_info.return_value = {"subscription": {"plan": "sandbox"}} + + with pytest.raises(TriggerNodeLimitExceededError): + workflow_service.publish_workflow(session=mock_sqlalchemy_session, app_model=app, account=account) + + # ==================== Version Management Tests ==================== + # These tests verify listing and managing published workflow versions + + def test_get_all_published_workflow_with_pagination(self, workflow_service): + """ + Test get_all_published_workflow returns paginated results. + + Apps can have many published versions over time. + Pagination prevents loading all versions at once, improving performance. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock(workflow_id="workflow-123") + + # Mock workflows + mock_workflows = [ + TestWorkflowAssociatedDataFactory.create_workflow_mock(workflow_id=f"workflow-{i}", version=f"v{i}") + for i in range(5) + ] + + mock_session = MagicMock() + mock_session.scalars.return_value.all.return_value = mock_workflows + + with patch("services.workflow_service.select") as mock_select: + mock_stmt = MagicMock() + mock_select.return_value = mock_stmt + mock_stmt.where.return_value = mock_stmt + mock_stmt.order_by.return_value = mock_stmt + mock_stmt.limit.return_value = mock_stmt + mock_stmt.offset.return_value = mock_stmt + + workflows, has_more = workflow_service.get_all_published_workflow( + session=mock_session, app_model=app, page=1, limit=10, user_id=None + ) + + assert len(workflows) == 5 + assert has_more is False + + def test_get_all_published_workflow_has_more(self, workflow_service): + """ + Test get_all_published_workflow indicates has_more when results exceed limit. + + The has_more flag tells the UI whether to show a "Load More" button. + This is determined by fetching limit+1 records and checking if we got that many. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock(workflow_id="workflow-123") + + # Mock 11 workflows (limit is 10, so has_more should be True) + mock_workflows = [ + TestWorkflowAssociatedDataFactory.create_workflow_mock(workflow_id=f"workflow-{i}", version=f"v{i}") + for i in range(11) + ] + + mock_session = MagicMock() + mock_session.scalars.return_value.all.return_value = mock_workflows + + with patch("services.workflow_service.select") as mock_select: + mock_stmt = MagicMock() + mock_select.return_value = mock_stmt + mock_stmt.where.return_value = mock_stmt + mock_stmt.order_by.return_value = mock_stmt + mock_stmt.limit.return_value = mock_stmt + mock_stmt.offset.return_value = mock_stmt + + workflows, has_more = workflow_service.get_all_published_workflow( + session=mock_session, app_model=app, page=1, limit=10, user_id=None + ) + + assert len(workflows) == 10 + assert has_more is True + + def test_get_all_published_workflow_no_workflow_id(self, workflow_service): + """Test get_all_published_workflow returns empty when app has no workflow_id.""" + app = TestWorkflowAssociatedDataFactory.create_app_mock(workflow_id=None) + mock_session = MagicMock() + + workflows, has_more = workflow_service.get_all_published_workflow( + session=mock_session, app_model=app, page=1, limit=10, user_id=None + ) + + assert workflows == [] + assert has_more is False + + # ==================== Update Workflow Tests ==================== + # These tests verify updating workflow metadata (name, comments, etc.) + + def test_update_workflow_success(self, workflow_service): + """ + Test update_workflow updates workflow attributes. + + Allows updating metadata like marked_name and marked_comment + without creating a new version. Only specific fields are allowed + to prevent accidental modification of workflow logic. + """ + workflow_id = "workflow-123" + tenant_id = "tenant-456" + account_id = "user-123" + mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock(workflow_id=workflow_id) + + mock_session = MagicMock() + mock_session.scalar.return_value = mock_workflow + + with patch("services.workflow_service.select") as mock_select: + mock_stmt = MagicMock() + mock_select.return_value = mock_stmt + mock_stmt.where.return_value = mock_stmt + + result = workflow_service.update_workflow( + session=mock_session, + workflow_id=workflow_id, + tenant_id=tenant_id, + account_id=account_id, + data={"marked_name": "Updated Name", "marked_comment": "Updated Comment"}, + ) + + assert result == mock_workflow + assert mock_workflow.marked_name == "Updated Name" + assert mock_workflow.marked_comment == "Updated Comment" + assert mock_workflow.updated_by == account_id + + def test_update_workflow_not_found(self, workflow_service): + """Test update_workflow returns None when workflow not found.""" + mock_session = MagicMock() + mock_session.scalar.return_value = None + + with patch("services.workflow_service.select") as mock_select: + mock_stmt = MagicMock() + mock_select.return_value = mock_stmt + mock_stmt.where.return_value = mock_stmt + + result = workflow_service.update_workflow( + session=mock_session, + workflow_id="nonexistent", + tenant_id="tenant-456", + account_id="user-123", + data={"marked_name": "Test"}, + ) + + assert result is None + + # ==================== Delete Workflow Tests ==================== + # These tests verify workflow deletion with safety checks + + def test_delete_workflow_success(self, workflow_service): + """ + Test delete_workflow successfully deletes a published workflow. + + Users can delete old published versions they no longer need. + This helps manage storage and keeps the version list clean. + """ + workflow_id = "workflow-123" + tenant_id = "tenant-456" + mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock(workflow_id=workflow_id, version="v1") + + mock_session = MagicMock() + # Mock successful deletion scenario: + # 1. Workflow exists + # 2. No app is currently using it + # 3. Not published as a tool + mock_session.scalar.side_effect = [mock_workflow, None] # workflow exists, no app using it + mock_session.query.return_value.where.return_value.first.return_value = None # no tool provider + + with patch("services.workflow_service.select") as mock_select: + mock_stmt = MagicMock() + mock_select.return_value = mock_stmt + mock_stmt.where.return_value = mock_stmt + + result = workflow_service.delete_workflow( + session=mock_session, workflow_id=workflow_id, tenant_id=tenant_id + ) + + assert result is True + mock_session.delete.assert_called_once_with(mock_workflow) + + def test_delete_workflow_draft_raises_error(self, workflow_service): + """ + Test delete_workflow raises error when trying to delete draft. + + Draft workflows cannot be deleted - they're the working copy. + Users can only delete published versions to clean up old snapshots. + """ + workflow_id = "workflow-123" + tenant_id = "tenant-456" + mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock( + workflow_id=workflow_id, version=Workflow.VERSION_DRAFT + ) + + mock_session = MagicMock() + mock_session.scalar.return_value = mock_workflow + + with patch("services.workflow_service.select") as mock_select: + mock_stmt = MagicMock() + mock_select.return_value = mock_stmt + mock_stmt.where.return_value = mock_stmt + + with pytest.raises(DraftWorkflowDeletionError, match="Cannot delete draft workflow"): + workflow_service.delete_workflow(session=mock_session, workflow_id=workflow_id, tenant_id=tenant_id) + + def test_delete_workflow_in_use_by_app_raises_error(self, workflow_service): + """ + Test delete_workflow raises error when workflow is in use by app. + + Cannot delete a workflow version that's currently published/active. + This would break the app for users. Must publish a different version first. + """ + workflow_id = "workflow-123" + tenant_id = "tenant-456" + mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock(workflow_id=workflow_id, version="v1") + mock_app = TestWorkflowAssociatedDataFactory.create_app_mock(workflow_id=workflow_id) + + mock_session = MagicMock() + mock_session.scalar.side_effect = [mock_workflow, mock_app] + + with patch("services.workflow_service.select") as mock_select: + mock_stmt = MagicMock() + mock_select.return_value = mock_stmt + mock_stmt.where.return_value = mock_stmt + + with pytest.raises(WorkflowInUseError, match="currently in use by app"): + workflow_service.delete_workflow(session=mock_session, workflow_id=workflow_id, tenant_id=tenant_id) + + def test_delete_workflow_published_as_tool_raises_error(self, workflow_service): + """ + Test delete_workflow raises error when workflow is published as tool. + + Workflows can be published as reusable tools for other workflows. + Cannot delete a version that's being used as a tool, as this would + break other workflows that depend on it. + """ + workflow_id = "workflow-123" + tenant_id = "tenant-456" + mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow_mock(workflow_id=workflow_id, version="v1") + mock_tool_provider = MagicMock() + + mock_session = MagicMock() + mock_session.scalar.side_effect = [mock_workflow, None] # workflow exists, no app using it + mock_session.query.return_value.where.return_value.first.return_value = mock_tool_provider + + with patch("services.workflow_service.select") as mock_select: + mock_stmt = MagicMock() + mock_select.return_value = mock_stmt + mock_stmt.where.return_value = mock_stmt + + with pytest.raises(WorkflowInUseError, match="published as a tool"): + workflow_service.delete_workflow(session=mock_session, workflow_id=workflow_id, tenant_id=tenant_id) + + def test_delete_workflow_not_found_raises_error(self, workflow_service): + """Test delete_workflow raises error when workflow not found.""" + workflow_id = "nonexistent" + tenant_id = "tenant-456" + + mock_session = MagicMock() + mock_session.scalar.return_value = None + + with patch("services.workflow_service.select") as mock_select: + mock_stmt = MagicMock() + mock_select.return_value = mock_stmt + mock_stmt.where.return_value = mock_stmt + + with pytest.raises(ValueError, match="not found"): + workflow_service.delete_workflow(session=mock_session, workflow_id=workflow_id, tenant_id=tenant_id) + + # ==================== Get Default Block Config Tests ==================== + # These tests verify retrieval of default node configurations + + def test_get_default_block_configs(self, workflow_service): + """ + Test get_default_block_configs returns list of default configs. + + Returns default configurations for all available node types. + Used by the UI to populate the node palette and provide sensible defaults + when users add new nodes to their workflow. + """ + with patch("services.workflow_service.NODE_TYPE_CLASSES_MAPPING") as mock_mapping: + # Mock node class with default config + mock_node_class = MagicMock() + mock_node_class.get_default_config.return_value = {"type": "llm", "config": {}} + + mock_mapping.values.return_value = [{"latest": mock_node_class}] + + with patch("services.workflow_service.LATEST_VERSION", "latest"): + result = workflow_service.get_default_block_configs() + + assert len(result) > 0 + + def test_get_default_block_config_for_node_type(self, workflow_service): + """ + Test get_default_block_config returns config for specific node type. + + Returns the default configuration for a specific node type (e.g., LLM, HTTP). + This includes default values for all required and optional parameters. + """ + with ( + patch("services.workflow_service.NODE_TYPE_CLASSES_MAPPING") as mock_mapping, + patch("services.workflow_service.LATEST_VERSION", "latest"), + ): + # Mock node class with default config + mock_node_class = MagicMock() + mock_config = {"type": "llm", "config": {"provider": "openai"}} + mock_node_class.get_default_config.return_value = mock_config + + # Create a mock mapping that includes NodeType.LLM + mock_mapping.__contains__.return_value = True + mock_mapping.__getitem__.return_value = {"latest": mock_node_class} + + result = workflow_service.get_default_block_config(NodeType.LLM.value) + + assert result == mock_config + mock_node_class.get_default_config.assert_called_once() + + def test_get_default_block_config_invalid_node_type(self, workflow_service): + """Test get_default_block_config returns empty dict for invalid node type.""" + with patch("services.workflow_service.NODE_TYPE_CLASSES_MAPPING") as mock_mapping: + # Mock mapping to not contain the node type + mock_mapping.__contains__.return_value = False + + # Use a valid NodeType but one that's not in the mapping + result = workflow_service.get_default_block_config(NodeType.LLM.value) + + assert result == {} + + # ==================== Workflow Conversion Tests ==================== + # These tests verify converting basic apps to workflow apps + + def test_convert_to_workflow_from_chat_app(self, workflow_service): + """ + Test convert_to_workflow converts chat app to workflow. + + Allows users to migrate from simple chat apps to advanced workflow apps. + The conversion creates equivalent workflow nodes from the chat configuration, + giving users more control and customization options. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock(mode=AppMode.CHAT.value) + account = TestWorkflowAssociatedDataFactory.create_account_mock() + args = { + "name": "Converted Workflow", + "icon_type": "emoji", + "icon": "🤖", + "icon_background": "#FFEAD5", + } + + with patch("services.workflow_service.WorkflowConverter") as MockConverter: + mock_converter = MockConverter.return_value + mock_new_app = TestWorkflowAssociatedDataFactory.create_app_mock(mode=AppMode.WORKFLOW.value) + mock_converter.convert_to_workflow.return_value = mock_new_app + + result = workflow_service.convert_to_workflow(app, account, args) + + assert result == mock_new_app + mock_converter.convert_to_workflow.assert_called_once() + + def test_convert_to_workflow_from_completion_app(self, workflow_service): + """ + Test convert_to_workflow converts completion app to workflow. + + Similar to chat conversion, but for completion-style apps. + Completion apps are simpler (single prompt-response), so the + conversion creates a basic workflow with fewer nodes. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock(mode=AppMode.COMPLETION.value) + account = TestWorkflowAssociatedDataFactory.create_account_mock() + args = {"name": "Converted Workflow"} + + with patch("services.workflow_service.WorkflowConverter") as MockConverter: + mock_converter = MockConverter.return_value + mock_new_app = TestWorkflowAssociatedDataFactory.create_app_mock(mode=AppMode.WORKFLOW.value) + mock_converter.convert_to_workflow.return_value = mock_new_app + + result = workflow_service.convert_to_workflow(app, account, args) + + assert result == mock_new_app + + def test_convert_to_workflow_invalid_mode_raises_error(self, workflow_service): + """ + Test convert_to_workflow raises error for invalid app mode. + + Only chat and completion apps can be converted to workflows. + Apps that are already workflows or have other modes cannot be converted. + """ + app = TestWorkflowAssociatedDataFactory.create_app_mock(mode=AppMode.WORKFLOW.value) + account = TestWorkflowAssociatedDataFactory.create_account_mock() + args = {} + + with pytest.raises(ValueError, match="not supported convert to workflow"): + workflow_service.convert_to_workflow(app, account, args) diff --git a/api/uv.lock b/api/uv.lock index 0c9f73ccf0..963591ac27 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -124,16 +124,16 @@ wheels = [ [[package]] name = "alembic" -version = "1.17.1" +version = "1.17.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mako" }, { name = "sqlalchemy" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6e/b6/2a81d7724c0c124edc5ec7a167e85858b6fd31b9611c6fb8ecf617b7e2d3/alembic-1.17.1.tar.gz", hash = "sha256:8a289f6778262df31571d29cca4c7fbacd2f0f582ea0816f4c399b6da7528486", size = 1981285, upload-time = "2025-10-29T00:23:16.667Z" } +sdist = { url = "https://files.pythonhosted.org/packages/02/a6/74c8cadc2882977d80ad756a13857857dbcf9bd405bc80b662eb10651282/alembic-1.17.2.tar.gz", hash = "sha256:bbe9751705c5e0f14877f02d46c53d10885e377e3d90eda810a016f9baa19e8e", size = 1988064, upload-time = "2025-11-14T20:35:04.057Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/32/7df1d81ec2e50fb661944a35183d87e62d3f6c6d9f8aff64a4f245226d55/alembic-1.17.1-py3-none-any.whl", hash = "sha256:cbc2386e60f89608bb63f30d2d6cc66c7aaed1fe105bd862828600e5ad167023", size = 247848, upload-time = "2025-10-29T00:23:18.79Z" }, + { url = "https://files.pythonhosted.org/packages/ba/88/6237e97e3385b57b5f1528647addea5cc03d4d65d5979ab24327d41fb00d/alembic-1.17.2-py3-none-any.whl", hash = "sha256:f483dd1fe93f6c5d49217055e4d15b905b425b6af906746abb35b69c1996c4e6", size = 248554, upload-time = "2025-11-14T20:35:05.699Z" }, ] [[package]] @@ -272,12 +272,15 @@ sdist = { url = "https://files.pythonhosted.org/packages/09/be/f594e79625e5ccfcf [[package]] name = "alibabacloud-tea-util" -version = "0.3.13" +version = "0.3.14" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "alibabacloud-tea" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/23/18/35be17103c8f40f9eebec3b1567f51b3eec09c3a47a5dd62bcb413f4e619/alibabacloud_tea_util-0.3.13.tar.gz", hash = "sha256:8cbdfd2a03fbbf622f901439fa08643898290dd40e1d928347f6346e43f63c90", size = 6535, upload-time = "2024-07-15T12:25:12.07Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/ee/ea90be94ad781a5055db29556744681fc71190ef444ae53adba45e1be5f3/alibabacloud_tea_util-0.3.14.tar.gz", hash = "sha256:708e7c9f64641a3c9e0e566365d2f23675f8d7c2a3e2971d9402ceede0408cdb", size = 7515, upload-time = "2025-11-19T06:01:08.504Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/9e/c394b4e2104766fb28a1e44e3ed36e4c7773b4d05c868e482be99d5635c9/alibabacloud_tea_util-0.3.14-py3-none-any.whl", hash = "sha256:10d3e5c340d8f7ec69dd27345eb2fc5a1dab07875742525edf07bbe86db93bfe", size = 6697, upload-time = "2025-11-19T06:01:07.355Z" }, +] [[package]] name = "alibabacloud-tea-xml" @@ -395,11 +398,11 @@ wheels = [ [[package]] name = "asgiref" -version = "3.10.0" +version = "3.11.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/46/08/4dfec9b90758a59acc6be32ac82e98d1fbfc321cb5cfa410436dbacf821c/asgiref-3.10.0.tar.gz", hash = "sha256:d89f2d8cd8b56dada7d52fa7dc8075baa08fb836560710d38c292a7a3f78c04e", size = 37483, upload-time = "2025-10-05T09:15:06.557Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/b9/4db2509eabd14b4a8c71d1b24c8d5734c52b8560a7b1e1a8b56c8d25568b/asgiref-3.11.0.tar.gz", hash = "sha256:13acff32519542a1736223fb79a715acdebe24286d98e8b164a73085f40da2c4", size = 37969, upload-time = "2025-11-19T15:32:20.106Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/9c/fc2331f538fbf7eedba64b2052e99ccf9ba9d6888e2f41441ee28847004b/asgiref-3.10.0-py3-none-any.whl", hash = "sha256:aef8a81283a34d0ab31630c9b7dfe70c812c95eba78171367ca8745e88124734", size = 24050, upload-time = "2025-10-05T09:15:05.11Z" }, + { url = "https://files.pythonhosted.org/packages/91/be/317c2c55b8bbec407257d45f5c8d1b6867abc76d12043f2d3d58c538a4ea/asgiref-3.11.0-py3-none-any.whl", hash = "sha256:1db9021efadb0d9512ce8ffaf72fcef601c7b73a8807a1bb2ef143dc6b14846d", size = 24096, upload-time = "2025-11-19T15:32:19.004Z" }, ] [[package]] @@ -498,16 +501,16 @@ wheels = [ [[package]] name = "bce-python-sdk" -version = "0.9.52" +version = "0.9.53" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "future" }, { name = "pycryptodome" }, { name = "six" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/83/0a/e49d7774ce186fd51c611a2533baff8e7db0d22baef12223773f389b06b1/bce_python_sdk-0.9.52.tar.gz", hash = "sha256:dd54213ac25b8b1260fb45f1fbc0f2b1c53bb0f9f594258ca0479f1fc85f7405", size = 275614, upload-time = "2025-11-12T09:09:28.227Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/8d/85ec18ca2dba624cb5932bda74e926c346a7a6403a628aeda45d848edb48/bce_python_sdk-0.9.53.tar.gz", hash = "sha256:fb14b09d1064a6987025648589c8245cb7e404acd38bb900f0775f396e3d9b3e", size = 275594, upload-time = "2025-11-21T03:48:58.869Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/d0/f57f75c96e8bb72144845f7208f712a54454f1d063d5ef02f1e9ea476b79/bce_python_sdk-0.9.52-py3-none-any.whl", hash = "sha256:f1ed39aa61c2d4a002cd2345e01dd92ac55c75960440d76163ead419b3b550e7", size = 390401, upload-time = "2025-11-12T09:09:26.663Z" }, + { url = "https://files.pythonhosted.org/packages/7d/e9/6fc142b5ac5b2e544bc155757dc28eee2b22a576ca9eaf968ac033b6dc45/bce_python_sdk-0.9.53-py3-none-any.whl", hash = "sha256:00fc46b0ff8d1700911aef82b7263533c52a63b1cc5a51449c4f715a116846a7", size = 390434, upload-time = "2025-11-21T03:48:57.201Z" }, ] [[package]] @@ -566,11 +569,11 @@ wheels = [ [[package]] name = "billiard" -version = "4.2.2" +version = "4.2.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b9/6a/1405343016bce8354b29d90aad6b0bf6485b5e60404516e4b9a3a9646cf0/billiard-4.2.2.tar.gz", hash = "sha256:e815017a062b714958463e07ba15981d802dc53d41c5b69d28c5a7c238f8ecf3", size = 155592, upload-time = "2025-09-20T14:44:40.456Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/50/cc2b8b6e6433918a6b9a3566483b743dcd229da1e974be9b5f259db3aad7/billiard-4.2.3.tar.gz", hash = "sha256:96486f0885afc38219d02d5f0ccd5bec8226a414b834ab244008cbb0025b8dcb", size = 156450, upload-time = "2025-11-16T17:47:30.281Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/80/ef8dff49aae0e4430f81842f7403e14e0ca59db7bbaf7af41245b67c6b25/billiard-4.2.2-py3-none-any.whl", hash = "sha256:4bc05dcf0d1cc6addef470723aac2a6232f3c7ed7475b0b580473a9145829457", size = 86896, upload-time = "2025-09-20T14:44:39.157Z" }, + { url = "https://files.pythonhosted.org/packages/b3/cc/38b6f87170908bd8aaf9e412b021d17e85f690abe00edf50192f1a4566b9/billiard-4.2.3-py3-none-any.whl", hash = "sha256:989e9b688e3abf153f307b68a1328dfacfb954e30a4f920005654e276c69236b", size = 87042, upload-time = "2025-11-16T17:47:29.005Z" }, ] [[package]] @@ -598,16 +601,16 @@ wheels = [ [[package]] name = "boto3-stubs" -version = "1.40.72" +version = "1.41.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore-stubs" }, { name = "types-s3transfer" }, { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4f/db/90881ac0b8afdfa9b95ae66b4094ed33f88b6086a8945229a95156257ca9/boto3_stubs-1.40.72.tar.gz", hash = "sha256:cbcf7b6e8a7f54e77fcb2b8d00041993fe4f76554c716b1d290e48650d569cd0", size = 99406, upload-time = "2025-11-12T20:36:23.685Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/5b/6d274aa25f7fa09f8b7defab5cb9389e6496a7d9b76c1efcf27b0b15e868/boto3_stubs-1.41.3.tar.gz", hash = "sha256:c7cc9706ac969c8ea284c2d45ec45b6371745666d087c6c5e7c9d39dafdd48bc", size = 100010, upload-time = "2025-11-24T20:34:27.052Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/ea/0f2814edc61c2e6fedd9b7a7fbc55149d1ffac7f7cd02d04cc51d1a3b1ca/boto3_stubs-1.40.72-py3-none-any.whl", hash = "sha256:4807f334b87914f75db3c6cd85f7eb706b5777e6ddaf117f8d63219cc01fb4b2", size = 68982, upload-time = "2025-11-12T20:36:12.855Z" }, + { url = "https://files.pythonhosted.org/packages/7e/d6/ef971013d1fc7333c6df322d98ebf4592df9c80e1966fb12732f91e9e71b/boto3_stubs-1.41.3-py3-none-any.whl", hash = "sha256:bec698419b31b499f3740f1dfb6dae6519167d9e3aa536f6f730ed280556230b", size = 69294, upload-time = "2025-11-24T20:34:23.1Z" }, ] [package.optional-dependencies] @@ -631,14 +634,14 @@ wheels = [ [[package]] name = "botocore-stubs" -version = "1.40.72" +version = "1.41.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "types-awscrt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/c9/17d5337cc81f107fd0a6d04b5b20c75bea0fe8b77bcc644de324487f8310/botocore_stubs-1.40.72.tar.gz", hash = "sha256:6d268d0dd9366dc15e7af52cbd0d3a3f3cd14e2191de0e280badc69f8d34708c", size = 42208, upload-time = "2025-11-12T21:23:53.344Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/8f/a42c3ae68d0b9916f6e067546d73e9a24a6af8793999a742e7af0b7bffa2/botocore_stubs-1.41.3.tar.gz", hash = "sha256:bacd1647cd95259aa8fc4ccdb5b1b3893f495270c120cda0d7d210e0ae6a4170", size = 42404, upload-time = "2025-11-24T20:29:27.47Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/99/9387b31ec1d980af83ca097366cc10714757d2c1390b4ac6b692c07a9e7f/botocore_stubs-1.40.72-py3-none-any.whl", hash = "sha256:1166a81074714312d3843be3f879d16966cbffdc440ab61ad6f0cd8922fde679", size = 66542, upload-time = "2025-11-12T21:23:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/57/b7/f4a051cefaf76930c77558b31646bcce7e9b3fbdcbc89e4073783e961519/botocore_stubs-1.41.3-py3-none-any.whl", hash = "sha256:6ab911bd9f7256f1dcea2e24a4af7ae0f9f07e83d0a760bba37f028f4a2e5589", size = 66749, upload-time = "2025-11-24T20:29:26.142Z" }, ] [[package]] @@ -696,19 +699,22 @@ wheels = [ [[package]] name = "brotlicffi" -version = "1.1.0.0" +version = "1.2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation == 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/9d/70caa61192f570fcf0352766331b735afa931b4c6bc9a348a0925cc13288/brotlicffi-1.1.0.0.tar.gz", hash = "sha256:b77827a689905143f87915310b93b273ab17888fd43ef350d4832c4a71083c13", size = 465192, upload-time = "2023-09-14T14:22:40.707Z" } +sdist = { url = "https://files.pythonhosted.org/packages/84/85/57c314a6b35336efbbdc13e5fc9ae13f6b60a0647cfa7c1221178ac6d8ae/brotlicffi-1.2.0.0.tar.gz", hash = "sha256:34345d8d1f9d534fcac2249e57a4c3c8801a33c9942ff9f8574f67a175e17adb", size = 476682, upload-time = "2025-11-21T18:17:57.334Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/11/7b96009d3dcc2c931e828ce1e157f03824a69fb728d06bfd7b2fc6f93718/brotlicffi-1.1.0.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9b7ae6bd1a3f0df532b6d67ff674099a96d22bc0948955cb338488c31bfb8851", size = 453786, upload-time = "2023-09-14T14:21:57.72Z" }, - { url = "https://files.pythonhosted.org/packages/d6/e6/a8f46f4a4ee7856fbd6ac0c6fb0dc65ed181ba46cd77875b8d9bbe494d9e/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19ffc919fa4fc6ace69286e0a23b3789b4219058313cf9b45625016bf7ff996b", size = 2911165, upload-time = "2023-09-14T14:21:59.613Z" }, - { url = "https://files.pythonhosted.org/packages/be/20/201559dff14e83ba345a5ec03335607e47467b6633c210607e693aefac40/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9feb210d932ffe7798ee62e6145d3a757eb6233aa9a4e7db78dd3690d7755814", size = 2927895, upload-time = "2023-09-14T14:22:01.22Z" }, - { url = "https://files.pythonhosted.org/packages/cd/15/695b1409264143be3c933f708a3f81d53c4a1e1ebbc06f46331decbf6563/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84763dbdef5dd5c24b75597a77e1b30c66604725707565188ba54bab4f114820", size = 2851834, upload-time = "2023-09-14T14:22:03.571Z" }, - { url = "https://files.pythonhosted.org/packages/b4/40/b961a702463b6005baf952794c2e9e0099bde657d0d7e007f923883b907f/brotlicffi-1.1.0.0-cp37-abi3-win32.whl", hash = "sha256:1b12b50e07c3911e1efa3a8971543e7648100713d4e0971b13631cce22c587eb", size = 341731, upload-time = "2023-09-14T14:22:05.74Z" }, - { url = "https://files.pythonhosted.org/packages/1c/fa/5408a03c041114ceab628ce21766a4ea882aa6f6f0a800e04ee3a30ec6b9/brotlicffi-1.1.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:994a4f0681bb6c6c3b0925530a1926b7a189d878e6e5e38fae8efa47c5d9c613", size = 366783, upload-time = "2023-09-14T14:22:07.096Z" }, + { url = "https://files.pythonhosted.org/packages/e4/df/a72b284d8c7bef0ed5756b41c2eb7d0219a1dd6ac6762f1c7bdbc31ef3af/brotlicffi-1.2.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:9458d08a7ccde8e3c0afedbf2c70a8263227a68dea5ab13590593f4c0a4fd5f4", size = 432340, upload-time = "2025-11-21T18:17:42.277Z" }, + { url = "https://files.pythonhosted.org/packages/74/2b/cc55a2d1d6fb4f5d458fba44a3d3f91fb4320aa14145799fd3a996af0686/brotlicffi-1.2.0.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:84e3d0020cf1bd8b8131f4a07819edee9f283721566fe044a20ec792ca8fd8b7", size = 1534002, upload-time = "2025-11-21T18:17:43.746Z" }, + { url = "https://files.pythonhosted.org/packages/e4/9c/d51486bf366fc7d6735f0e46b5b96ca58dc005b250263525a1eea3cd5d21/brotlicffi-1.2.0.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:33cfb408d0cff64cd50bef268c0fed397c46fbb53944aa37264148614a62e990", size = 1536547, upload-time = "2025-11-21T18:17:45.729Z" }, + { url = "https://files.pythonhosted.org/packages/1b/37/293a9a0a7caf17e6e657668bebb92dfe730305999fe8c0e2703b8888789c/brotlicffi-1.2.0.0-cp38-abi3-win32.whl", hash = "sha256:23e5c912fdc6fd37143203820230374d24babd078fc054e18070a647118158f6", size = 343085, upload-time = "2025-11-21T18:17:48.887Z" }, + { url = "https://files.pythonhosted.org/packages/07/6b/6e92009df3b8b7272f85a0992b306b61c34b7ea1c4776643746e61c380ac/brotlicffi-1.2.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:f139a7cdfe4ae7859513067b736eb44d19fae1186f9e99370092f6915216451b", size = 378586, upload-time = "2025-11-21T18:17:50.531Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ec/52488a0563f1663e2ccc75834b470650f4b8bcdea3132aef3bf67219c661/brotlicffi-1.2.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fa102a60e50ddbd08de86a63431a722ea216d9bc903b000bf544149cc9b823dc", size = 402002, upload-time = "2025-11-21T18:17:51.76Z" }, + { url = "https://files.pythonhosted.org/packages/e4/63/d4aea4835fd97da1401d798d9b8ba77227974de565faea402f520b37b10f/brotlicffi-1.2.0.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7d3c4332fc808a94e8c1035950a10d04b681b03ab585ce897ae2a360d479037c", size = 406447, upload-time = "2025-11-21T18:17:53.614Z" }, + { url = "https://files.pythonhosted.org/packages/62/4e/5554ecb2615ff035ef8678d4e419549a0f7a28b3f096b272174d656749fb/brotlicffi-1.2.0.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fb4eb5830026b79a93bf503ad32b2c5257315e9ffc49e76b2715cffd07c8e3db", size = 402521, upload-time = "2025-11-21T18:17:54.875Z" }, + { url = "https://files.pythonhosted.org/packages/b5/d3/b07f8f125ac52bbee5dc00ef0d526f820f67321bf4184f915f17f50a4657/brotlicffi-1.2.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:3832c66e00d6d82087f20a972b2fc03e21cd99ef22705225a6f8f418a9158ecc", size = 374730, upload-time = "2025-11-21T18:17:56.334Z" }, ] [[package]] @@ -942,14 +948,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.0" +version = "8.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/46/61/de6cd827efad202d7057d93e0fed9294b96952e188f7384832791c7b2254/click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4", size = 276943, upload-time = "2025-09-18T17:32:23.696Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/d3/9dcc0f5797f070ec8edf30fbadfb200e71d9db6b84d211e3b2085a7589a0/click-8.3.0-py3-none-any.whl", hash = "sha256:9b9f285302c6e3064f4330c05f05b81945b2a39544279343e6e7c5f27a9baddc", size = 107295, upload-time = "2025-09-18T17:32:22.42Z" }, + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, ] [[package]] @@ -1331,7 +1337,7 @@ wheels = [ [[package]] name = "dify-api" -version = "1.10.0" +version = "1.10.1" source = { virtual = "." } dependencies = [ { name = "apscheduler" }, @@ -1844,11 +1850,11 @@ wheels = [ [[package]] name = "eval-type-backport" -version = "0.2.2" +version = "0.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/30/ea/8b0ac4469d4c347c6a385ff09dc3c048c2d021696664e26c7ee6791631b5/eval_type_backport-0.2.2.tar.gz", hash = "sha256:f0576b4cf01ebb5bd358d02314d31846af5e07678387486e2c798af0e7d849c1", size = 9079, upload-time = "2024-12-21T20:09:46.005Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/23/079e39571d6dd8d90d7a369ecb55ad766efb6bae4e77389629e14458c280/eval_type_backport-0.3.0.tar.gz", hash = "sha256:1638210401e184ff17f877e9a2fa076b60b5838790f4532a21761cc2be67aea1", size = 9272, upload-time = "2025-11-13T20:56:50.845Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/31/55cd413eaccd39125368be33c46de24a1f639f2e12349b0361b4678f3915/eval_type_backport-0.2.2-py3-none-any.whl", hash = "sha256:cb6ad7c393517f476f96d456d0412ea80f0a8cf96f6892834cd9340149111b0a", size = 5830, upload-time = "2024-12-21T20:09:44.175Z" }, + { url = "https://files.pythonhosted.org/packages/19/d8/2a1c638d9e0aa7e269269a1a1bf423ddd94267f1a01bbe3ad03432b67dd4/eval_type_backport-0.3.0-py3-none-any.whl", hash = "sha256:975a10a0fe333c8b6260d7fdb637698c9a16c3a9e3b6eb943fee6a6f67a37fe8", size = 6061, upload-time = "2025-11-13T20:56:49.499Z" }, ] [[package]] @@ -1866,7 +1872,7 @@ wheels = [ [[package]] name = "fastapi" -version = "0.121.1" +version = "0.122.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, @@ -1874,9 +1880,9 @@ dependencies = [ { name = "starlette" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6b/a4/29e1b861fc9017488ed02ff1052feffa40940cb355ed632a8845df84ce84/fastapi-0.121.1.tar.gz", hash = "sha256:b6dba0538fd15dab6fe4d3e5493c3957d8a9e1e9257f56446b5859af66f32441", size = 342523, upload-time = "2025-11-08T21:48:14.068Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/de/3ee97a4f6ffef1fb70bf20561e4f88531633bb5045dc6cebc0f8471f764d/fastapi-0.122.0.tar.gz", hash = "sha256:cd9b5352031f93773228af8b4c443eedc2ac2aa74b27780387b853c3726fb94b", size = 346436, upload-time = "2025-11-24T19:17:47.95Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/fd/2e6f7d706899cc08690c5f6641e2ffbfffe019e8f16ce77104caa5730910/fastapi-0.121.1-py3-none-any.whl", hash = "sha256:2c5c7028bc3a58d8f5f09aecd3fd88a000ccc0c5ad627693264181a3c33aa1fc", size = 109192, upload-time = "2025-11-08T21:48:12.458Z" }, + { url = "https://files.pythonhosted.org/packages/7a/93/aa8072af4ff37b795f6bbf43dcaf61115f40f49935c7dbb180c9afc3f421/fastapi-0.122.0-py3-none-any.whl", hash = "sha256:a456e8915dfc6c8914a50d9651133bd47ec96d331c5b44600baa635538a30d67", size = 110671, upload-time = "2025-11-24T19:17:45.96Z" }, ] [[package]] @@ -1911,14 +1917,14 @@ wheels = [ [[package]] name = "fickling" -version = "0.1.4" +version = "0.1.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "stdlib-list" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/df/23/0a03d2d01c004ab3f0181bbda3642c7d88226b4a25f47675ef948326504f/fickling-0.1.4.tar.gz", hash = "sha256:cb06bbb7b6a1c443eacf230ab7e212d8b4f3bb2333f307a8c94a144537018888", size = 40956, upload-time = "2025-07-07T13:17:59.572Z" } +sdist = { url = "https://files.pythonhosted.org/packages/41/94/0d0ce455952c036cfee235637f786c1d1d07d1b90f6a4dfb50e0eff929d6/fickling-0.1.5.tar.gz", hash = "sha256:92f9b49e717fa8dbc198b4b7b685587adb652d85aa9ede8131b3e44494efca05", size = 282462, upload-time = "2025-11-18T05:04:30.748Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/40/059cd7c6913cc20b029dd5c8f38578d185f71737c5a62387df4928cd10fe/fickling-0.1.4-py3-none-any.whl", hash = "sha256:110522385a30b7936c50c3860ba42b0605254df9d0ef6cbdaf0ad8fb455a6672", size = 42573, upload-time = "2025-07-07T13:17:58.071Z" }, + { url = "https://files.pythonhosted.org/packages/bf/a7/d25912b2e3a5b0a37e6f460050bbc396042b5906a6563a1962c484abc3c6/fickling-0.1.5-py3-none-any.whl", hash = "sha256:6aed7270bfa276e188b0abe043a27b3a042129d28ec1fa6ff389bdcc5ad178bb", size = 46240, upload-time = "2025-11-18T05:04:29.048Z" }, ] [[package]] @@ -2384,14 +2390,14 @@ wheels = [ [[package]] name = "google-resumable-media" -version = "2.7.2" +version = "2.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-crc32c" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/58/5a/0efdc02665dca14e0837b62c8a1a93132c264bd02054a15abb2218afe0ae/google_resumable_media-2.7.2.tar.gz", hash = "sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0", size = 2163099, upload-time = "2024-08-07T22:20:38.555Z" } +sdist = { url = "https://files.pythonhosted.org/packages/64/d7/520b62a35b23038ff005e334dba3ffc75fcf583bee26723f1fd8fd4b6919/google_resumable_media-2.8.0.tar.gz", hash = "sha256:f1157ed8b46994d60a1bc432544db62352043113684d4e030ee02e77ebe9a1ae", size = 2163265, upload-time = "2025-11-17T15:38:06.659Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/82/35/b8d3baf8c46695858cb9d8835a53baa1eeb9906ddaf2f728a5f5b640fd1e/google_resumable_media-2.7.2-py2.py3-none-any.whl", hash = "sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa", size = 81251, upload-time = "2024-08-07T22:20:36.409Z" }, + { url = "https://files.pythonhosted.org/packages/1f/0b/93afde9cfe012260e9fe1522f35c9b72d6ee222f316586b1f23ecf44d518/google_resumable_media-2.8.0-py3-none-any.whl", hash = "sha256:dd14a116af303845a8d932ddae161a26e86cc229645bc98b39f026f9b1717582", size = 81340, upload-time = "2025-11-17T15:38:05.594Z" }, ] [[package]] @@ -2847,14 +2853,14 @@ wheels = [ [[package]] name = "hypothesis" -version = "6.147.0" +version = "6.148.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sortedcontainers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/53/e19fe74671fd60db86344a4623c818fac58b813cc3efbb7ea3b3074dcb71/hypothesis-6.147.0.tar.gz", hash = "sha256:72e6004ea3bd1460bdb4640b6389df23b87ba7a4851893fd84d1375635d3e507", size = 468587, upload-time = "2025-11-06T20:27:29.682Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/99/a3c6eb3fdd6bfa01433d674b0f12cd9102aa99630689427422d920aea9c6/hypothesis-6.148.2.tar.gz", hash = "sha256:07e65d34d687ddff3e92a3ac6b43966c193356896813aec79f0a611c5018f4b1", size = 469984, upload-time = "2025-11-18T20:21:17.047Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/1b/932eddc3d55c4ed6c585006cffe6c6a133b5e1797d873de0bcf5208e4fed/hypothesis-6.147.0-py3-none-any.whl", hash = "sha256:de588807b6da33550d32f47bcd42b1a86d061df85673aa73e6443680249d185e", size = 535595, upload-time = "2025-11-06T20:27:23.536Z" }, + { url = "https://files.pythonhosted.org/packages/b1/d2/c2673aca0127e204965e0e9b3b7a0e91e9b12993859ac8758abd22669b89/hypothesis-6.148.2-py3-none-any.whl", hash = "sha256:bf8ddc829009da73b321994b902b1964bcc3e5c3f0ed9a1c1e6a1631ab97c5fa", size = 536986, upload-time = "2025-11-18T20:21:15.212Z" }, ] [[package]] @@ -2868,16 +2874,17 @@ wheels = [ [[package]] name = "import-linter" -version = "2.6" +version = "2.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "grimp" }, + { name = "rich" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1d/20/37f3661ccbdba41072a74cb7a57a932b6884ab6c489318903d2d870c6c07/import_linter-2.6.tar.gz", hash = "sha256:60429a450eb6ebeed536f6d2b83428b026c5747ca69d029812e2f1360b136f85", size = 161294, upload-time = "2025-11-10T09:59:20.977Z" } +sdist = { url = "https://files.pythonhosted.org/packages/50/20/cc371a35123cd6afe4c8304cf199a53530a05f7437eda79ce84d9c6f6949/import_linter-2.7.tar.gz", hash = "sha256:7bea754fac9cde54182c81eeb48f649eea20b865219c39f7ac2abd23775d07d2", size = 219914, upload-time = "2025-11-19T11:44:28.193Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/44/df/02389e13d340229baa687bd0b9be4878e13668ce0beadbe531fb2b597386/import_linter-2.6-py3-none-any.whl", hash = "sha256:4e835141294b803325a619b8c789398320b81f0bde7771e0dd36f34524e51b1e", size = 46488, upload-time = "2025-11-10T09:59:19.611Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b5/26a1d198f3de0676354a628f6e2a65334b744855d77e25eea739287eea9a/import_linter-2.7-py3-none-any.whl", hash = "sha256:be03bbd467b3f0b4535fb3ee12e07995d9837864b307df2e78888364e0ba012d", size = 46197, upload-time = "2025-11-19T11:44:27.023Z" }, ] [[package]] @@ -3017,11 +3024,11 @@ wheels = [ [[package]] name = "json-repair" -version = "0.53.0" +version = "0.54.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/69/9c/be1d84106529aeacbe6151c1e1dc202f5a5cfa0d9bac748d4a1039ebb913/json_repair-0.53.0.tar.gz", hash = "sha256:97fcbf1eea0bbcf6d5cc94befc573623ab4bbba6abdc394cfd3b933a2571266d", size = 36204, upload-time = "2025-11-08T13:45:15.807Z" } +sdist = { url = "https://files.pythonhosted.org/packages/00/46/d3a4d9a3dad39bb4a2ad16b8adb9fe2e8611b20b71197fe33daa6768e85d/json_repair-0.54.1.tar.gz", hash = "sha256:d010bc31f1fc66e7c36dc33bff5f8902674498ae5cb8e801ad455a53b455ad1d", size = 38555, upload-time = "2025-11-19T14:55:24.265Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/49/e588ec59b64222c8d38585f9ceffbf71870c3cbfb2873e53297c4f4afd0b/json_repair-0.53.0-py3-none-any.whl", hash = "sha256:17f7439e41ae39964e1d678b1def38cb8ec43d607340564acf3e62d8ce47a727", size = 27404, upload-time = "2025-11-08T13:45:14.464Z" }, + { url = "https://files.pythonhosted.org/packages/db/96/c9aad7ee949cc1bf15df91f347fbc2d3bd10b30b80c7df689ce6fe9332b5/json_repair-0.54.1-py3-none-any.whl", hash = "sha256:016160c5db5d5fe443164927bb58d2dfbba5f43ad85719fa9bc51c713a443ab1", size = 29311, upload-time = "2025-11-19T14:55:22.886Z" }, ] [[package]] @@ -3551,14 +3558,14 @@ wheels = [ [[package]] name = "mypy-boto3-bedrock-runtime" -version = "1.40.62" +version = "1.41.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/d0/ca3c58a1284f9142959fb00889322d4889278c2e4b165350d8e294c07d9c/mypy_boto3_bedrock_runtime-1.40.62.tar.gz", hash = "sha256:5505a60e2b5f9c845ee4778366d49c93c3723f6790d0cec116d8fc5f5609d846", size = 28611, upload-time = "2025-10-29T21:43:02.599Z" } +sdist = { url = "https://files.pythonhosted.org/packages/af/f1/00aea4f91501728e7af7e899ce3a75d48d6df97daa720db11e46730fa123/mypy_boto3_bedrock_runtime-1.41.2.tar.gz", hash = "sha256:ba2c11f2f18116fd69e70923389ce68378fa1620f70e600efb354395a1a9e0e5", size = 28890, upload-time = "2025-11-21T20:35:30.074Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/c5/ad62e5f80684ce5fe878d320634189ef29d00ee294cd62a37f3e51719f47/mypy_boto3_bedrock_runtime-1.40.62-py3-none-any.whl", hash = "sha256:e383e70b5dffb0b335b49fc1b2772f0d35118f99994bc7e731445ba0ab237831", size = 34497, upload-time = "2025-10-29T21:43:01.591Z" }, + { url = "https://files.pythonhosted.org/packages/a7/cc/96a2af58c632701edb5be1dda95434464da43df40ae868a1ab1ddf033839/mypy_boto3_bedrock_runtime-1.41.2-py3-none-any.whl", hash = "sha256:a720ff1e98cf10723c37a61a46cff220b190c55b8fb57d4397e6cf286262cf02", size = 34967, upload-time = "2025-11-21T20:35:27.655Z" }, ] [[package]] @@ -3591,11 +3598,11 @@ wheels = [ [[package]] name = "networkx" -version = "3.5" +version = "3.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/fc/7b6fd4d22c8c4dc5704430140d8b3f520531d4fe7328b8f8d03f5a7950e8/networkx-3.6.tar.gz", hash = "sha256:285276002ad1f7f7da0f7b42f004bcba70d381e936559166363707fdad3d72ad", size = 2511464, upload-time = "2025-11-24T03:03:47.158Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, + { url = "https://files.pythonhosted.org/packages/07/c7/d64168da60332c17d24c0d2f08bdf3987e8d1ae9d84b5bbd0eec2eb26a55/networkx-3.6-py3-none-any.whl", hash = "sha256:cdb395b105806062473d3be36458d8f1459a4e4b98e236a66c3a48996e07684f", size = 2063713, upload-time = "2025-11-24T03:03:45.21Z" }, ] [[package]] @@ -3615,18 +3622,18 @@ wheels = [ [[package]] name = "nodejs-wheel-binaries" -version = "22.20.0" +version = "24.11.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/54/02f58c8119e2f1984e2572cc77a7b469dbaf4f8d171ad376e305749ef48e/nodejs_wheel_binaries-22.20.0.tar.gz", hash = "sha256:a62d47c9fd9c32191dff65bbe60261504f26992a0a19fe8b4d523256a84bd351", size = 8058, upload-time = "2025-09-26T09:48:00.906Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/89/da307731fdbb05a5f640b26de5b8ac0dc463fef059162accfc89e32f73bc/nodejs_wheel_binaries-24.11.1.tar.gz", hash = "sha256:413dfffeadfb91edb4d8256545dea797c237bba9b3faefea973cde92d96bb922", size = 8059, upload-time = "2025-11-18T18:21:58.207Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/24/6d/333e5458422f12318e3c3e6e7f194353aa68b0d633217c7e89833427ca01/nodejs_wheel_binaries-22.20.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:455add5ac4f01c9c830ab6771dbfad0fdf373f9b040d3aabe8cca9b6c56654fb", size = 53246314, upload-time = "2025-09-26T09:47:32.536Z" }, - { url = "https://files.pythonhosted.org/packages/56/30/dcd6879d286a35b3c4c8f9e5e0e1bcf4f9e25fe35310fc77ecf97f915a23/nodejs_wheel_binaries-22.20.0-py2.py3-none-macosx_11_0_x86_64.whl", hash = "sha256:5d8c12f97eea7028b34a84446eb5ca81829d0c428dfb4e647e09ac617f4e21fa", size = 53644391, upload-time = "2025-09-26T09:47:36.093Z" }, - { url = "https://files.pythonhosted.org/packages/58/be/c7b2e7aa3bb281d380a1c531f84d0ccfe225832dfc3bed1ca171753b9630/nodejs_wheel_binaries-22.20.0-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a2b0989194148f66e9295d8f11bc463bde02cbe276517f4d20a310fb84780ae", size = 60282516, upload-time = "2025-09-26T09:47:39.88Z" }, - { url = "https://files.pythonhosted.org/packages/3e/c5/8befacf4190e03babbae54cb0809fb1a76e1600ec3967ab8ee9f8fc85b65/nodejs_wheel_binaries-22.20.0-py2.py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5c500aa4dc046333ecb0a80f183e069e5c30ce637f1c1a37166b2c0b642dc21", size = 60347290, upload-time = "2025-09-26T09:47:43.712Z" }, - { url = "https://files.pythonhosted.org/packages/c0/bd/cfffd1e334277afa0714962c6ec432b5fe339340a6bca2e5fa8e678e7590/nodejs_wheel_binaries-22.20.0-py2.py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3279eb1b99521f0d20a850bbfc0159a658e0e85b843b3cf31b090d7da9f10dfc", size = 62178798, upload-time = "2025-09-26T09:47:47.752Z" }, - { url = "https://files.pythonhosted.org/packages/08/14/10b83a9c02faac985b3e9f5e65d63a34fc0f46b48d8a2c3e4caa3e1e7318/nodejs_wheel_binaries-22.20.0-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:d29705797b33bade62d79d8f106c2453c8a26442a9b2a5576610c0f7e7c351ed", size = 62772957, upload-time = "2025-09-26T09:47:51.266Z" }, - { url = "https://files.pythonhosted.org/packages/b4/a9/c6a480259aa0d6b270aac2c6ba73a97444b9267adde983a5b7e34f17e45a/nodejs_wheel_binaries-22.20.0-py2.py3-none-win_amd64.whl", hash = "sha256:4bd658962f24958503541963e5a6f2cc512a8cb301e48a69dc03c879f40a28ae", size = 40120431, upload-time = "2025-09-26T09:47:54.363Z" }, - { url = "https://files.pythonhosted.org/packages/42/b1/6a4eb2c6e9efa028074b0001b61008c9d202b6b46caee9e5d1b18c088216/nodejs_wheel_binaries-22.20.0-py2.py3-none-win_arm64.whl", hash = "sha256:1fccac931faa210d22b6962bcdbc99269d16221d831b9a118bbb80fe434a60b8", size = 38844133, upload-time = "2025-09-26T09:47:57.357Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5f/be5a4112e678143d4c15264d918f9a2dc086905c6426eb44515cf391a958/nodejs_wheel_binaries-24.11.1-py2.py3-none-macosx_13_0_arm64.whl", hash = "sha256:0e14874c3579def458245cdbc3239e37610702b0aa0975c1dc55e2cb80e42102", size = 55114309, upload-time = "2025-11-18T18:21:21.697Z" }, + { url = "https://files.pythonhosted.org/packages/fa/1c/2e9d6af2ea32b65928c42b3e5baa7a306870711d93c3536cb25fc090a80d/nodejs_wheel_binaries-24.11.1-py2.py3-none-macosx_13_0_x86_64.whl", hash = "sha256:c2741525c9874b69b3e5a6d6c9179a6fe484ea0c3d5e7b7c01121c8e5d78b7e2", size = 55285957, upload-time = "2025-11-18T18:21:27.177Z" }, + { url = "https://files.pythonhosted.org/packages/d0/79/35696d7ba41b1bd35ef8682f13d46ba38c826c59e58b86b267458eb53d87/nodejs_wheel_binaries-24.11.1-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:5ef598101b0fb1c2bf643abb76dfbf6f76f1686198ed17ae46009049ee83c546", size = 59645875, upload-time = "2025-11-18T18:21:33.004Z" }, + { url = "https://files.pythonhosted.org/packages/b4/98/2a9694adee0af72bc602a046b0632a0c89e26586090c558b1c9199b187cc/nodejs_wheel_binaries-24.11.1-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:cde41d5e4705266688a8d8071debf4f8a6fcea264c61292782672ee75a6905f9", size = 60140941, upload-time = "2025-11-18T18:21:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d6/573e5e2cba9d934f5f89d0beab00c3315e2e6604eb4df0fcd1d80c5a07a8/nodejs_wheel_binaries-24.11.1-py2.py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:78bc5bb889313b565df8969bb7423849a9c7fc218bf735ff0ce176b56b3e96f0", size = 61644243, upload-time = "2025-11-18T18:21:43.325Z" }, + { url = "https://files.pythonhosted.org/packages/c7/e6/643234d5e94067df8ce8d7bba10f3804106668f7a1050aeb10fdd226ead4/nodejs_wheel_binaries-24.11.1-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:c79a7e43869ccecab1cae8183778249cceb14ca2de67b5650b223385682c6239", size = 62225657, upload-time = "2025-11-18T18:21:47.708Z" }, + { url = "https://files.pythonhosted.org/packages/4d/1c/2fb05127102a80225cab7a75c0e9edf88a0a1b79f912e1e36c7c1aaa8f4e/nodejs_wheel_binaries-24.11.1-py2.py3-none-win_amd64.whl", hash = "sha256:10197b1c9c04d79403501766f76508b0dac101ab34371ef8a46fcf51773497d0", size = 41322308, upload-time = "2025-11-18T18:21:51.347Z" }, + { url = "https://files.pythonhosted.org/packages/ad/b7/bc0cdbc2cc3a66fcac82c79912e135a0110b37b790a14c477f18e18d90cd/nodejs_wheel_binaries-24.11.1-py2.py3-none-win_arm64.whl", hash = "sha256:376b9ea1c4bc1207878975dfeb604f7aa5668c260c6154dcd2af9d42f7734116", size = 39026497, upload-time = "2025-11-18T18:21:54.634Z" }, ] [[package]] @@ -3768,7 +3775,7 @@ wheels = [ [[package]] name = "openai" -version = "2.7.2" +version = "2.8.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -3780,9 +3787,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/71/e3/cec27fa28ef36c4ccea71e9e8c20be9b8539618732989a82027575aab9d4/openai-2.7.2.tar.gz", hash = "sha256:082ef61163074d8efad0035dd08934cf5e3afd37254f70fc9165dd6a8c67dcbd", size = 595732, upload-time = "2025-11-10T16:42:31.108Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/e4/42591e356f1d53c568418dc7e30dcda7be31dd5a4d570bca22acb0525862/openai-2.8.1.tar.gz", hash = "sha256:cb1b79eef6e809f6da326a7ef6038719e35aa944c42d081807bfa1be8060f15f", size = 602490, upload-time = "2025-11-17T22:39:59.549Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/25/66/22cfe4b695b5fd042931b32c67d685e867bfd169ebf46036b95b57314c33/openai-2.7.2-py3-none-any.whl", hash = "sha256:116f522f4427f8a0a59b51655a356da85ce092f3ed6abeca65f03c8be6e073d9", size = 1008375, upload-time = "2025-11-10T16:42:28.574Z" }, + { url = "https://files.pythonhosted.org/packages/55/4f/dbc0c124c40cb390508a82770fb9f6e3ed162560181a85089191a851c59a/openai-2.8.1-py3-none-any.whl", hash = "sha256:c6c3b5a04994734386e8dad3c00a393f56d3b68a27cd2e8acae91a59e4122463", size = 1022688, upload-time = "2025-11-17T22:39:57.675Z" }, ] [[package]] @@ -4504,7 +4511,7 @@ wheels = [ [[package]] name = "posthog" -version = "7.0.0" +version = "7.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, @@ -4514,9 +4521,9 @@ dependencies = [ { name = "six" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/15/4d/16d777528149cd0e06306973081b5b070506abcd0fe831c6cb6966260d59/posthog-7.0.0.tar.gz", hash = "sha256:94973227f5fe5e7d656d305ff48c8bff3d505fd1e78b6fcd7ccc9dfe8d3401c2", size = 126504, upload-time = "2025-11-11T18:13:06.986Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/d4/b9afe855a8a7a1bf4459c28ae4c300b40338122dc850acabefcf2c3df24d/posthog-7.0.1.tar.gz", hash = "sha256:21150562c2630a599c1d7eac94bc5c64eb6f6acbf3ff52ccf1e57345706db05a", size = 126985, upload-time = "2025-11-15T12:44:22.465Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/9a/dc29b9ff4e5233a3c071b6b4c85dba96f4fcb9169c460bc81abd98555fb3/posthog-7.0.0-py3-none-any.whl", hash = "sha256:676d8a5197a17bf7bd00e31020a5f232988f249f57aab532f0d01c6243835934", size = 144727, upload-time = "2025-11-11T18:13:05.444Z" }, + { url = "https://files.pythonhosted.org/packages/05/0c/8b6b20b0be71725e6e8a32dcd460cdbf62fe6df9bc656a650150dc98fedd/posthog-7.0.1-py3-none-any.whl", hash = "sha256:efe212d8d88a9ba80a20c588eab4baf4b1a5e90e40b551160a5603bb21e96904", size = 145234, upload-time = "2025-11-15T12:44:21.247Z" }, ] [[package]] @@ -4893,7 +4900,7 @@ wheels = [ [[package]] name = "pyobvector" -version = "0.2.19" +version = "0.2.20" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiomysql" }, @@ -4903,17 +4910,18 @@ dependencies = [ { name = "sqlalchemy" }, { name = "sqlglot" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/19/9a/03da0d77f6033694ab7e7214abdd48c372102a185142db880ba00d6a6172/pyobvector-0.2.19.tar.gz", hash = "sha256:5e6847f08679cf6ded800b5b8ae89353173c33f5d90fd1392f55e5fafa4fb886", size = 46314, upload-time = "2025-11-10T08:30:10.186Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/6f/24ae2d4ba811e5e112c89bb91ba7c50eb79658563650c8fc65caa80655f8/pyobvector-0.2.20.tar.gz", hash = "sha256:72a54044632ba3bb27d340fb660c50b22548d34c6a9214b6653bc18eee4287c4", size = 46648, upload-time = "2025-11-20T09:30:16.354Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/72/48/d6b60ae86a2a2c0c607a33e0c8fc9e469500e06e5bb07ea7e9417910f458/pyobvector-0.2.19-py3-none-any.whl", hash = "sha256:0a6b93c950722ecbab72571e0ab81d0f8f4d1f52df9c25c00693392477e45e4b", size = 59886, upload-time = "2025-11-10T08:30:08.627Z" }, + { url = "https://files.pythonhosted.org/packages/ae/21/630c4e9f0d30b7a6eebe0590cd97162e82a2d3ac4ed3a33259d0a67e0861/pyobvector-0.2.20-py3-none-any.whl", hash = "sha256:9a3c1d3eb5268eae64185f8807b10fd182f271acf33323ee731c2ad554d1c076", size = 60131, upload-time = "2025-11-20T09:30:14.88Z" }, ] [[package]] name = "pypandoc" -version = "1.16" +version = "1.16.2" source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/18/9f5f70567b97758625335209b98d5cb857e19aa1a9306e9749567a240634/pypandoc-1.16.2.tar.gz", hash = "sha256:7a72a9fbf4a5dc700465e384c3bb333d22220efc4e972cb98cf6fc723cdca86b", size = 31477, upload-time = "2025-11-13T16:30:29.608Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/24/77/af1fc54740a0712988f9518e629d38edc7b8ffccd7549203f19c3d8a2db6/pypandoc-1.16-py3-none-any.whl", hash = "sha256:868f390d48388743e7a5885915cbbaa005dea36a825ecdfd571f8c523416c822", size = 19425, upload-time = "2025-11-08T15:44:38.429Z" }, + { url = "https://files.pythonhosted.org/packages/bb/e9/b145683854189bba84437ea569bfa786f408c8dc5bc16d8eb0753f5583bf/pypandoc-1.16.2-py3-none-any.whl", hash = "sha256:c200c1139c8e3247baf38d1e9279e85d9f162499d1999c6aa8418596558fe79b", size = 19451, upload-time = "2025-11-13T16:30:07.66Z" }, ] [[package]] @@ -4927,11 +4935,11 @@ wheels = [ [[package]] name = "pypdf" -version = "6.2.0" +version = "6.4.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4e/2b/8795ec0378384000b0a37a2b5e6d67fa3d84802945aa2c612a78a784d7d4/pypdf-6.2.0.tar.gz", hash = "sha256:46b4d8495d68ae9c818e7964853cd9984e6a04c19fe7112760195395992dce48", size = 5272001, upload-time = "2025-11-09T11:10:41.911Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/01/f7510cc6124f494cfbec2e8d3c2e1a20d4f6c18622b0c03a3a70e968bacb/pypdf-6.4.0.tar.gz", hash = "sha256:4769d471f8ddc3341193ecc5d6560fa44cf8cd0abfabf21af4e195cc0c224072", size = 5276661, upload-time = "2025-11-23T14:04:43.185Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/de/ba/743ddcaf1a8fb439342399645921e2cf2c600464cba5531a11f1cc0822b6/pypdf-6.2.0-py3-none-any.whl", hash = "sha256:4c0f3e62677217a777ab79abe22bf1285442d70efabf552f61c7a03b6f5c569f", size = 326592, upload-time = "2025-11-09T11:10:39.941Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f2/9c9429411c91ac1dd5cd66780f22b6df20c64c3646cdd1e6d67cf38579c4/pypdf-6.4.0-py3-none-any.whl", hash = "sha256:55ab9837ed97fd7fcc5c131d52fcc2223bc5c6b8a1488bbf7c0e27f1f0023a79", size = 329497, upload-time = "2025-11-23T14:04:41.448Z" }, ] [[package]] @@ -5144,11 +5152,11 @@ wheels = [ [[package]] name = "python-iso639" -version = "2025.11.11" +version = "2025.11.16" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/89/6f/45bc5ae1c132ab7852a8642d66d25ffff6e4b398195127ac66158d3b5f4d/python_iso639-2025.11.11.tar.gz", hash = "sha256:75fab30f1a0f46b4e8161eafb84afe4ecd07eaada05e2c5364f14b0f9c864477", size = 173897, upload-time = "2025-11-11T15:23:00.893Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/3b/3e07aadeeb7bbb2574d6aa6ccacbc58b17bd2b1fb6c7196bf96ab0e45129/python_iso639-2025.11.16.tar.gz", hash = "sha256:aabe941267898384415a509f5236d7cfc191198c84c5c6f73dac73d9783f5169", size = 174186, upload-time = "2025-11-16T21:53:37.031Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/69/081960288e4cd541cbdb90e1768373e1198b040bf2ae40cd25b9c9799205/python_iso639-2025.11.11-py3-none-any.whl", hash = "sha256:02ea4cfca2c189b5665e4e8adc8c17c62ab6e4910932541a23baddea33207ea2", size = 167723, upload-time = "2025-11-11T15:22:59.819Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2d/563849c31e58eb2e273fa0c391a7d9987db32f4d9152fe6ecdac0a8ffe93/python_iso639-2025.11.16-py3-none-any.whl", hash = "sha256:65f6ac6c6d8e8207f6175f8bf7fff7db486c6dc5c1d8866c2b77d2a923370896", size = 167818, upload-time = "2025-11-16T21:53:35.36Z" }, ] [[package]] @@ -5477,52 +5485,52 @@ wheels = [ [[package]] name = "rpds-py" -version = "0.28.0" +version = "0.29.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/48/dc/95f074d43452b3ef5d06276696ece4b3b5d696e7c9ad7173c54b1390cd70/rpds_py-0.28.0.tar.gz", hash = "sha256:abd4df20485a0983e2ca334a216249b6186d6e3c1627e106651943dbdb791aea", size = 27419, upload-time = "2025-10-22T22:24:29.327Z" } +sdist = { url = "https://files.pythonhosted.org/packages/98/33/23b3b3419b6a3e0f559c7c0d2ca8fc1b9448382b25245033788785921332/rpds_py-0.29.0.tar.gz", hash = "sha256:fe55fe686908f50154d1dc599232016e50c243b438c3b7432f24e2895b0e5359", size = 69359, upload-time = "2025-11-16T14:50:39.532Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/34/058d0db5471c6be7bef82487ad5021ff8d1d1d27794be8730aad938649cf/rpds_py-0.28.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:03065002fd2e287725d95fbc69688e0c6daf6c6314ba38bdbaa3895418e09296", size = 362344, upload-time = "2025-10-22T22:21:39.713Z" }, - { url = "https://files.pythonhosted.org/packages/5d/67/9503f0ec8c055a0782880f300c50a2b8e5e72eb1f94dfc2053da527444dd/rpds_py-0.28.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28ea02215f262b6d078daec0b45344c89e161eab9526b0d898221d96fdda5f27", size = 348440, upload-time = "2025-10-22T22:21:41.056Z" }, - { url = "https://files.pythonhosted.org/packages/68/2e/94223ee9b32332a41d75b6f94b37b4ce3e93878a556fc5f152cbd856a81f/rpds_py-0.28.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25dbade8fbf30bcc551cb352376c0ad64b067e4fc56f90e22ba70c3ce205988c", size = 379068, upload-time = "2025-10-22T22:21:42.593Z" }, - { url = "https://files.pythonhosted.org/packages/b4/25/54fd48f9f680cfc44e6a7f39a5fadf1d4a4a1fd0848076af4a43e79f998c/rpds_py-0.28.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c03002f54cc855860bfdc3442928ffdca9081e73b5b382ed0b9e8efe6e5e205", size = 390518, upload-time = "2025-10-22T22:21:43.998Z" }, - { url = "https://files.pythonhosted.org/packages/1b/85/ac258c9c27f2ccb1bd5d0697e53a82ebcf8088e3186d5d2bf8498ee7ed44/rpds_py-0.28.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9699fa7990368b22032baf2b2dce1f634388e4ffc03dfefaaac79f4695edc95", size = 525319, upload-time = "2025-10-22T22:21:45.645Z" }, - { url = "https://files.pythonhosted.org/packages/40/cb/c6734774789566d46775f193964b76627cd5f42ecf246d257ce84d1912ed/rpds_py-0.28.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b9b06fe1a75e05e0713f06ea0c89ecb6452210fd60e2f1b6ddc1067b990e08d9", size = 404896, upload-time = "2025-10-22T22:21:47.544Z" }, - { url = "https://files.pythonhosted.org/packages/1f/53/14e37ce83202c632c89b0691185dca9532288ff9d390eacae3d2ff771bae/rpds_py-0.28.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9f83e7b326a3f9ec3ef84cda98fb0a74c7159f33e692032233046e7fd15da2", size = 382862, upload-time = "2025-10-22T22:21:49.176Z" }, - { url = "https://files.pythonhosted.org/packages/6a/83/f3642483ca971a54d60caa4449f9d6d4dbb56a53e0072d0deff51b38af74/rpds_py-0.28.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:0d3259ea9ad8743a75a43eb7819324cdab393263c91be86e2d1901ee65c314e0", size = 398848, upload-time = "2025-10-22T22:21:51.024Z" }, - { url = "https://files.pythonhosted.org/packages/44/09/2d9c8b2f88e399b4cfe86efdf2935feaf0394e4f14ab30c6c5945d60af7d/rpds_py-0.28.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a7548b345f66f6695943b4ef6afe33ccd3f1b638bd9afd0f730dd255c249c9e", size = 412030, upload-time = "2025-10-22T22:21:52.665Z" }, - { url = "https://files.pythonhosted.org/packages/dd/f5/e1cec473d4bde6df1fd3738be8e82d64dd0600868e76e92dfeaebbc2d18f/rpds_py-0.28.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9a40040aa388b037eb39416710fbcce9443498d2eaab0b9b45ae988b53f5c67", size = 559700, upload-time = "2025-10-22T22:21:54.123Z" }, - { url = "https://files.pythonhosted.org/packages/8d/be/73bb241c1649edbf14e98e9e78899c2c5e52bbe47cb64811f44d2cc11808/rpds_py-0.28.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8f60c7ea34e78c199acd0d3cda37a99be2c861dd2b8cf67399784f70c9f8e57d", size = 584581, upload-time = "2025-10-22T22:21:56.102Z" }, - { url = "https://files.pythonhosted.org/packages/9c/9c/ffc6e9218cd1eb5c2c7dbd276c87cd10e8c2232c456b554169eb363381df/rpds_py-0.28.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1571ae4292649100d743b26d5f9c63503bb1fedf538a8f29a98dce2d5ba6b4e6", size = 549981, upload-time = "2025-10-22T22:21:58.253Z" }, - { url = "https://files.pythonhosted.org/packages/5f/50/da8b6d33803a94df0149345ee33e5d91ed4d25fc6517de6a25587eae4133/rpds_py-0.28.0-cp311-cp311-win32.whl", hash = "sha256:5cfa9af45e7c1140af7321fa0bef25b386ee9faa8928c80dc3a5360971a29e8c", size = 214729, upload-time = "2025-10-22T22:21:59.625Z" }, - { url = "https://files.pythonhosted.org/packages/12/fd/b0f48c4c320ee24c8c20df8b44acffb7353991ddf688af01eef5f93d7018/rpds_py-0.28.0-cp311-cp311-win_amd64.whl", hash = "sha256:dd8d86b5d29d1b74100982424ba53e56033dc47720a6de9ba0259cf81d7cecaa", size = 223977, upload-time = "2025-10-22T22:22:01.092Z" }, - { url = "https://files.pythonhosted.org/packages/b4/21/c8e77a2ac66e2ec4e21f18a04b4e9a0417ecf8e61b5eaeaa9360a91713b4/rpds_py-0.28.0-cp311-cp311-win_arm64.whl", hash = "sha256:4e27d3a5709cc2b3e013bf93679a849213c79ae0573f9b894b284b55e729e120", size = 217326, upload-time = "2025-10-22T22:22:02.944Z" }, - { url = "https://files.pythonhosted.org/packages/b8/5c/6c3936495003875fe7b14f90ea812841a08fca50ab26bd840e924097d9c8/rpds_py-0.28.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6b4f28583a4f247ff60cd7bdda83db8c3f5b05a7a82ff20dd4b078571747708f", size = 366439, upload-time = "2025-10-22T22:22:04.525Z" }, - { url = "https://files.pythonhosted.org/packages/56/f9/a0f1ca194c50aa29895b442771f036a25b6c41a35e4f35b1a0ea713bedae/rpds_py-0.28.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d678e91b610c29c4b3d52a2c148b641df2b4676ffe47c59f6388d58b99cdc424", size = 348170, upload-time = "2025-10-22T22:22:06.397Z" }, - { url = "https://files.pythonhosted.org/packages/18/ea/42d243d3a586beb72c77fa5def0487daf827210069a95f36328e869599ea/rpds_py-0.28.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e819e0e37a44a78e1383bf1970076e2ccc4dc8c2bbaa2f9bd1dc987e9afff628", size = 378838, upload-time = "2025-10-22T22:22:07.932Z" }, - { url = "https://files.pythonhosted.org/packages/e7/78/3de32e18a94791af8f33601402d9d4f39613136398658412a4e0b3047327/rpds_py-0.28.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5ee514e0f0523db5d3fb171f397c54875dbbd69760a414dccf9d4d7ad628b5bd", size = 393299, upload-time = "2025-10-22T22:22:09.435Z" }, - { url = "https://files.pythonhosted.org/packages/13/7e/4bdb435afb18acea2eb8a25ad56b956f28de7c59f8a1d32827effa0d4514/rpds_py-0.28.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f3fa06d27fdcee47f07a39e02862da0100cb4982508f5ead53ec533cd5fe55e", size = 518000, upload-time = "2025-10-22T22:22:11.326Z" }, - { url = "https://files.pythonhosted.org/packages/31/d0/5f52a656875cdc60498ab035a7a0ac8f399890cc1ee73ebd567bac4e39ae/rpds_py-0.28.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46959ef2e64f9e4a41fc89aa20dbca2b85531f9a72c21099a3360f35d10b0d5a", size = 408746, upload-time = "2025-10-22T22:22:13.143Z" }, - { url = "https://files.pythonhosted.org/packages/3e/cd/49ce51767b879cde77e7ad9fae164ea15dce3616fe591d9ea1df51152706/rpds_py-0.28.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8455933b4bcd6e83fde3fefc987a023389c4b13f9a58c8d23e4b3f6d13f78c84", size = 386379, upload-time = "2025-10-22T22:22:14.602Z" }, - { url = "https://files.pythonhosted.org/packages/6a/99/e4e1e1ee93a98f72fc450e36c0e4d99c35370220e815288e3ecd2ec36a2a/rpds_py-0.28.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:ad50614a02c8c2962feebe6012b52f9802deec4263946cddea37aaf28dd25a66", size = 401280, upload-time = "2025-10-22T22:22:16.063Z" }, - { url = "https://files.pythonhosted.org/packages/61/35/e0c6a57488392a8b319d2200d03dad2b29c0db9996f5662c3b02d0b86c02/rpds_py-0.28.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e5deca01b271492553fdb6c7fd974659dce736a15bae5dad7ab8b93555bceb28", size = 412365, upload-time = "2025-10-22T22:22:17.504Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6a/841337980ea253ec797eb084665436007a1aad0faac1ba097fb906c5f69c/rpds_py-0.28.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:735f8495a13159ce6a0d533f01e8674cec0c57038c920495f87dcb20b3ddb48a", size = 559573, upload-time = "2025-10-22T22:22:19.108Z" }, - { url = "https://files.pythonhosted.org/packages/e7/5e/64826ec58afd4c489731f8b00729c5f6afdb86f1df1df60bfede55d650bb/rpds_py-0.28.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:961ca621ff10d198bbe6ba4957decca61aa2a0c56695384c1d6b79bf61436df5", size = 583973, upload-time = "2025-10-22T22:22:20.768Z" }, - { url = "https://files.pythonhosted.org/packages/b6/ee/44d024b4843f8386a4eeaa4c171b3d31d55f7177c415545fd1a24c249b5d/rpds_py-0.28.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2374e16cc9131022e7d9a8f8d65d261d9ba55048c78f3b6e017971a4f5e6353c", size = 553800, upload-time = "2025-10-22T22:22:22.25Z" }, - { url = "https://files.pythonhosted.org/packages/7d/89/33e675dccff11a06d4d85dbb4d1865f878d5020cbb69b2c1e7b2d3f82562/rpds_py-0.28.0-cp312-cp312-win32.whl", hash = "sha256:d15431e334fba488b081d47f30f091e5d03c18527c325386091f31718952fe08", size = 216954, upload-time = "2025-10-22T22:22:24.105Z" }, - { url = "https://files.pythonhosted.org/packages/af/36/45f6ebb3210887e8ee6dbf1bc710ae8400bb417ce165aaf3024b8360d999/rpds_py-0.28.0-cp312-cp312-win_amd64.whl", hash = "sha256:a410542d61fc54710f750d3764380b53bf09e8c4edbf2f9141a82aa774a04f7c", size = 227844, upload-time = "2025-10-22T22:22:25.551Z" }, - { url = "https://files.pythonhosted.org/packages/57/91/f3fb250d7e73de71080f9a221d19bd6a1c1eb0d12a1ea26513f6c1052ad6/rpds_py-0.28.0-cp312-cp312-win_arm64.whl", hash = "sha256:1f0cfd1c69e2d14f8c892b893997fa9a60d890a0c8a603e88dca4955f26d1edd", size = 217624, upload-time = "2025-10-22T22:22:26.914Z" }, - { url = "https://files.pythonhosted.org/packages/ae/bc/b43f2ea505f28119bd551ae75f70be0c803d2dbcd37c1b3734909e40620b/rpds_py-0.28.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f5e7101145427087e493b9c9b959da68d357c28c562792300dd21a095118ed16", size = 363913, upload-time = "2025-10-22T22:24:07.129Z" }, - { url = "https://files.pythonhosted.org/packages/28/f2/db318195d324c89a2c57dc5195058cbadd71b20d220685c5bd1da79ee7fe/rpds_py-0.28.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:31eb671150b9c62409a888850aaa8e6533635704fe2b78335f9aaf7ff81eec4d", size = 350452, upload-time = "2025-10-22T22:24:08.754Z" }, - { url = "https://files.pythonhosted.org/packages/ae/f2/1391c819b8573a4898cedd6b6c5ec5bc370ce59e5d6bdcebe3c9c1db4588/rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48b55c1f64482f7d8bd39942f376bfdf2f6aec637ee8c805b5041e14eeb771db", size = 380957, upload-time = "2025-10-22T22:24:10.826Z" }, - { url = "https://files.pythonhosted.org/packages/5a/5c/e5de68ee7eb7248fce93269833d1b329a196d736aefb1a7481d1e99d1222/rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24743a7b372e9a76171f6b69c01aedf927e8ac3e16c474d9fe20d552a8cb45c7", size = 391919, upload-time = "2025-10-22T22:24:12.559Z" }, - { url = "https://files.pythonhosted.org/packages/fb/4f/2376336112cbfeb122fd435d608ad8d5041b3aed176f85a3cb32c262eb80/rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:389c29045ee8bbb1627ea190b4976a310a295559eaf9f1464a1a6f2bf84dde78", size = 528541, upload-time = "2025-10-22T22:24:14.197Z" }, - { url = "https://files.pythonhosted.org/packages/68/53/5ae232e795853dd20da7225c5dd13a09c0a905b1a655e92bdf8d78a99fd9/rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23690b5827e643150cf7b49569679ec13fe9a610a15949ed48b85eb7f98f34ec", size = 405629, upload-time = "2025-10-22T22:24:16.001Z" }, - { url = "https://files.pythonhosted.org/packages/b9/2d/351a3b852b683ca9b6b8b38ed9efb2347596973849ba6c3a0e99877c10aa/rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f0c9266c26580e7243ad0d72fc3e01d6b33866cfab5084a6da7576bcf1c4f72", size = 384123, upload-time = "2025-10-22T22:24:17.585Z" }, - { url = "https://files.pythonhosted.org/packages/e0/15/870804daa00202728cc91cb8e2385fa9f1f4eb49857c49cfce89e304eae6/rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:4c6c4db5d73d179746951486df97fd25e92396be07fc29ee8ff9a8f5afbdfb27", size = 400923, upload-time = "2025-10-22T22:24:19.512Z" }, - { url = "https://files.pythonhosted.org/packages/53/25/3706b83c125fa2a0bccceac951de3f76631f6bd0ee4d02a0ed780712ef1b/rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3b695a8fa799dd2cfdb4804b37096c5f6dba1ac7f48a7fbf6d0485bcd060316", size = 413767, upload-time = "2025-10-22T22:24:21.316Z" }, - { url = "https://files.pythonhosted.org/packages/ef/f9/ce43dbe62767432273ed2584cef71fef8411bddfb64125d4c19128015018/rpds_py-0.28.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:6aa1bfce3f83baf00d9c5fcdbba93a3ab79958b4c7d7d1f55e7fe68c20e63912", size = 561530, upload-time = "2025-10-22T22:24:22.958Z" }, - { url = "https://files.pythonhosted.org/packages/46/c9/ffe77999ed8f81e30713dd38fd9ecaa161f28ec48bb80fa1cd9118399c27/rpds_py-0.28.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:7b0f9dceb221792b3ee6acb5438eb1f02b0cb2c247796a72b016dcc92c6de829", size = 585453, upload-time = "2025-10-22T22:24:24.779Z" }, - { url = "https://files.pythonhosted.org/packages/ed/d2/4a73b18821fd4669762c855fd1f4e80ceb66fb72d71162d14da58444a763/rpds_py-0.28.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:5d0145edba8abd3db0ab22b5300c99dc152f5c9021fab861be0f0544dc3cbc5f", size = 552199, upload-time = "2025-10-22T22:24:26.54Z" }, + { url = "https://files.pythonhosted.org/packages/36/ab/7fb95163a53ab122c74a7c42d2d2f012819af2cf3deb43fb0d5acf45cc1a/rpds_py-0.29.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:9b9c764a11fd637e0322a488560533112837f5334ffeb48b1be20f6d98a7b437", size = 372344, upload-time = "2025-11-16T14:47:57.279Z" }, + { url = "https://files.pythonhosted.org/packages/b3/45/f3c30084c03b0d0f918cb4c5ae2c20b0a148b51ba2b3f6456765b629bedd/rpds_py-0.29.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3fd2164d73812026ce970d44c3ebd51e019d2a26a4425a5dcbdfa93a34abc383", size = 363041, upload-time = "2025-11-16T14:47:58.908Z" }, + { url = "https://files.pythonhosted.org/packages/e3/e9/4d044a1662608c47a87cbb37b999d4d5af54c6d6ebdda93a4d8bbf8b2a10/rpds_py-0.29.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a097b7f7f7274164566ae90a221fd725363c0e9d243e2e9ed43d195ccc5495c", size = 391775, upload-time = "2025-11-16T14:48:00.197Z" }, + { url = "https://files.pythonhosted.org/packages/50/c9/7616d3ace4e6731aeb6e3cd85123e03aec58e439044e214b9c5c60fd8eb1/rpds_py-0.29.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7cdc0490374e31cedefefaa1520d5fe38e82fde8748cbc926e7284574c714d6b", size = 405624, upload-time = "2025-11-16T14:48:01.496Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e2/6d7d6941ca0843609fd2d72c966a438d6f22617baf22d46c3d2156c31350/rpds_py-0.29.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89ca2e673ddd5bde9b386da9a0aac0cab0e76f40c8f0aaf0d6311b6bbf2aa311", size = 527894, upload-time = "2025-11-16T14:48:03.167Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f7/aee14dc2db61bb2ae1e3068f134ca9da5f28c586120889a70ff504bb026f/rpds_py-0.29.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5d9da3ff5af1ca1249b1adb8ef0573b94c76e6ae880ba1852f033bf429d4588", size = 412720, upload-time = "2025-11-16T14:48:04.413Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e2/2293f236e887c0360c2723d90c00d48dee296406994d6271faf1712e94ec/rpds_py-0.29.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8238d1d310283e87376c12f658b61e1ee23a14c0e54c7c0ce953efdbdc72deed", size = 392945, upload-time = "2025-11-16T14:48:06.252Z" }, + { url = "https://files.pythonhosted.org/packages/14/cd/ceea6147acd3bd1fd028d1975228f08ff19d62098078d5ec3eed49703797/rpds_py-0.29.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:2d6fb2ad1c36f91c4646989811e84b1ea5e0c3cf9690b826b6e32b7965853a63", size = 406385, upload-time = "2025-11-16T14:48:07.575Z" }, + { url = "https://files.pythonhosted.org/packages/52/36/fe4dead19e45eb77a0524acfdbf51e6cda597b26fc5b6dddbff55fbbb1a5/rpds_py-0.29.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:534dc9df211387547267ccdb42253aa30527482acb38dd9b21c5c115d66a96d2", size = 423943, upload-time = "2025-11-16T14:48:10.175Z" }, + { url = "https://files.pythonhosted.org/packages/a1/7b/4551510803b582fa4abbc8645441a2d15aa0c962c3b21ebb380b7e74f6a1/rpds_py-0.29.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d456e64724a075441e4ed648d7f154dc62e9aabff29bcdf723d0c00e9e1d352f", size = 574204, upload-time = "2025-11-16T14:48:11.499Z" }, + { url = "https://files.pythonhosted.org/packages/64/ba/071ccdd7b171e727a6ae079f02c26f75790b41555f12ca8f1151336d2124/rpds_py-0.29.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a738f2da2f565989401bd6fd0b15990a4d1523c6d7fe83f300b7e7d17212feca", size = 600587, upload-time = "2025-11-16T14:48:12.822Z" }, + { url = "https://files.pythonhosted.org/packages/03/09/96983d48c8cf5a1e03c7d9cc1f4b48266adfb858ae48c7c2ce978dbba349/rpds_py-0.29.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a110e14508fd26fd2e472bb541f37c209409876ba601cf57e739e87d8a53cf95", size = 562287, upload-time = "2025-11-16T14:48:14.108Z" }, + { url = "https://files.pythonhosted.org/packages/40/f0/8c01aaedc0fa92156f0391f39ea93b5952bc0ec56b897763858f95da8168/rpds_py-0.29.0-cp311-cp311-win32.whl", hash = "sha256:923248a56dd8d158389a28934f6f69ebf89f218ef96a6b216a9be6861804d3f4", size = 221394, upload-time = "2025-11-16T14:48:15.374Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a5/a8b21c54c7d234efdc83dc034a4d7cd9668e3613b6316876a29b49dece71/rpds_py-0.29.0-cp311-cp311-win_amd64.whl", hash = "sha256:539eb77eb043afcc45314d1be09ea6d6cafb3addc73e0547c171c6d636957f60", size = 235713, upload-time = "2025-11-16T14:48:16.636Z" }, + { url = "https://files.pythonhosted.org/packages/a7/1f/df3c56219523947b1be402fa12e6323fe6d61d883cf35d6cb5d5bb6db9d9/rpds_py-0.29.0-cp311-cp311-win_arm64.whl", hash = "sha256:bdb67151ea81fcf02d8f494703fb728d4d34d24556cbff5f417d74f6f5792e7c", size = 229157, upload-time = "2025-11-16T14:48:17.891Z" }, + { url = "https://files.pythonhosted.org/packages/3c/50/bc0e6e736d94e420df79be4deb5c9476b63165c87bb8f19ef75d100d21b3/rpds_py-0.29.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0891cfd8db43e085c0ab93ab7e9b0c8fee84780d436d3b266b113e51e79f954", size = 376000, upload-time = "2025-11-16T14:48:19.141Z" }, + { url = "https://files.pythonhosted.org/packages/3e/3a/46676277160f014ae95f24de53bed0e3b7ea66c235e7de0b9df7bd5d68ba/rpds_py-0.29.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3897924d3f9a0361472d884051f9a2460358f9a45b1d85a39a158d2f8f1ad71c", size = 360575, upload-time = "2025-11-16T14:48:20.443Z" }, + { url = "https://files.pythonhosted.org/packages/75/ba/411d414ed99ea1afdd185bbabeeaac00624bd1e4b22840b5e9967ade6337/rpds_py-0.29.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21deb8e0d1571508c6491ce5ea5e25669b1dd4adf1c9d64b6314842f708b5d", size = 392159, upload-time = "2025-11-16T14:48:22.12Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b1/e18aa3a331f705467a48d0296778dc1fea9d7f6cf675bd261f9a846c7e90/rpds_py-0.29.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9efe71687d6427737a0a2de9ca1c0a216510e6cd08925c44162be23ed7bed2d5", size = 410602, upload-time = "2025-11-16T14:48:23.563Z" }, + { url = "https://files.pythonhosted.org/packages/2f/6c/04f27f0c9f2299274c76612ac9d2c36c5048bb2c6c2e52c38c60bf3868d9/rpds_py-0.29.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:40f65470919dc189c833e86b2c4bd21bd355f98436a2cef9e0a9a92aebc8e57e", size = 515808, upload-time = "2025-11-16T14:48:24.949Z" }, + { url = "https://files.pythonhosted.org/packages/83/56/a8412aa464fb151f8bc0d91fb0bb888adc9039bd41c1c6ba8d94990d8cf8/rpds_py-0.29.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:def48ff59f181130f1a2cb7c517d16328efac3ec03951cca40c1dc2049747e83", size = 416015, upload-time = "2025-11-16T14:48:26.782Z" }, + { url = "https://files.pythonhosted.org/packages/04/4c/f9b8a05faca3d9e0a6397c90d13acb9307c9792b2bff621430c58b1d6e76/rpds_py-0.29.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad7bd570be92695d89285a4b373006930715b78d96449f686af422debb4d3949", size = 395325, upload-time = "2025-11-16T14:48:28.055Z" }, + { url = "https://files.pythonhosted.org/packages/34/60/869f3bfbf8ed7b54f1ad9a5543e0fdffdd40b5a8f587fe300ee7b4f19340/rpds_py-0.29.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:5a572911cd053137bbff8e3a52d31c5d2dba51d3a67ad902629c70185f3f2181", size = 410160, upload-time = "2025-11-16T14:48:29.338Z" }, + { url = "https://files.pythonhosted.org/packages/91/aa/e5b496334e3aba4fe4c8a80187b89f3c1294c5c36f2a926da74338fa5a73/rpds_py-0.29.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d583d4403bcbf10cffc3ab5cee23d7643fcc960dff85973fd3c2d6c86e8dbb0c", size = 425309, upload-time = "2025-11-16T14:48:30.691Z" }, + { url = "https://files.pythonhosted.org/packages/85/68/4e24a34189751ceb6d66b28f18159922828dd84155876551f7ca5b25f14f/rpds_py-0.29.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:070befbb868f257d24c3bb350dbd6e2f645e83731f31264b19d7231dd5c396c7", size = 574644, upload-time = "2025-11-16T14:48:31.964Z" }, + { url = "https://files.pythonhosted.org/packages/8c/cf/474a005ea4ea9c3b4f17b6108b6b13cebfc98ebaff11d6e1b193204b3a93/rpds_py-0.29.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:fc935f6b20b0c9f919a8ff024739174522abd331978f750a74bb68abd117bd19", size = 601605, upload-time = "2025-11-16T14:48:33.252Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b1/c56f6a9ab8c5f6bb5c65c4b5f8229167a3a525245b0773f2c0896686b64e/rpds_py-0.29.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8c5a8ecaa44ce2d8d9d20a68a2483a74c07f05d72e94a4dff88906c8807e77b0", size = 564593, upload-time = "2025-11-16T14:48:34.643Z" }, + { url = "https://files.pythonhosted.org/packages/b3/13/0494cecce4848f68501e0a229432620b4b57022388b071eeff95f3e1e75b/rpds_py-0.29.0-cp312-cp312-win32.whl", hash = "sha256:ba5e1aeaf8dd6d8f6caba1f5539cddda87d511331714b7b5fc908b6cfc3636b7", size = 223853, upload-time = "2025-11-16T14:48:36.419Z" }, + { url = "https://files.pythonhosted.org/packages/1f/6a/51e9aeb444a00cdc520b032a28b07e5f8dc7bc328b57760c53e7f96997b4/rpds_py-0.29.0-cp312-cp312-win_amd64.whl", hash = "sha256:b5f6134faf54b3cb83375db0f113506f8b7770785be1f95a631e7e2892101977", size = 239895, upload-time = "2025-11-16T14:48:37.956Z" }, + { url = "https://files.pythonhosted.org/packages/d1/d4/8bce56cdad1ab873e3f27cb31c6a51d8f384d66b022b820525b879f8bed1/rpds_py-0.29.0-cp312-cp312-win_arm64.whl", hash = "sha256:b016eddf00dca7944721bf0cd85b6af7f6c4efaf83ee0b37c4133bd39757a8c7", size = 230321, upload-time = "2025-11-16T14:48:39.71Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ac/b97e80bf107159e5b9ba9c91df1ab95f69e5e41b435f27bdd737f0d583ac/rpds_py-0.29.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:acd82a9e39082dc5f4492d15a6b6c8599aa21db5c35aaf7d6889aea16502c07d", size = 373963, upload-time = "2025-11-16T14:50:16.205Z" }, + { url = "https://files.pythonhosted.org/packages/40/5a/55e72962d5d29bd912f40c594e68880d3c7a52774b0f75542775f9250712/rpds_py-0.29.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:715b67eac317bf1c7657508170a3e011a1ea6ccb1c9d5f296e20ba14196be6b3", size = 364644, upload-time = "2025-11-16T14:50:18.22Z" }, + { url = "https://files.pythonhosted.org/packages/99/2a/6b6524d0191b7fc1351c3c0840baac42250515afb48ae40c7ed15499a6a2/rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3b1b87a237cb2dba4db18bcfaaa44ba4cd5936b91121b62292ff21df577fc43", size = 393847, upload-time = "2025-11-16T14:50:20.012Z" }, + { url = "https://files.pythonhosted.org/packages/1c/b8/c5692a7df577b3c0c7faed7ac01ee3c608b81750fc5d89f84529229b6873/rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1c3c3e8101bb06e337c88eb0c0ede3187131f19d97d43ea0e1c5407ea74c0cbf", size = 407281, upload-time = "2025-11-16T14:50:21.64Z" }, + { url = "https://files.pythonhosted.org/packages/f0/57/0546c6f84031b7ea08b76646a8e33e45607cc6bd879ff1917dc077bb881e/rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2b8e54d6e61f3ecd3abe032065ce83ea63417a24f437e4a3d73d2f85ce7b7cfe", size = 529213, upload-time = "2025-11-16T14:50:23.219Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c1/01dd5f444233605555bc11fe5fed6a5c18f379f02013870c176c8e630a23/rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3fbd4e9aebf110473a420dea85a238b254cf8a15acb04b22a5a6b5ce8925b760", size = 413808, upload-time = "2025-11-16T14:50:25.262Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0a/60f98b06156ea2a7af849fb148e00fbcfdb540909a5174a5ed10c93745c7/rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80fdf53d36e6c72819993e35d1ebeeb8e8fc688d0c6c2b391b55e335b3afba5a", size = 394600, upload-time = "2025-11-16T14:50:26.956Z" }, + { url = "https://files.pythonhosted.org/packages/37/f1/dc9312fc9bec040ece08396429f2bd9e0977924ba7a11c5ad7056428465e/rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:ea7173df5d86f625f8dde6d5929629ad811ed8decda3b60ae603903839ac9ac0", size = 408634, upload-time = "2025-11-16T14:50:28.989Z" }, + { url = "https://files.pythonhosted.org/packages/ed/41/65024c9fd40c89bb7d604cf73beda4cbdbcebe92d8765345dd65855b6449/rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:76054d540061eda273274f3d13a21a4abdde90e13eaefdc205db37c05230efce", size = 426064, upload-time = "2025-11-16T14:50:30.674Z" }, + { url = "https://files.pythonhosted.org/packages/a2/e0/cf95478881fc88ca2fdbf56381d7df36567cccc39a05394beac72182cd62/rpds_py-0.29.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:9f84c549746a5be3bc7415830747a3a0312573afc9f95785eb35228bb17742ec", size = 575871, upload-time = "2025-11-16T14:50:33.428Z" }, + { url = "https://files.pythonhosted.org/packages/ea/c0/df88097e64339a0218b57bd5f9ca49898e4c394db756c67fccc64add850a/rpds_py-0.29.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:0ea962671af5cb9a260489e311fa22b2e97103e3f9f0caaea6f81390af96a9ed", size = 601702, upload-time = "2025-11-16T14:50:36.051Z" }, + { url = "https://files.pythonhosted.org/packages/87/f4/09ffb3ebd0cbb9e2c7c9b84d252557ecf434cd71584ee1e32f66013824df/rpds_py-0.29.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:f7728653900035fb7b8d06e1e5900545d8088efc9d5d4545782da7df03ec803f", size = 564054, upload-time = "2025-11-16T14:50:37.733Z" }, ] [[package]] @@ -5539,28 +5547,28 @@ wheels = [ [[package]] name = "ruff" -version = "0.14.4" +version = "0.14.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/df/55/cccfca45157a2031dcbb5a462a67f7cf27f8b37d4b3b1cd7438f0f5c1df6/ruff-0.14.4.tar.gz", hash = "sha256:f459a49fe1085a749f15414ca76f61595f1a2cc8778ed7c279b6ca2e1fd19df3", size = 5587844, upload-time = "2025-11-06T22:07:45.033Z" } +sdist = { url = "https://files.pythonhosted.org/packages/52/f0/62b5a1a723fe183650109407fa56abb433b00aa1c0b9ba555f9c4efec2c6/ruff-0.14.6.tar.gz", hash = "sha256:6f0c742ca6a7783a736b867a263b9a7a80a45ce9bee391eeda296895f1b4e1cc", size = 5669501, upload-time = "2025-11-21T14:26:17.903Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/b9/67240254166ae1eaa38dec32265e9153ac53645a6c6670ed36ad00722af8/ruff-0.14.4-py3-none-linux_armv6l.whl", hash = "sha256:e6604613ffbcf2297cd5dcba0e0ac9bd0c11dc026442dfbb614504e87c349518", size = 12606781, upload-time = "2025-11-06T22:07:01.841Z" }, - { url = "https://files.pythonhosted.org/packages/46/c8/09b3ab245d8652eafe5256ab59718641429f68681ee713ff06c5c549f156/ruff-0.14.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d99c0b52b6f0598acede45ee78288e5e9b4409d1ce7f661f0fa36d4cbeadf9a4", size = 12946765, upload-time = "2025-11-06T22:07:05.858Z" }, - { url = "https://files.pythonhosted.org/packages/14/bb/1564b000219144bf5eed2359edc94c3590dd49d510751dad26202c18a17d/ruff-0.14.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9358d490ec030f1b51d048a7fd6ead418ed0826daf6149e95e30aa67c168af33", size = 11928120, upload-time = "2025-11-06T22:07:08.023Z" }, - { url = "https://files.pythonhosted.org/packages/a3/92/d5f1770e9988cc0742fefaa351e840d9aef04ec24ae1be36f333f96d5704/ruff-0.14.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81b40d27924f1f02dfa827b9c0712a13c0e4b108421665322218fc38caf615c2", size = 12370877, upload-time = "2025-11-06T22:07:10.015Z" }, - { url = "https://files.pythonhosted.org/packages/e2/29/e9282efa55f1973d109faf839a63235575519c8ad278cc87a182a366810e/ruff-0.14.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f5e649052a294fe00818650712083cddc6cc02744afaf37202c65df9ea52efa5", size = 12408538, upload-time = "2025-11-06T22:07:13.085Z" }, - { url = "https://files.pythonhosted.org/packages/8e/01/930ed6ecfce130144b32d77d8d69f5c610e6d23e6857927150adf5d7379a/ruff-0.14.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa082a8f878deeba955531f975881828fd6afd90dfa757c2b0808aadb437136e", size = 13141942, upload-time = "2025-11-06T22:07:15.386Z" }, - { url = "https://files.pythonhosted.org/packages/6a/46/a9c89b42b231a9f487233f17a89cbef9d5acd538d9488687a02ad288fa6b/ruff-0.14.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:1043c6811c2419e39011890f14d0a30470f19d47d197c4858b2787dfa698f6c8", size = 14544306, upload-time = "2025-11-06T22:07:17.631Z" }, - { url = "https://files.pythonhosted.org/packages/78/96/9c6cf86491f2a6d52758b830b89b78c2ae61e8ca66b86bf5a20af73d20e6/ruff-0.14.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a9f3a936ac27fb7c2a93e4f4b943a662775879ac579a433291a6f69428722649", size = 14210427, upload-time = "2025-11-06T22:07:19.832Z" }, - { url = "https://files.pythonhosted.org/packages/71/f4/0666fe7769a54f63e66404e8ff698de1dcde733e12e2fd1c9c6efb689cb5/ruff-0.14.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:95643ffd209ce78bc113266b88fba3d39e0461f0cbc8b55fb92505030fb4a850", size = 13658488, upload-time = "2025-11-06T22:07:22.32Z" }, - { url = "https://files.pythonhosted.org/packages/ee/79/6ad4dda2cfd55e41ac9ed6d73ef9ab9475b1eef69f3a85957210c74ba12c/ruff-0.14.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:456daa2fa1021bc86ca857f43fe29d5d8b3f0e55e9f90c58c317c1dcc2afc7b5", size = 13354908, upload-time = "2025-11-06T22:07:24.347Z" }, - { url = "https://files.pythonhosted.org/packages/b5/60/f0b6990f740bb15c1588601d19d21bcc1bd5de4330a07222041678a8e04f/ruff-0.14.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:f911bba769e4a9f51af6e70037bb72b70b45a16db5ce73e1f72aefe6f6d62132", size = 13587803, upload-time = "2025-11-06T22:07:26.327Z" }, - { url = "https://files.pythonhosted.org/packages/c9/da/eaaada586f80068728338e0ef7f29ab3e4a08a692f92eb901a4f06bbff24/ruff-0.14.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:76158a7369b3979fa878612c623a7e5430c18b2fd1c73b214945c2d06337db67", size = 12279654, upload-time = "2025-11-06T22:07:28.46Z" }, - { url = "https://files.pythonhosted.org/packages/66/d4/b1d0e82cf9bf8aed10a6d45be47b3f402730aa2c438164424783ac88c0ed/ruff-0.14.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f3b8f3b442d2b14c246e7aeca2e75915159e06a3540e2f4bed9f50d062d24469", size = 12357520, upload-time = "2025-11-06T22:07:31.468Z" }, - { url = "https://files.pythonhosted.org/packages/04/f4/53e2b42cc82804617e5c7950b7079d79996c27e99c4652131c6a1100657f/ruff-0.14.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c62da9a06779deecf4d17ed04939ae8b31b517643b26370c3be1d26f3ef7dbde", size = 12719431, upload-time = "2025-11-06T22:07:33.831Z" }, - { url = "https://files.pythonhosted.org/packages/a2/94/80e3d74ed9a72d64e94a7b7706b1c1ebaa315ef2076fd33581f6a1cd2f95/ruff-0.14.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5a443a83a1506c684e98acb8cb55abaf3ef725078be40237463dae4463366349", size = 13464394, upload-time = "2025-11-06T22:07:35.905Z" }, - { url = "https://files.pythonhosted.org/packages/54/1a/a49f071f04c42345c793d22f6cf5e0920095e286119ee53a64a3a3004825/ruff-0.14.4-py3-none-win32.whl", hash = "sha256:643b69cb63cd996f1fc7229da726d07ac307eae442dd8974dbc7cf22c1e18fff", size = 12493429, upload-time = "2025-11-06T22:07:38.43Z" }, - { url = "https://files.pythonhosted.org/packages/bc/22/e58c43e641145a2b670328fb98bc384e20679b5774258b1e540207580266/ruff-0.14.4-py3-none-win_amd64.whl", hash = "sha256:26673da283b96fe35fa0c939bf8411abec47111644aa9f7cfbd3c573fb125d2c", size = 13635380, upload-time = "2025-11-06T22:07:40.496Z" }, - { url = "https://files.pythonhosted.org/packages/30/bd/4168a751ddbbf43e86544b4de8b5c3b7be8d7167a2a5cb977d274e04f0a1/ruff-0.14.4-py3-none-win_arm64.whl", hash = "sha256:dd09c292479596b0e6fec8cd95c65c3a6dc68e9ad17b8f2382130f87ff6a75bb", size = 12663065, upload-time = "2025-11-06T22:07:42.603Z" }, + { url = "https://files.pythonhosted.org/packages/67/d2/7dd544116d107fffb24a0064d41a5d2ed1c9d6372d142f9ba108c8e39207/ruff-0.14.6-py3-none-linux_armv6l.whl", hash = "sha256:d724ac2f1c240dbd01a2ae98db5d1d9a5e1d9e96eba999d1c48e30062df578a3", size = 13326119, upload-time = "2025-11-21T14:25:24.2Z" }, + { url = "https://files.pythonhosted.org/packages/36/6a/ad66d0a3315d6327ed6b01f759d83df3c4d5f86c30462121024361137b6a/ruff-0.14.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9f7539ea257aa4d07b7ce87aed580e485c40143f2473ff2f2b75aee003186004", size = 13526007, upload-time = "2025-11-21T14:25:26.906Z" }, + { url = "https://files.pythonhosted.org/packages/a3/9d/dae6db96df28e0a15dea8e986ee393af70fc97fd57669808728080529c37/ruff-0.14.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7f6007e55b90a2a7e93083ba48a9f23c3158c433591c33ee2e99a49b889c6332", size = 12676572, upload-time = "2025-11-21T14:25:29.826Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/f319e87759949062cfee1b26245048e92e2acce900ad3a909285f9db1859/ruff-0.14.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a8e7b9d73d8728b68f632aa8e824ef041d068d231d8dbc7808532d3629a6bef", size = 13140745, upload-time = "2025-11-21T14:25:32.788Z" }, + { url = "https://files.pythonhosted.org/packages/95/d3/248c1efc71a0a8ed4e8e10b4b2266845d7dfc7a0ab64354afe049eaa1310/ruff-0.14.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d50d45d4553a3ebcbd33e7c5e0fe6ca4aafd9a9122492de357205c2c48f00775", size = 13076486, upload-time = "2025-11-21T14:25:35.601Z" }, + { url = "https://files.pythonhosted.org/packages/a5/19/b68d4563fe50eba4b8c92aa842149bb56dd24d198389c0ed12e7faff4f7d/ruff-0.14.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:118548dd121f8a21bfa8ab2c5b80e5b4aed67ead4b7567790962554f38e598ce", size = 13727563, upload-time = "2025-11-21T14:25:38.514Z" }, + { url = "https://files.pythonhosted.org/packages/47/ac/943169436832d4b0e867235abbdb57ce3a82367b47e0280fa7b4eabb7593/ruff-0.14.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:57256efafbfefcb8748df9d1d766062f62b20150691021f8ab79e2d919f7c11f", size = 15199755, upload-time = "2025-11-21T14:25:41.516Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b9/288bb2399860a36d4bb0541cb66cce3c0f4156aaff009dc8499be0c24bf2/ruff-0.14.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff18134841e5c68f8e5df1999a64429a02d5549036b394fafbe410f886e1989d", size = 14850608, upload-time = "2025-11-21T14:25:44.428Z" }, + { url = "https://files.pythonhosted.org/packages/ee/b1/a0d549dd4364e240f37e7d2907e97ee80587480d98c7799d2d8dc7a2f605/ruff-0.14.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29c4b7ec1e66a105d5c27bd57fa93203637d66a26d10ca9809dc7fc18ec58440", size = 14118754, upload-time = "2025-11-21T14:25:47.214Z" }, + { url = "https://files.pythonhosted.org/packages/13/ac/9b9fe63716af8bdfddfacd0882bc1586f29985d3b988b3c62ddce2e202c3/ruff-0.14.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:167843a6f78680746d7e226f255d920aeed5e4ad9c03258094a2d49d3028b105", size = 13949214, upload-time = "2025-11-21T14:25:50.002Z" }, + { url = "https://files.pythonhosted.org/packages/12/27/4dad6c6a77fede9560b7df6802b1b697e97e49ceabe1f12baf3ea20862e9/ruff-0.14.6-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:16a33af621c9c523b1ae006b1b99b159bf5ac7e4b1f20b85b2572455018e0821", size = 14106112, upload-time = "2025-11-21T14:25:52.841Z" }, + { url = "https://files.pythonhosted.org/packages/6a/db/23e322d7177873eaedea59a7932ca5084ec5b7e20cb30f341ab594130a71/ruff-0.14.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1432ab6e1ae2dc565a7eea707d3b03a0c234ef401482a6f1621bc1f427c2ff55", size = 13035010, upload-time = "2025-11-21T14:25:55.536Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9c/20e21d4d69dbb35e6a1df7691e02f363423658a20a2afacf2a2c011800dc/ruff-0.14.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:4c55cfbbe7abb61eb914bfd20683d14cdfb38a6d56c6c66efa55ec6570ee4e71", size = 13054082, upload-time = "2025-11-21T14:25:58.625Z" }, + { url = "https://files.pythonhosted.org/packages/66/25/906ee6a0464c3125c8d673c589771a974965c2be1a1e28b5c3b96cb6ef88/ruff-0.14.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:efea3c0f21901a685fff4befda6d61a1bf4cb43de16da87e8226a281d614350b", size = 13303354, upload-time = "2025-11-21T14:26:01.816Z" }, + { url = "https://files.pythonhosted.org/packages/4c/58/60577569e198d56922b7ead07b465f559002b7b11d53f40937e95067ca1c/ruff-0.14.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:344d97172576d75dc6afc0e9243376dbe1668559c72de1864439c4fc95f78185", size = 14054487, upload-time = "2025-11-21T14:26:05.058Z" }, + { url = "https://files.pythonhosted.org/packages/67/0b/8e4e0639e4cc12547f41cb771b0b44ec8225b6b6a93393176d75fe6f7d40/ruff-0.14.6-py3-none-win32.whl", hash = "sha256:00169c0c8b85396516fdd9ce3446c7ca20c2a8f90a77aa945ba6b8f2bfe99e85", size = 13013361, upload-time = "2025-11-21T14:26:08.152Z" }, + { url = "https://files.pythonhosted.org/packages/fb/02/82240553b77fd1341f80ebb3eaae43ba011c7a91b4224a9f317d8e6591af/ruff-0.14.6-py3-none-win_amd64.whl", hash = "sha256:390e6480c5e3659f8a4c8d6a0373027820419ac14fa0d2713bd8e6c3e125b8b9", size = 14432087, upload-time = "2025-11-21T14:26:10.891Z" }, + { url = "https://files.pythonhosted.org/packages/a5/1f/93f9b0fad9470e4c829a5bb678da4012f0c710d09331b860ee555216f4ea/ruff-0.14.6-py3-none-win_arm64.whl", hash = "sha256:d43c81fbeae52cfa8728d8766bbf46ee4298c888072105815b392da70ca836b2", size = 13520930, upload-time = "2025-11-21T14:26:13.951Z" }, ] [[package]] @@ -5577,36 +5585,36 @@ wheels = [ [[package]] name = "safetensors" -version = "0.6.2" +version = "0.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ac/cc/738f3011628920e027a11754d9cae9abec1aed00f7ae860abbf843755233/safetensors-0.6.2.tar.gz", hash = "sha256:43ff2aa0e6fa2dc3ea5524ac7ad93a9839256b8703761e76e2d0b2a3fa4f15d9", size = 197968, upload-time = "2025-08-08T13:13:58.654Z" } +sdist = { url = "https://files.pythonhosted.org/packages/29/9c/6e74567782559a63bd040a236edca26fd71bc7ba88de2ef35d75df3bca5e/safetensors-0.7.0.tar.gz", hash = "sha256:07663963b67e8bd9f0b8ad15bb9163606cd27cc5a1b96235a50d8369803b96b0", size = 200878, upload-time = "2025-11-19T15:18:43.199Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/b1/3f5fd73c039fc87dba3ff8b5d528bfc5a32b597fea8e7a6a4800343a17c7/safetensors-0.6.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9c85ede8ec58f120bad982ec47746981e210492a6db876882aa021446af8ffba", size = 454797, upload-time = "2025-08-08T13:13:52.066Z" }, - { url = "https://files.pythonhosted.org/packages/8c/c9/bb114c158540ee17907ec470d01980957fdaf87b4aa07914c24eba87b9c6/safetensors-0.6.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d6675cf4b39c98dbd7d940598028f3742e0375a6b4d4277e76beb0c35f4b843b", size = 432206, upload-time = "2025-08-08T13:13:50.931Z" }, - { url = "https://files.pythonhosted.org/packages/d3/8e/f70c34e47df3110e8e0bb268d90db8d4be8958a54ab0336c9be4fe86dac8/safetensors-0.6.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d2d2b3ce1e2509c68932ca03ab8f20570920cd9754b05063d4368ee52833ecd", size = 473261, upload-time = "2025-08-08T13:13:41.259Z" }, - { url = "https://files.pythonhosted.org/packages/2a/f5/be9c6a7c7ef773e1996dc214e73485286df1836dbd063e8085ee1976f9cb/safetensors-0.6.2-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:93de35a18f46b0f5a6a1f9e26d91b442094f2df02e9fd7acf224cfec4238821a", size = 485117, upload-time = "2025-08-08T13:13:43.506Z" }, - { url = "https://files.pythonhosted.org/packages/c9/55/23f2d0a2c96ed8665bf17a30ab4ce5270413f4d74b6d87dd663258b9af31/safetensors-0.6.2-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89a89b505f335640f9120fac65ddeb83e40f1fd081cb8ed88b505bdccec8d0a1", size = 616154, upload-time = "2025-08-08T13:13:45.096Z" }, - { url = "https://files.pythonhosted.org/packages/98/c6/affb0bd9ce02aa46e7acddbe087912a04d953d7a4d74b708c91b5806ef3f/safetensors-0.6.2-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc4d0d0b937e04bdf2ae6f70cd3ad51328635fe0e6214aa1fc811f3b576b3bda", size = 520713, upload-time = "2025-08-08T13:13:46.25Z" }, - { url = "https://files.pythonhosted.org/packages/fe/5d/5a514d7b88e310c8b146e2404e0dc161282e78634d9358975fd56dfd14be/safetensors-0.6.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8045db2c872db8f4cbe3faa0495932d89c38c899c603f21e9b6486951a5ecb8f", size = 485835, upload-time = "2025-08-08T13:13:49.373Z" }, - { url = "https://files.pythonhosted.org/packages/7a/7b/4fc3b2ba62c352b2071bea9cfbad330fadda70579f617506ae1a2f129cab/safetensors-0.6.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:81e67e8bab9878bb568cffbc5f5e655adb38d2418351dc0859ccac158f753e19", size = 521503, upload-time = "2025-08-08T13:13:47.651Z" }, - { url = "https://files.pythonhosted.org/packages/5a/50/0057e11fe1f3cead9254315a6c106a16dd4b1a19cd247f7cc6414f6b7866/safetensors-0.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b0e4d029ab0a0e0e4fdf142b194514695b1d7d3735503ba700cf36d0fc7136ce", size = 652256, upload-time = "2025-08-08T13:13:53.167Z" }, - { url = "https://files.pythonhosted.org/packages/e9/29/473f789e4ac242593ac1656fbece6e1ecd860bb289e635e963667807afe3/safetensors-0.6.2-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:fa48268185c52bfe8771e46325a1e21d317207bcabcb72e65c6e28e9ffeb29c7", size = 747281, upload-time = "2025-08-08T13:13:54.656Z" }, - { url = "https://files.pythonhosted.org/packages/68/52/f7324aad7f2df99e05525c84d352dc217e0fa637a4f603e9f2eedfbe2c67/safetensors-0.6.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:d83c20c12c2d2f465997c51b7ecb00e407e5f94d7dec3ea0cc11d86f60d3fde5", size = 692286, upload-time = "2025-08-08T13:13:55.884Z" }, - { url = "https://files.pythonhosted.org/packages/ad/fe/cad1d9762868c7c5dc70c8620074df28ebb1a8e4c17d4c0cb031889c457e/safetensors-0.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d944cea65fad0ead848b6ec2c37cc0b197194bec228f8020054742190e9312ac", size = 655957, upload-time = "2025-08-08T13:13:57.029Z" }, - { url = "https://files.pythonhosted.org/packages/59/a7/e2158e17bbe57d104f0abbd95dff60dda916cf277c9f9663b4bf9bad8b6e/safetensors-0.6.2-cp38-abi3-win32.whl", hash = "sha256:cab75ca7c064d3911411461151cb69380c9225798a20e712b102edda2542ddb1", size = 308926, upload-time = "2025-08-08T13:14:01.095Z" }, - { url = "https://files.pythonhosted.org/packages/2c/c3/c0be1135726618dc1e28d181b8c442403d8dbb9e273fd791de2d4384bcdd/safetensors-0.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:c7b214870df923cbc1593c3faee16bec59ea462758699bd3fee399d00aac072c", size = 320192, upload-time = "2025-08-08T13:13:59.467Z" }, + { url = "https://files.pythonhosted.org/packages/fa/47/aef6c06649039accf914afef490268e1067ed82be62bcfa5b7e886ad15e8/safetensors-0.7.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c82f4d474cf725255d9e6acf17252991c3c8aac038d6ef363a4bf8be2f6db517", size = 467781, upload-time = "2025-11-19T15:18:35.84Z" }, + { url = "https://files.pythonhosted.org/packages/e8/00/374c0c068e30cd31f1e1b46b4b5738168ec79e7689ca82ee93ddfea05109/safetensors-0.7.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:94fd4858284736bb67a897a41608b5b0c2496c9bdb3bf2af1fa3409127f20d57", size = 447058, upload-time = "2025-11-19T15:18:34.416Z" }, + { url = "https://files.pythonhosted.org/packages/f1/06/578ffed52c2296f93d7fd2d844cabfa92be51a587c38c8afbb8ae449ca89/safetensors-0.7.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e07d91d0c92a31200f25351f4acb2bc6aff7f48094e13ebb1d0fb995b54b6542", size = 491748, upload-time = "2025-11-19T15:18:09.79Z" }, + { url = "https://files.pythonhosted.org/packages/ae/33/1debbbb70e4791dde185edb9413d1fe01619255abb64b300157d7f15dddd/safetensors-0.7.0-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8469155f4cb518bafb4acf4865e8bb9d6804110d2d9bdcaa78564b9fd841e104", size = 503881, upload-time = "2025-11-19T15:18:16.145Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1c/40c2ca924d60792c3be509833df711b553c60effbd91da6f5284a83f7122/safetensors-0.7.0-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:54bef08bf00a2bff599982f6b08e8770e09cc012d7bba00783fc7ea38f1fb37d", size = 623463, upload-time = "2025-11-19T15:18:21.11Z" }, + { url = "https://files.pythonhosted.org/packages/9b/3a/13784a9364bd43b0d61eef4bea2845039bc2030458b16594a1bd787ae26e/safetensors-0.7.0-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:42cb091236206bb2016d245c377ed383aa7f78691748f3bb6ee1bfa51ae2ce6a", size = 532855, upload-time = "2025-11-19T15:18:25.719Z" }, + { url = "https://files.pythonhosted.org/packages/a0/60/429e9b1cb3fc651937727befe258ea24122d9663e4d5709a48c9cbfceecb/safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac7252938f0696ddea46f5e855dd3138444e82236e3be475f54929f0c510d48", size = 507152, upload-time = "2025-11-19T15:18:33.023Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a8/4b45e4e059270d17af60359713ffd83f97900d45a6afa73aaa0d737d48b6/safetensors-0.7.0-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1d060c70284127fa805085d8f10fbd0962792aed71879d00864acda69dbab981", size = 541856, upload-time = "2025-11-19T15:18:31.075Z" }, + { url = "https://files.pythonhosted.org/packages/06/87/d26d8407c44175d8ae164a95b5a62707fcc445f3c0c56108e37d98070a3d/safetensors-0.7.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cdab83a366799fa730f90a4ebb563e494f28e9e92c4819e556152ad55e43591b", size = 674060, upload-time = "2025-11-19T15:18:37.211Z" }, + { url = "https://files.pythonhosted.org/packages/11/f5/57644a2ff08dc6325816ba7217e5095f17269dada2554b658442c66aed51/safetensors-0.7.0-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:672132907fcad9f2aedcb705b2d7b3b93354a2aec1b2f706c4db852abe338f85", size = 771715, upload-time = "2025-11-19T15:18:38.689Z" }, + { url = "https://files.pythonhosted.org/packages/86/31/17883e13a814bd278ae6e266b13282a01049b0c81341da7fd0e3e71a80a3/safetensors-0.7.0-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:5d72abdb8a4d56d4020713724ba81dac065fedb7f3667151c4a637f1d3fb26c0", size = 714377, upload-time = "2025-11-19T15:18:40.162Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d8/0c8a7dc9b41dcac53c4cbf9df2b9c83e0e0097203de8b37a712b345c0be5/safetensors-0.7.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b0f6d66c1c538d5a94a73aa9ddca8ccc4227e6c9ff555322ea40bdd142391dd4", size = 677368, upload-time = "2025-11-19T15:18:41.627Z" }, + { url = "https://files.pythonhosted.org/packages/05/e5/cb4b713c8a93469e3c5be7c3f8d77d307e65fe89673e731f5c2bfd0a9237/safetensors-0.7.0-cp38-abi3-win32.whl", hash = "sha256:c74af94bf3ac15ac4d0f2a7c7b4663a15f8c2ab15ed0fc7531ca61d0835eccba", size = 326423, upload-time = "2025-11-19T15:18:45.74Z" }, + { url = "https://files.pythonhosted.org/packages/5d/e6/ec8471c8072382cb91233ba7267fd931219753bb43814cbc71757bfd4dab/safetensors-0.7.0-cp38-abi3-win_amd64.whl", hash = "sha256:d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755", size = 341380, upload-time = "2025-11-19T15:18:44.427Z" }, ] [[package]] name = "scipy-stubs" -version = "1.16.3.0" +version = "1.16.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "optype", extra = ["numpy"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bd/68/c53c3bce6bd069a164015be1be2671c968b526be4af1e85db64c88f04546/scipy_stubs-1.16.3.0.tar.gz", hash = "sha256:d6943c085e47a1ed431309f9ca582b6a206a9db808a036132a0bf01ebc34b506", size = 356462, upload-time = "2025-10-28T22:05:31.198Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/3e/8baf960c68f012b8297930d4686b235813974833a417db8d0af798b0b93d/scipy_stubs-1.16.3.1.tar.gz", hash = "sha256:0738d55a7f8b0c94cdb8063f711d53330ebefe166f7d48dec9ffd932a337226d", size = 359990, upload-time = "2025-11-23T23:05:21.274Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/86/1c/0ba7305fa01cfe7a6f1b8c86ccdd1b7a0d43fa9bd769c059995311e291a2/scipy_stubs-1.16.3.0-py3-none-any.whl", hash = "sha256:90e5d82ced2183ef3c5c0a28a77df8cc227458624364fa0ff975ad24fa89d6ad", size = 557713, upload-time = "2025-10-28T22:05:29.454Z" }, + { url = "https://files.pythonhosted.org/packages/0c/39/e2a69866518f88dc01940c9b9b044db97c3387f2826bd2a173e49a5c0469/scipy_stubs-1.16.3.1-py3-none-any.whl", hash = "sha256:69bc52ef6c3f8e09208abdfaf32291eb51e9ddf8fa4389401ccd9473bdd2a26d", size = 560397, upload-time = "2025-11-23T23:05:19.432Z" }, ] [[package]] @@ -5773,11 +5781,11 @@ wheels = [ [[package]] name = "sqlglot" -version = "27.29.0" +version = "28.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d1/50/766692a83468adb1bde9e09ea524a01719912f6bc4fdb47ec18368320f6e/sqlglot-27.29.0.tar.gz", hash = "sha256:2270899694663acef94fa93497971837e6fadd712f4a98b32aee1e980bc82722", size = 5503507, upload-time = "2025-10-29T13:50:24.594Z" } +sdist = { url = "https://files.pythonhosted.org/packages/52/8d/9ce5904aca760b81adf821c77a1dcf07c98f9caaa7e3b5c991c541ff89d2/sqlglot-28.0.0.tar.gz", hash = "sha256:cc9a651ef4182e61dac58aa955e5fb21845a5865c6a4d7d7b5a7857450285ad4", size = 5520798, upload-time = "2025-11-17T10:34:57.016Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/70/20c1912bc0bfebf516d59d618209443b136c58a7cff141afa7cf30969988/sqlglot-27.29.0-py3-none-any.whl", hash = "sha256:9a5ea8ac61826a7763de10cad45a35f0aa9bfcf7b96ee74afb2314de9089e1cb", size = 526060, upload-time = "2025-10-29T13:50:22.061Z" }, + { url = "https://files.pythonhosted.org/packages/56/6d/86de134f40199105d2fee1b066741aa870b3ce75ee74018d9c8508bbb182/sqlglot-28.0.0-py3-none-any.whl", hash = "sha256:ac1778e7fa4812f4f7e5881b260632fc167b00ca4c1226868891fb15467122e4", size = 536127, upload-time = "2025-11-17T10:34:55.192Z" }, ] [[package]] @@ -5971,7 +5979,7 @@ wheels = [ [[package]] name = "testcontainers" -version = "4.13.2" +version = "4.13.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "docker" }, @@ -5980,9 +5988,9 @@ dependencies = [ { name = "urllib3" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/18/51/edac83edab339d8b4dce9a7b659163afb1ea7e011bfed1d5573d495a4485/testcontainers-4.13.2.tar.gz", hash = "sha256:2315f1e21b059427a9d11e8921f85fef322fbe0d50749bcca4eaa11271708ba4", size = 78692, upload-time = "2025-10-07T21:53:07.531Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/b3/c272537f3ea2f312555efeb86398cc382cd07b740d5f3c730918c36e64e1/testcontainers-4.13.3.tar.gz", hash = "sha256:9d82a7052c9a53c58b69e1dc31da8e7a715e8b3ec1c4df5027561b47e2efe646", size = 79064, upload-time = "2025-11-14T05:08:47.584Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/5e/73aa94770f1df0595364aed526f31d54440db5492911e2857318ed326e51/testcontainers-4.13.2-py3-none-any.whl", hash = "sha256:0209baf8f4274b568cde95bef2cadf7b1d33b375321f793790462e235cd684ee", size = 124771, upload-time = "2025-10-07T21:53:05.937Z" }, + { url = "https://files.pythonhosted.org/packages/73/27/c2f24b19dafa197c514abe70eda69bc031c5152c6b1f1e5b20099e2ceedd/testcontainers-4.13.3-py3-none-any.whl", hash = "sha256:063278c4805ffa6dd85e56648a9da3036939e6c0ac1001e851c9276b19b05970", size = 124784, upload-time = "2025-11-14T05:08:46.053Z" }, ] [[package]] @@ -6128,27 +6136,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.1a26" +version = "0.0.1a27" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/39/39/b4b4ecb6ca6d7e937fa56f0b92a8f48d7719af8fe55bdbf667638e9f93e2/ty-0.0.1a26.tar.gz", hash = "sha256:65143f8efeb2da1644821b710bf6b702a31ddcf60a639d5a576db08bded91db4", size = 4432154, upload-time = "2025-11-10T18:02:30.142Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/65/3592d7c73d80664378fc90d0a00c33449a99cbf13b984433c883815245f3/ty-0.0.1a27.tar.gz", hash = "sha256:d34fe04979f2c912700cbf0919e8f9b4eeaa10c4a2aff7450e5e4c90f998bc28", size = 4516059, upload-time = "2025-11-18T21:55:18.381Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/6a/661833ecacc4d994f7e30a7f1307bfd3a4a91392a6b03fb6a018723e75b8/ty-0.0.1a26-py3-none-linux_armv6l.whl", hash = "sha256:09208dca99bb548e9200136d4d42618476bfe1f4d2066511f2c8e2e4dfeced5e", size = 9173869, upload-time = "2025-11-10T18:01:46.012Z" }, - { url = "https://files.pythonhosted.org/packages/66/a8/32ea50f064342de391a7267f84349287e2f1c2eb0ad4811d6110916179d6/ty-0.0.1a26-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:91d12b66c91a1b82e698a2aa73fe043a1a9da83ff0dfd60b970500bee0963b91", size = 8973420, upload-time = "2025-11-10T18:01:49.32Z" }, - { url = "https://files.pythonhosted.org/packages/d1/f6/6659d55940cd5158a6740ae46a65be84a7ee9167738033a9b1259c36eef5/ty-0.0.1a26-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5bc6dfcea5477c81ad01d6a29ebc9bfcbdb21c34664f79c9e1b84be7aa8f289", size = 8528888, upload-time = "2025-11-10T18:01:51.511Z" }, - { url = "https://files.pythonhosted.org/packages/79/c9/4cbe7295013cc412b4f100b509aaa21982c08c59764a2efa537ead049345/ty-0.0.1a26-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40e5d15635e9918924138e8d3fb1cbf80822dfb8dc36ea8f3e72df598c0c4bea", size = 8801867, upload-time = "2025-11-10T18:01:53.888Z" }, - { url = "https://files.pythonhosted.org/packages/ed/b3/25099b219a6444c4b29f175784a275510c1cd85a23a926d687ab56915027/ty-0.0.1a26-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:86dc147ed0790c7c8fd3f0d6c16c3c5135b01e99c440e89c6ca1e0e592bb6682", size = 8975519, upload-time = "2025-11-10T18:01:56.231Z" }, - { url = "https://files.pythonhosted.org/packages/73/3e/3ad570f4f592cb1d11982dd2c426c90d2aa9f3d38bf77a7e2ce8aa614302/ty-0.0.1a26-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fbe0e07c9d5e624edfc79a468f2ef191f9435581546a5bb6b92713ddc86ad4a6", size = 9331932, upload-time = "2025-11-10T18:01:58.476Z" }, - { url = "https://files.pythonhosted.org/packages/04/fa/62c72eead0302787f9cc0d613fc671107afeecdaf76ebb04db8f91bb9f7e/ty-0.0.1a26-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:0dcebbfe9f24b43d98a078f4a41321ae7b08bea40f5c27d81394b3f54e9f7fb5", size = 9921353, upload-time = "2025-11-10T18:02:00.749Z" }, - { url = "https://files.pythonhosted.org/packages/6c/1f/3b329c4b60d878704e09eb9d05467f911f188e699961c044b75932893e0a/ty-0.0.1a26-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0901b75afc7738224ffc98bbc8ea03a20f167a2a83a4b23a6550115e8b3ddbc6", size = 9700800, upload-time = "2025-11-10T18:02:03.544Z" }, - { url = "https://files.pythonhosted.org/packages/92/24/13fcba20dd86a7c3f83c814279aa3eb6a29c5f1b38a3b3a4a0fd22159189/ty-0.0.1a26-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4788f34d384c132977958d76fef7f274f8d181b22e33933c4d16cff2bb5ca3b9", size = 9728289, upload-time = "2025-11-10T18:02:06.386Z" }, - { url = "https://files.pythonhosted.org/packages/40/7a/798894ff0b948425570b969be35e672693beeb6b852815b7340bc8de1575/ty-0.0.1a26-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b98851c11c560ce63cd972ed9728aa079d9cf40483f2cdcf3626a55849bfe107", size = 9279735, upload-time = "2025-11-10T18:02:09.425Z" }, - { url = "https://files.pythonhosted.org/packages/1a/54/71261cc1b8dc7d3c4ad92a83b4d1681f5cb7ea5965ebcbc53311ae8c6424/ty-0.0.1a26-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c20b4625a20059adecd86fe2c4df87cd6115fea28caee45d3bdcf8fb83d29510", size = 8767428, upload-time = "2025-11-10T18:02:11.956Z" }, - { url = "https://files.pythonhosted.org/packages/8e/07/b248b73a640badba2b301e6845699b7dd241f40a321b9b1bce684d440f70/ty-0.0.1a26-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d9909e96276f8d16382d285db92ae902174cae842aa953003ec0c06642db2f8a", size = 9009170, upload-time = "2025-11-10T18:02:14.878Z" }, - { url = "https://files.pythonhosted.org/packages/f8/35/ec8353f2bb7fd2f41bca6070b29ecb58e2de9af043e649678b8c132d5439/ty-0.0.1a26-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a76d649ceefe9baa9bbae97d217bee076fd8eeb2a961f66f1dff73cc70af4ac8", size = 9119215, upload-time = "2025-11-10T18:02:18.329Z" }, - { url = "https://files.pythonhosted.org/packages/70/48/db49fe1b7e66edf90dc285869043f99c12aacf7a99c36ee760e297bac6d5/ty-0.0.1a26-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a0ee0f6366bcf70fae114e714d45335cacc8daa936037441e02998a9110b7a29", size = 9398655, upload-time = "2025-11-10T18:02:21.031Z" }, - { url = "https://files.pythonhosted.org/packages/10/f8/d869492bdbb21ae8cf4c99b02f20812bbbf49aa187cfeb387dfaa03036a8/ty-0.0.1a26-py3-none-win32.whl", hash = "sha256:86689b90024810cac7750bf0c6e1652e4b4175a9de7b82b8b1583202aeb47287", size = 8645669, upload-time = "2025-11-10T18:02:23.23Z" }, - { url = "https://files.pythonhosted.org/packages/b4/18/8a907575d2b335afee7556cb92233ebb5efcefe17752fc9dcab21cffb23b/ty-0.0.1a26-py3-none-win_amd64.whl", hash = "sha256:829e6e6dbd7d9d370f97b2398b4804552554bdcc2d298114fed5e2ea06cbc05c", size = 9442975, upload-time = "2025-11-10T18:02:25.68Z" }, - { url = "https://files.pythonhosted.org/packages/e9/22/af92dcfdd84b78dd97ac6b7154d6a763781f04a400140444885c297cc213/ty-0.0.1a26-py3-none-win_arm64.whl", hash = "sha256:b8f431c784d4cf5b4195a3521b2eca9c15902f239b91154cb920da33f943c62b", size = 8958958, upload-time = "2025-11-10T18:02:28.071Z" }, + { url = "https://files.pythonhosted.org/packages/e6/05/7945aa97356446fd53ed3ddc7ee02a88d8ad394217acd9428f472d6b109d/ty-0.0.1a27-py3-none-linux_armv6l.whl", hash = "sha256:3cbb735f5ecb3a7a5f5b82fb24da17912788c109086df4e97d454c8fb236fbc5", size = 9375047, upload-time = "2025-11-18T21:54:31.577Z" }, + { url = "https://files.pythonhosted.org/packages/69/4e/89b167a03de0e9ec329dc89bc02e8694768e4576337ef6c0699987681342/ty-0.0.1a27-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:4a6367236dc456ba2416563301d498aef8c6f8959be88777ef7ba5ac1bf15f0b", size = 9169540, upload-time = "2025-11-18T21:54:34.036Z" }, + { url = "https://files.pythonhosted.org/packages/38/07/e62009ab9cc242e1becb2bd992097c80a133fce0d4f055fba6576150d08a/ty-0.0.1a27-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8e93e231a1bcde964cdb062d2d5e549c24493fb1638eecae8fcc42b81e9463a4", size = 8711942, upload-time = "2025-11-18T21:54:36.3Z" }, + { url = "https://files.pythonhosted.org/packages/b5/43/f35716ec15406f13085db52e762a3cc663c651531a8124481d0ba602eca0/ty-0.0.1a27-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5b6a8166b60117da1179851a3d719cc798bf7e61f91b35d76242f0059e9ae1d", size = 8984208, upload-time = "2025-11-18T21:54:39.453Z" }, + { url = "https://files.pythonhosted.org/packages/2d/79/486a3374809523172379768de882c7a369861165802990177fe81489b85f/ty-0.0.1a27-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bfbe8b0e831c072b79a078d6c126d7f4d48ca17f64a103de1b93aeda32265dc5", size = 9157209, upload-time = "2025-11-18T21:54:42.664Z" }, + { url = "https://files.pythonhosted.org/packages/ff/08/9a7c8efcb327197d7d347c548850ef4b54de1c254981b65e8cd0672dc327/ty-0.0.1a27-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:90e09678331552e7c25d7eb47868b0910dc5b9b212ae22c8ce71a52d6576ddbb", size = 9519207, upload-time = "2025-11-18T21:54:45.311Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9d/7b4680683e83204b9edec551bb91c21c789ebc586b949c5218157ee474b7/ty-0.0.1a27-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:88c03e4beeca79d85a5618921e44b3a6ea957e0453e08b1cdd418b51da645939", size = 10148794, upload-time = "2025-11-18T21:54:48.329Z" }, + { url = "https://files.pythonhosted.org/packages/89/21/8b961b0ab00c28223f06b33222427a8e31aa04f39d1b236acc93021c626c/ty-0.0.1a27-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ece5811322789fefe22fc088ed36c5879489cd39e913f9c1ff2a7678f089c61", size = 9900563, upload-time = "2025-11-18T21:54:51.214Z" }, + { url = "https://files.pythonhosted.org/packages/85/eb/95e1f0b426c2ea8d443aa923fcab509059c467bbe64a15baaf573fea1203/ty-0.0.1a27-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f2ccb4f0fddcd6e2017c268dfce2489e9a36cb82a5900afe6425835248b1086", size = 9926355, upload-time = "2025-11-18T21:54:53.927Z" }, + { url = "https://files.pythonhosted.org/packages/f5/78/40e7f072049e63c414f2845df780be3a494d92198c87c2ffa65e63aecf3f/ty-0.0.1a27-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33450528312e41d003e96a1647780b2783ab7569bbc29c04fc76f2d1908061e3", size = 9480580, upload-time = "2025-11-18T21:54:56.617Z" }, + { url = "https://files.pythonhosted.org/packages/18/da/f4a2dfedab39096808ddf7475f35ceb750d9a9da840bee4afd47b871742f/ty-0.0.1a27-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a0a9ac635deaa2b15947701197ede40cdecd13f89f19351872d16f9ccd773fa1", size = 8957524, upload-time = "2025-11-18T21:54:59.085Z" }, + { url = "https://files.pythonhosted.org/packages/21/ea/26fee9a20cf77a157316fd3ab9c6db8ad5a0b20b2d38a43f3452622587ac/ty-0.0.1a27-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:797fb2cd49b6b9b3ac9f2f0e401fb02d3aa155badc05a8591d048d38d28f1e0c", size = 9201098, upload-time = "2025-11-18T21:55:01.845Z" }, + { url = "https://files.pythonhosted.org/packages/b0/53/e14591d1275108c9ae28f97ac5d4b93adcc2c8a4b1b9a880dfa9d07c15f8/ty-0.0.1a27-py3-none-musllinux_1_2_i686.whl", hash = "sha256:7fe81679a0941f85e98187d444604e24b15bde0a85874957c945751756314d03", size = 9275470, upload-time = "2025-11-18T21:55:04.23Z" }, + { url = "https://files.pythonhosted.org/packages/37/44/e2c9acecac70bf06fb41de285e7be2433c2c9828f71e3bf0e886fc85c4fd/ty-0.0.1a27-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:355f651d0cdb85535a82bd9f0583f77b28e3fd7bba7b7da33dcee5a576eff28b", size = 9592394, upload-time = "2025-11-18T21:55:06.542Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a7/4636369731b24ed07c2b4c7805b8d990283d677180662c532d82e4ef1a36/ty-0.0.1a27-py3-none-win32.whl", hash = "sha256:61782e5f40e6df622093847b34c366634b75d53f839986f1bf4481672ad6cb55", size = 8783816, upload-time = "2025-11-18T21:55:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/a7/1d/b76487725628d9e81d9047dc0033a5e167e0d10f27893d04de67fe1a9763/ty-0.0.1a27-py3-none-win_amd64.whl", hash = "sha256:c682b238085d3191acddcf66ef22641562946b1bba2a7f316012d5b2a2f4de11", size = 9616833, upload-time = "2025-11-18T21:55:12.457Z" }, + { url = "https://files.pythonhosted.org/packages/3a/db/c7cd5276c8f336a3cf87992b75ba9d486a7cf54e753fcd42495b3bc56fb7/ty-0.0.1a27-py3-none-win_arm64.whl", hash = "sha256:e146dfa32cbb0ac6afb0cb65659e87e4e313715e68d76fe5ae0a4b3d5b912ce8", size = 9137796, upload-time = "2025-11-18T21:55:15.897Z" }, ] [[package]] @@ -6177,11 +6185,11 @@ wheels = [ [[package]] name = "types-awscrt" -version = "0.28.4" +version = "0.29.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f7/6f/d4f2adb086e8f5cd2ae83cf8dbb192057d8b5025120e5b372468292db67f/types_awscrt-0.28.4.tar.gz", hash = "sha256:15929da84802f27019ee8e4484fb1c102e1f6d4cf22eb48688c34a5a86d02eb6", size = 17692, upload-time = "2025-11-11T02:56:53.516Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/77/c25c0fbdd3b269b13139c08180bcd1521957c79bd133309533384125810c/types_awscrt-0.29.0.tar.gz", hash = "sha256:7f81040846095cbaf64e6b79040434750d4f2f487544d7748b778c349d393510", size = 17715, upload-time = "2025-11-21T21:01:24.223Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/ae/9acc4adf1d5d7bb7d09b6f9ff5d4d04a72eb64700d104106dd517665cd57/types_awscrt-0.28.4-py3-none-any.whl", hash = "sha256:2d453f9e27583fcc333771b69a5255a5a4e2c52f86e70f65f3c5a6789d3443d0", size = 42307, upload-time = "2025-11-11T02:56:52.231Z" }, + { url = "https://files.pythonhosted.org/packages/37/a9/6b7a0ceb8e6f2396cc290ae2f1520a1598842119f09b943d83d6ff01bc49/types_awscrt-0.29.0-py3-none-any.whl", hash = "sha256:ece1906d5708b51b6603b56607a702ed1e5338a2df9f31950e000f03665ac387", size = 42343, upload-time = "2025-11-21T21:01:22.979Z" }, ] [[package]] @@ -6302,11 +6310,14 @@ wheels = [ [[package]] name = "types-html5lib" -version = "1.1.11.20251014" +version = "1.1.11.20251117" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8a/b8/0ce98d9b20a4e8bdac4f4914054acadf5b3a36a7a832e11e0d1938e4c3ce/types_html5lib-1.1.11.20251014.tar.gz", hash = "sha256:cc628d626e0111a2426a64f5f061ecfd113958b69ff6b3dc0eaaed2347ba9455", size = 16895, upload-time = "2025-10-14T02:54:50.003Z" } +dependencies = [ + { name = "types-webencodings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/f3/d9a1bbba7b42b5558a3f9fe017d967f5338cf8108d35991d9b15fdea3e0d/types_html5lib-1.1.11.20251117.tar.gz", hash = "sha256:1a6a3ac5394aa12bf547fae5d5eff91dceec46b6d07c4367d9b39a37f42f201a", size = 18100, upload-time = "2025-11-17T03:08:00.78Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/cb/df12640506b8dbd2f2bd0643c5ef4a72fa6285ec4cd7f4b20457842e7fd5/types_html5lib-1.1.11.20251014-py3-none-any.whl", hash = "sha256:4ff2cf18dfc547009ab6fa4190fc3de464ba815c9090c3dd4a5b65f664bfa76c", size = 22916, upload-time = "2025-10-14T02:54:48.686Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ab/f5606db367c1f57f7400d3cb3bead6665ee2509621439af1b29c35ef6f9e/types_html5lib-1.1.11.20251117-py3-none-any.whl", hash = "sha256:2a3fc935de788a4d2659f4535002a421e05bea5e172b649d33232e99d4272d08", size = 24302, upload-time = "2025-11-17T03:07:59.996Z" }, ] [[package]] @@ -6395,11 +6406,11 @@ wheels = [ [[package]] name = "types-psutil" -version = "7.0.0.20251111" +version = "7.0.0.20251116" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1a/ba/4f48c927f38c7a4d6f7ff65cde91c49d28a95a56e00ec19b2813e1e0b1c1/types_psutil-7.0.0.20251111.tar.gz", hash = "sha256:d109ee2da4c0a9b69b8cefc46e195db8cf0fc0200b6641480df71e7f3f51a239", size = 20287, upload-time = "2025-11-11T03:06:37.482Z" } +sdist = { url = "https://files.pythonhosted.org/packages/47/ec/c1e9308b91582cad1d7e7d3007fd003ef45a62c2500f8219313df5fc3bba/types_psutil-7.0.0.20251116.tar.gz", hash = "sha256:92b5c78962e55ce1ed7b0189901a4409ece36ab9fd50c3029cca7e681c606c8a", size = 22192, upload-time = "2025-11-16T03:10:32.859Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/bc/b081d10fbd933cdf839109707a693c668a174e2276d64159a582a9cebd3f/types_psutil-7.0.0.20251111-py3-none-any.whl", hash = "sha256:85ba00205dcfa3c73685122e5a360205d2fbc9b56f942b591027bf401ce0cc47", size = 23052, upload-time = "2025-11-11T03:06:36.011Z" }, + { url = "https://files.pythonhosted.org/packages/c3/0e/11ba08a5375c21039ed5f8e6bba41e9452fb69f0e2f7ee05ed5cca2a2cdf/types_psutil-7.0.0.20251116-py3-none-any.whl", hash = "sha256:74c052de077c2024b85cd435e2cba971165fe92a5eace79cbeb821e776dbc047", size = 25376, upload-time = "2025-11-16T03:10:31.813Z" }, ] [[package]] @@ -6413,14 +6424,14 @@ wheels = [ [[package]] name = "types-pygments" -version = "2.19.0.20250809" +version = "2.19.0.20251121" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "types-docutils" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/1b/a6317763a8f2de01c425644273e5fbe3145d648a081f3bad590b3c34e000/types_pygments-2.19.0.20250809.tar.gz", hash = "sha256:01366fd93ef73c792e6ee16498d3abf7a184f1624b50b77f9506a47ed85974c2", size = 18454, upload-time = "2025-08-09T03:17:14.322Z" } +sdist = { url = "https://files.pythonhosted.org/packages/90/3b/cd650700ce9e26b56bd1a6aa4af397bbbc1784e22a03971cb633cdb0b601/types_pygments-2.19.0.20251121.tar.gz", hash = "sha256:eef114fde2ef6265365522045eac0f8354978a566852f69e75c531f0553822b1", size = 18590, upload-time = "2025-11-21T03:03:46.623Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/c4/d9f0923a941159664d664a0b714242fbbd745046db2d6c8de6fe1859c572/types_pygments-2.19.0.20250809-py3-none-any.whl", hash = "sha256:8e813e5fc25f741b81cadc1e181d402ebd288e34a9812862ddffee2f2b57db7c", size = 25407, upload-time = "2025-08-09T03:17:13.223Z" }, + { url = "https://files.pythonhosted.org/packages/99/8a/9244b21f1d60dcc62e261435d76b02f1853b4771663d7ec7d287e47a9ba9/types_pygments-2.19.0.20251121-py3-none-any.whl", hash = "sha256:cb3bfde34eb75b984c98fb733ce4f795213bd3378f855c32e75b49318371bb25", size = 25674, upload-time = "2025-11-21T03:03:45.72Z" }, ] [[package]] @@ -6447,11 +6458,11 @@ wheels = [ [[package]] name = "types-python-dateutil" -version = "2.9.0.20251108" +version = "2.9.0.20251115" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b0/42/18dff855130c3551d2b5159165bd24466f374dcb78670e5259d2ed51f55c/types_python_dateutil-2.9.0.20251108.tar.gz", hash = "sha256:d8a6687e197f2fa71779ce36176c666841f811368710ab8d274b876424ebfcaa", size = 16220, upload-time = "2025-11-08T02:55:53.393Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/36/06d01fb52c0d57e9ad0c237654990920fa41195e4b3d640830dabf9eeb2f/types_python_dateutil-2.9.0.20251115.tar.gz", hash = "sha256:8a47f2c3920f52a994056b8786309b43143faa5a64d4cbb2722d6addabdf1a58", size = 16363, upload-time = "2025-11-15T03:00:13.717Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/25/dd/9fb1f5ef742cab1ea390582f407c967677704d2f5362b48c09de0d0dc8d4/types_python_dateutil-2.9.0.20251108-py3-none-any.whl", hash = "sha256:a4a537f0ea7126f8ccc2763eec9aa31ac8609e3c8e530eb2ddc5ee234b3cd764", size = 18127, upload-time = "2025-11-08T02:55:52.291Z" }, + { url = "https://files.pythonhosted.org/packages/43/0b/56961d3ba517ed0df9b3a27bfda6514f3d01b28d499d1bce9068cfe4edd1/types_python_dateutil-2.9.0.20251115-py3-none-any.whl", hash = "sha256:9cf9c1c582019753b8639a081deefd7e044b9fa36bd8217f565c6c4e36ee0624", size = 18251, upload-time = "2025-11-15T03:00:12.317Z" }, ] [[package]] @@ -6526,11 +6537,11 @@ wheels = [ [[package]] name = "types-s3transfer" -version = "0.14.0" +version = "0.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/9b/8913198b7fc700acc1dcb84827137bb2922052e43dde0f4fb0ed2dc6f118/types_s3transfer-0.14.0.tar.gz", hash = "sha256:17f800a87c7eafab0434e9d87452c809c290ae906c2024c24261c564479e9c95", size = 14218, upload-time = "2025-10-11T21:11:27.892Z" } +sdist = { url = "https://files.pythonhosted.org/packages/79/bf/b00dcbecb037c4999b83c8109b8096fe78f87f1266cadc4f95d4af196292/types_s3transfer-0.15.0.tar.gz", hash = "sha256:43a523e0c43a88e447dfda5f4f6b63bf3da85316fdd2625f650817f2b170b5f7", size = 14236, upload-time = "2025-11-21T21:16:26.553Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/92/c3/4dfb2e87c15ca582b7d956dfb7e549de1d005c758eb9a305e934e1b83fda/types_s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:108134854069a38b048e9b710b9b35904d22a9d0f37e4e1889c2e6b58e5b3253", size = 19697, upload-time = "2025-10-11T21:11:26.749Z" }, + { url = "https://files.pythonhosted.org/packages/8a/39/39a322d7209cc259e3e27c4d498129e9583a2f3a8aea57eb1a9941cb5e9e/types_s3transfer-0.15.0-py3-none-any.whl", hash = "sha256:1e617b14a9d3ce5be565f4b187fafa1d96075546b52072121f8fda8e0a444aed", size = 19702, upload-time = "2025-11-21T21:16:25.146Z" }, ] [[package]] @@ -6607,6 +6618,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/f2/d812543c350674d8b3f6e17c8922248ee3bb752c2a76f64beb8c538b40cf/types_ujson-5.10.0.20250822-py3-none-any.whl", hash = "sha256:3e9e73a6dc62ccc03449d9ac2c580cd1b7a8e4873220db498f7dd056754be080", size = 7657, upload-time = "2025-08-22T03:02:18.699Z" }, ] +[[package]] +name = "types-webencodings" +version = "0.5.0.20251108" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/d6/75e381959a2706644f02f7527d264de3216cf6ed333f98eff95954d78e07/types_webencodings-0.5.0.20251108.tar.gz", hash = "sha256:2378e2ceccced3d41bb5e21387586e7b5305e11519fc6b0659c629f23b2e5de4", size = 7470, upload-time = "2025-11-08T02:56:00.132Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/4e/8fcf33e193ce4af03c19d0e08483cf5f0838e883f800909c6bc61cb361be/types_webencodings-0.5.0.20251108-py3-none-any.whl", hash = "sha256:e21f81ff750795faffddaffd70a3d8bfff77d006f22c27e393eb7812586249d8", size = 8715, upload-time = "2025-11-08T02:55:59.456Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -6741,7 +6761,7 @@ pptx = [ [[package]] name = "unstructured-client" -version = "0.42.3" +version = "0.42.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, @@ -6752,9 +6772,9 @@ dependencies = [ { name = "pypdf" }, { name = "requests-toolbelt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/96/45/0d605c1c4ed6e38845e9e7d95758abddc7d66e1d096ef9acdf2ecdeaf009/unstructured_client-0.42.3.tar.gz", hash = "sha256:a568d8b281fafdf452647d874060cd0647e33e4a19e811b4db821eb1f3051163", size = 91379, upload-time = "2025-08-12T20:48:04.937Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/8f/43c9a936a153e62f18e7629128698feebd81d2cfff2835febc85377b8eb8/unstructured_client-0.42.4.tar.gz", hash = "sha256:144ecd231a11d091cdc76acf50e79e57889269b8c9d8b9df60e74cf32ac1ba5e", size = 91404, upload-time = "2025-11-14T16:59:25.131Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/1c/137993fff771efc3d5c31ea6b6d126c635c7b124ea641531bca1fd8ea815/unstructured_client-0.42.3-py3-none-any.whl", hash = "sha256:14e9a6a44ed58c64bacd32c62d71db19bf9c2f2b46a2401830a8dfff48249d39", size = 207814, upload-time = "2025-08-12T20:48:03.638Z" }, + { url = "https://files.pythonhosted.org/packages/5e/6c/7c69e4353e5bdd05fc247c2ec1d840096eb928975697277b015c49405b0f/unstructured_client-0.42.4-py3-none-any.whl", hash = "sha256:fc6341344dd2f2e2aed793636b5f4e6204cad741ff2253d5a48ff2f2bccb8e9a", size = 207863, upload-time = "2025-11-14T16:59:23.674Z" }, ] [[package]] @@ -6957,7 +6977,7 @@ wheels = [ [[package]] name = "weave" -version = "0.52.16" +version = "0.52.17" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -6973,9 +6993,9 @@ dependencies = [ { name = "tzdata", marker = "sys_platform == 'win32'" }, { name = "wandb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/be/30/b795b5a857e8a908e68f3ed969587bb2bc63527ef2260f72ac1a6fd983e8/weave-0.52.16.tar.gz", hash = "sha256:7bb8fdce0393007e9c40fb1769d0606bfe55401c4cd13146457ccac4b49c695d", size = 607024, upload-time = "2025-11-07T19:45:30.898Z" } +sdist = { url = "https://files.pythonhosted.org/packages/09/95/27e05d954972a83372a3ceb6b5db6136bc4f649fa69d8009b27c144ca111/weave-0.52.17.tar.gz", hash = "sha256:940aaf892b65c72c67cb893e97ed5339136a4b33a7ea85d52ed36671111826ef", size = 609149, upload-time = "2025-11-13T22:09:51.045Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/87/a54513796605dfaef2c3c23c2733bcb4b24866a623635c057b2ffdb74052/weave-0.52.16-py3-none-any.whl", hash = "sha256:85985b8cf233032c6d915dfac95b3bcccb1304444d99a6b4a61f3666b58146ce", size = 764366, upload-time = "2025-11-07T19:45:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/ed/0b/ae7860d2b0c02e7efab26815a9a5286d3b0f9f4e0356446f2896351bf770/weave-0.52.17-py3-none-any.whl", hash = "sha256:5772ef82521a033829c921115c5779399581a7ae06d81dfd527126e2115d16d4", size = 765887, upload-time = "2025-11-13T22:09:49.161Z" }, ] [[package]] @@ -7202,20 +7222,22 @@ wheels = [ [[package]] name = "zope-interface" -version = "8.1" +version = "8.1.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/7d/b5b85e09f87be5f33decde2626347123696fc6d9d655cb16f5a986b60a97/zope_interface-8.1.tar.gz", hash = "sha256:a02ee40770c6a2f3d168a8f71f09b62aec3e4fb366da83f8e849dbaa5b38d12f", size = 253831, upload-time = "2025-11-10T07:56:24.825Z" } +sdist = { url = "https://files.pythonhosted.org/packages/71/c9/5ec8679a04d37c797d343f650c51ad67d178f0001c363e44b6ac5f97a9da/zope_interface-8.1.1.tar.gz", hash = "sha256:51b10e6e8e238d719636a401f44f1e366146912407b58453936b781a19be19ec", size = 254748, upload-time = "2025-11-15T08:32:52.404Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/a5/92e53d4d67c127d3ed0e002b90e758a28b4dacb9d81da617c3bae28d2907/zope_interface-8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:db263a60c728c86e6a74945f3f74cfe0ede252e726cf71e05a0c7aca8d9d5432", size = 207891, upload-time = "2025-11-10T07:58:53.189Z" }, - { url = "https://files.pythonhosted.org/packages/b3/76/a100cc050aa76df9bcf8bbd51000724465e2336fd4c786b5904c6c6dfc55/zope_interface-8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cfa89e5b05b7a79ab34e368293ad008321231e321b3ce4430487407b4fe3450a", size = 208335, upload-time = "2025-11-10T07:58:54.232Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ae/37c3e964c599c57323e02ca92a6bf81b4bc9848b88fb5eb3f6fc26320af2/zope_interface-8.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:87eaf011912a06ef86da70aba2ca0ddb68b8ab84a7d1da6b144a586b70a61bca", size = 255011, upload-time = "2025-11-10T07:58:30.304Z" }, - { url = "https://files.pythonhosted.org/packages/b6/9b/b693b6021d83177db2f5237fc3917921c7f497bac9a062eba422435ee172/zope_interface-8.1-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:10f06d128f1c181ded3af08c5004abcb3719c13a976ce9163124e7eeded6899a", size = 259780, upload-time = "2025-11-10T07:58:33.306Z" }, - { url = "https://files.pythonhosted.org/packages/c3/e2/0d1783563892ad46fedd0b1369e8d60ff8fcec0cd6859ab2d07e36f4f0ff/zope_interface-8.1-cp311-cp311-win_amd64.whl", hash = "sha256:17fb5382a4b9bd2ea05648a457c583e5a69f0bfa3076ed1963d48bc42a2da81f", size = 212143, upload-time = "2025-11-10T07:59:56.744Z" }, - { url = "https://files.pythonhosted.org/packages/02/6f/0bfb2beb373b7ca1c3d12807678f20bac1a07f62892f84305a1b544da785/zope_interface-8.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a8aee385282ab2a9813171b15f41317e22ab0a96cf05e9e9e16b29f4af8b6feb", size = 208596, upload-time = "2025-11-10T07:58:09.945Z" }, - { url = "https://files.pythonhosted.org/packages/49/50/169981a42812a2e21bc33fb48640ad01a790ed93c179a9854fe66f901641/zope_interface-8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:af651a87f950a13e45fd49510111f582717fb106a63d6a0c2d3ba86b29734f07", size = 208787, upload-time = "2025-11-10T07:58:11.4Z" }, - { url = "https://files.pythonhosted.org/packages/f8/fb/cb9cb9591a7c78d0878b280b3d3cea42ec17c69c2219b655521b9daa36e8/zope_interface-8.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:80ed7683cf337f3b295e4b96153e2e87f12595c218323dc237c0147a6cc9da26", size = 259224, upload-time = "2025-11-10T07:58:31.882Z" }, - { url = "https://files.pythonhosted.org/packages/18/28/aa89afcefbb93b26934bb5cf030774804b267de2d9300f8bd8e0c6f20bc4/zope_interface-8.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb9a7a45944b28c16d25df7a91bf2b9bdb919fa2b9e11782366a1e737d266ec1", size = 264671, upload-time = "2025-11-10T07:58:36.283Z" }, - { url = "https://files.pythonhosted.org/packages/de/7a/9cea2b9e64d74f27484c59b9a42d6854506673eb0b90c3b8cd088f652d5b/zope_interface-8.1-cp312-cp312-win_amd64.whl", hash = "sha256:fc5e120e3618741714c474b2427d08d36bd292855208b4397e325bd50d81439d", size = 212257, upload-time = "2025-11-10T07:59:54.691Z" }, + { url = "https://files.pythonhosted.org/packages/77/fc/d84bac27332bdefe8c03f7289d932aeb13a5fd6aeedba72b0aa5b18276ff/zope_interface-8.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e8a0fdd5048c1bb733e4693eae9bc4145a19419ea6a1c95299318a93fe9f3d72", size = 207955, upload-time = "2025-11-15T08:36:45.902Z" }, + { url = "https://files.pythonhosted.org/packages/52/02/e1234eb08b10b5cf39e68372586acc7f7bbcd18176f6046433a8f6b8b263/zope_interface-8.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a4cb0ea75a26b606f5bc8524fbce7b7d8628161b6da002c80e6417ce5ec757c0", size = 208398, upload-time = "2025-11-15T08:36:47.016Z" }, + { url = "https://files.pythonhosted.org/packages/3c/be/aabda44d4bc490f9966c2b77fa7822b0407d852cb909b723f2d9e05d2427/zope_interface-8.1.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:c267b00b5a49a12743f5e1d3b4beef45479d696dab090f11fe3faded078a5133", size = 255079, upload-time = "2025-11-15T08:36:48.157Z" }, + { url = "https://files.pythonhosted.org/packages/d8/7f/4fbc7c2d7cb310e5a91b55db3d98e98d12b262014c1fcad9714fe33c2adc/zope_interface-8.1.1-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e25d3e2b9299e7ec54b626573673bdf0d740cf628c22aef0a3afef85b438aa54", size = 259850, upload-time = "2025-11-15T08:36:49.544Z" }, + { url = "https://files.pythonhosted.org/packages/fe/2c/dc573fffe59cdbe8bbbdd2814709bdc71c4870893e7226700bc6a08c5e0c/zope_interface-8.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:63db1241804417aff95ac229c13376c8c12752b83cc06964d62581b493e6551b", size = 261033, upload-time = "2025-11-15T08:36:51.061Z" }, + { url = "https://files.pythonhosted.org/packages/0e/51/1ac50e5ee933d9e3902f3400bda399c128a5c46f9f209d16affe3d4facc5/zope_interface-8.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:9639bf4ed07b5277fb231e54109117c30d608254685e48a7104a34618bcbfc83", size = 212215, upload-time = "2025-11-15T08:36:52.553Z" }, + { url = "https://files.pythonhosted.org/packages/08/3d/f5b8dd2512f33bfab4faba71f66f6873603d625212206dd36f12403ae4ca/zope_interface-8.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a16715808408db7252b8c1597ed9008bdad7bf378ed48eb9b0595fad4170e49d", size = 208660, upload-time = "2025-11-15T08:36:53.579Z" }, + { url = "https://files.pythonhosted.org/packages/e5/41/c331adea9b11e05ff9ac4eb7d3032b24c36a3654ae9f2bf4ef2997048211/zope_interface-8.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce6b58752acc3352c4aa0b55bbeae2a941d61537e6afdad2467a624219025aae", size = 208851, upload-time = "2025-11-15T08:36:54.854Z" }, + { url = "https://files.pythonhosted.org/packages/25/00/7a8019c3bb8b119c5f50f0a4869183a4b699ca004a7f87ce98382e6b364c/zope_interface-8.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:807778883d07177713136479de7fd566f9056a13aef63b686f0ab4807c6be259", size = 259292, upload-time = "2025-11-15T08:36:56.409Z" }, + { url = "https://files.pythonhosted.org/packages/1a/fc/b70e963bf89345edffdd5d16b61e789fdc09365972b603e13785360fea6f/zope_interface-8.1.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50e5eb3b504a7d63dc25211b9298071d5b10a3eb754d6bf2f8ef06cb49f807ab", size = 264741, upload-time = "2025-11-15T08:36:57.675Z" }, + { url = "https://files.pythonhosted.org/packages/96/fe/7d0b5c0692b283901b34847f2b2f50d805bfff4b31de4021ac9dfb516d2a/zope_interface-8.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eee6f93b2512ec9466cf30c37548fd3ed7bc4436ab29cd5943d7a0b561f14f0f", size = 264281, upload-time = "2025-11-15T08:36:58.968Z" }, + { url = "https://files.pythonhosted.org/packages/2b/2c/a7cebede1cf2757be158bcb151fe533fa951038cfc5007c7597f9f86804b/zope_interface-8.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:80edee6116d569883c58ff8efcecac3b737733d646802036dc337aa839a5f06b", size = 212327, upload-time = "2025-11-15T08:37:00.4Z" }, ] [[package]] diff --git a/dev/start-web b/dev/start-web index dc06d6a59f..31c5e168f9 100755 --- a/dev/start-web +++ b/dev/start-web @@ -5,4 +5,4 @@ set -x SCRIPT_DIR="$(dirname "$(realpath "$0")")" cd "$SCRIPT_DIR/../web" -pnpm install && pnpm build && pnpm start +pnpm install && pnpm dev diff --git a/docker/.env.example b/docker/.env.example index 7e2e9aa26d..c9981baaba 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -133,6 +133,8 @@ ACCESS_TOKEN_EXPIRE_MINUTES=60 # Refresh token expiration time in days REFRESH_TOKEN_EXPIRE_DAYS=30 +# The default number of active requests for the application, where 0 means unlimited, should be a non-negative integer. +APP_DEFAULT_ACTIVE_REQUESTS=0 # The maximum number of active requests for the application, where 0 means unlimited, should be a non-negative integer. APP_MAX_ACTIVE_REQUESTS=0 APP_MAX_EXECUTION_TIME=1200 @@ -525,6 +527,7 @@ VECTOR_INDEX_NAME_PREFIX=Vector_index WEAVIATE_ENDPOINT=http://weaviate:8080 WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih WEAVIATE_GRPC_ENDPOINT=grpc://weaviate:50051 +WEAVIATE_TOKENIZATION=word # For OceanBase metadata database configuration, available when `DB_TYPE` is `mysql` and `COMPOSE_PROFILES` includes `oceanbase`. # For OceanBase vector database configuration, available when `VECTOR_STORE` is `oceanbase` diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index eb0733e414..703a60ef67 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:1.10.0 + image: langgenius/dify-api:1.10.1 restart: always environment: # Use the shared environment variables. @@ -41,7 +41,7 @@ services: # worker service # The Celery worker for processing all queues (dataset, workflow, mail, etc.) worker: - image: langgenius/dify-api:1.10.0 + image: langgenius/dify-api:1.10.1 restart: always environment: # Use the shared environment variables. @@ -78,7 +78,7 @@ services: # worker_beat service # Celery beat for scheduling periodic tasks. worker_beat: - image: langgenius/dify-api:1.10.0 + image: langgenius/dify-api:1.10.1 restart: always environment: # Use the shared environment variables. @@ -106,7 +106,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:1.10.0 + image: langgenius/dify-web:1.10.1 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -131,7 +131,7 @@ services: ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} - + # The PostgreSQL database. db_postgres: image: postgres:15-alpine @@ -459,7 +459,7 @@ services: timeout: 10s # seekdb vector database - seekdb: + seekdb: image: oceanbase/seekdb:latest container_name: seekdb profiles: @@ -486,7 +486,7 @@ services: # Qdrant vector store. # (if used, you need to set VECTOR_STORE to qdrant in the api & worker service.) qdrant: - image: langgenius/qdrant:v1.7.3 + image: langgenius/qdrant:v1.8.3 profiles: - qdrant restart: always @@ -676,7 +676,7 @@ services: milvus-standalone: container_name: milvus-standalone - image: milvusdb/milvus:v2.5.15 + image: milvusdb/milvus:v2.6.3 profiles: - milvus command: ["milvus", "run", "standalone"] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index d1e970719c..de2e3943fe 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -34,6 +34,7 @@ x-shared-env: &shared-api-worker-env FILES_ACCESS_TIMEOUT: ${FILES_ACCESS_TIMEOUT:-300} ACCESS_TOKEN_EXPIRE_MINUTES: ${ACCESS_TOKEN_EXPIRE_MINUTES:-60} REFRESH_TOKEN_EXPIRE_DAYS: ${REFRESH_TOKEN_EXPIRE_DAYS:-30} + APP_DEFAULT_ACTIVE_REQUESTS: ${APP_DEFAULT_ACTIVE_REQUESTS:-0} APP_MAX_ACTIVE_REQUESTS: ${APP_MAX_ACTIVE_REQUESTS:-0} APP_MAX_EXECUTION_TIME: ${APP_MAX_EXECUTION_TIME:-1200} DIFY_BIND_ADDRESS: ${DIFY_BIND_ADDRESS:-0.0.0.0} @@ -164,6 +165,7 @@ x-shared-env: &shared-api-worker-env WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://weaviate:8080} WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} WEAVIATE_GRPC_ENDPOINT: ${WEAVIATE_GRPC_ENDPOINT:-grpc://weaviate:50051} + WEAVIATE_TOKENIZATION: ${WEAVIATE_TOKENIZATION:-word} OCEANBASE_VECTOR_HOST: ${OCEANBASE_VECTOR_HOST:-oceanbase} OCEANBASE_VECTOR_PORT: ${OCEANBASE_VECTOR_PORT:-2881} OCEANBASE_VECTOR_USER: ${OCEANBASE_VECTOR_USER:-root@test} @@ -635,7 +637,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:1.10.0 + image: langgenius/dify-api:1.10.1 restart: always environment: # Use the shared environment variables. @@ -674,7 +676,7 @@ services: # worker service # The Celery worker for processing all queues (dataset, workflow, mail, etc.) worker: - image: langgenius/dify-api:1.10.0 + image: langgenius/dify-api:1.10.1 restart: always environment: # Use the shared environment variables. @@ -711,7 +713,7 @@ services: # worker_beat service # Celery beat for scheduling periodic tasks. worker_beat: - image: langgenius/dify-api:1.10.0 + image: langgenius/dify-api:1.10.1 restart: always environment: # Use the shared environment variables. @@ -739,7 +741,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:1.10.0 + image: langgenius/dify-web:1.10.1 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -764,7 +766,7 @@ services: ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} - + # The PostgreSQL database. db_postgres: image: postgres:15-alpine @@ -1092,7 +1094,7 @@ services: timeout: 10s # seekdb vector database - seekdb: + seekdb: image: oceanbase/seekdb:latest container_name: seekdb profiles: @@ -1119,7 +1121,7 @@ services: # Qdrant vector store. # (if used, you need to set VECTOR_STORE to qdrant in the api & worker service.) qdrant: - image: langgenius/qdrant:v1.7.3 + image: langgenius/qdrant:v1.8.3 profiles: - qdrant restart: always @@ -1309,7 +1311,7 @@ services: milvus-standalone: container_name: milvus-standalone - image: milvusdb/milvus:v2.5.15 + image: milvusdb/milvus:v2.6.3 profiles: - milvus command: ["milvus", "run", "standalone"] diff --git a/docs/ar-SA/README.md b/docs/ar-SA/README.md index 30920ed983..99e3e3567e 100644 --- a/docs/ar-SA/README.md +++ b/docs/ar-SA/README.md @@ -32,6 +32,12 @@ Issues closed Discussion posts + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/bn-BD/README.md b/docs/bn-BD/README.md index 5430364ef9..f3fa68b466 100644 --- a/docs/bn-BD/README.md +++ b/docs/bn-BD/README.md @@ -36,6 +36,12 @@ Issues closed Discussion posts + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/de-DE/README.md b/docs/de-DE/README.md index 6c49fbdfc3..c71a0bfccf 100644 --- a/docs/de-DE/README.md +++ b/docs/de-DE/README.md @@ -36,6 +36,12 @@ Issues closed Discussion posts + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/es-ES/README.md b/docs/es-ES/README.md index ae83d416e3..da81b51d6a 100644 --- a/docs/es-ES/README.md +++ b/docs/es-ES/README.md @@ -32,6 +32,12 @@ Issues cerrados Publicaciones de discusión + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/fr-FR/README.md b/docs/fr-FR/README.md index b7d006a927..03f3221798 100644 --- a/docs/fr-FR/README.md +++ b/docs/fr-FR/README.md @@ -32,6 +32,12 @@ Problèmes fermés Messages de discussion + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/hi-IN/README.md b/docs/hi-IN/README.md index 7c4fc70db0..bedeaa6246 100644 --- a/docs/hi-IN/README.md +++ b/docs/hi-IN/README.md @@ -36,6 +36,12 @@ Issues closed Discussion posts + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/it-IT/README.md b/docs/it-IT/README.md index 598e87ec25..2e96335d3e 100644 --- a/docs/it-IT/README.md +++ b/docs/it-IT/README.md @@ -36,6 +36,12 @@ Issues closed Discussion posts + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/ja-JP/README.md b/docs/ja-JP/README.md index f9e700d1df..659ffbda51 100644 --- a/docs/ja-JP/README.md +++ b/docs/ja-JP/README.md @@ -32,6 +32,12 @@ クローズされた問題 ディスカッション投稿 + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/ko-KR/README.md b/docs/ko-KR/README.md index 4e4b82e920..2f6c526ef2 100644 --- a/docs/ko-KR/README.md +++ b/docs/ko-KR/README.md @@ -32,6 +32,12 @@ Issues closed Discussion posts + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/pt-BR/README.md b/docs/pt-BR/README.md index 444faa0a67..ed29ec0294 100644 --- a/docs/pt-BR/README.md +++ b/docs/pt-BR/README.md @@ -36,6 +36,12 @@ Issues closed Discussion posts + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/sl-SI/README.md b/docs/sl-SI/README.md index 04dc3b5dff..caef2c303c 100644 --- a/docs/sl-SI/README.md +++ b/docs/sl-SI/README.md @@ -33,6 +33,12 @@ Issues closed Discussion posts + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/tlh/README.md b/docs/tlh/README.md index b1e3016efd..a25849c443 100644 --- a/docs/tlh/README.md +++ b/docs/tlh/README.md @@ -32,6 +32,12 @@ Issues closed Discussion posts + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/tr-TR/README.md b/docs/tr-TR/README.md index 965a1704be..6361ca5dd9 100644 --- a/docs/tr-TR/README.md +++ b/docs/tr-TR/README.md @@ -32,6 +32,12 @@ Kapatılan sorunlar Tartışma gönderileri + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/vi-VN/README.md b/docs/vi-VN/README.md index 07329e84cd..3042a98d95 100644 --- a/docs/vi-VN/README.md +++ b/docs/vi-VN/README.md @@ -32,6 +32,12 @@ Vấn đề đã đóng Bài thảo luận + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/zh-CN/README.md b/docs/zh-CN/README.md index 888a0d7f12..15bb447ad8 100644 --- a/docs/zh-CN/README.md +++ b/docs/zh-CN/README.md @@ -32,6 +32,12 @@ Issues closed Discussion posts + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/docs/zh-TW/README.md b/docs/zh-TW/README.md index d8c484a6d4..14b343ba29 100644 --- a/docs/zh-TW/README.md +++ b/docs/zh-TW/README.md @@ -36,6 +36,12 @@ Issues closed Discussion posts + + LFX Health Score + + LFX Contributors + + LFX Active Contributors

diff --git a/web/README.md b/web/README.md index 6daf1e922e..1855ebc3b8 100644 --- a/web/README.md +++ b/web/README.md @@ -99,9 +99,9 @@ If your IDE is VSCode, rename `web/.vscode/settings.example.json` to `web/.vscod ## Test -We start to use [Jest](https://jestjs.io/) and [React Testing Library](https://testing-library.com/docs/react-testing-library/intro/) for Unit Testing. +We use [Jest](https://jestjs.io/) and [React Testing Library](https://testing-library.com/docs/react-testing-library/intro/) for Unit Testing. -You can create a test file with a suffix of `.spec` beside the file that to be tested. For example, if you want to test a file named `util.ts`. The test file name should be `util.spec.ts`. +**📖 Complete Testing Guide**: See [web/testing/testing.md](./testing/testing.md) for detailed testing specifications, best practices, and examples. Run test: @@ -109,10 +109,22 @@ Run test: pnpm run test ``` -If you are not familiar with writing tests, here is some code to refer to: +### Example Code -- [classnames.spec.ts](./utils/classnames.spec.ts) -- [index.spec.tsx](./app/components/base/button/index.spec.tsx) +If you are not familiar with writing tests, refer to: + +- [classnames.spec.ts](./utils/classnames.spec.ts) - Utility function test example +- [index.spec.tsx](./app/components/base/button/index.spec.tsx) - Component test example + +### Analyze Component Complexity + +Before writing tests, use the script to analyze component complexity: + +```bash +pnpm analyze-component app/components/your-component/index.tsx +``` + +This will help you determine the testing strategy. See [web/testing/testing.md](./testing/testing.md) for details. ## Documentation diff --git a/web/__tests__/workflow-onboarding-integration.test.tsx b/web/__tests__/workflow-onboarding-integration.test.tsx index c1a922bb1f..ded8c75bd1 100644 --- a/web/__tests__/workflow-onboarding-integration.test.tsx +++ b/web/__tests__/workflow-onboarding-integration.test.tsx @@ -1,6 +1,24 @@ import { BlockEnum } from '@/app/components/workflow/types' import { useWorkflowStore } from '@/app/components/workflow/store' +// Type for mocked store +type MockWorkflowStore = { + showOnboarding: boolean + setShowOnboarding: jest.Mock + hasShownOnboarding: boolean + setHasShownOnboarding: jest.Mock + hasSelectedStartNode: boolean + setHasSelectedStartNode: jest.Mock + setShouldAutoOpenStartNodeSelector: jest.Mock + notInitialWorkflow: boolean +} + +// Type for mocked node +type MockNode = { + id: string + data: { type?: BlockEnum } +} + // Mock zustand store jest.mock('@/app/components/workflow/store') @@ -39,7 +57,7 @@ describe('Workflow Onboarding Integration Logic', () => { describe('Onboarding State Management', () => { it('should initialize onboarding state correctly', () => { - const store = useWorkflowStore() + const store = useWorkflowStore() as unknown as MockWorkflowStore expect(store.showOnboarding).toBe(false) expect(store.hasSelectedStartNode).toBe(false) @@ -47,7 +65,7 @@ describe('Workflow Onboarding Integration Logic', () => { }) it('should update onboarding visibility', () => { - const store = useWorkflowStore() + const store = useWorkflowStore() as unknown as MockWorkflowStore store.setShowOnboarding(true) expect(mockSetShowOnboarding).toHaveBeenCalledWith(true) @@ -57,14 +75,14 @@ describe('Workflow Onboarding Integration Logic', () => { }) it('should track node selection state', () => { - const store = useWorkflowStore() + const store = useWorkflowStore() as unknown as MockWorkflowStore store.setHasSelectedStartNode(true) expect(mockSetHasSelectedStartNode).toHaveBeenCalledWith(true) }) it('should track onboarding show state', () => { - const store = useWorkflowStore() + const store = useWorkflowStore() as unknown as MockWorkflowStore store.setHasShownOnboarding(true) expect(mockSetHasShownOnboarding).toHaveBeenCalledWith(true) @@ -205,60 +223,44 @@ describe('Workflow Onboarding Integration Logic', () => { it('should auto-expand for TriggerSchedule in new workflow', () => { const shouldAutoOpenStartNodeSelector = true - const nodeType = BlockEnum.TriggerSchedule + const nodeType: BlockEnum = BlockEnum.TriggerSchedule const isChatMode = false + const validStartTypes = [BlockEnum.Start, BlockEnum.TriggerSchedule, BlockEnum.TriggerWebhook, BlockEnum.TriggerPlugin] - const shouldAutoExpand = shouldAutoOpenStartNodeSelector && ( - nodeType === BlockEnum.Start - || nodeType === BlockEnum.TriggerSchedule - || nodeType === BlockEnum.TriggerWebhook - || nodeType === BlockEnum.TriggerPlugin - ) && !isChatMode + const shouldAutoExpand = shouldAutoOpenStartNodeSelector && validStartTypes.includes(nodeType) && !isChatMode expect(shouldAutoExpand).toBe(true) }) it('should auto-expand for TriggerWebhook in new workflow', () => { const shouldAutoOpenStartNodeSelector = true - const nodeType = BlockEnum.TriggerWebhook + const nodeType: BlockEnum = BlockEnum.TriggerWebhook const isChatMode = false + const validStartTypes = [BlockEnum.Start, BlockEnum.TriggerSchedule, BlockEnum.TriggerWebhook, BlockEnum.TriggerPlugin] - const shouldAutoExpand = shouldAutoOpenStartNodeSelector && ( - nodeType === BlockEnum.Start - || nodeType === BlockEnum.TriggerSchedule - || nodeType === BlockEnum.TriggerWebhook - || nodeType === BlockEnum.TriggerPlugin - ) && !isChatMode + const shouldAutoExpand = shouldAutoOpenStartNodeSelector && validStartTypes.includes(nodeType) && !isChatMode expect(shouldAutoExpand).toBe(true) }) it('should auto-expand for TriggerPlugin in new workflow', () => { const shouldAutoOpenStartNodeSelector = true - const nodeType = BlockEnum.TriggerPlugin + const nodeType: BlockEnum = BlockEnum.TriggerPlugin const isChatMode = false + const validStartTypes = [BlockEnum.Start, BlockEnum.TriggerSchedule, BlockEnum.TriggerWebhook, BlockEnum.TriggerPlugin] - const shouldAutoExpand = shouldAutoOpenStartNodeSelector && ( - nodeType === BlockEnum.Start - || nodeType === BlockEnum.TriggerSchedule - || nodeType === BlockEnum.TriggerWebhook - || nodeType === BlockEnum.TriggerPlugin - ) && !isChatMode + const shouldAutoExpand = shouldAutoOpenStartNodeSelector && validStartTypes.includes(nodeType) && !isChatMode expect(shouldAutoExpand).toBe(true) }) it('should not auto-expand for non-trigger nodes', () => { const shouldAutoOpenStartNodeSelector = true - const nodeType = BlockEnum.LLM + const nodeType: BlockEnum = BlockEnum.LLM const isChatMode = false + const validStartTypes = [BlockEnum.Start, BlockEnum.TriggerSchedule, BlockEnum.TriggerWebhook, BlockEnum.TriggerPlugin] - const shouldAutoExpand = shouldAutoOpenStartNodeSelector && ( - nodeType === BlockEnum.Start - || nodeType === BlockEnum.TriggerSchedule - || nodeType === BlockEnum.TriggerWebhook - || nodeType === BlockEnum.TriggerPlugin - ) && !isChatMode + const shouldAutoExpand = shouldAutoOpenStartNodeSelector && validStartTypes.includes(nodeType) && !isChatMode expect(shouldAutoExpand).toBe(false) }) @@ -321,7 +323,7 @@ describe('Workflow Onboarding Integration Logic', () => { const nodeData = { type: BlockEnum.Start, title: 'Start' } // Simulate node creation logic from workflow-children.tsx - const createdNodeData = { + const createdNodeData: Record = { ...nodeData, // Note: 'selected: true' should NOT be added } @@ -334,7 +336,7 @@ describe('Workflow Onboarding Integration Logic', () => { const nodeData = { type: BlockEnum.TriggerWebhook, title: 'Webhook Trigger' } const toolConfig = { webhook_url: 'https://example.com/webhook' } - const createdNodeData = { + const createdNodeData: Record = { ...nodeData, ...toolConfig, // Note: 'selected: true' should NOT be added @@ -352,7 +354,7 @@ describe('Workflow Onboarding Integration Logic', () => { config: { interval: '1h' }, } - const createdNodeData = { + const createdNodeData: Record = { ...nodeData, } @@ -495,7 +497,7 @@ describe('Workflow Onboarding Integration Logic', () => { BlockEnum.TriggerWebhook, BlockEnum.TriggerPlugin, ] - const hasStartNode = nodes.some(node => startNodeTypes.includes(node.data?.type)) + const hasStartNode = nodes.some((node: MockNode) => startNodeTypes.includes(node.data?.type as BlockEnum)) const isEmpty = nodes.length === 0 || !hasStartNode expect(isEmpty).toBe(true) @@ -516,7 +518,7 @@ describe('Workflow Onboarding Integration Logic', () => { BlockEnum.TriggerWebhook, BlockEnum.TriggerPlugin, ] - const hasStartNode = nodes.some(node => startNodeTypes.includes(node.data.type)) + const hasStartNode = nodes.some((node: MockNode) => startNodeTypes.includes(node.data.type as BlockEnum)) const isEmpty = nodes.length === 0 || !hasStartNode expect(isEmpty).toBe(true) @@ -536,7 +538,7 @@ describe('Workflow Onboarding Integration Logic', () => { BlockEnum.TriggerWebhook, BlockEnum.TriggerPlugin, ] - const hasStartNode = nodes.some(node => startNodeTypes.includes(node.data.type)) + const hasStartNode = nodes.some((node: MockNode) => startNodeTypes.includes(node.data.type as BlockEnum)) const isEmpty = nodes.length === 0 || !hasStartNode expect(isEmpty).toBe(false) @@ -571,7 +573,7 @@ describe('Workflow Onboarding Integration Logic', () => { }) // Simulate the check logic with hasShownOnboarding = true - const store = useWorkflowStore() + const store = useWorkflowStore() as unknown as MockWorkflowStore const shouldTrigger = !store.hasShownOnboarding && !store.showOnboarding && !store.notInitialWorkflow expect(shouldTrigger).toBe(false) @@ -605,7 +607,7 @@ describe('Workflow Onboarding Integration Logic', () => { }) // Simulate the check logic with notInitialWorkflow = true - const store = useWorkflowStore() + const store = useWorkflowStore() as unknown as MockWorkflowStore const shouldTrigger = !store.hasShownOnboarding && !store.showOnboarding && !store.notInitialWorkflow expect(shouldTrigger).toBe(false) diff --git a/web/app/components/app/app-publisher/index.tsx b/web/app/components/app/app-publisher/index.tsx index a11af3b816..bba5ebfa21 100644 --- a/web/app/components/app/app-publisher/index.tsx +++ b/web/app/components/app/app-publisher/index.tsx @@ -38,7 +38,7 @@ import { PortalToFollowElemTrigger, } from '@/app/components/base/portal-to-follow-elem' import WorkflowToolConfigureButton from '@/app/components/tools/workflow-tool/configure-button' -import type { InputVar } from '@/app/components/workflow/types' +import type { InputVar, Variable } from '@/app/components/workflow/types' import { appDefaultIconBackground } from '@/config' import { useGlobalPublicStore } from '@/context/global-public-context' import { useFormatTimeFromNow } from '@/hooks/use-format-time-from-now' @@ -103,6 +103,7 @@ export type AppPublisherProps = { crossAxisOffset?: number toolPublished?: boolean inputs?: InputVar[] + outputs?: Variable[] onRefreshData?: () => void workflowToolAvailable?: boolean missingStartNode?: boolean @@ -125,6 +126,7 @@ const AppPublisher = ({ crossAxisOffset = 0, toolPublished, inputs, + outputs, onRefreshData, workflowToolAvailable = true, missingStartNode = false, @@ -457,6 +459,7 @@ const AppPublisher = ({ name={appDetail?.name} description={appDetail?.description} inputs={inputs} + outputs={outputs} handlePublish={handlePublish} onRefreshData={onRefreshData} disabledReason={workflowToolMessage} diff --git a/web/app/components/app/configuration/config-var/config-modal/index.tsx b/web/app/components/app/configuration/config-var/config-modal/index.tsx index bab77e61b0..17df1558d8 100644 --- a/web/app/components/app/configuration/config-var/config-modal/index.tsx +++ b/web/app/components/app/configuration/config-var/config-modal/index.tsx @@ -109,6 +109,13 @@ const ConfigModal: FC = ({ [key]: value, } + // Clear default value if modified options no longer include current default + if (key === 'options' && prev.default) { + const optionsArray = Array.isArray(value) ? value : [] + if (!optionsArray.includes(prev.default)) + newPayload.default = undefined + } + return newPayload }) } diff --git a/web/app/components/app/configuration/config-var/config-select/index.tsx b/web/app/components/app/configuration/config-var/config-select/index.tsx index 40ddaef78f..713a715f1c 100644 --- a/web/app/components/app/configuration/config-var/config-select/index.tsx +++ b/web/app/components/app/configuration/config-var/config-select/index.tsx @@ -71,6 +71,7 @@ const ConfigSelect: FC = ({ className='absolute right-1.5 top-1/2 block translate-y-[-50%] cursor-pointer rounded-md p-1 text-text-tertiary hover:bg-state-destructive-hover hover:text-text-destructive' onClick={() => { onChange(options.filter((_, i) => index !== i)) + setDeletingID(null) }} onMouseEnter={() => setDeletingID(index)} onMouseLeave={() => setDeletingID(null)} diff --git a/web/app/components/app/configuration/debug/chat-user-input.tsx b/web/app/components/app/configuration/debug/chat-user-input.tsx index b1161de075..16666d514e 100644 --- a/web/app/components/app/configuration/debug/chat-user-input.tsx +++ b/web/app/components/app/configuration/debug/chat-user-input.tsx @@ -1,4 +1,4 @@ -import React from 'react' +import React, { useEffect } from 'react' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' import ConfigContext from '@/context/debug-configuration' @@ -32,6 +32,24 @@ const ChatUserInput = ({ return obj })() + // Initialize inputs with default values from promptVariables + useEffect(() => { + const newInputs = { ...inputs } + let hasChanges = false + + promptVariables.forEach((variable) => { + const { key, default: defaultValue } = variable + // Only set default value if the field is empty and a default exists + if (defaultValue !== undefined && defaultValue !== null && defaultValue !== '' && (inputs[key] === undefined || inputs[key] === null || inputs[key] === '')) { + newInputs[key] = defaultValue + hasChanges = true + } + }) + + if (hasChanges) + setInputs(newInputs) + }, [promptVariables, inputs, setInputs]) + const handleInputValueChange = (key: string, value: string | boolean) => { if (!(key in promptVariableObj)) return diff --git a/web/app/components/app/configuration/prompt-value-panel/index.tsx b/web/app/components/app/configuration/prompt-value-panel/index.tsx index e8b988767c..005f7f938f 100644 --- a/web/app/components/app/configuration/prompt-value-panel/index.tsx +++ b/web/app/components/app/configuration/prompt-value-panel/index.tsx @@ -1,6 +1,6 @@ 'use client' import type { FC } from 'react' -import React, { useMemo, useState } from 'react' +import React, { useEffect, useMemo, useState } from 'react' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' import { @@ -54,6 +54,24 @@ const PromptValuePanel: FC = ({ return obj }, [promptVariables]) + // Initialize inputs with default values from promptVariables + useEffect(() => { + const newInputs = { ...inputs } + let hasChanges = false + + promptVariables.forEach((variable) => { + const { key, default: defaultValue } = variable + // Only set default value if the field is empty and a default exists + if (defaultValue !== undefined && defaultValue !== null && defaultValue !== '' && (inputs[key] === undefined || inputs[key] === null || inputs[key] === '')) { + newInputs[key] = defaultValue + hasChanges = true + } + }) + + if (hasChanges) + setInputs(newInputs) + }, [promptVariables, inputs, setInputs]) + const canNotRun = useMemo(() => { if (mode !== AppModeEnum.COMPLETION) return true diff --git a/web/app/components/app/configuration/tools/external-data-tool-modal.tsx b/web/app/components/app/configuration/tools/external-data-tool-modal.tsx index d438618a93..990e679c79 100644 --- a/web/app/components/app/configuration/tools/external-data-tool-modal.tsx +++ b/web/app/components/app/configuration/tools/external-data-tool-modal.tsx @@ -191,11 +191,11 @@ const ExternalDataToolModal: FC = ({ onClose={noop} className='!w-[640px] !max-w-none !p-8 !pb-6' > -

+
{`${action} ${t('appDebug.variableConfig.apiBasedVar')}`}
-
+
{t('common.apiBasedExtension.type')}
= ({ />
-
+
{t('appDebug.feature.tools.modal.name.title')}
handleValueChange({ label: e.target.value })} - className='mr-2 block h-9 grow appearance-none rounded-lg bg-gray-100 px-3 text-sm text-gray-900 outline-none' + className='mr-2 block h-9 grow appearance-none rounded-lg bg-components-input-bg-normal px-3 text-sm text-components-input-text-filled outline-none' placeholder={t('appDebug.feature.tools.modal.name.placeholder') || ''} /> { setShowEmojiPicker(true) }} - className='!h-9 !w-9 cursor-pointer rounded-lg border-[0.5px] border-black/5 ' + className='!h-9 !w-9 cursor-pointer rounded-lg border-[0.5px] border-components-panel-border ' icon={localeData.icon} background={localeData.icon_background} />
-
+
{t('appDebug.feature.tools.modal.variableName.title')}
handleValueChange({ variable: e.target.value })} - className='block h-9 w-full appearance-none rounded-lg bg-gray-100 px-3 text-sm text-gray-900 outline-none' + className='block h-9 w-full appearance-none rounded-lg bg-components-input-bg-normal px-3 text-sm text-components-input-text-filled outline-none' placeholder={t('appDebug.feature.tools.modal.variableName.placeholder') || ''} />
{ localeData.type === 'api' && (
-
+
{t('common.apiBasedExtension.selector.title')} - + {t('common.apiBasedExtension.link')}
diff --git a/web/app/components/app/log/index.tsx b/web/app/components/app/log/index.tsx index 55a3f7d12d..cedf2de74d 100644 --- a/web/app/components/app/log/index.tsx +++ b/web/app/components/app/log/index.tsx @@ -1,11 +1,12 @@ 'use client' import type { FC } from 'react' -import React, { useState } from 'react' +import React, { useCallback, useEffect, useState } from 'react' import useSWR from 'swr' import { useDebounce } from 'ahooks' import { omit } from 'lodash-es' import dayjs from 'dayjs' import { useTranslation } from 'react-i18next' +import { usePathname, useRouter, useSearchParams } from 'next/navigation' import List from './list' import Filter, { TIME_PERIOD_MAPPING } from './filter' import EmptyElement from './empty-element' @@ -28,15 +29,29 @@ export type QueryParam = { const Logs: FC = ({ appDetail }) => { const { t } = useTranslation() + const router = useRouter() + const pathname = usePathname() + const searchParams = useSearchParams() const [queryParams, setQueryParams] = useState({ period: '2', annotation_status: 'all', sort_by: '-created_at', }) - const [currPage, setCurrPage] = React.useState(0) + const getPageFromParams = useCallback(() => { + const pageParam = Number.parseInt(searchParams.get('page') || '1', 10) + if (Number.isNaN(pageParam) || pageParam < 1) + return 0 + return pageParam - 1 + }, [searchParams]) + const [currPage, setCurrPage] = React.useState(() => getPageFromParams()) const [limit, setLimit] = React.useState(APP_PAGE_LIMIT) const debouncedQueryParams = useDebounce(queryParams, { wait: 500 }) + useEffect(() => { + const pageFromParams = getPageFromParams() + setCurrPage(prev => (prev === pageFromParams ? prev : pageFromParams)) + }, [getPageFromParams]) + // Get the app type first const isChatMode = appDetail.mode !== AppModeEnum.COMPLETION @@ -70,6 +85,18 @@ const Logs: FC = ({ appDetail }) => { const total = isChatMode ? chatConversations?.total : completionConversations?.total + const handlePageChange = useCallback((page: number) => { + setCurrPage(page) + const params = new URLSearchParams(searchParams.toString()) + const nextPageValue = page + 1 + if (nextPageValue === 1) + params.delete('page') + else + params.set('page', String(nextPageValue)) + const queryString = params.toString() + router.replace(queryString ? `${pathname}?${queryString}` : pathname, { scroll: false }) + }, [pathname, router, searchParams]) + return (

{t('appLog.description')}

@@ -85,7 +112,7 @@ const Logs: FC = ({ appDetail }) => { {(total && total > APP_PAGE_LIMIT) ? ({ @@ -7,6 +8,9 @@ jest.mock('@/app/components/workflow/utils/workflow-entry', () => ({ const mockGetWorkflowEntryNode = getWorkflowEntryNode as jest.MockedFunction +// Mock entry node for testing (truthy value) +const mockEntryNode = { id: 'start-node', data: { type: 'start' } } as Node + describe('App Card Toggle Logic', () => { beforeEach(() => { jest.clearAllMocks() @@ -39,7 +43,7 @@ describe('App Card Toggle Logic', () => { describe('Entry Node Detection Logic', () => { it('should disable toggle when workflow missing entry node', () => { - mockGetWorkflowEntryNode.mockReturnValue(false) + mockGetWorkflowEntryNode.mockReturnValue(undefined) const result = calculateToggleState( 'workflow', @@ -55,7 +59,7 @@ describe('App Card Toggle Logic', () => { }) it('should enable toggle when workflow has entry node', () => { - mockGetWorkflowEntryNode.mockReturnValue(true) + mockGetWorkflowEntryNode.mockReturnValue(mockEntryNode) const result = calculateToggleState( 'workflow', @@ -101,7 +105,7 @@ describe('App Card Toggle Logic', () => { }) it('should consider published state when workflow has graph', () => { - mockGetWorkflowEntryNode.mockReturnValue(true) + mockGetWorkflowEntryNode.mockReturnValue(mockEntryNode) const result = calculateToggleState( 'workflow', @@ -117,7 +121,7 @@ describe('App Card Toggle Logic', () => { describe('Permissions Logic', () => { it('should disable webapp toggle when user lacks editor permissions', () => { - mockGetWorkflowEntryNode.mockReturnValue(true) + mockGetWorkflowEntryNode.mockReturnValue(mockEntryNode) const result = calculateToggleState( 'workflow', @@ -132,7 +136,7 @@ describe('App Card Toggle Logic', () => { }) it('should disable api toggle when user lacks manager permissions', () => { - mockGetWorkflowEntryNode.mockReturnValue(true) + mockGetWorkflowEntryNode.mockReturnValue(mockEntryNode) const result = calculateToggleState( 'workflow', @@ -147,7 +151,7 @@ describe('App Card Toggle Logic', () => { }) it('should enable toggle when user has proper permissions', () => { - mockGetWorkflowEntryNode.mockReturnValue(true) + mockGetWorkflowEntryNode.mockReturnValue(mockEntryNode) const webappResult = calculateToggleState( 'workflow', @@ -172,7 +176,7 @@ describe('App Card Toggle Logic', () => { describe('Combined Conditions Logic', () => { it('should handle multiple disable conditions correctly', () => { - mockGetWorkflowEntryNode.mockReturnValue(false) + mockGetWorkflowEntryNode.mockReturnValue(undefined) const result = calculateToggleState( 'workflow', @@ -191,7 +195,7 @@ describe('App Card Toggle Logic', () => { }) it('should enable when all conditions are satisfied', () => { - mockGetWorkflowEntryNode.mockReturnValue(true) + mockGetWorkflowEntryNode.mockReturnValue(mockEntryNode) const result = calculateToggleState( 'workflow', diff --git a/web/app/components/base/audio-gallery/AudioPlayer.tsx b/web/app/components/base/audio-gallery/AudioPlayer.tsx index cad7adac02..399f055161 100644 --- a/web/app/components/base/audio-gallery/AudioPlayer.tsx +++ b/web/app/components/base/audio-gallery/AudioPlayer.tsx @@ -10,10 +10,11 @@ import { Theme } from '@/types/app' import cn from '@/utils/classnames' type AudioPlayerProps = { - src: string + src?: string // Keep backward compatibility + srcs?: string[] // Support multiple sources } -const AudioPlayer: React.FC = ({ src }) => { +const AudioPlayer: React.FC = ({ src, srcs }) => { const [isPlaying, setIsPlaying] = useState(false) const [currentTime, setCurrentTime] = useState(0) const [duration, setDuration] = useState(0) @@ -61,19 +62,22 @@ const AudioPlayer: React.FC = ({ src }) => { // Preload audio metadata audio.load() - // Delayed generation of waveform data - // eslint-disable-next-line ts/no-use-before-define - const timer = setTimeout(() => generateWaveformData(src), 1000) - - return () => { - audio.removeEventListener('loadedmetadata', setAudioData) - audio.removeEventListener('timeupdate', setAudioTime) - audio.removeEventListener('progress', handleProgress) - audio.removeEventListener('ended', handleEnded) - audio.removeEventListener('error', handleError) - clearTimeout(timer) + // Use the first source or src to generate waveform + const primarySrc = srcs?.[0] || src + if (primarySrc) { + // Delayed generation of waveform data + // eslint-disable-next-line ts/no-use-before-define + const timer = setTimeout(() => generateWaveformData(primarySrc), 1000) + return () => { + audio.removeEventListener('loadedmetadata', setAudioData) + audio.removeEventListener('timeupdate', setAudioTime) + audio.removeEventListener('progress', handleProgress) + audio.removeEventListener('ended', handleEnded) + audio.removeEventListener('error', handleError) + clearTimeout(timer) + } } - }, [src]) + }, [src, srcs]) const generateWaveformData = async (audioSrc: string) => { if (!window.AudioContext && !(window as any).webkitAudioContext) { @@ -85,8 +89,9 @@ const AudioPlayer: React.FC = ({ src }) => { return null } - const url = new URL(src) - const isHttp = url.protocol === 'http:' || url.protocol === 'https:' + const primarySrc = srcs?.[0] || src + const url = primarySrc ? new URL(primarySrc) : null + const isHttp = url ? (url.protocol === 'http:' || url.protocol === 'https:') : false if (!isHttp) { setIsAudioAvailable(false) return null @@ -286,8 +291,13 @@ const AudioPlayer: React.FC = ({ src }) => { }, [duration]) return ( -
-