diff --git a/api/commands.py b/api/commands.py index 82efe34611..8ca19e1dac 100644 --- a/api/commands.py +++ b/api/commands.py @@ -1521,6 +1521,14 @@ def transform_datasource_credentials(): auth_count = 0 for firecrawl_tenant_credential in firecrawl_tenant_credentials: auth_count += 1 + if not firecrawl_tenant_credential.credentials: + click.echo( + click.style( + f"Skipping firecrawl credential for tenant {tenant_id} due to missing credentials.", + fg="yellow", + ) + ) + continue # get credential api key credentials_json = json.loads(firecrawl_tenant_credential.credentials) api_key = credentials_json.get("config", {}).get("api_key") @@ -1576,6 +1584,14 @@ def transform_datasource_credentials(): auth_count = 0 for jina_tenant_credential in jina_tenant_credentials: auth_count += 1 + if not jina_tenant_credential.credentials: + click.echo( + click.style( + f"Skipping jina credential for tenant {tenant_id} due to missing credentials.", + fg="yellow", + ) + ) + continue # get credential api key credentials_json = json.loads(jina_tenant_credential.credentials) api_key = credentials_json.get("config", {}).get("api_key") diff --git a/api/controllers/console/apikey.py b/api/controllers/console/apikey.py index fec527e4cb..b1e3813f33 100644 --- a/api/controllers/console/apikey.py +++ b/api/controllers/console/apikey.py @@ -1,5 +1,4 @@ import flask_restx -from flask_login import current_user from flask_restx import Resource, fields, marshal_with from flask_restx._http import HTTPStatus from sqlalchemy import select @@ -8,7 +7,8 @@ from werkzeug.exceptions import Forbidden from extensions.ext_database import db from libs.helper import TimestampField -from libs.login import login_required +from libs.login import current_user, login_required +from models.account import Account from models.dataset import Dataset from models.model import ApiToken, App @@ -57,6 +57,8 @@ class BaseApiKeyListResource(Resource): def get(self, resource_id): assert self.resource_id_field is not None, "resource_id_field must be set" resource_id = str(resource_id) + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None _get_resource(resource_id, current_user.current_tenant_id, self.resource_model) keys = db.session.scalars( select(ApiToken).where( @@ -69,8 +71,10 @@ class BaseApiKeyListResource(Resource): def post(self, resource_id): assert self.resource_id_field is not None, "resource_id_field must be set" resource_id = str(resource_id) + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None _get_resource(resource_id, current_user.current_tenant_id, self.resource_model) - if not current_user.is_editor: + if not current_user.has_edit_permission: raise Forbidden() current_key_count = ( @@ -108,6 +112,8 @@ class BaseApiKeyResource(Resource): assert self.resource_id_field is not None, "resource_id_field must be set" resource_id = str(resource_id) api_key_id = str(api_key_id) + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None _get_resource(resource_id, current_user.current_tenant_id, self.resource_model) # The role of the current user in the ta table must be admin or owner diff --git a/api/controllers/console/billing/compliance.py b/api/controllers/console/billing/compliance.py index e489b48c82..c0d104e0d4 100644 --- a/api/controllers/console/billing/compliance.py +++ b/api/controllers/console/billing/compliance.py @@ -1,9 +1,9 @@ from flask import request -from flask_login import current_user from flask_restx import Resource, reqparse from libs.helper import extract_remote_ip -from libs.login import login_required +from libs.login import current_user, login_required +from models.account import Account from services.billing_service import BillingService from .. import console_ns @@ -17,6 +17,8 @@ class ComplianceApi(Resource): @account_initialization_required @only_edition_cloud def get(self): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None parser = reqparse.RequestParser() parser.add_argument("doc_name", type=str, required=True, location="args") args = parser.parse_args() diff --git a/api/controllers/console/datasets/hit_testing_base.py b/api/controllers/console/datasets/hit_testing_base.py index a68e337135..6113f1fd17 100644 --- a/api/controllers/console/datasets/hit_testing_base.py +++ b/api/controllers/console/datasets/hit_testing_base.py @@ -1,7 +1,5 @@ import logging -from typing import cast -from flask_login import current_user from flask_restx import marshal, reqparse from werkzeug.exceptions import Forbidden, InternalServerError, NotFound @@ -21,6 +19,7 @@ from core.errors.error import ( ) from core.model_runtime.errors.invoke import InvokeError from fields.hit_testing_fields import hit_testing_record_fields +from libs.login import current_user from models.account import Account from services.dataset_service import DatasetService from services.hit_testing_service import HitTestingService @@ -31,6 +30,7 @@ logger = logging.getLogger(__name__) class DatasetsHitTestingBase: @staticmethod def get_and_validate_dataset(dataset_id: str): + assert isinstance(current_user, Account) dataset = DatasetService.get_dataset(dataset_id) if dataset is None: raise NotFound("Dataset not found.") @@ -57,11 +57,12 @@ class DatasetsHitTestingBase: @staticmethod def perform_hit_testing(dataset, args): + assert isinstance(current_user, Account) try: response = HitTestingService.retrieve( dataset=dataset, query=args["query"], - account=cast(Account, current_user), + account=current_user, retrieval_model=args["retrieval_model"], external_retrieval_model=args["external_retrieval_model"], limit=10, diff --git a/api/controllers/console/explore/wraps.py b/api/controllers/console/explore/wraps.py index 3a8ba64a03..5956eb52c4 100644 --- a/api/controllers/console/explore/wraps.py +++ b/api/controllers/console/explore/wraps.py @@ -2,15 +2,15 @@ from collections.abc import Callable from functools import wraps from typing import Concatenate, ParamSpec, TypeVar -from flask_login import current_user from flask_restx import Resource from werkzeug.exceptions import NotFound from controllers.console.explore.error import AppAccessDeniedError from controllers.console.wraps import account_initialization_required from extensions.ext_database import db -from libs.login import login_required +from libs.login import current_user, login_required from models import InstalledApp +from models.account import Account from services.app_service import AppService from services.enterprise.enterprise_service import EnterpriseService from services.feature_service import FeatureService @@ -24,6 +24,8 @@ def installed_app_required(view: Callable[Concatenate[InstalledApp, P], R] | Non def decorator(view: Callable[Concatenate[InstalledApp, P], R]): @wraps(view) def decorated(installed_app_id: str, *args: P.args, **kwargs: P.kwargs): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None installed_app = ( db.session.query(InstalledApp) .where( @@ -56,6 +58,7 @@ def user_allowed_to_access_app(view: Callable[Concatenate[InstalledApp, P], R] | def decorated(installed_app: InstalledApp, *args: P.args, **kwargs: P.kwargs): feature = FeatureService.get_system_features() if feature.webapp_auth.enabled: + assert isinstance(current_user, Account) app_id = installed_app.app_id app_code = AppService.get_app_code_by_id(app_id) res = EnterpriseService.WebAppAuth.is_user_allowed_to_access_webapp( diff --git a/api/controllers/console/extension.py b/api/controllers/console/extension.py index 57f5ab191e..c6b3cf7515 100644 --- a/api/controllers/console/extension.py +++ b/api/controllers/console/extension.py @@ -1,11 +1,11 @@ -from flask_login import current_user from flask_restx import Resource, fields, marshal_with, reqparse from constants import HIDDEN_VALUE from controllers.console import api, console_ns from controllers.console.wraps import account_initialization_required, setup_required from fields.api_based_extension_fields import api_based_extension_fields -from libs.login import login_required +from libs.login import current_user, login_required +from models.account import Account from models.api_based_extension import APIBasedExtension from services.api_based_extension_service import APIBasedExtensionService from services.code_based_extension_service import CodeBasedExtensionService @@ -47,6 +47,8 @@ class APIBasedExtensionAPI(Resource): @account_initialization_required @marshal_with(api_based_extension_fields) def get(self): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None tenant_id = current_user.current_tenant_id return APIBasedExtensionService.get_all_by_tenant_id(tenant_id) @@ -68,6 +70,8 @@ class APIBasedExtensionAPI(Resource): @account_initialization_required @marshal_with(api_based_extension_fields) def post(self): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None parser = reqparse.RequestParser() parser.add_argument("name", type=str, required=True, location="json") parser.add_argument("api_endpoint", type=str, required=True, location="json") @@ -95,6 +99,8 @@ class APIBasedExtensionDetailAPI(Resource): @account_initialization_required @marshal_with(api_based_extension_fields) def get(self, id): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None api_based_extension_id = str(id) tenant_id = current_user.current_tenant_id @@ -119,6 +125,8 @@ class APIBasedExtensionDetailAPI(Resource): @account_initialization_required @marshal_with(api_based_extension_fields) def post(self, id): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None api_based_extension_id = str(id) tenant_id = current_user.current_tenant_id @@ -146,6 +154,8 @@ class APIBasedExtensionDetailAPI(Resource): @login_required @account_initialization_required def delete(self, id): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None api_based_extension_id = str(id) tenant_id = current_user.current_tenant_id diff --git a/api/controllers/console/feature.py b/api/controllers/console/feature.py index d43b839291..80847b8fef 100644 --- a/api/controllers/console/feature.py +++ b/api/controllers/console/feature.py @@ -1,7 +1,7 @@ -from flask_login import current_user from flask_restx import Resource, fields -from libs.login import login_required +from libs.login import current_user, login_required +from models.account import Account from services.feature_service import FeatureService from . import api, console_ns @@ -23,6 +23,8 @@ class FeatureApi(Resource): @cloud_utm_record def get(self): """Get feature configuration for current tenant""" + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None return FeatureService.get_features(current_user.current_tenant_id).model_dump() diff --git a/api/controllers/console/remote_files.py b/api/controllers/console/remote_files.py index 7aaf807fb0..4d4bb5d779 100644 --- a/api/controllers/console/remote_files.py +++ b/api/controllers/console/remote_files.py @@ -1,8 +1,6 @@ import urllib.parse -from typing import cast import httpx -from flask_login import current_user from flask_restx import Resource, marshal_with, reqparse import services @@ -16,6 +14,7 @@ from core.file import helpers as file_helpers from core.helper import ssrf_proxy from extensions.ext_database import db from fields.file_fields import file_fields_with_signed_url, remote_file_info_fields +from libs.login import current_user from models.account import Account from services.file_service import FileService @@ -65,7 +64,8 @@ class RemoteFileUploadApi(Resource): content = resp.content if resp.request.method == "GET" else ssrf_proxy.get(url).content try: - user = cast(Account, current_user) + assert isinstance(current_user, Account) + user = current_user upload_file = FileService(db.engine).upload_file( filename=file_info.filename, content=content, diff --git a/api/controllers/console/tag/tags.py b/api/controllers/console/tag/tags.py index 3d29b3ee61..b6086c5766 100644 --- a/api/controllers/console/tag/tags.py +++ b/api/controllers/console/tag/tags.py @@ -1,12 +1,12 @@ from flask import request -from flask_login import current_user from flask_restx import Resource, marshal_with, reqparse from werkzeug.exceptions import Forbidden from controllers.console import console_ns from controllers.console.wraps import account_initialization_required, setup_required from fields.tag_fields import dataset_tag_fields -from libs.login import login_required +from libs.login import current_user, login_required +from models.account import Account from models.model import Tag from services.tag_service import TagService @@ -24,6 +24,8 @@ class TagListApi(Resource): @account_initialization_required @marshal_with(dataset_tag_fields) def get(self): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None tag_type = request.args.get("type", type=str, default="") keyword = request.args.get("keyword", default=None, type=str) tags = TagService.get_tags(tag_type, current_user.current_tenant_id, keyword) @@ -34,8 +36,10 @@ class TagListApi(Resource): @login_required @account_initialization_required def post(self): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None # The role of the current user in the ta table must be admin, owner, or editor - if not (current_user.is_editor or current_user.is_dataset_editor): + if not (current_user.has_edit_permission or current_user.is_dataset_editor): raise Forbidden() parser = reqparse.RequestParser() @@ -59,9 +63,11 @@ class TagUpdateDeleteApi(Resource): @login_required @account_initialization_required def patch(self, tag_id): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None tag_id = str(tag_id) # The role of the current user in the ta table must be admin, owner, or editor - if not (current_user.is_editor or current_user.is_dataset_editor): + if not (current_user.has_edit_permission or current_user.is_dataset_editor): raise Forbidden() parser = reqparse.RequestParser() @@ -81,9 +87,11 @@ class TagUpdateDeleteApi(Resource): @login_required @account_initialization_required def delete(self, tag_id): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None tag_id = str(tag_id) # The role of the current user in the ta table must be admin, owner, or editor - if not current_user.is_editor: + if not current_user.has_edit_permission: raise Forbidden() TagService.delete_tag(tag_id) @@ -97,8 +105,10 @@ class TagBindingCreateApi(Resource): @login_required @account_initialization_required def post(self): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator - if not (current_user.is_editor or current_user.is_dataset_editor): + if not (current_user.has_edit_permission or current_user.is_dataset_editor): raise Forbidden() parser = reqparse.RequestParser() @@ -123,8 +133,10 @@ class TagBindingDeleteApi(Resource): @login_required @account_initialization_required def post(self): + assert isinstance(current_user, Account) + assert current_user.current_tenant_id is not None # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator - if not (current_user.is_editor or current_user.is_dataset_editor): + if not (current_user.has_edit_permission or current_user.is_dataset_editor): raise Forbidden() parser = reqparse.RequestParser() diff --git a/api/controllers/console/workspace/agent_providers.py b/api/controllers/console/workspace/agent_providers.py index 0a2c8fcfb4..e044b2db5b 100644 --- a/api/controllers/console/workspace/agent_providers.py +++ b/api/controllers/console/workspace/agent_providers.py @@ -1,10 +1,10 @@ -from flask_login import current_user from flask_restx import Resource, fields from controllers.console import api, console_ns from controllers.console.wraps import account_initialization_required, setup_required from core.model_runtime.utils.encoders import jsonable_encoder -from libs.login import login_required +from libs.login import current_user, login_required +from models.account import Account from services.agent_service import AgentService @@ -21,7 +21,9 @@ class AgentProviderListApi(Resource): @login_required @account_initialization_required def get(self): + assert isinstance(current_user, Account) user = current_user + assert user.current_tenant_id is not None user_id = user.id tenant_id = user.current_tenant_id @@ -43,7 +45,9 @@ class AgentProviderApi(Resource): @login_required @account_initialization_required def get(self, provider_name: str): + assert isinstance(current_user, Account) user = current_user + assert user.current_tenant_id is not None user_id = user.id tenant_id = user.current_tenant_id return jsonable_encoder(AgentService.get_agent_provider(user_id, tenant_id, provider_name)) diff --git a/api/controllers/console/workspace/endpoint.py b/api/controllers/console/workspace/endpoint.py index 0657b764cc..782bd72565 100644 --- a/api/controllers/console/workspace/endpoint.py +++ b/api/controllers/console/workspace/endpoint.py @@ -1,4 +1,3 @@ -from flask_login import current_user from flask_restx import Resource, fields, reqparse from werkzeug.exceptions import Forbidden @@ -6,10 +5,18 @@ from controllers.console import api, console_ns from controllers.console.wraps import account_initialization_required, setup_required from core.model_runtime.utils.encoders import jsonable_encoder from core.plugin.impl.exc import PluginPermissionDeniedError -from libs.login import login_required +from libs.login import current_user, login_required +from models.account import Account from services.plugin.endpoint_service import EndpointService +def _current_account_with_tenant() -> tuple[Account, str]: + assert isinstance(current_user, Account) + tenant_id = current_user.current_tenant_id + assert tenant_id is not None + return current_user, tenant_id + + @console_ns.route("/workspaces/current/endpoints/create") class EndpointCreateApi(Resource): @api.doc("create_endpoint") @@ -34,7 +41,7 @@ class EndpointCreateApi(Resource): @login_required @account_initialization_required def post(self): - user = current_user + user, tenant_id = _current_account_with_tenant() if not user.is_admin_or_owner: raise Forbidden() @@ -51,7 +58,7 @@ class EndpointCreateApi(Resource): try: return { "success": EndpointService.create_endpoint( - tenant_id=user.current_tenant_id, + tenant_id=tenant_id, user_id=user.id, plugin_unique_identifier=plugin_unique_identifier, name=name, @@ -80,7 +87,7 @@ class EndpointListApi(Resource): @login_required @account_initialization_required def get(self): - user = current_user + user, tenant_id = _current_account_with_tenant() parser = reqparse.RequestParser() parser.add_argument("page", type=int, required=True, location="args") @@ -93,7 +100,7 @@ class EndpointListApi(Resource): return jsonable_encoder( { "endpoints": EndpointService.list_endpoints( - tenant_id=user.current_tenant_id, + tenant_id=tenant_id, user_id=user.id, page=page, page_size=page_size, @@ -123,7 +130,7 @@ class EndpointListForSinglePluginApi(Resource): @login_required @account_initialization_required def get(self): - user = current_user + user, tenant_id = _current_account_with_tenant() parser = reqparse.RequestParser() parser.add_argument("page", type=int, required=True, location="args") @@ -138,7 +145,7 @@ class EndpointListForSinglePluginApi(Resource): return jsonable_encoder( { "endpoints": EndpointService.list_endpoints_for_single_plugin( - tenant_id=user.current_tenant_id, + tenant_id=tenant_id, user_id=user.id, plugin_id=plugin_id, page=page, @@ -165,7 +172,7 @@ class EndpointDeleteApi(Resource): @login_required @account_initialization_required def post(self): - user = current_user + user, tenant_id = _current_account_with_tenant() parser = reqparse.RequestParser() parser.add_argument("endpoint_id", type=str, required=True) @@ -177,9 +184,7 @@ class EndpointDeleteApi(Resource): endpoint_id = args["endpoint_id"] return { - "success": EndpointService.delete_endpoint( - tenant_id=user.current_tenant_id, user_id=user.id, endpoint_id=endpoint_id - ) + "success": EndpointService.delete_endpoint(tenant_id=tenant_id, user_id=user.id, endpoint_id=endpoint_id) } @@ -207,7 +212,7 @@ class EndpointUpdateApi(Resource): @login_required @account_initialization_required def post(self): - user = current_user + user, tenant_id = _current_account_with_tenant() parser = reqparse.RequestParser() parser.add_argument("endpoint_id", type=str, required=True) @@ -224,7 +229,7 @@ class EndpointUpdateApi(Resource): return { "success": EndpointService.update_endpoint( - tenant_id=user.current_tenant_id, + tenant_id=tenant_id, user_id=user.id, endpoint_id=endpoint_id, name=name, @@ -250,7 +255,7 @@ class EndpointEnableApi(Resource): @login_required @account_initialization_required def post(self): - user = current_user + user, tenant_id = _current_account_with_tenant() parser = reqparse.RequestParser() parser.add_argument("endpoint_id", type=str, required=True) @@ -262,9 +267,7 @@ class EndpointEnableApi(Resource): raise Forbidden() return { - "success": EndpointService.enable_endpoint( - tenant_id=user.current_tenant_id, user_id=user.id, endpoint_id=endpoint_id - ) + "success": EndpointService.enable_endpoint(tenant_id=tenant_id, user_id=user.id, endpoint_id=endpoint_id) } @@ -285,7 +288,7 @@ class EndpointDisableApi(Resource): @login_required @account_initialization_required def post(self): - user = current_user + user, tenant_id = _current_account_with_tenant() parser = reqparse.RequestParser() parser.add_argument("endpoint_id", type=str, required=True) @@ -297,7 +300,5 @@ class EndpointDisableApi(Resource): raise Forbidden() return { - "success": EndpointService.disable_endpoint( - tenant_id=user.current_tenant_id, user_id=user.id, endpoint_id=endpoint_id - ) + "success": EndpointService.disable_endpoint(tenant_id=tenant_id, user_id=user.id, endpoint_id=endpoint_id) } diff --git a/api/controllers/console/workspace/members.py b/api/controllers/console/workspace/members.py index 8b89853bd9..dd6a878d87 100644 --- a/api/controllers/console/workspace/members.py +++ b/api/controllers/console/workspace/members.py @@ -1,7 +1,6 @@ from urllib import parse from flask import abort, request -from flask_login import current_user from flask_restx import Resource, marshal_with, reqparse import services @@ -26,7 +25,7 @@ from controllers.console.wraps import ( from extensions.ext_database import db from fields.member_fields import account_with_role_list_fields from libs.helper import extract_remote_ip -from libs.login import login_required +from libs.login import current_user, login_required from models.account import Account, TenantAccountRole from services.account_service import AccountService, RegisterService, TenantService from services.errors.account import AccountAlreadyInTenantError diff --git a/api/controllers/console/workspace/workspace.py b/api/controllers/console/workspace/workspace.py index bc748ac3d2..4a0539785a 100644 --- a/api/controllers/console/workspace/workspace.py +++ b/api/controllers/console/workspace/workspace.py @@ -1,7 +1,6 @@ import logging from flask import request -from flask_login import current_user from flask_restx import Resource, fields, inputs, marshal, marshal_with, reqparse from sqlalchemy import select from werkzeug.exceptions import Unauthorized @@ -24,7 +23,7 @@ from controllers.console.wraps import ( ) from extensions.ext_database import db from libs.helper import TimestampField -from libs.login import login_required +from libs.login import current_user, login_required from models.account import Account, Tenant, TenantStatus from services.account_service import TenantService from services.feature_service import FeatureService diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index 914d386c78..9e903d9286 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -7,13 +7,13 @@ from functools import wraps from typing import ParamSpec, TypeVar from flask import abort, request -from flask_login import current_user from configs import dify_config from controllers.console.workspace.error import AccountNotInitializedError from extensions.ext_database import db from extensions.ext_redis import redis_client -from models.account import AccountStatus +from libs.login import current_user +from models.account import Account, AccountStatus from models.dataset import RateLimitLog from models.model import DifySetup from services.feature_service import FeatureService, LicenseStatus @@ -25,11 +25,16 @@ P = ParamSpec("P") R = TypeVar("R") +def _current_account() -> Account: + assert isinstance(current_user, Account) + return current_user + + def account_initialization_required(view: Callable[P, R]): @wraps(view) def decorated(*args: P.args, **kwargs: P.kwargs): # check account initialization - account = current_user + account = _current_account() if account.status == AccountStatus.UNINITIALIZED: raise AccountNotInitializedError() @@ -75,7 +80,9 @@ def only_edition_self_hosted(view: Callable[P, R]): def cloud_edition_billing_enabled(view: Callable[P, R]): @wraps(view) def decorated(*args: P.args, **kwargs: P.kwargs): - features = FeatureService.get_features(current_user.current_tenant_id) + account = _current_account() + assert account.current_tenant_id is not None + features = FeatureService.get_features(account.current_tenant_id) if not features.billing.enabled: abort(403, "Billing feature is not enabled.") return view(*args, **kwargs) @@ -87,7 +94,10 @@ def cloud_edition_billing_resource_check(resource: str): def interceptor(view: Callable[P, R]): @wraps(view) def decorated(*args: P.args, **kwargs: P.kwargs): - features = FeatureService.get_features(current_user.current_tenant_id) + account = _current_account() + assert account.current_tenant_id is not None + tenant_id = account.current_tenant_id + features = FeatureService.get_features(tenant_id) if features.billing.enabled: members = features.members apps = features.apps @@ -128,7 +138,9 @@ def cloud_edition_billing_knowledge_limit_check(resource: str): def interceptor(view: Callable[P, R]): @wraps(view) def decorated(*args: P.args, **kwargs: P.kwargs): - features = FeatureService.get_features(current_user.current_tenant_id) + account = _current_account() + assert account.current_tenant_id is not None + features = FeatureService.get_features(account.current_tenant_id) if features.billing.enabled: if resource == "add_segment": if features.billing.subscription.plan == "sandbox": @@ -151,10 +163,13 @@ def cloud_edition_billing_rate_limit_check(resource: str): @wraps(view) def decorated(*args: P.args, **kwargs: P.kwargs): if resource == "knowledge": - knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(current_user.current_tenant_id) + account = _current_account() + assert account.current_tenant_id is not None + tenant_id = account.current_tenant_id + knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(tenant_id) if knowledge_rate_limit.enabled: current_time = int(time.time() * 1000) - key = f"rate_limit_{current_user.current_tenant_id}" + key = f"rate_limit_{tenant_id}" redis_client.zadd(key, {current_time: current_time}) @@ -165,7 +180,7 @@ def cloud_edition_billing_rate_limit_check(resource: str): if request_count > knowledge_rate_limit.limit: # add ratelimit record rate_limit_log = RateLimitLog( - tenant_id=current_user.current_tenant_id, + tenant_id=tenant_id, subscription_plan=knowledge_rate_limit.subscription_plan, operation="knowledge", ) @@ -185,14 +200,17 @@ def cloud_utm_record(view: Callable[P, R]): @wraps(view) def decorated(*args: P.args, **kwargs: P.kwargs): with contextlib.suppress(Exception): - features = FeatureService.get_features(current_user.current_tenant_id) + account = _current_account() + assert account.current_tenant_id is not None + tenant_id = account.current_tenant_id + features = FeatureService.get_features(tenant_id) if features.billing.enabled: utm_info = request.cookies.get("utm_info") if utm_info: utm_info_dict: dict = json.loads(utm_info) - OperationService.record_utm(current_user.current_tenant_id, utm_info_dict) + OperationService.record_utm(tenant_id, utm_info_dict) return view(*args, **kwargs) @@ -271,7 +289,9 @@ def enable_change_email(view: Callable[P, R]): def is_allow_transfer_owner(view: Callable[P, R]): @wraps(view) def decorated(*args: P.args, **kwargs: P.kwargs): - features = FeatureService.get_features(current_user.current_tenant_id) + account = _current_account() + assert account.current_tenant_id is not None + features = FeatureService.get_features(account.current_tenant_id) if features.is_allow_transfer_workspace: return view(*args, **kwargs) @@ -284,7 +304,9 @@ def is_allow_transfer_owner(view: Callable[P, R]): def knowledge_pipeline_publish_enabled(view): @wraps(view) def decorated(*args, **kwargs): - features = FeatureService.get_features(current_user.current_tenant_id) + account = _current_account() + assert account.current_tenant_id is not None + features = FeatureService.get_features(account.current_tenant_id) if features.knowledge_pipeline.publish_enabled: return view(*args, **kwargs) abort(403) diff --git a/api/core/datasource/entities/datasource_entities.py b/api/core/datasource/entities/datasource_entities.py index 7f503b963f..260dcf04f5 100644 --- a/api/core/datasource/entities/datasource_entities.py +++ b/api/core/datasource/entities/datasource_entities.py @@ -1,5 +1,5 @@ import enum -from enum import Enum +from enum import StrEnum from typing import Any from pydantic import BaseModel, Field, ValidationInfo, field_validator @@ -218,7 +218,7 @@ class DatasourceLabel(BaseModel): icon: str = Field(..., description="The icon of the tool") -class DatasourceInvokeFrom(Enum): +class DatasourceInvokeFrom(StrEnum): """ Enum class for datasource invoke """ diff --git a/api/core/model_runtime/entities/provider_entities.py b/api/core/model_runtime/entities/provider_entities.py index 831fb9d4db..0508116962 100644 --- a/api/core/model_runtime/entities/provider_entities.py +++ b/api/core/model_runtime/entities/provider_entities.py @@ -1,5 +1,5 @@ from collections.abc import Sequence -from enum import Enum, StrEnum, auto +from enum import StrEnum, auto from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -7,7 +7,7 @@ from core.model_runtime.entities.common_entities import I18nObject from core.model_runtime.entities.model_entities import AIModelEntity, ModelType -class ConfigurateMethod(Enum): +class ConfigurateMethod(StrEnum): """ Enum class for configurate method of provider model. """ diff --git a/api/core/plugin/impl/base.py b/api/core/plugin/impl/base.py index c791b35161..952fefdbbc 100644 --- a/api/core/plugin/impl/base.py +++ b/api/core/plugin/impl/base.py @@ -255,7 +255,7 @@ class BasePluginClient: except Exception: raise PluginDaemonInnerError(code=rep.code, message=rep.message) - logger.error("Error in stream reponse for plugin %s", rep.__dict__) + logger.error("Error in stream response for plugin %s", rep.__dict__) self._handle_plugin_daemon_error(error.error_type, error.message) raise ValueError(f"plugin daemon: {rep.message}, code: {rep.code}") if rep.data is None: diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.py index 6e9e2b4527..2290de19bc 100644 --- a/api/core/rag/datasource/retrieval_service.py +++ b/api/core/rag/datasource/retrieval_service.py @@ -34,7 +34,7 @@ class RetrievalService: @classmethod def retrieve( cls, - retrieval_method: str, + retrieval_method: RetrievalMethod, dataset_id: str, query: str, top_k: int, @@ -56,7 +56,7 @@ class RetrievalService: # Optimize multithreading with thread pools with ThreadPoolExecutor(max_workers=dify_config.RETRIEVAL_SERVICE_EXECUTORS) as executor: # type: ignore futures = [] - if retrieval_method == "keyword_search": + if retrieval_method == RetrievalMethod.KEYWORD_SEARCH: futures.append( executor.submit( cls.keyword_search, @@ -220,7 +220,7 @@ class RetrievalService: score_threshold: float | None, reranking_model: dict | None, all_documents: list, - retrieval_method: str, + retrieval_method: RetrievalMethod, exceptions: list, document_ids_filter: list[str] | None = None, ): diff --git a/api/core/rag/entities/event.py b/api/core/rag/entities/event.py index a61b17ddb8..2d8d4060dd 100644 --- a/api/core/rag/entities/event.py +++ b/api/core/rag/entities/event.py @@ -1,11 +1,11 @@ from collections.abc import Mapping -from enum import Enum +from enum import StrEnum from typing import Any from pydantic import BaseModel, Field -class DatasourceStreamEvent(Enum): +class DatasourceStreamEvent(StrEnum): """ Datasource Stream event """ diff --git a/api/core/rag/extractor/unstructured/unstructured_doc_extractor.py b/api/core/rag/extractor/unstructured/unstructured_doc_extractor.py index 5199208f70..7dd8beaa46 100644 --- a/api/core/rag/extractor/unstructured/unstructured_doc_extractor.py +++ b/api/core/rag/extractor/unstructured/unstructured_doc_extractor.py @@ -1,6 +1,7 @@ import logging import os +from configs import dify_config from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document @@ -49,7 +50,8 @@ class UnstructuredWordExtractor(BaseExtractor): from unstructured.chunking.title import chunk_by_title - chunks = chunk_by_title(elements, max_characters=2000, combine_text_under_n_chars=2000) + max_characters = dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH + chunks = chunk_by_title(elements, max_characters=max_characters, combine_text_under_n_chars=max_characters) documents = [] for chunk in chunks: text = chunk.text.strip() diff --git a/api/core/rag/extractor/unstructured/unstructured_eml_extractor.py b/api/core/rag/extractor/unstructured/unstructured_eml_extractor.py index ad04bd0bd1..d97d4c3a48 100644 --- a/api/core/rag/extractor/unstructured/unstructured_eml_extractor.py +++ b/api/core/rag/extractor/unstructured/unstructured_eml_extractor.py @@ -4,6 +4,7 @@ import logging from bs4 import BeautifulSoup +from configs import dify_config from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document @@ -46,7 +47,8 @@ class UnstructuredEmailExtractor(BaseExtractor): from unstructured.chunking.title import chunk_by_title - chunks = chunk_by_title(elements, max_characters=2000, combine_text_under_n_chars=2000) + max_characters = dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH + chunks = chunk_by_title(elements, max_characters=max_characters, combine_text_under_n_chars=max_characters) documents = [] for chunk in chunks: text = chunk.text.strip() diff --git a/api/core/rag/extractor/unstructured/unstructured_epub_extractor.py b/api/core/rag/extractor/unstructured/unstructured_epub_extractor.py index fc14ee6275..3061d957ac 100644 --- a/api/core/rag/extractor/unstructured/unstructured_epub_extractor.py +++ b/api/core/rag/extractor/unstructured/unstructured_epub_extractor.py @@ -2,6 +2,7 @@ import logging import pypandoc # type: ignore +from configs import dify_config from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document @@ -40,7 +41,8 @@ class UnstructuredEpubExtractor(BaseExtractor): from unstructured.chunking.title import chunk_by_title - chunks = chunk_by_title(elements, max_characters=2000, combine_text_under_n_chars=2000) + max_characters = dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH + chunks = chunk_by_title(elements, max_characters=max_characters, combine_text_under_n_chars=max_characters) documents = [] for chunk in chunks: text = chunk.text.strip() diff --git a/api/core/rag/extractor/unstructured/unstructured_markdown_extractor.py b/api/core/rag/extractor/unstructured/unstructured_markdown_extractor.py index 23030d7739..b6d8c47111 100644 --- a/api/core/rag/extractor/unstructured/unstructured_markdown_extractor.py +++ b/api/core/rag/extractor/unstructured/unstructured_markdown_extractor.py @@ -1,5 +1,6 @@ import logging +from configs import dify_config from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document @@ -32,7 +33,8 @@ class UnstructuredMarkdownExtractor(BaseExtractor): elements = partition_md(filename=self._file_path) from unstructured.chunking.title import chunk_by_title - chunks = chunk_by_title(elements, max_characters=2000, combine_text_under_n_chars=2000) + max_characters = dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH + chunks = chunk_by_title(elements, max_characters=max_characters, combine_text_under_n_chars=max_characters) documents = [] for chunk in chunks: text = chunk.text.strip() diff --git a/api/core/rag/extractor/unstructured/unstructured_msg_extractor.py b/api/core/rag/extractor/unstructured/unstructured_msg_extractor.py index f29e639d1b..ae60fc7981 100644 --- a/api/core/rag/extractor/unstructured/unstructured_msg_extractor.py +++ b/api/core/rag/extractor/unstructured/unstructured_msg_extractor.py @@ -1,5 +1,6 @@ import logging +from configs import dify_config from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document @@ -31,7 +32,8 @@ class UnstructuredMsgExtractor(BaseExtractor): elements = partition_msg(filename=self._file_path) from unstructured.chunking.title import chunk_by_title - chunks = chunk_by_title(elements, max_characters=2000, combine_text_under_n_chars=2000) + max_characters = dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH + chunks = chunk_by_title(elements, max_characters=max_characters, combine_text_under_n_chars=max_characters) documents = [] for chunk in chunks: text = chunk.text.strip() diff --git a/api/core/rag/extractor/unstructured/unstructured_xml_extractor.py b/api/core/rag/extractor/unstructured/unstructured_xml_extractor.py index d75e166f1b..2d4846d85e 100644 --- a/api/core/rag/extractor/unstructured/unstructured_xml_extractor.py +++ b/api/core/rag/extractor/unstructured/unstructured_xml_extractor.py @@ -1,5 +1,6 @@ import logging +from configs import dify_config from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document @@ -32,7 +33,8 @@ class UnstructuredXmlExtractor(BaseExtractor): from unstructured.chunking.title import chunk_by_title - chunks = chunk_by_title(elements, max_characters=2000, combine_text_under_n_chars=2000) + max_characters = dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH + chunks = chunk_by_title(elements, max_characters=max_characters, combine_text_under_n_chars=max_characters) documents = [] for chunk in chunks: text = chunk.text.strip() diff --git a/api/core/rag/index_processor/index_processor_base.py b/api/core/rag/index_processor/index_processor_base.py index 05cffb5a55..d4eff53204 100644 --- a/api/core/rag/index_processor/index_processor_base.py +++ b/api/core/rag/index_processor/index_processor_base.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Optional from configs import dify_config from core.rag.extractor.entity.extract_setting import ExtractSetting from core.rag.models.document import Document +from core.rag.retrieval.retrieval_methods import RetrievalMethod from core.rag.splitter.fixed_text_splitter import ( EnhanceRecursiveCharacterTextSplitter, FixedRecursiveCharacterTextSplitter, @@ -49,7 +50,7 @@ class BaseIndexProcessor(ABC): @abstractmethod def retrieve( self, - retrieval_method: str, + retrieval_method: RetrievalMethod, query: str, dataset: Dataset, top_k: int, diff --git a/api/core/rag/index_processor/processor/paragraph_index_processor.py b/api/core/rag/index_processor/processor/paragraph_index_processor.py index 4fcffbcc77..5e5fea7ea9 100644 --- a/api/core/rag/index_processor/processor/paragraph_index_processor.py +++ b/api/core/rag/index_processor/processor/paragraph_index_processor.py @@ -14,6 +14,7 @@ from core.rag.extractor.extract_processor import ExtractProcessor from core.rag.index_processor.constant.index_type import IndexType from core.rag.index_processor.index_processor_base import BaseIndexProcessor from core.rag.models.document import Document +from core.rag.retrieval.retrieval_methods import RetrievalMethod from core.tools.utils.text_processing_utils import remove_leading_symbols from libs import helper from models.dataset import Dataset, DatasetProcessRule @@ -106,7 +107,7 @@ class ParagraphIndexProcessor(BaseIndexProcessor): def retrieve( self, - retrieval_method: str, + retrieval_method: RetrievalMethod, query: str, dataset: Dataset, top_k: int, diff --git a/api/core/rag/index_processor/processor/parent_child_index_processor.py b/api/core/rag/index_processor/processor/parent_child_index_processor.py index 7bdde286f5..4fa78e2f95 100644 --- a/api/core/rag/index_processor/processor/parent_child_index_processor.py +++ b/api/core/rag/index_processor/processor/parent_child_index_processor.py @@ -16,6 +16,7 @@ from core.rag.extractor.extract_processor import ExtractProcessor from core.rag.index_processor.constant.index_type import IndexType from core.rag.index_processor.index_processor_base import BaseIndexProcessor from core.rag.models.document import ChildDocument, Document, ParentChildStructureChunk +from core.rag.retrieval.retrieval_methods import RetrievalMethod from extensions.ext_database import db from libs import helper from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentSegment @@ -161,7 +162,7 @@ class ParentChildIndexProcessor(BaseIndexProcessor): def retrieve( self, - retrieval_method: str, + retrieval_method: RetrievalMethod, query: str, dataset: Dataset, top_k: int, diff --git a/api/core/rag/index_processor/processor/qa_index_processor.py b/api/core/rag/index_processor/processor/qa_index_processor.py index 9c8f70dba8..3e3deb0180 100644 --- a/api/core/rag/index_processor/processor/qa_index_processor.py +++ b/api/core/rag/index_processor/processor/qa_index_processor.py @@ -21,6 +21,7 @@ from core.rag.extractor.extract_processor import ExtractProcessor from core.rag.index_processor.constant.index_type import IndexType from core.rag.index_processor.index_processor_base import BaseIndexProcessor from core.rag.models.document import Document, QAStructureChunk +from core.rag.retrieval.retrieval_methods import RetrievalMethod from core.tools.utils.text_processing_utils import remove_leading_symbols from libs import helper from models.dataset import Dataset @@ -141,7 +142,7 @@ class QAIndexProcessor(BaseIndexProcessor): def retrieve( self, - retrieval_method: str, + retrieval_method: RetrievalMethod, query: str, dataset: Dataset, top_k: int, diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index 0a702d2902..99bbe615fb 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -364,7 +364,7 @@ class DatasetRetrieval: top_k = retrieval_model_config["top_k"] # get retrieval method if dataset.indexing_technique == "economy": - retrieval_method = "keyword_search" + retrieval_method = RetrievalMethod.KEYWORD_SEARCH else: retrieval_method = retrieval_model_config["search_method"] # get reranking model @@ -623,7 +623,7 @@ class DatasetRetrieval: if dataset.indexing_technique == "economy": # use keyword table query documents = RetrievalService.retrieve( - retrieval_method="keyword_search", + retrieval_method=RetrievalMethod.KEYWORD_SEARCH, dataset_id=dataset.id, query=query, top_k=top_k, diff --git a/api/core/rag/retrieval/retrieval_methods.py b/api/core/rag/retrieval/retrieval_methods.py index 5f0f2a9d33..c77a026351 100644 --- a/api/core/rag/retrieval/retrieval_methods.py +++ b/api/core/rag/retrieval/retrieval_methods.py @@ -1,7 +1,7 @@ -from enum import Enum +from enum import StrEnum -class RetrievalMethod(Enum): +class RetrievalMethod(StrEnum): SEMANTIC_SEARCH = "semantic_search" FULL_TEXT_SEARCH = "full_text_search" HYBRID_SEARCH = "hybrid_search" diff --git a/api/core/tools/mcp_tool/provider.py b/api/core/tools/mcp_tool/provider.py index f269b8db9b..0c2870727e 100644 --- a/api/core/tools/mcp_tool/provider.py +++ b/api/core/tools/mcp_tool/provider.py @@ -76,7 +76,8 @@ class MCPToolProviderController(ToolProviderController): ) for remote_mcp_tool in remote_mcp_tools ] - + if not db_provider.icon: + raise ValueError("Database provider icon is required") return cls( entity=ToolProviderEntityWithPlugin( identity=ToolProviderIdentity( diff --git a/api/core/tools/utils/dataset_retriever/dataset_multi_retriever_tool.py b/api/core/tools/utils/dataset_retriever/dataset_multi_retriever_tool.py index b5bc4d3c00..20e10be075 100644 --- a/api/core/tools/utils/dataset_retriever/dataset_multi_retriever_tool.py +++ b/api/core/tools/utils/dataset_retriever/dataset_multi_retriever_tool.py @@ -172,7 +172,7 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool): if dataset.indexing_technique == "economy": # use keyword table query documents = RetrievalService.retrieve( - retrieval_method="keyword_search", + retrieval_method=RetrievalMethod.KEYWORD_SEARCH, dataset_id=dataset.id, query=query, top_k=retrieval_model.get("top_k") or 4, diff --git a/api/core/tools/utils/dataset_retriever/dataset_retriever_tool.py b/api/core/tools/utils/dataset_retriever/dataset_retriever_tool.py index 1eae582f67..915a22dd0f 100644 --- a/api/core/tools/utils/dataset_retriever/dataset_retriever_tool.py +++ b/api/core/tools/utils/dataset_retriever/dataset_retriever_tool.py @@ -130,7 +130,7 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool): if dataset.indexing_technique == "economy": # use keyword table query documents = RetrievalService.retrieve( - retrieval_method="keyword_search", + retrieval_method=RetrievalMethod.KEYWORD_SEARCH, dataset_id=dataset.id, query=query, top_k=self.top_k, diff --git a/api/core/tools/utils/model_invocation_utils.py b/api/core/tools/utils/model_invocation_utils.py index 526f5c8b9a..b4bae08a9b 100644 --- a/api/core/tools/utils/model_invocation_utils.py +++ b/api/core/tools/utils/model_invocation_utils.py @@ -5,6 +5,7 @@ Therefore, a model manager is needed to list/invoke/validate models. """ import json +from decimal import Decimal from typing import cast from core.model_manager import ModelManager @@ -118,10 +119,10 @@ class ModelInvocationUtils: model_response="", prompt_tokens=prompt_tokens, answer_tokens=0, - answer_unit_price=0, - answer_price_unit=0, + answer_unit_price=Decimal(), + answer_price_unit=Decimal(), provider_response_latency=0, - total_price=0, + total_price=Decimal(), currency="USD", ) @@ -152,7 +153,7 @@ class ModelInvocationUtils: raise InvokeModelError(f"Invoke error: {e}") # update tool model invoke - tool_model_invoke.model_response = response.message.content + tool_model_invoke.model_response = str(response.message.content) if response.usage: tool_model_invoke.answer_tokens = response.usage.completion_tokens tool_model_invoke.answer_unit_price = response.usage.completion_unit_price diff --git a/api/core/workflow/enums.py b/api/core/workflow/enums.py index 00a125660a..eb88bb67ee 100644 --- a/api/core/workflow/enums.py +++ b/api/core/workflow/enums.py @@ -1,7 +1,7 @@ -from enum import Enum, StrEnum +from enum import StrEnum -class NodeState(Enum): +class NodeState(StrEnum): """State of a node or edge during workflow execution.""" UNKNOWN = "unknown" diff --git a/api/core/workflow/graph_engine/layers/execution_limits.py b/api/core/workflow/graph_engine/layers/execution_limits.py index e39af89837..a2d36d142d 100644 --- a/api/core/workflow/graph_engine/layers/execution_limits.py +++ b/api/core/workflow/graph_engine/layers/execution_limits.py @@ -10,7 +10,7 @@ When limits are exceeded, the layer automatically aborts execution. import logging import time -from enum import Enum +from enum import StrEnum from typing import final from typing_extensions import override @@ -24,7 +24,7 @@ from core.workflow.graph_events import ( from core.workflow.graph_events.node import NodeRunFailedEvent, NodeRunSucceededEvent -class LimitType(Enum): +class LimitType(StrEnum): """Types of execution limits that can be exceeded.""" STEP_LIMIT = "step_limit" diff --git a/api/core/workflow/nodes/knowledge_index/entities.py b/api/core/workflow/nodes/knowledge_index/entities.py index c79373afd5..3daca90b9b 100644 --- a/api/core/workflow/nodes/knowledge_index/entities.py +++ b/api/core/workflow/nodes/knowledge_index/entities.py @@ -2,6 +2,7 @@ from typing import Literal, Union from pydantic import BaseModel +from core.rag.retrieval.retrieval_methods import RetrievalMethod from core.workflow.nodes.base import BaseNodeData @@ -63,7 +64,7 @@ class RetrievalSetting(BaseModel): Retrieval Setting. """ - search_method: Literal["semantic_search", "keyword_search", "full_text_search", "hybrid_search"] + search_method: RetrievalMethod top_k: int score_threshold: float | None = 0.5 score_threshold_enabled: bool = False diff --git a/api/migrations/env.py b/api/migrations/env.py index a5d815dcfd..66a4614e80 100644 --- a/api/migrations/env.py +++ b/api/migrations/env.py @@ -37,10 +37,11 @@ config.set_main_option('sqlalchemy.url', get_engine_url()) # my_important_option = config.get_main_option("my_important_option") # ... etc. -from models.base import Base +from models.base import TypeBase + def get_metadata(): - return Base.metadata + return TypeBase.metadata def include_object(object, name, type_, reflected, compare_to): if type_ == "foreign_key_constraint": diff --git a/api/models/source.py b/api/models/source.py index 5b4c486bc4..0ed7c4c70e 100644 --- a/api/models/source.py +++ b/api/models/source.py @@ -6,12 +6,12 @@ from sqlalchemy import DateTime, String, func from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column -from models.base import Base +from models.base import TypeBase from .types import StringUUID -class DataSourceOauthBinding(Base): +class DataSourceOauthBinding(TypeBase): __tablename__ = "data_source_oauth_bindings" __table_args__ = ( sa.PrimaryKeyConstraint("id", name="source_binding_pkey"), @@ -19,17 +19,25 @@ class DataSourceOauthBinding(Base): sa.Index("source_info_idx", "source_info", postgresql_using="gin"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) - tenant_id = mapped_column(StringUUID, nullable=False) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) + tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) access_token: Mapped[str] = mapped_column(String(255), nullable=False) provider: Mapped[str] = mapped_column(String(255), nullable=False) - source_info = mapped_column(JSONB, nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp()) - updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp()) - disabled: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false")) + source_info: Mapped[dict] = mapped_column(JSONB, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime, nullable=False, server_default=func.current_timestamp(), init=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime, + nullable=False, + server_default=func.current_timestamp(), + onupdate=func.current_timestamp(), + init=False, + ) + disabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"), default=False) -class DataSourceApiKeyAuthBinding(Base): +class DataSourceApiKeyAuthBinding(TypeBase): __tablename__ = "data_source_api_key_auth_bindings" __table_args__ = ( sa.PrimaryKeyConstraint("id", name="data_source_api_key_auth_binding_pkey"), @@ -37,14 +45,22 @@ class DataSourceApiKeyAuthBinding(Base): sa.Index("data_source_api_key_auth_binding_provider_idx", "provider"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) - tenant_id = mapped_column(StringUUID, nullable=False) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) + tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) category: Mapped[str] = mapped_column(String(255), nullable=False) provider: Mapped[str] = mapped_column(String(255), nullable=False) - credentials = mapped_column(sa.Text, nullable=True) # JSON - created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp()) - updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp()) - disabled: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false")) + credentials: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None) # JSON + created_at: Mapped[datetime] = mapped_column( + DateTime, nullable=False, server_default=func.current_timestamp(), init=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime, + nullable=False, + server_default=func.current_timestamp(), + onupdate=func.current_timestamp(), + init=False, + ) + disabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"), default=False) def to_dict(self): return { @@ -52,7 +68,7 @@ class DataSourceApiKeyAuthBinding(Base): "tenant_id": self.tenant_id, "category": self.category, "provider": self.provider, - "credentials": json.loads(self.credentials), + "credentials": json.loads(self.credentials) if self.credentials else None, "created_at": self.created_at.timestamp(), "updated_at": self.updated_at.timestamp(), "disabled": self.disabled, diff --git a/api/models/task.py b/api/models/task.py index 4e49254dbd..513f167cce 100644 --- a/api/models/task.py +++ b/api/models/task.py @@ -6,41 +6,43 @@ from sqlalchemy import DateTime, String from sqlalchemy.orm import Mapped, mapped_column from libs.datetime_utils import naive_utc_now -from models.base import Base +from models.base import TypeBase -class CeleryTask(Base): +class CeleryTask(TypeBase): """Task result/status.""" __tablename__ = "celery_taskmeta" - id = mapped_column(sa.Integer, sa.Sequence("task_id_sequence"), primary_key=True, autoincrement=True) - task_id = mapped_column(String(155), unique=True) - status = mapped_column(String(50), default=states.PENDING) - result = mapped_column(sa.PickleType, nullable=True) - date_done = mapped_column( + id: Mapped[int] = mapped_column( + sa.Integer, sa.Sequence("task_id_sequence"), primary_key=True, autoincrement=True, init=False + ) + task_id: Mapped[str] = mapped_column(String(155), unique=True) + status: Mapped[str] = mapped_column(String(50), default=states.PENDING) + result: Mapped[bytes | None] = mapped_column(sa.PickleType, nullable=True, default=None) + date_done: Mapped[datetime | None] = mapped_column( DateTime, - default=lambda: naive_utc_now(), - onupdate=lambda: naive_utc_now(), + default=naive_utc_now, + onupdate=naive_utc_now, nullable=True, ) - traceback = mapped_column(sa.Text, nullable=True) - name = mapped_column(String(155), nullable=True) - args = mapped_column(sa.LargeBinary, nullable=True) - kwargs = mapped_column(sa.LargeBinary, nullable=True) - worker = mapped_column(String(155), nullable=True) - retries: Mapped[int | None] = mapped_column(sa.Integer, nullable=True) - queue = mapped_column(String(155), nullable=True) + traceback: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None) + name: Mapped[str | None] = mapped_column(String(155), nullable=True, default=None) + args: Mapped[bytes | None] = mapped_column(sa.LargeBinary, nullable=True, default=None) + kwargs: Mapped[bytes | None] = mapped_column(sa.LargeBinary, nullable=True, default=None) + worker: Mapped[str | None] = mapped_column(String(155), nullable=True, default=None) + retries: Mapped[int | None] = mapped_column(sa.Integer, nullable=True, default=None) + queue: Mapped[str | None] = mapped_column(String(155), nullable=True, default=None) -class CeleryTaskSet(Base): +class CeleryTaskSet(TypeBase): """TaskSet result.""" __tablename__ = "celery_tasksetmeta" id: Mapped[int] = mapped_column( - sa.Integer, sa.Sequence("taskset_id_sequence"), autoincrement=True, primary_key=True + sa.Integer, sa.Sequence("taskset_id_sequence"), autoincrement=True, primary_key=True, init=False ) - taskset_id = mapped_column(String(155), unique=True) - result = mapped_column(sa.PickleType, nullable=True) - date_done: Mapped[datetime | None] = mapped_column(DateTime, default=lambda: naive_utc_now(), nullable=True) + taskset_id: Mapped[str] = mapped_column(String(155), unique=True) + result: Mapped[bytes | None] = mapped_column(sa.PickleType, nullable=True, default=None) + date_done: Mapped[datetime | None] = mapped_column(DateTime, default=naive_utc_now, nullable=True) diff --git a/api/models/tools.py b/api/models/tools.py index d581d588a4..aec53da50c 100644 --- a/api/models/tools.py +++ b/api/models/tools.py @@ -1,6 +1,7 @@ import json from collections.abc import Mapping from datetime import datetime +from decimal import Decimal from typing import TYPE_CHECKING, Any, cast from urllib.parse import urlparse @@ -13,7 +14,7 @@ from core.helper import encrypter from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_bundle import ApiToolBundle from core.tools.entities.tool_entities import ApiProviderSchemaType, WorkflowToolParameterConfiguration -from models.base import Base, TypeBase +from models.base import TypeBase from .engine import db from .model import Account, App, Tenant @@ -42,28 +43,28 @@ class ToolOAuthSystemClient(TypeBase): # tenant level tool oauth client params (client_id, client_secret, etc.) -class ToolOAuthTenantClient(Base): +class ToolOAuthTenantClient(TypeBase): __tablename__ = "tool_oauth_tenant_clients" __table_args__ = ( sa.PrimaryKeyConstraint("id", name="tool_oauth_tenant_client_pkey"), sa.UniqueConstraint("tenant_id", "plugin_id", "provider", name="unique_tool_oauth_tenant_client"), ) - id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # tenant id tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) plugin_id: Mapped[str] = mapped_column(String(512), nullable=False) provider: Mapped[str] = mapped_column(String(255), nullable=False) - enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true")) + enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"), init=False) # oauth params of the tool provider - encrypted_oauth_params: Mapped[str] = mapped_column(sa.Text, nullable=False) + encrypted_oauth_params: Mapped[str] = mapped_column(sa.Text, nullable=False, init=False) @property def oauth_params(self) -> dict[str, Any]: return cast(dict[str, Any], json.loads(self.encrypted_oauth_params or "{}")) -class BuiltinToolProvider(Base): +class BuiltinToolProvider(TypeBase): """ This table stores the tool provider information for built-in tools for each tenant. """ @@ -75,37 +76,45 @@ class BuiltinToolProvider(Base): ) # id of the tool provider - id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) name: Mapped[str] = mapped_column( - String(256), nullable=False, server_default=sa.text("'API KEY 1'::character varying") + String(256), + nullable=False, + server_default=sa.text("'API KEY 1'::character varying"), ) # id of the tenant - tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=True) + tenant_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True) # who created this tool provider user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # name of the tool provider provider: Mapped[str] = mapped_column(String(256), nullable=False) # credential of the tool provider - encrypted_credentials: Mapped[str] = mapped_column(sa.Text, nullable=True) + encrypted_credentials: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None) created_at: Mapped[datetime] = mapped_column( - sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)") + sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False ) updated_at: Mapped[datetime] = mapped_column( - sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)") + sa.DateTime, + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP(0)"), + onupdate=func.current_timestamp(), + init=False, ) - is_default: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false")) + is_default: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"), default=False) # credential type, e.g., "api-key", "oauth2" credential_type: Mapped[str] = mapped_column( - String(32), nullable=False, server_default=sa.text("'api-key'::character varying") + String(32), nullable=False, server_default=sa.text("'api-key'::character varying"), default="api-key" ) - expires_at: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("-1")) + expires_at: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("-1"), default=-1) @property def credentials(self) -> dict[str, Any]: + if not self.encrypted_credentials: + return {} return cast(dict[str, Any], json.loads(self.encrypted_credentials)) -class ApiToolProvider(Base): +class ApiToolProvider(TypeBase): """ The table stores the api providers. """ @@ -116,31 +125,43 @@ class ApiToolProvider(Base): sa.UniqueConstraint("name", "tenant_id", name="unique_api_tool_provider"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # name of the api provider - name = mapped_column(String(255), nullable=False, server_default=sa.text("'API KEY 1'::character varying")) + name: Mapped[str] = mapped_column( + String(255), + nullable=False, + server_default=sa.text("'API KEY 1'::character varying"), + ) # icon icon: Mapped[str] = mapped_column(String(255), nullable=False) # original schema - schema = mapped_column(sa.Text, nullable=False) + schema: Mapped[str] = mapped_column(sa.Text, nullable=False) schema_type_str: Mapped[str] = mapped_column(String(40), nullable=False) # who created this tool - user_id = mapped_column(StringUUID, nullable=False) + user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # tenant id - tenant_id = mapped_column(StringUUID, nullable=False) + tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # description of the provider - description = mapped_column(sa.Text, nullable=False) + description: Mapped[str] = mapped_column(sa.Text, nullable=False) # json format tools - tools_str = mapped_column(sa.Text, nullable=False) + tools_str: Mapped[str] = mapped_column(sa.Text, nullable=False) # json format credentials - credentials_str = mapped_column(sa.Text, nullable=False) + credentials_str: Mapped[str] = mapped_column(sa.Text, nullable=False) # privacy policy - privacy_policy = mapped_column(String(255), nullable=True) + privacy_policy: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None) # custom_disclaimer custom_disclaimer: Mapped[str] = mapped_column(sa.TEXT, default="") - created_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) - updated_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) + created_at: Mapped[datetime] = mapped_column( + sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False + ) + updated_at: Mapped[datetime] = mapped_column( + sa.DateTime, + nullable=False, + server_default=func.current_timestamp(), + onupdate=func.current_timestamp(), + init=False, + ) @property def schema_type(self) -> "ApiProviderSchemaType": @@ -189,7 +210,7 @@ class ToolLabelBinding(TypeBase): label_name: Mapped[str] = mapped_column(String(40), nullable=False) -class WorkflowToolProvider(Base): +class WorkflowToolProvider(TypeBase): """ The table stores the workflow providers. """ @@ -201,7 +222,7 @@ class WorkflowToolProvider(Base): sa.UniqueConstraint("tenant_id", "app_id", name="unique_workflow_tool_provider_app_id"), ) - id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # name of the workflow provider name: Mapped[str] = mapped_column(String(255), nullable=False) # label of the workflow provider @@ -219,15 +240,19 @@ class WorkflowToolProvider(Base): # description of the provider description: Mapped[str] = mapped_column(sa.Text, nullable=False) # parameter configuration - parameter_configuration: Mapped[str] = mapped_column(sa.Text, nullable=False, server_default="[]") + parameter_configuration: Mapped[str] = mapped_column(sa.Text, nullable=False, server_default="[]", default="[]") # privacy policy - privacy_policy: Mapped[str] = mapped_column(String(255), nullable=True, server_default="") + privacy_policy: Mapped[str | None] = mapped_column(String(255), nullable=True, server_default="", default=None) created_at: Mapped[datetime] = mapped_column( - sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)") + sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False ) updated_at: Mapped[datetime] = mapped_column( - sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)") + sa.DateTime, + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP(0)"), + onupdate=func.current_timestamp(), + init=False, ) @property @@ -252,7 +277,7 @@ class WorkflowToolProvider(Base): return db.session.query(App).where(App.id == self.app_id).first() -class MCPToolProvider(Base): +class MCPToolProvider(TypeBase): """ The table stores the mcp providers. """ @@ -265,7 +290,7 @@ class MCPToolProvider(Base): sa.UniqueConstraint("tenant_id", "server_identifier", name="unique_mcp_provider_server_identifier"), ) - id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # name of the mcp provider name: Mapped[str] = mapped_column(String(40), nullable=False) # server identifier of the mcp provider @@ -275,27 +300,33 @@ class MCPToolProvider(Base): # hash of server_url for uniqueness check server_url_hash: Mapped[str] = mapped_column(String(64), nullable=False) # icon of the mcp provider - icon: Mapped[str] = mapped_column(String(255), nullable=True) + icon: Mapped[str | None] = mapped_column(String(255), nullable=True) # tenant id tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # who created this tool user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # encrypted credentials - encrypted_credentials: Mapped[str] = mapped_column(sa.Text, nullable=True) + encrypted_credentials: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None) # authed authed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=False) # tools tools: Mapped[str] = mapped_column(sa.Text, nullable=False, default="[]") created_at: Mapped[datetime] = mapped_column( - sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)") + sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False ) updated_at: Mapped[datetime] = mapped_column( - sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)") + sa.DateTime, + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP(0)"), + onupdate=func.current_timestamp(), + init=False, + ) + timeout: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default=sa.text("30"), default=30.0) + sse_read_timeout: Mapped[float] = mapped_column( + sa.Float, nullable=False, server_default=sa.text("300"), default=300.0 ) - timeout: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default=sa.text("30")) - sse_read_timeout: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default=sa.text("300")) # encrypted headers for MCP server requests - encrypted_headers: Mapped[str | None] = mapped_column(sa.Text, nullable=True) + encrypted_headers: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None) def load_user(self) -> Account | None: return db.session.query(Account).where(Account.id == self.user_id).first() @@ -306,9 +337,11 @@ class MCPToolProvider(Base): @property def credentials(self) -> dict[str, Any]: + if not self.encrypted_credentials: + return {} try: return cast(dict[str, Any], json.loads(self.encrypted_credentials)) or {} - except Exception: + except json.JSONDecodeError: return {} @property @@ -321,6 +354,7 @@ class MCPToolProvider(Base): def provider_icon(self) -> Mapping[str, str] | str: from core.file import helpers as file_helpers + assert self.icon try: return json.loads(self.icon) except json.JSONDecodeError: @@ -419,7 +453,7 @@ class MCPToolProvider(Base): return encrypter.decrypt(self.credentials) -class ToolModelInvoke(Base): +class ToolModelInvoke(TypeBase): """ store the invoke logs from tool invoke """ @@ -427,37 +461,47 @@ class ToolModelInvoke(Base): __tablename__ = "tool_model_invokes" __table_args__ = (sa.PrimaryKeyConstraint("id", name="tool_model_invoke_pkey"),) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # who invoke this tool - user_id = mapped_column(StringUUID, nullable=False) + user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # tenant id - tenant_id = mapped_column(StringUUID, nullable=False) + tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # provider provider: Mapped[str] = mapped_column(String(255), nullable=False) # type - tool_type = mapped_column(String(40), nullable=False) + tool_type: Mapped[str] = mapped_column(String(40), nullable=False) # tool name - tool_name = mapped_column(String(128), nullable=False) + tool_name: Mapped[str] = mapped_column(String(128), nullable=False) # invoke parameters - model_parameters = mapped_column(sa.Text, nullable=False) + model_parameters: Mapped[str] = mapped_column(sa.Text, nullable=False) # prompt messages - prompt_messages = mapped_column(sa.Text, nullable=False) + prompt_messages: Mapped[str] = mapped_column(sa.Text, nullable=False) # invoke response - model_response = mapped_column(sa.Text, nullable=False) + model_response: Mapped[str] = mapped_column(sa.Text, nullable=False) prompt_tokens: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0")) answer_tokens: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0")) - answer_unit_price = mapped_column(sa.Numeric(10, 4), nullable=False) - answer_price_unit = mapped_column(sa.Numeric(10, 7), nullable=False, server_default=sa.text("0.001")) - provider_response_latency = mapped_column(sa.Float, nullable=False, server_default=sa.text("0")) - total_price = mapped_column(sa.Numeric(10, 7)) + answer_unit_price: Mapped[Decimal] = mapped_column(sa.Numeric(10, 4), nullable=False) + answer_price_unit: Mapped[Decimal] = mapped_column( + sa.Numeric(10, 7), nullable=False, server_default=sa.text("0.001") + ) + provider_response_latency: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default=sa.text("0")) + total_price: Mapped[Decimal | None] = mapped_column(sa.Numeric(10, 7)) currency: Mapped[str] = mapped_column(String(255), nullable=False) - created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) - updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) + created_at: Mapped[datetime] = mapped_column( + sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False + ) + updated_at: Mapped[datetime] = mapped_column( + sa.DateTime, + nullable=False, + server_default=func.current_timestamp(), + onupdate=func.current_timestamp(), + init=False, + ) @deprecated -class ToolConversationVariables(Base): +class ToolConversationVariables(TypeBase): """ store the conversation variables from tool invoke """ @@ -470,18 +514,26 @@ class ToolConversationVariables(Base): sa.Index("conversation_id_idx", "conversation_id"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # conversation user id - user_id = mapped_column(StringUUID, nullable=False) + user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # tenant id - tenant_id = mapped_column(StringUUID, nullable=False) + tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # conversation id - conversation_id = mapped_column(StringUUID, nullable=False) + conversation_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # variables pool - variables_str = mapped_column(sa.Text, nullable=False) + variables_str: Mapped[str] = mapped_column(sa.Text, nullable=False) - created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) - updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) + created_at: Mapped[datetime] = mapped_column( + sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False + ) + updated_at: Mapped[datetime] = mapped_column( + sa.DateTime, + nullable=False, + server_default=func.current_timestamp(), + onupdate=func.current_timestamp(), + init=False, + ) @property def variables(self): @@ -519,7 +571,7 @@ class ToolFile(TypeBase): @deprecated -class DeprecatedPublishedAppTool(Base): +class DeprecatedPublishedAppTool(TypeBase): """ The table stores the apps published as a tool for each person. """ @@ -530,26 +582,34 @@ class DeprecatedPublishedAppTool(Base): sa.UniqueConstraint("app_id", "user_id", name="unique_published_app_tool"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # id of the app - app_id = mapped_column(StringUUID, ForeignKey("apps.id"), nullable=False) + app_id: Mapped[str] = mapped_column(StringUUID, ForeignKey("apps.id"), nullable=False) user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # who published this tool - description = mapped_column(sa.Text, nullable=False) + description: Mapped[str] = mapped_column(sa.Text, nullable=False) # llm_description of the tool, for LLM - llm_description = mapped_column(sa.Text, nullable=False) + llm_description: Mapped[str] = mapped_column(sa.Text, nullable=False) # query description, query will be seem as a parameter of the tool, # to describe this parameter to llm, we need this field - query_description = mapped_column(sa.Text, nullable=False) + query_description: Mapped[str] = mapped_column(sa.Text, nullable=False) # query name, the name of the query parameter - query_name = mapped_column(String(40), nullable=False) + query_name: Mapped[str] = mapped_column(String(40), nullable=False) # name of the tool provider - tool_name = mapped_column(String(40), nullable=False) + tool_name: Mapped[str] = mapped_column(String(40), nullable=False) # author - author = mapped_column(String(40), nullable=False) - created_at = mapped_column(sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")) - updated_at = mapped_column(sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")) + author: Mapped[str] = mapped_column(String(40), nullable=False) + created_at: Mapped[datetime] = mapped_column( + sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False + ) + updated_at: Mapped[datetime] = mapped_column( + sa.DateTime, + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP(0)"), + onupdate=func.current_timestamp(), + init=False, + ) @property def description_i18n(self) -> "I18nObject": diff --git a/api/models/web.py b/api/models/web.py index 74f99e187b..7df5bd6e87 100644 --- a/api/models/web.py +++ b/api/models/web.py @@ -4,46 +4,58 @@ import sqlalchemy as sa from sqlalchemy import DateTime, String, func from sqlalchemy.orm import Mapped, mapped_column -from models.base import Base +from models.base import TypeBase from .engine import db from .model import Message from .types import StringUUID -class SavedMessage(Base): +class SavedMessage(TypeBase): __tablename__ = "saved_messages" __table_args__ = ( sa.PrimaryKeyConstraint("id", name="saved_message_pkey"), sa.Index("saved_message_message_idx", "app_id", "message_id", "created_by_role", "created_by"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) - app_id = mapped_column(StringUUID, nullable=False) - message_id = mapped_column(StringUUID, nullable=False) - created_by_role = mapped_column( + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) + app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) + message_id: Mapped[str] = mapped_column(StringUUID, nullable=False) + created_by_role: Mapped[str] = mapped_column( String(255), nullable=False, server_default=sa.text("'end_user'::character varying") ) - created_by = mapped_column(StringUUID, nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp()) + created_by: Mapped[str] = mapped_column(StringUUID, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime, + nullable=False, + server_default=func.current_timestamp(), + init=False, + ) @property def message(self): return db.session.query(Message).where(Message.id == self.message_id).first() -class PinnedConversation(Base): +class PinnedConversation(TypeBase): __tablename__ = "pinned_conversations" __table_args__ = ( sa.PrimaryKeyConstraint("id", name="pinned_conversation_pkey"), sa.Index("pinned_conversation_conversation_idx", "app_id", "conversation_id", "created_by_role", "created_by"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) - app_id = mapped_column(StringUUID, nullable=False) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) + app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) conversation_id: Mapped[str] = mapped_column(StringUUID) - created_by_role = mapped_column( - String(255), nullable=False, server_default=sa.text("'end_user'::character varying") + created_by_role: Mapped[str] = mapped_column( + String(255), + nullable=False, + server_default=sa.text("'end_user'::character varying"), + ) + created_by: Mapped[str] = mapped_column(StringUUID, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime, + nullable=False, + server_default=func.current_timestamp(), + init=False, ) - created_by = mapped_column(StringUUID, nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp()) diff --git a/api/pyproject.toml b/api/pyproject.toml index 8b1d3e1537..267c4b7b97 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "markdown~=3.5.1", "numpy~=1.26.4", "openpyxl~=3.1.5", - "opik~=1.7.25", + "opik~=1.8.72", "opentelemetry-api==1.27.0", "opentelemetry-distro==0.48b0", "opentelemetry-exporter-otlp==1.27.0", diff --git a/api/services/auth/api_key_auth_service.py b/api/services/auth/api_key_auth_service.py index 055cf65816..56aaf407ee 100644 --- a/api/services/auth/api_key_auth_service.py +++ b/api/services/auth/api_key_auth_service.py @@ -26,10 +26,9 @@ class ApiKeyAuthService: api_key = encrypter.encrypt_token(tenant_id, args["credentials"]["config"]["api_key"]) args["credentials"]["config"]["api_key"] = api_key - data_source_api_key_binding = DataSourceApiKeyAuthBinding() - data_source_api_key_binding.tenant_id = tenant_id - data_source_api_key_binding.category = args["category"] - data_source_api_key_binding.provider = args["provider"] + data_source_api_key_binding = DataSourceApiKeyAuthBinding( + tenant_id=tenant_id, category=args["category"], provider=args["provider"] + ) data_source_api_key_binding.credentials = json.dumps(args["credentials"], ensure_ascii=False) db.session.add(data_source_api_key_binding) db.session.commit() @@ -48,6 +47,8 @@ class ApiKeyAuthService: ) if not data_source_api_key_bindings: return None + if not data_source_api_key_bindings.credentials: + return None credentials = json.loads(data_source_api_key_bindings.credentials) return credentials diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 87861ada87..53216e4fdd 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -1470,7 +1470,7 @@ class DocumentService: dataset.collection_binding_id = dataset_collection_binding.id if not dataset.retrieval_model: default_retrieval_model = { - "search_method": RetrievalMethod.SEMANTIC_SEARCH.value, + "search_method": RetrievalMethod.SEMANTIC_SEARCH, "reranking_enable": False, "reranking_model": {"reranking_provider_name": "", "reranking_model_name": ""}, "top_k": 4, @@ -1752,7 +1752,7 @@ class DocumentService: # dataset.collection_binding_id = dataset_collection_binding.id # if not dataset.retrieval_model: # default_retrieval_model = { - # "search_method": RetrievalMethod.SEMANTIC_SEARCH.value, + # "search_method": RetrievalMethod.SEMANTIC_SEARCH, # "reranking_enable": False, # "reranking_model": {"reranking_provider_name": "", "reranking_model_name": ""}, # "top_k": 2, @@ -2205,7 +2205,7 @@ class DocumentService: retrieval_model = knowledge_config.retrieval_model else: retrieval_model = RetrievalModel( - search_method=RetrievalMethod.SEMANTIC_SEARCH.value, + search_method=RetrievalMethod.SEMANTIC_SEARCH, reranking_enable=False, reranking_model=RerankingModel(reranking_provider_name="", reranking_model_name=""), top_k=4, diff --git a/api/services/entities/knowledge_entities/knowledge_entities.py b/api/services/entities/knowledge_entities/knowledge_entities.py index 33f65bde58..b9a210740d 100644 --- a/api/services/entities/knowledge_entities/knowledge_entities.py +++ b/api/services/entities/knowledge_entities/knowledge_entities.py @@ -3,6 +3,8 @@ from typing import Literal from pydantic import BaseModel +from core.rag.retrieval.retrieval_methods import RetrievalMethod + class ParentMode(StrEnum): FULL_DOC = "full-doc" @@ -95,7 +97,7 @@ class WeightModel(BaseModel): class RetrievalModel(BaseModel): - search_method: Literal["hybrid_search", "semantic_search", "full_text_search", "keyword_search"] + search_method: RetrievalMethod reranking_enable: bool reranking_model: RerankingModel | None = None reranking_mode: str | None = None diff --git a/api/services/entities/knowledge_entities/rag_pipeline_entities.py b/api/services/entities/knowledge_entities/rag_pipeline_entities.py index 860bfde401..a97ccab914 100644 --- a/api/services/entities/knowledge_entities/rag_pipeline_entities.py +++ b/api/services/entities/knowledge_entities/rag_pipeline_entities.py @@ -2,6 +2,8 @@ from typing import Literal from pydantic import BaseModel, field_validator +from core.rag.retrieval.retrieval_methods import RetrievalMethod + class IconInfo(BaseModel): icon: str @@ -83,7 +85,7 @@ class RetrievalSetting(BaseModel): Retrieval Setting. """ - search_method: Literal["semantic_search", "full_text_search", "keyword_search", "hybrid_search"] + search_method: RetrievalMethod top_k: int score_threshold: float | None = 0.5 score_threshold_enabled: bool = False diff --git a/api/services/entities/model_provider_entities.py b/api/services/entities/model_provider_entities.py index 0f5151919f..d07badefa7 100644 --- a/api/services/entities/model_provider_entities.py +++ b/api/services/entities/model_provider_entities.py @@ -1,5 +1,5 @@ from collections.abc import Sequence -from enum import Enum +from enum import StrEnum from pydantic import BaseModel, ConfigDict, model_validator @@ -27,7 +27,7 @@ from core.model_runtime.entities.provider_entities import ( from models.provider import ProviderType -class CustomConfigurationStatus(Enum): +class CustomConfigurationStatus(StrEnum): """ Enum class for custom configuration status. """ diff --git a/api/services/external_knowledge_service.py b/api/services/external_knowledge_service.py index b6ba3bafea..5cd3b471f9 100644 --- a/api/services/external_knowledge_service.py +++ b/api/services/external_knowledge_service.py @@ -88,9 +88,9 @@ class ExternalDatasetService: else: raise ValueError(f"invalid endpoint: {endpoint}") try: - response = httpx.post(endpoint, headers={"Authorization": f"Bearer {api_key}"}) - except Exception: - raise ValueError(f"failed to connect to the endpoint: {endpoint}") + response = ssrf_proxy.post(endpoint, headers={"Authorization": f"Bearer {api_key}"}) + except Exception as e: + raise ValueError(f"failed to connect to the endpoint: {endpoint}") from e if response.status_code == 502: raise ValueError(f"Bad Gateway: failed to connect to the endpoint: {endpoint}") if response.status_code == 404: diff --git a/api/services/hit_testing_service.py b/api/services/hit_testing_service.py index aa29354a6e..c6ea35076e 100644 --- a/api/services/hit_testing_service.py +++ b/api/services/hit_testing_service.py @@ -63,7 +63,7 @@ class HitTestingService: if metadata_condition and not document_ids_filter: return cls.compact_retrieve_response(query, []) all_documents = RetrievalService.retrieve( - retrieval_method=retrieval_model.get("search_method", "semantic_search"), + retrieval_method=RetrievalMethod(retrieval_model.get("search_method", RetrievalMethod.SEMANTIC_SEARCH)), dataset_id=dataset.id, query=query, top_k=retrieval_model.get("top_k", 4), diff --git a/api/services/rag_pipeline/rag_pipeline_transform_service.py b/api/services/rag_pipeline/rag_pipeline_transform_service.py index 39f426a2b0..d79ab71668 100644 --- a/api/services/rag_pipeline/rag_pipeline_transform_service.py +++ b/api/services/rag_pipeline/rag_pipeline_transform_service.py @@ -9,6 +9,7 @@ from flask_login import current_user from constants import DOCUMENT_EXTENSIONS from core.plugin.impl.plugin import PluginInstaller +from core.rag.retrieval.retrieval_methods import RetrievalMethod from extensions.ext_database import db from factories import variable_factory from models.dataset import Dataset, Document, DocumentPipelineExecutionLog, Pipeline @@ -164,7 +165,7 @@ class RagPipelineTransformService: if retrieval_model: retrieval_setting = RetrievalSetting.model_validate(retrieval_model) if indexing_technique == "economy": - retrieval_setting.search_method = "keyword_search" + retrieval_setting.search_method = RetrievalMethod.KEYWORD_SEARCH knowledge_configuration.retrieval_model = retrieval_setting else: dataset.retrieval_model = knowledge_configuration.retrieval_model.model_dump() diff --git a/api/services/tools/api_tools_manage_service.py b/api/services/tools/api_tools_manage_service.py index 2c0c63f634..bb024cc846 100644 --- a/api/services/tools/api_tools_manage_service.py +++ b/api/services/tools/api_tools_manage_service.py @@ -148,7 +148,7 @@ class ApiToolManageService: description=extra_info.get("description", ""), schema_type_str=schema_type, tools_str=json.dumps(jsonable_encoder(tool_bundles)), - credentials_str={}, + credentials_str="{}", privacy_policy=privacy_policy, custom_disclaimer=custom_disclaimer, ) diff --git a/api/services/tools/builtin_tools_manage_service.py b/api/services/tools/builtin_tools_manage_service.py index cab4a5c6ab..b5dcec17d0 100644 --- a/api/services/tools/builtin_tools_manage_service.py +++ b/api/services/tools/builtin_tools_manage_service.py @@ -683,7 +683,7 @@ class BuiltinToolManageService: cache=NoOpProviderCredentialCache(), ) original_params = encrypter.decrypt(custom_client_params.oauth_params) - new_params: dict = { + new_params = { key: value if value != HIDDEN_VALUE else original_params.get(key, UNKNOWN_VALUE) for key, value in client_params.items() } diff --git a/api/services/tools/mcp_tools_manage_service.py b/api/services/tools/mcp_tools_manage_service.py index 605ad8379b..54133d3801 100644 --- a/api/services/tools/mcp_tools_manage_service.py +++ b/api/services/tools/mcp_tools_manage_service.py @@ -188,6 +188,8 @@ class MCPToolManageService: raise user = mcp_provider.load_user() + if not mcp_provider.icon: + raise ValueError("MCP provider icon is required") return ToolProviderApiEntity( id=mcp_provider.id, name=mcp_provider.name, diff --git a/api/services/tools/tools_transform_service.py b/api/services/tools/tools_transform_service.py index 81b4d6993a..b7850ea150 100644 --- a/api/services/tools/tools_transform_service.py +++ b/api/services/tools/tools_transform_service.py @@ -152,7 +152,8 @@ class ToolTransformService: if decrypt_credentials: credentials = db_provider.credentials - + if not db_provider.tenant_id: + raise ValueError(f"Required tenant_id is missing for BuiltinToolProvider with id {db_provider.id}") # init tool configuration encrypter, _ = create_provider_encrypter( tenant_id=db_provider.tenant_id, diff --git a/api/tests/unit_tests/controllers/console/test_wraps.py b/api/tests/unit_tests/controllers/console/test_wraps.py index 9742368f04..5d132cb787 100644 --- a/api/tests/unit_tests/controllers/console/test_wraps.py +++ b/api/tests/unit_tests/controllers/console/test_wraps.py @@ -60,7 +60,7 @@ class TestAccountInitialization: return "success" # Act - with patch("controllers.console.wraps.current_user", mock_user): + with patch("controllers.console.wraps._current_account", return_value=mock_user): result = protected_view() # Assert @@ -77,7 +77,7 @@ class TestAccountInitialization: return "success" # Act & Assert - with patch("controllers.console.wraps.current_user", mock_user): + with patch("controllers.console.wraps._current_account", return_value=mock_user): with pytest.raises(AccountNotInitializedError): protected_view() @@ -163,7 +163,7 @@ class TestBillingResourceLimits: return "member_added" # Act - with patch("controllers.console.wraps.current_user"): + with patch("controllers.console.wraps._current_account", return_value=MockUser("test_user")): with patch("controllers.console.wraps.FeatureService.get_features", return_value=mock_features): result = add_member() @@ -185,7 +185,7 @@ class TestBillingResourceLimits: # Act & Assert with app.test_request_context(): - with patch("controllers.console.wraps.current_user", MockUser("test_user")): + with patch("controllers.console.wraps._current_account", return_value=MockUser("test_user")): with patch("controllers.console.wraps.FeatureService.get_features", return_value=mock_features): with pytest.raises(Exception) as exc_info: add_member() @@ -207,7 +207,7 @@ class TestBillingResourceLimits: # Test 1: Should reject when source is datasets with app.test_request_context("/?source=datasets"): - with patch("controllers.console.wraps.current_user", MockUser("test_user")): + with patch("controllers.console.wraps._current_account", return_value=MockUser("test_user")): with patch("controllers.console.wraps.FeatureService.get_features", return_value=mock_features): with pytest.raises(Exception) as exc_info: upload_document() @@ -215,7 +215,7 @@ class TestBillingResourceLimits: # Test 2: Should allow when source is not datasets with app.test_request_context("/?source=other"): - with patch("controllers.console.wraps.current_user", MockUser("test_user")): + with patch("controllers.console.wraps._current_account", return_value=MockUser("test_user")): with patch("controllers.console.wraps.FeatureService.get_features", return_value=mock_features): result = upload_document() assert result == "document_uploaded" @@ -239,7 +239,7 @@ class TestRateLimiting: return "knowledge_success" # Act - with patch("controllers.console.wraps.current_user"): + with patch("controllers.console.wraps._current_account", return_value=MockUser("test_user")): with patch( "controllers.console.wraps.FeatureService.get_knowledge_rate_limit", return_value=mock_rate_limit ): @@ -271,7 +271,7 @@ class TestRateLimiting: # Act & Assert with app.test_request_context(): - with patch("controllers.console.wraps.current_user", MockUser("test_user")): + with patch("controllers.console.wraps._current_account", return_value=MockUser("test_user")): with patch( "controllers.console.wraps.FeatureService.get_knowledge_rate_limit", return_value=mock_rate_limit ): diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py index e5ead6ff66..b4ee1b91b4 100644 --- a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py +++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py @@ -1,10 +1,12 @@ import os +from pytest_mock import MockerFixture + from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp from tests.unit_tests.core.rag.extractor.test_notion_extractor import _mock_response -def test_firecrawl_web_extractor_crawl_mode(mocker): +def test_firecrawl_web_extractor_crawl_mode(mocker: MockerFixture): url = "https://firecrawl.dev" api_key = os.getenv("FIRECRAWL_API_KEY") or "fc-" base_url = "https://api.firecrawl.dev" diff --git a/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py index f1e1820acc..58bec7d19e 100644 --- a/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py +++ b/api/tests/unit_tests/core/rag/extractor/test_notion_extractor.py @@ -1,5 +1,7 @@ from unittest import mock +from pytest_mock import MockerFixture + from core.rag.extractor import notion_extractor user_id = "user1" @@ -57,7 +59,7 @@ def _remove_multiple_new_lines(text): return text.strip() -def test_notion_page(mocker): +def test_notion_page(mocker: MockerFixture): texts = ["Head 1", "1.1", "paragraph 1", "1.1.1"] mocked_notion_page = { "object": "list", @@ -77,7 +79,7 @@ def test_notion_page(mocker): assert content == "# Head 1\n## 1.1\nparagraph 1\n### 1.1.1" -def test_notion_database(mocker): +def test_notion_database(mocker: MockerFixture): page_title_list = ["page1", "page2", "page3"] mocked_notion_database = { "object": "list", diff --git a/api/tests/unit_tests/core/test_model_manager.py b/api/tests/unit_tests/core/test_model_manager.py index d98e9f6bad..5a7547e85c 100644 --- a/api/tests/unit_tests/core/test_model_manager.py +++ b/api/tests/unit_tests/core/test_model_manager.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock, patch import pytest import redis +from pytest_mock import MockerFixture from core.entities.provider_entities import ModelLoadBalancingConfiguration from core.model_manager import LBModelManager @@ -39,7 +40,7 @@ def lb_model_manager(): return lb_model_manager -def test_lb_model_manager_fetch_next(mocker, lb_model_manager): +def test_lb_model_manager_fetch_next(mocker: MockerFixture, lb_model_manager: LBModelManager): # initialize redis client redis_client.initialize(redis.Redis()) diff --git a/api/tests/unit_tests/core/test_provider_manager.py b/api/tests/unit_tests/core/test_provider_manager.py index 2dab394029..0c3887beab 100644 --- a/api/tests/unit_tests/core/test_provider_manager.py +++ b/api/tests/unit_tests/core/test_provider_manager.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from core.entities.provider_entities import ModelSettings from core.model_runtime.entities.model_entities import ModelType @@ -7,19 +8,25 @@ from models.provider import LoadBalancingModelConfig, ProviderModelSetting @pytest.fixture -def mock_provider_entity(mocker): +def mock_provider_entity(mocker: MockerFixture): mock_entity = mocker.Mock() mock_entity.provider = "openai" mock_entity.configurate_methods = ["predefined-model"] mock_entity.supported_model_types = [ModelType.LLM] - mock_entity.model_credential_schema = mocker.Mock() - mock_entity.model_credential_schema.credential_form_schemas = [] + # Use PropertyMock to ensure credential_form_schemas is iterable + provider_credential_schema = mocker.Mock() + type(provider_credential_schema).credential_form_schemas = mocker.PropertyMock(return_value=[]) + mock_entity.provider_credential_schema = provider_credential_schema + + model_credential_schema = mocker.Mock() + type(model_credential_schema).credential_form_schemas = mocker.PropertyMock(return_value=[]) + mock_entity.model_credential_schema = model_credential_schema return mock_entity -def test__to_model_settings(mocker, mock_provider_entity): +def test__to_model_settings(mocker: MockerFixture, mock_provider_entity): # Mocking the inputs provider_model_settings = [ ProviderModelSetting( @@ -79,7 +86,7 @@ def test__to_model_settings(mocker, mock_provider_entity): assert result[0].load_balancing_configs[1].name == "first" -def test__to_model_settings_only_one_lb(mocker, mock_provider_entity): +def test__to_model_settings_only_one_lb(mocker: MockerFixture, mock_provider_entity): # Mocking the inputs provider_model_settings = [ ProviderModelSetting( @@ -127,7 +134,7 @@ def test__to_model_settings_only_one_lb(mocker, mock_provider_entity): assert len(result[0].load_balancing_configs) == 0 -def test__to_model_settings_lb_disabled(mocker, mock_provider_entity): +def test__to_model_settings_lb_disabled(mocker: MockerFixture, mock_provider_entity): # Mocking the inputs provider_model_settings = [ ProviderModelSetting( diff --git a/api/uv.lock b/api/uv.lock index 5791da0891..c5289c6343 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -1532,7 +1532,7 @@ requires-dist = [ { name = "opentelemetry-sdk", specifier = "==1.27.0" }, { name = "opentelemetry-semantic-conventions", specifier = "==0.48b0" }, { name = "opentelemetry-util-http", specifier = "==0.48b0" }, - { name = "opik", specifier = "~=1.7.25" }, + { name = "opik", specifier = "~=1.8.72" }, { name = "packaging", specifier = "~=23.2" }, { name = "pandas", extras = ["excel", "output-formatting", "performance"], specifier = "~=2.2.2" }, { name = "psycogreen", specifier = "~=1.0.2" }, @@ -4044,7 +4044,7 @@ wheels = [ [[package]] name = "opik" -version = "1.7.43" +version = "1.8.72" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "boto3-stubs", extra = ["bedrock-runtime"] }, @@ -4063,9 +4063,9 @@ dependencies = [ { name = "tqdm" }, { name = "uuid6" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ba/52/cea0317bc3207bc967b48932781995d9cdb2c490e7e05caa00ff660f7205/opik-1.7.43.tar.gz", hash = "sha256:0b02522b0b74d0a67b141939deda01f8bb69690eda6b04a7cecb1c7f0649ccd0", size = 326886, upload-time = "2025-07-07T10:30:07.715Z" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/08/679b60db21994cf3318d4cdd1d08417c1877b79ac20971a8d80f118c9455/opik-1.8.72.tar.gz", hash = "sha256:26fcb003dc609d96b52eaf6a12fb16eb2b69eb0d1b35d88279ec612925d23944", size = 409774, upload-time = "2025-10-10T13:22:38.2Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/ae/f3566bdc3c49a1a8f795b1b6e726ef211c87e31f92d870ca6d63999c9bbf/opik-1.7.43-py3-none-any.whl", hash = "sha256:a66395c8b5ea7c24846f72dafc70c74d5b8f24ffbc4c8a1b3a7f9456e550568d", size = 625356, upload-time = "2025-07-07T10:30:06.389Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f5/04d35af828d127de65a36286ce5b53e7310087a6b55a56f398daa7f0c9a6/opik-1.8.72-py3-none-any.whl", hash = "sha256:697e361a8364666f36aeb197aaba7ffa0696b49f04d2257b733d436749c90a8c", size = 768233, upload-time = "2025-10-10T13:22:36.352Z" }, ] [[package]] diff --git a/web/app/components/plugins/install-plugin/hooks/use-refresh-plugin-list.tsx b/web/app/components/plugins/install-plugin/hooks/use-refresh-plugin-list.tsx index 6294887356..024444cd6a 100644 --- a/web/app/components/plugins/install-plugin/hooks/use-refresh-plugin-list.tsx +++ b/web/app/components/plugins/install-plugin/hooks/use-refresh-plugin-list.tsx @@ -7,6 +7,7 @@ import { useInvalidateStrategyProviders } from '@/service/use-strategy' import type { Plugin, PluginDeclaration, PluginManifestInMarket } from '../../types' import { PluginType } from '../../types' import { useInvalidDataSourceList } from '@/service/use-pipeline' +import { useInvalidDataSourceListAuth } from '@/service/use-datasource' const useRefreshPluginList = () => { const invalidateInstalledPluginList = useInvalidateInstalledPluginList() @@ -19,6 +20,8 @@ const useRefreshPluginList = () => { const invalidateAllBuiltInTools = useInvalidateAllBuiltInTools() const invalidateAllDataSources = useInvalidDataSourceList() + const invalidateDataSourceListAuth = useInvalidDataSourceListAuth() + const invalidateStrategyProviders = useInvalidateStrategyProviders() return { refreshPluginList: (manifest?: PluginManifestInMarket | Plugin | PluginDeclaration | null, refreshAllType?: boolean) => { @@ -32,8 +35,10 @@ const useRefreshPluginList = () => { // TODO: update suggested tools. It's a function in hook useMarketplacePlugins,handleUpdatePlugins } - if ((manifest && PluginType.datasource.includes(manifest.category)) || refreshAllType) + if ((manifest && PluginType.datasource.includes(manifest.category)) || refreshAllType) { invalidateAllDataSources() + invalidateDataSourceListAuth() + } // model select if ((manifest && PluginType.model.includes(manifest.category)) || refreshAllType) { diff --git a/web/app/components/workflow/block-selector/data-sources.tsx b/web/app/components/workflow/block-selector/data-sources.tsx index 294c7c1c79..441ede2334 100644 --- a/web/app/components/workflow/block-selector/data-sources.tsx +++ b/web/app/components/workflow/block-selector/data-sources.tsx @@ -1,10 +1,9 @@ import { useCallback, + useEffect, + useMemo, useRef, } from 'react' -import Link from 'next/link' -import { useTranslation } from 'react-i18next' -import { RiArrowRightUpLine } from '@remixicon/react' import { BlockEnum } from '../types' import type { OnSelectBlock, @@ -14,10 +13,12 @@ import type { DataSourceDefaultValue, ToolDefaultValue } from './types' import Tools from './tools' import { ViewType } from './view-type-select' import cn from '@/utils/classnames' -import type { ListRef } from '@/app/components/workflow/block-selector/market-place-plugin/list' -import { getMarketplaceUrl } from '@/utils/var' +import PluginList, { type ListRef } from '@/app/components/workflow/block-selector/market-place-plugin/list' import { useGlobalPublicStore } from '@/context/global-public-context' import { DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE } from './constants' +import { useMarketplacePlugins } from '../../plugins/marketplace/hooks' +import { PluginType } from '../../plugins/types' +import { useGetLanguage } from '@/context/i18n' type AllToolsProps = { className?: string @@ -34,9 +35,26 @@ const DataSources = ({ onSelect, dataSources, }: AllToolsProps) => { - const { t } = useTranslation() + const language = useGetLanguage() const pluginRef = useRef(null) const wrapElemRef = useRef(null) + + const isMatchingKeywords = (text: string, keywords: string) => { + return text.toLowerCase().includes(keywords.toLowerCase()) + } + + const filteredDatasources = useMemo(() => { + const hasFilter = searchText + if (!hasFilter) + return dataSources.filter(toolWithProvider => toolWithProvider.tools.length > 0) + + return dataSources.filter((toolWithProvider) => { + return isMatchingKeywords(toolWithProvider.name, searchText) || toolWithProvider.tools.some((tool) => { + return tool.label[language].toLowerCase().includes(searchText.toLowerCase()) || tool.name.toLowerCase().includes(searchText.toLowerCase()) + }) + }) + }, [searchText, dataSources, language]) + const handleSelect = useCallback((_: any, toolDefaultValue: ToolDefaultValue) => { let defaultValue: DataSourceDefaultValue = { plugin_id: toolDefaultValue?.provider_id, @@ -55,8 +73,24 @@ const DataSources = ({ } onSelect(BlockEnum.DataSource, toolDefaultValue && defaultValue) }, [onSelect]) + const { enable_marketplace } = useGlobalPublicStore(s => s.systemFeatures) + const { + queryPluginsWithDebounced: fetchPlugins, + plugins: notInstalledPlugins = [], + } = useMarketplacePlugins() + + useEffect(() => { + if (!enable_marketplace) return + if (searchText) { + fetchPlugins({ + query: searchText, + category: PluginType.datasource, + }) + } + }, [searchText, enable_marketplace]) + return (
- { - enable_marketplace && ( - - {t('plugin.findMoreInMarketplace')} - - - ) - } + {/* Plugins from marketplace */} + {enable_marketplace && ( + + )}
) diff --git a/web/app/components/workflow/nodes/components.ts b/web/app/components/workflow/nodes/components.ts new file mode 100644 index 0000000000..cdf3a21598 --- /dev/null +++ b/web/app/components/workflow/nodes/components.ts @@ -0,0 +1,94 @@ +import type { ComponentType } from 'react' +import { BlockEnum } from '../types' +import StartNode from './start/node' +import StartPanel from './start/panel' +import EndNode from './end/node' +import EndPanel from './end/panel' +import AnswerNode from './answer/node' +import AnswerPanel from './answer/panel' +import LLMNode from './llm/node' +import LLMPanel from './llm/panel' +import KnowledgeRetrievalNode from './knowledge-retrieval/node' +import KnowledgeRetrievalPanel from './knowledge-retrieval/panel' +import QuestionClassifierNode from './question-classifier/node' +import QuestionClassifierPanel from './question-classifier/panel' +import IfElseNode from './if-else/node' +import IfElsePanel from './if-else/panel' +import CodeNode from './code/node' +import CodePanel from './code/panel' +import TemplateTransformNode from './template-transform/node' +import TemplateTransformPanel from './template-transform/panel' +import HttpNode from './http/node' +import HttpPanel from './http/panel' +import ToolNode from './tool/node' +import ToolPanel from './tool/panel' +import VariableAssignerNode from './variable-assigner/node' +import VariableAssignerPanel from './variable-assigner/panel' +import AssignerNode from './assigner/node' +import AssignerPanel from './assigner/panel' +import ParameterExtractorNode from './parameter-extractor/node' +import ParameterExtractorPanel from './parameter-extractor/panel' +import IterationNode from './iteration/node' +import IterationPanel from './iteration/panel' +import LoopNode from './loop/node' +import LoopPanel from './loop/panel' +import DocExtractorNode from './document-extractor/node' +import DocExtractorPanel from './document-extractor/panel' +import ListFilterNode from './list-operator/node' +import ListFilterPanel from './list-operator/panel' +import AgentNode from './agent/node' +import AgentPanel from './agent/panel' +import DataSourceNode from './data-source/node' +import DataSourcePanel from './data-source/panel' +import KnowledgeBaseNode from './knowledge-base/node' +import KnowledgeBasePanel from './knowledge-base/panel' + +export const NodeComponentMap: Record> = { + [BlockEnum.Start]: StartNode, + [BlockEnum.End]: EndNode, + [BlockEnum.Answer]: AnswerNode, + [BlockEnum.LLM]: LLMNode, + [BlockEnum.KnowledgeRetrieval]: KnowledgeRetrievalNode, + [BlockEnum.QuestionClassifier]: QuestionClassifierNode, + [BlockEnum.IfElse]: IfElseNode, + [BlockEnum.Code]: CodeNode, + [BlockEnum.TemplateTransform]: TemplateTransformNode, + [BlockEnum.HttpRequest]: HttpNode, + [BlockEnum.Tool]: ToolNode, + [BlockEnum.VariableAssigner]: VariableAssignerNode, + [BlockEnum.Assigner]: AssignerNode, + [BlockEnum.VariableAggregator]: VariableAssignerNode, + [BlockEnum.ParameterExtractor]: ParameterExtractorNode, + [BlockEnum.Iteration]: IterationNode, + [BlockEnum.Loop]: LoopNode, + [BlockEnum.DocExtractor]: DocExtractorNode, + [BlockEnum.ListFilter]: ListFilterNode, + [BlockEnum.Agent]: AgentNode, + [BlockEnum.DataSource]: DataSourceNode, + [BlockEnum.KnowledgeBase]: KnowledgeBaseNode, +} + +export const PanelComponentMap: Record> = { + [BlockEnum.Start]: StartPanel, + [BlockEnum.End]: EndPanel, + [BlockEnum.Answer]: AnswerPanel, + [BlockEnum.LLM]: LLMPanel, + [BlockEnum.KnowledgeRetrieval]: KnowledgeRetrievalPanel, + [BlockEnum.QuestionClassifier]: QuestionClassifierPanel, + [BlockEnum.IfElse]: IfElsePanel, + [BlockEnum.Code]: CodePanel, + [BlockEnum.TemplateTransform]: TemplateTransformPanel, + [BlockEnum.HttpRequest]: HttpPanel, + [BlockEnum.Tool]: ToolPanel, + [BlockEnum.VariableAssigner]: VariableAssignerPanel, + [BlockEnum.VariableAggregator]: VariableAssignerPanel, + [BlockEnum.Assigner]: AssignerPanel, + [BlockEnum.ParameterExtractor]: ParameterExtractorPanel, + [BlockEnum.Iteration]: IterationPanel, + [BlockEnum.Loop]: LoopPanel, + [BlockEnum.DocExtractor]: DocExtractorPanel, + [BlockEnum.ListFilter]: ListFilterPanel, + [BlockEnum.Agent]: AgentPanel, + [BlockEnum.DataSource]: DataSourcePanel, + [BlockEnum.KnowledgeBase]: KnowledgeBasePanel, +} diff --git a/web/app/components/workflow/nodes/constants.ts b/web/app/components/workflow/nodes/constants.ts index 3efc7189ed..78684577f2 100644 --- a/web/app/components/workflow/nodes/constants.ts +++ b/web/app/components/workflow/nodes/constants.ts @@ -1,101 +1,5 @@ -import type { ComponentType } from 'react' -import { BlockEnum } from '../types' -import StartNode from './start/node' -import StartPanel from './start/panel' -import EndNode from './end/node' -import EndPanel from './end/panel' -import AnswerNode from './answer/node' -import AnswerPanel from './answer/panel' -import LLMNode from './llm/node' -import LLMPanel from './llm/panel' -import KnowledgeRetrievalNode from './knowledge-retrieval/node' -import KnowledgeRetrievalPanel from './knowledge-retrieval/panel' -import QuestionClassifierNode from './question-classifier/node' -import QuestionClassifierPanel from './question-classifier/panel' -import IfElseNode from './if-else/node' -import IfElsePanel from './if-else/panel' -import CodeNode from './code/node' -import CodePanel from './code/panel' -import TemplateTransformNode from './template-transform/node' -import TemplateTransformPanel from './template-transform/panel' -import HttpNode from './http/node' -import HttpPanel from './http/panel' -import ToolNode from './tool/node' -import ToolPanel from './tool/panel' -import VariableAssignerNode from './variable-assigner/node' -import VariableAssignerPanel from './variable-assigner/panel' -import AssignerNode from './assigner/node' -import AssignerPanel from './assigner/panel' -import ParameterExtractorNode from './parameter-extractor/node' -import ParameterExtractorPanel from './parameter-extractor/panel' -import IterationNode from './iteration/node' -import IterationPanel from './iteration/panel' -import LoopNode from './loop/node' -import LoopPanel from './loop/panel' -import DocExtractorNode from './document-extractor/node' -import DocExtractorPanel from './document-extractor/panel' -import ListFilterNode from './list-operator/node' -import ListFilterPanel from './list-operator/panel' -import AgentNode from './agent/node' -import AgentPanel from './agent/panel' -import DataSourceNode from './data-source/node' -import DataSourcePanel from './data-source/panel' -import KnowledgeBaseNode from './knowledge-base/node' -import KnowledgeBasePanel from './knowledge-base/panel' import { TransferMethod } from '@/types/app' -export const NodeComponentMap: Record> = { - [BlockEnum.Start]: StartNode, - [BlockEnum.End]: EndNode, - [BlockEnum.Answer]: AnswerNode, - [BlockEnum.LLM]: LLMNode, - [BlockEnum.KnowledgeRetrieval]: KnowledgeRetrievalNode, - [BlockEnum.QuestionClassifier]: QuestionClassifierNode, - [BlockEnum.IfElse]: IfElseNode, - [BlockEnum.Code]: CodeNode, - [BlockEnum.TemplateTransform]: TemplateTransformNode, - [BlockEnum.HttpRequest]: HttpNode, - [BlockEnum.Tool]: ToolNode, - [BlockEnum.VariableAssigner]: VariableAssignerNode, - [BlockEnum.Assigner]: AssignerNode, - [BlockEnum.VariableAggregator]: VariableAssignerNode, - [BlockEnum.ParameterExtractor]: ParameterExtractorNode, - [BlockEnum.Iteration]: IterationNode, - [BlockEnum.Loop]: LoopNode, - [BlockEnum.DocExtractor]: DocExtractorNode, - [BlockEnum.ListFilter]: ListFilterNode, - [BlockEnum.Agent]: AgentNode, - [BlockEnum.DataSource]: DataSourceNode, - [BlockEnum.KnowledgeBase]: KnowledgeBaseNode, -} - -export const PanelComponentMap: Record> = { - [BlockEnum.Start]: StartPanel, - [BlockEnum.End]: EndPanel, - [BlockEnum.Answer]: AnswerPanel, - [BlockEnum.LLM]: LLMPanel, - [BlockEnum.KnowledgeRetrieval]: KnowledgeRetrievalPanel, - [BlockEnum.QuestionClassifier]: QuestionClassifierPanel, - [BlockEnum.IfElse]: IfElsePanel, - [BlockEnum.Code]: CodePanel, - [BlockEnum.TemplateTransform]: TemplateTransformPanel, - [BlockEnum.HttpRequest]: HttpPanel, - [BlockEnum.Tool]: ToolPanel, - [BlockEnum.VariableAssigner]: VariableAssignerPanel, - [BlockEnum.VariableAggregator]: VariableAssignerPanel, - [BlockEnum.Assigner]: AssignerPanel, - [BlockEnum.ParameterExtractor]: ParameterExtractorPanel, - [BlockEnum.Iteration]: IterationPanel, - [BlockEnum.Loop]: LoopPanel, - [BlockEnum.DocExtractor]: DocExtractorPanel, - [BlockEnum.ListFilter]: ListFilterPanel, - [BlockEnum.Agent]: AgentPanel, - [BlockEnum.DataSource]: DataSourcePanel, - [BlockEnum.KnowledgeBase]: KnowledgeBasePanel, -} - -export const CUSTOM_NODE_TYPE = 'custom' - export const FILE_TYPE_OPTIONS = [ { value: 'image', i18nKey: 'image' }, { value: 'document', i18nKey: 'doc' }, diff --git a/web/app/components/workflow/nodes/index.tsx b/web/app/components/workflow/nodes/index.tsx index 8458051da2..ba880b398b 100644 --- a/web/app/components/workflow/nodes/index.tsx +++ b/web/app/components/workflow/nodes/index.tsx @@ -8,7 +8,7 @@ import { CUSTOM_NODE } from '../constants' import { NodeComponentMap, PanelComponentMap, -} from './constants' +} from './components' import BaseNode from './_base/node' import BasePanel from './_base/components/workflow-panel' diff --git a/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/index.tsx b/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/index.tsx index 6410ab706f..60aa3d5590 100644 --- a/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/index.tsx +++ b/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/index.tsx @@ -29,7 +29,7 @@ const ChunkStructure = ({ =21.0.0} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-linux-arm64@0.34.3': resolution: {integrity: sha512-QdrKe3EvQrqwkDrtuTIjI0bu6YEJHTgEeqdzI3uWJOH6G1O8Nl1iEeVYRGdj1h5I21CqxSvQp1Yv7xeU3ZewbA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-linux-arm@0.33.5': resolution: {integrity: sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-linux-arm@0.34.3': resolution: {integrity: sha512-oBK9l+h6KBN0i3dC8rYntLiVfW8D8wH+NPNT3O/WBHeW0OQWCjfWksLUaPidsrDKpJgXp3G3/hkmhptAW0I3+A==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-linux-ppc64@0.34.3': resolution: {integrity: sha512-GLtbLQMCNC5nxuImPR2+RgrviwKwVql28FWZIW1zWruy6zLgA5/x2ZXk3mxj58X/tszVF69KK0Is83V8YgWhLA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [ppc64] os: [linux] - libc: [glibc] '@img/sharp-linux-s390x@0.33.5': resolution: {integrity: sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-linux-s390x@0.34.3': resolution: {integrity: sha512-3gahT+A6c4cdc2edhsLHmIOXMb17ltffJlxR0aC2VPZfwKoTGZec6u5GrFgdR7ciJSsHT27BD3TIuGcuRT0KmQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-linux-x64@0.33.5': resolution: {integrity: sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-linux-x64@0.34.3': resolution: {integrity: sha512-8kYso8d806ypnSq3/Ly0QEw90V5ZoHh10yH0HnrzOCr6DKAPI6QVHvwleqMkVQ0m+fc7EH8ah0BB0QPuWY6zJQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-linuxmusl-arm64@0.33.5': resolution: {integrity: sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-linuxmusl-arm64@0.34.3': resolution: {integrity: sha512-vAjbHDlr4izEiXM1OTggpCcPg9tn4YriK5vAjowJsHwdBIdx0fYRsURkxLG2RLm9gyBq66gwtWI8Gx0/ov+JKQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-linuxmusl-x64@0.33.5': resolution: {integrity: sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-linuxmusl-x64@0.34.3': resolution: {integrity: sha512-gCWUn9547K5bwvOn9l5XGAEjVTTRji4aPTqLzGXHvIr6bIDZKNTA34seMPgM0WmSf+RYBH411VavCejp3PkOeQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-wasm32@0.33.5': resolution: {integrity: sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==} @@ -2058,6 +2032,9 @@ packages: '@lexical/clipboard@0.36.2': resolution: {integrity: sha512-l7z52jltlMz1HmJRmG7ZdxySPjheRRxdV/75QEnzalMtqfLPgh4G5IpycISjbX+95PgEaC6rXbcjPix0CyHDJg==} + '@lexical/clipboard@0.37.0': + resolution: {integrity: sha512-hRwASFX/ilaI5r8YOcZuQgONFshRgCPfdxfofNL7uruSFYAO6LkUhsjzZwUgf0DbmCJmbBADFw15FSthgCUhGA==} + '@lexical/code@0.36.2': resolution: {integrity: sha512-dfS62rNo3uKwNAJQ39zC+8gYX0k8UAoW7u+JPIqx+K2VPukZlvpsPLNGft15pdWBkHc7Pv+o9gJlB6gGv+EBfA==} @@ -2073,6 +2050,9 @@ packages: '@lexical/extension@0.36.2': resolution: {integrity: sha512-NWxtqMFMzScq4Eemqp1ST2KREIfj57fUbn7qHv+mMnYgQZK4iIhrHKo5klonxi1oBURcxUZMIbdtH7MJ4BdisA==} + '@lexical/extension@0.37.0': + resolution: {integrity: sha512-Z58f2tIdz9bn8gltUu5cVg37qROGha38dUZv20gI2GeNugXAkoPzJYEcxlI1D/26tkevJ/7VaFUr9PTk+iKmaA==} + '@lexical/hashtag@0.36.2': resolution: {integrity: sha512-WdmKtzXFcahQT3ShFDeHF6LCR5C8yvFCj3ImI09rZwICrYeonbMrzsBUxS1joBz0HQ+ufF9Tx+RxLvGWx6WxzQ==} @@ -2082,12 +2062,18 @@ packages: '@lexical/html@0.36.2': resolution: {integrity: sha512-fgqALzgKnoy93G0yFyYD4C4qJTSMZyUt4JE5kj/POFwWNOnXThIqJhQGwBvH/ibImpIfOeds2TrSr8PbStlrNg==} + '@lexical/html@0.37.0': + resolution: {integrity: sha512-oTsBc45eL8/lmF7fqGR+UCjrJYP04gumzf5nk4TczrxWL2pM4GIMLLKG1mpQI2H1MDiRLzq3T/xdI7Gh74z7Zw==} + '@lexical/link@0.36.2': resolution: {integrity: sha512-Zb+DeHA1po8VMiOAAXsBmAHhfWmQttsUkI5oiZUmOXJruRuQ2rVr01NoxHpoEpLwHOABVNzD3PMbwov+g3c7lg==} '@lexical/list@0.36.2': resolution: {integrity: sha512-JpaIaE0lgNUrAR7iaCaIoETcCKG9EvZjM3G71VxiexTs7PltmEMq36LUlO2goafWurP7knG2rUpVnTcuSbYYeA==} + '@lexical/list@0.37.0': + resolution: {integrity: sha512-AOC6yAA3mfNvJKbwo+kvAbPJI+13yF2ISA65vbA578CugvJ08zIVgM+pSzxquGhD0ioJY3cXVW7+gdkCP1qu5g==} + '@lexical/mark@0.36.2': resolution: {integrity: sha512-n0MNXtGH+1i43hglgHjpQV0093HmIiFR7Budg2BJb8ZNzO1KZRqeXAHlA5ZzJ698FkAnS4R5bqG9tZ0JJHgAuA==} @@ -2115,15 +2101,24 @@ packages: '@lexical/selection@0.36.2': resolution: {integrity: sha512-n96joW3HCKBmPeESR172BxVE+m8V9SdidQm4kKb9jOZ1Ota+tnam2386TeI6795TWwgjDQJPK3HZNKcX6Gb+Bg==} + '@lexical/selection@0.37.0': + resolution: {integrity: sha512-Lix1s2r71jHfsTEs4q/YqK2s3uXKOnyA3fd1VDMWysO+bZzRwEO5+qyDvENZ0WrXSDCnlibNFV1HttWX9/zqyw==} + '@lexical/table@0.36.2': resolution: {integrity: sha512-96rNNPiVbC65i+Jn1QzIsehCS7UVUc69ovrh9Bt4+pXDebZSdZai153Q7RUq8q3AQ5ocK4/SA2kLQfMu0grj3Q==} + '@lexical/table@0.37.0': + resolution: {integrity: sha512-g7S8ml8kIujEDLWlzYKETgPCQ2U9oeWqdytRuHjHGi/rjAAGHSej5IRqTPIMxNP3VVQHnBoQ+Y9hBtjiuddhgQ==} + '@lexical/text@0.36.2': resolution: {integrity: sha512-IbbqgRdMAD6Uk9b2+qSVoy+8RVcczrz6OgXvg39+EYD+XEC7Rbw7kDTWzuNSJJpP7vxSO8YDZSaIlP5gNH3qKA==} '@lexical/utils@0.36.2': resolution: {integrity: sha512-P9+t2Ob10YNGYT/PWEER+1EqH8SAjCNRn+7SBvKbr0IdleGF2JvzbJwAWaRwZs1c18P11XdQZ779dGvWlfwBIw==} + '@lexical/utils@0.37.0': + resolution: {integrity: sha512-CFp4diY/kR5RqhzQSl/7SwsMod1sgLpI1FBifcOuJ6L/S6YywGpEB4B7aV5zqW21A/jU2T+2NZtxSUn6S+9gMg==} + '@lexical/yjs@0.36.2': resolution: {integrity: sha512-gZ66Mw+uKXTO8KeX/hNKAinXbFg3gnNYraG76lBXCwb/Ka3q34upIY9FUeGOwGVaau3iIDQhE49I+6MugAX2FQ==} peerDependencies: @@ -2203,28 +2198,24 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [glibc] '@next/swc-linux-arm64-musl@15.5.4': resolution: {integrity: sha512-TOK7iTxmXFc45UrtKqWdZ1shfxuL4tnVAOuuJK4S88rX3oyVV4ZkLjtMT85wQkfBrOOvU55aLty+MV8xmcJR8A==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [musl] '@next/swc-linux-x64-gnu@15.5.4': resolution: {integrity: sha512-7HKolaj+481FSW/5lL0BcTkA4Ueam9SPYWyN/ib/WGAFZf0DGAN8frNpNZYFHtM4ZstrHZS3LY3vrwlIQfsiMA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [glibc] '@next/swc-linux-x64-musl@15.5.4': resolution: {integrity: sha512-nlQQ6nfgN0nCO/KuyEUwwOdwQIGjOs4WNMjEUtpIQJPR2NUfmGpW2wkJln1d4nJ7oUzd1g4GivH5GoEPBgfsdw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [musl] '@next/swc-win32-arm64-msvc@15.5.4': resolution: {integrity: sha512-PcR2bN7FlM32XM6eumklmyWLLbu2vs+D7nJX8OAIoWy69Kef8mfiN4e8TUv2KohprwifdpFKPzIP1njuCjD0YA==} @@ -2401,49 +2392,41 @@ packages: resolution: {integrity: sha512-TWq+y2psMzbMtZB9USAq2bSA7NV1TMmh9lhAFbMGQ8Yp2YV4BRC/HilD6qF++efQl6shueGBFOv0LVe9BUXaIA==} cpu: [arm64] os: [linux] - libc: [glibc] '@oxc-resolver/binding-linux-arm64-musl@11.9.0': resolution: {integrity: sha512-8WwGLfXk7yttc6rD6g53+RnYfX5B8xOot1ffthLn8oCXzVRO4cdChlmeHStxwLD/MWx8z8BGeyfyINNrsh9N2w==} cpu: [arm64] os: [linux] - libc: [musl] '@oxc-resolver/binding-linux-ppc64-gnu@11.9.0': resolution: {integrity: sha512-ZWiAXfan6actlSzayaFS/kYO2zD6k1k0fmLb1opbujXYMKepEnjjVOvKdzCIYR/zKzudqI39dGc+ywqVdsPIpQ==} cpu: [ppc64] os: [linux] - libc: [glibc] '@oxc-resolver/binding-linux-riscv64-gnu@11.9.0': resolution: {integrity: sha512-p9mCSb+Bym+eycNo9k+81wQ5SAE31E+/rtfbDmF4/7krPotkEjPsEBSc3rqunRwO+FtsUn7H68JLY7hlai49eQ==} cpu: [riscv64] os: [linux] - libc: [glibc] '@oxc-resolver/binding-linux-riscv64-musl@11.9.0': resolution: {integrity: sha512-/SePuVxgFhLPciRwsJ8kLVltr+rxh0b6riGFuoPnFXBbHFclKnjNIt3TfqzUj0/vOnslXw3cVGPpmtkm2TgCgg==} cpu: [riscv64] os: [linux] - libc: [musl] '@oxc-resolver/binding-linux-s390x-gnu@11.9.0': resolution: {integrity: sha512-zLuEjlYIzfnr1Ei2UZYQBbCTa/9deh+BEjO9rh1ai8BfEq4uj6RupTtNpgHfgAsEYdqOBVExw9EU1S6SW3RCAw==} cpu: [s390x] os: [linux] - libc: [glibc] '@oxc-resolver/binding-linux-x64-gnu@11.9.0': resolution: {integrity: sha512-cxdg73WG+aVlPu/k4lEQPRVOhWunYOUglW6OSzclZLJJAXZU0tSZ5ymKaqPRkfTsyNSAafj1cA1XYd+P9UxBgw==} cpu: [x64] os: [linux] - libc: [glibc] '@oxc-resolver/binding-linux-x64-musl@11.9.0': resolution: {integrity: sha512-sy5nkVdMvNgqcx9sIY7G6U9TYZUZC4cmMGw/wKhJNuuD2/HFGtbje62ttXSwBAbVbmJ2GgZ4ZUo/S1OMyU+/OA==} cpu: [x64] os: [linux] - libc: [musl] '@oxc-resolver/binding-wasm32-wasi@11.9.0': resolution: {integrity: sha512-dfi/a0Xh6o6nOLbJdaYuy7txncEcwkRHp9DGGZaAP7zxDiepkBZ6ewSJODQrWwhjVmMteXo+XFzEOMjsC7WUtQ==} @@ -2494,42 +2477,36 @@ packages: engines: {node: '>= 10.0.0'} cpu: [arm] os: [linux] - libc: [glibc] '@parcel/watcher-linux-arm-musl@2.5.1': resolution: {integrity: sha512-6E+m/Mm1t1yhB8X412stiKFG3XykmgdIOqhjWj+VL8oHkKABfu/gjFj8DvLrYVHSBNC+/u5PeNrujiSQ1zwd1Q==} engines: {node: '>= 10.0.0'} cpu: [arm] os: [linux] - libc: [musl] '@parcel/watcher-linux-arm64-glibc@2.5.1': resolution: {integrity: sha512-LrGp+f02yU3BN9A+DGuY3v3bmnFUggAITBGriZHUREfNEzZh/GO06FF5u2kx8x+GBEUYfyTGamol4j3m9ANe8w==} engines: {node: '>= 10.0.0'} cpu: [arm64] os: [linux] - libc: [glibc] '@parcel/watcher-linux-arm64-musl@2.5.1': resolution: {integrity: sha512-cFOjABi92pMYRXS7AcQv9/M1YuKRw8SZniCDw0ssQb/noPkRzA+HBDkwmyOJYp5wXcsTrhxO0zq1U11cK9jsFg==} engines: {node: '>= 10.0.0'} cpu: [arm64] os: [linux] - libc: [musl] '@parcel/watcher-linux-x64-glibc@2.5.1': resolution: {integrity: sha512-GcESn8NZySmfwlTsIur+49yDqSny2IhPeZfXunQi48DMugKeZ7uy1FX83pO0X22sHntJ4Ub+9k34XQCX+oHt2A==} engines: {node: '>= 10.0.0'} cpu: [x64] os: [linux] - libc: [glibc] '@parcel/watcher-linux-x64-musl@2.5.1': resolution: {integrity: sha512-n0E2EQbatQ3bXhcH2D1XIAANAcTZkQICBPVaxMeaCVBtOpBZpWJuf7LwyWPSBDITb7In8mqQgJ7gH8CILCURXg==} engines: {node: '>= 10.0.0'} cpu: [x64] os: [linux] - libc: [musl] '@parcel/watcher-win32-arm64@2.5.1': resolution: {integrity: sha512-RFzklRvmc3PkjKjry3hLF9wD7ppR4AKcWNzH7kXR7GUe0Igb3Nz8fyPwtZCSquGrhU5HhUNDr/mKBqj7tqA2Vw==} @@ -6189,8 +6166,8 @@ packages: resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} engines: {node: '>=6'} - knip@5.64.1: - resolution: {integrity: sha512-80XnLsyeXuyxj1F4+NBtQFHxaRH0xWRw8EKwfQ6EkVZZ0bSz/kqqan08k/Qg8ajWsFPhFq+0S2RbLCBGIQtuOg==} + knip@5.64.3: + resolution: {integrity: sha512-P9dZetEZfSBwNBFwj55CAnPAMdzVLTTscWx6rdB8eBmPqXPji8F3L+hhWi+Xp+u9O6Xp2ClRDq2JENSK8Z04Qg==} engines: {node: '>=18.18.0'} hasBin: true peerDependencies: @@ -6231,6 +6208,9 @@ packages: lexical@0.36.2: resolution: {integrity: sha512-gIDJCmSAhtxD7h95WK17Nz19wCZu92Zn0p1/R45X01S/KAsLCwEtVJ2fTvIJNFTyx3QNJTuGcm5mYgRMUwq8rg==} + lexical@0.37.0: + resolution: {integrity: sha512-r5VJR2TioQPAsZATfktnJFrGIiy6gjQN8b/+0a2u1d7/QTH7lhbB7byhGSvcq1iaa1TV/xcf/pFV55a5V5hTDQ==} + lib0@0.2.114: resolution: {integrity: sha512-gcxmNFzA4hv8UYi8j43uPlQ7CGcyMJ2KQb5kZASw6SnAKAf10hK12i2fjrS3Cl/ugZa5Ui6WwIu1/6MIXiHttQ==} engines: {node: '>=16'} @@ -10573,6 +10553,14 @@ snapshots: '@lexical/utils': 0.36.2 lexical: 0.36.2 + '@lexical/clipboard@0.37.0': + dependencies: + '@lexical/html': 0.37.0 + '@lexical/list': 0.37.0 + '@lexical/selection': 0.37.0 + '@lexical/utils': 0.37.0 + lexical: 0.37.0 + '@lexical/code@0.36.2': dependencies: '@lexical/utils': 0.36.2 @@ -10601,6 +10589,12 @@ snapshots: '@preact/signals-core': 1.12.1 lexical: 0.36.2 + '@lexical/extension@0.37.0': + dependencies: + '@lexical/utils': 0.37.0 + '@preact/signals-core': 1.12.1 + lexical: 0.37.0 + '@lexical/hashtag@0.36.2': dependencies: '@lexical/text': 0.36.2 @@ -10619,6 +10613,12 @@ snapshots: '@lexical/utils': 0.36.2 lexical: 0.36.2 + '@lexical/html@0.37.0': + dependencies: + '@lexical/selection': 0.37.0 + '@lexical/utils': 0.37.0 + lexical: 0.37.0 + '@lexical/link@0.36.2': dependencies: '@lexical/extension': 0.36.2 @@ -10632,6 +10632,13 @@ snapshots: '@lexical/utils': 0.36.2 lexical: 0.36.2 + '@lexical/list@0.37.0': + dependencies: + '@lexical/extension': 0.37.0 + '@lexical/selection': 0.37.0 + '@lexical/utils': 0.37.0 + lexical: 0.37.0 + '@lexical/mark@0.36.2': dependencies: '@lexical/utils': 0.36.2 @@ -10701,6 +10708,10 @@ snapshots: dependencies: lexical: 0.36.2 + '@lexical/selection@0.37.0': + dependencies: + lexical: 0.37.0 + '@lexical/table@0.36.2': dependencies: '@lexical/clipboard': 0.36.2 @@ -10708,6 +10719,13 @@ snapshots: '@lexical/utils': 0.36.2 lexical: 0.36.2 + '@lexical/table@0.37.0': + dependencies: + '@lexical/clipboard': 0.37.0 + '@lexical/extension': 0.37.0 + '@lexical/utils': 0.37.0 + lexical: 0.37.0 + '@lexical/text@0.36.2': dependencies: lexical: 0.36.2 @@ -10719,6 +10737,13 @@ snapshots: '@lexical/table': 0.36.2 lexical: 0.36.2 + '@lexical/utils@0.37.0': + dependencies: + '@lexical/list': 0.37.0 + '@lexical/selection': 0.37.0 + '@lexical/table': 0.37.0 + lexical: 0.37.0 + '@lexical/yjs@0.36.2(yjs@13.6.27)': dependencies: '@lexical/offset': 0.36.2 @@ -15557,7 +15582,7 @@ snapshots: kleur@3.0.3: {} - knip@5.64.1(@types/node@18.15.0)(typescript@5.8.3): + knip@5.64.3(@types/node@18.15.0)(typescript@5.8.3): dependencies: '@nodelib/fs.walk': 1.2.8 '@types/node': 18.15.0 @@ -15608,6 +15633,8 @@ snapshots: lexical@0.36.2: {} + lexical@0.37.0: {} + lib0@0.2.114: dependencies: isomorphic.js: 0.2.5