From 0a56d655818f773568f554aae74ccb5f05a60993 Mon Sep 17 00:00:00 2001 From: Asuka Minato Date: Mon, 13 Oct 2025 11:16:12 +0900 Subject: [PATCH] Issue 23579 (#26777) Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> --- api/commands.py | 16 ++ api/core/tools/mcp_tool/provider.py | 3 +- .../tools/utils/model_invocation_utils.py | 9 +- api/migrations/env.py | 5 +- api/models/source.py | 48 ++-- api/models/task.py | 44 ++-- api/models/tools.py | 214 +++++++++++------- api/models/web.py | 42 ++-- api/services/auth/api_key_auth_service.py | 9 +- .../tools/api_tools_manage_service.py | 2 +- .../tools/builtin_tools_manage_service.py | 2 +- .../tools/mcp_tools_manage_service.py | 2 + api/services/tools/tools_transform_service.py | 3 +- 13 files changed, 256 insertions(+), 143 deletions(-) diff --git a/api/commands.py b/api/commands.py index 82efe34611..8ca19e1dac 100644 --- a/api/commands.py +++ b/api/commands.py @@ -1521,6 +1521,14 @@ def transform_datasource_credentials(): auth_count = 0 for firecrawl_tenant_credential in firecrawl_tenant_credentials: auth_count += 1 + if not firecrawl_tenant_credential.credentials: + click.echo( + click.style( + f"Skipping firecrawl credential for tenant {tenant_id} due to missing credentials.", + fg="yellow", + ) + ) + continue # get credential api key credentials_json = json.loads(firecrawl_tenant_credential.credentials) api_key = credentials_json.get("config", {}).get("api_key") @@ -1576,6 +1584,14 @@ def transform_datasource_credentials(): auth_count = 0 for jina_tenant_credential in jina_tenant_credentials: auth_count += 1 + if not jina_tenant_credential.credentials: + click.echo( + click.style( + f"Skipping jina credential for tenant {tenant_id} due to missing credentials.", + fg="yellow", + ) + ) + continue # get credential api key credentials_json = json.loads(jina_tenant_credential.credentials) api_key = credentials_json.get("config", {}).get("api_key") diff --git a/api/core/tools/mcp_tool/provider.py b/api/core/tools/mcp_tool/provider.py index f269b8db9b..0c2870727e 100644 --- a/api/core/tools/mcp_tool/provider.py +++ b/api/core/tools/mcp_tool/provider.py @@ -76,7 +76,8 @@ class MCPToolProviderController(ToolProviderController): ) for remote_mcp_tool in remote_mcp_tools ] - + if not db_provider.icon: + raise ValueError("Database provider icon is required") return cls( entity=ToolProviderEntityWithPlugin( identity=ToolProviderIdentity( diff --git a/api/core/tools/utils/model_invocation_utils.py b/api/core/tools/utils/model_invocation_utils.py index 526f5c8b9a..b4bae08a9b 100644 --- a/api/core/tools/utils/model_invocation_utils.py +++ b/api/core/tools/utils/model_invocation_utils.py @@ -5,6 +5,7 @@ Therefore, a model manager is needed to list/invoke/validate models. """ import json +from decimal import Decimal from typing import cast from core.model_manager import ModelManager @@ -118,10 +119,10 @@ class ModelInvocationUtils: model_response="", prompt_tokens=prompt_tokens, answer_tokens=0, - answer_unit_price=0, - answer_price_unit=0, + answer_unit_price=Decimal(), + answer_price_unit=Decimal(), provider_response_latency=0, - total_price=0, + total_price=Decimal(), currency="USD", ) @@ -152,7 +153,7 @@ class ModelInvocationUtils: raise InvokeModelError(f"Invoke error: {e}") # update tool model invoke - tool_model_invoke.model_response = response.message.content + tool_model_invoke.model_response = str(response.message.content) if response.usage: tool_model_invoke.answer_tokens = response.usage.completion_tokens tool_model_invoke.answer_unit_price = response.usage.completion_unit_price diff --git a/api/migrations/env.py b/api/migrations/env.py index a5d815dcfd..66a4614e80 100644 --- a/api/migrations/env.py +++ b/api/migrations/env.py @@ -37,10 +37,11 @@ config.set_main_option('sqlalchemy.url', get_engine_url()) # my_important_option = config.get_main_option("my_important_option") # ... etc. -from models.base import Base +from models.base import TypeBase + def get_metadata(): - return Base.metadata + return TypeBase.metadata def include_object(object, name, type_, reflected, compare_to): if type_ == "foreign_key_constraint": diff --git a/api/models/source.py b/api/models/source.py index 5b4c486bc4..0ed7c4c70e 100644 --- a/api/models/source.py +++ b/api/models/source.py @@ -6,12 +6,12 @@ from sqlalchemy import DateTime, String, func from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column -from models.base import Base +from models.base import TypeBase from .types import StringUUID -class DataSourceOauthBinding(Base): +class DataSourceOauthBinding(TypeBase): __tablename__ = "data_source_oauth_bindings" __table_args__ = ( sa.PrimaryKeyConstraint("id", name="source_binding_pkey"), @@ -19,17 +19,25 @@ class DataSourceOauthBinding(Base): sa.Index("source_info_idx", "source_info", postgresql_using="gin"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) - tenant_id = mapped_column(StringUUID, nullable=False) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) + tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) access_token: Mapped[str] = mapped_column(String(255), nullable=False) provider: Mapped[str] = mapped_column(String(255), nullable=False) - source_info = mapped_column(JSONB, nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp()) - updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp()) - disabled: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false")) + source_info: Mapped[dict] = mapped_column(JSONB, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime, nullable=False, server_default=func.current_timestamp(), init=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime, + nullable=False, + server_default=func.current_timestamp(), + onupdate=func.current_timestamp(), + init=False, + ) + disabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"), default=False) -class DataSourceApiKeyAuthBinding(Base): +class DataSourceApiKeyAuthBinding(TypeBase): __tablename__ = "data_source_api_key_auth_bindings" __table_args__ = ( sa.PrimaryKeyConstraint("id", name="data_source_api_key_auth_binding_pkey"), @@ -37,14 +45,22 @@ class DataSourceApiKeyAuthBinding(Base): sa.Index("data_source_api_key_auth_binding_provider_idx", "provider"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) - tenant_id = mapped_column(StringUUID, nullable=False) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) + tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) category: Mapped[str] = mapped_column(String(255), nullable=False) provider: Mapped[str] = mapped_column(String(255), nullable=False) - credentials = mapped_column(sa.Text, nullable=True) # JSON - created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp()) - updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp()) - disabled: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false")) + credentials: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None) # JSON + created_at: Mapped[datetime] = mapped_column( + DateTime, nullable=False, server_default=func.current_timestamp(), init=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime, + nullable=False, + server_default=func.current_timestamp(), + onupdate=func.current_timestamp(), + init=False, + ) + disabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"), default=False) def to_dict(self): return { @@ -52,7 +68,7 @@ class DataSourceApiKeyAuthBinding(Base): "tenant_id": self.tenant_id, "category": self.category, "provider": self.provider, - "credentials": json.loads(self.credentials), + "credentials": json.loads(self.credentials) if self.credentials else None, "created_at": self.created_at.timestamp(), "updated_at": self.updated_at.timestamp(), "disabled": self.disabled, diff --git a/api/models/task.py b/api/models/task.py index 4e49254dbd..513f167cce 100644 --- a/api/models/task.py +++ b/api/models/task.py @@ -6,41 +6,43 @@ from sqlalchemy import DateTime, String from sqlalchemy.orm import Mapped, mapped_column from libs.datetime_utils import naive_utc_now -from models.base import Base +from models.base import TypeBase -class CeleryTask(Base): +class CeleryTask(TypeBase): """Task result/status.""" __tablename__ = "celery_taskmeta" - id = mapped_column(sa.Integer, sa.Sequence("task_id_sequence"), primary_key=True, autoincrement=True) - task_id = mapped_column(String(155), unique=True) - status = mapped_column(String(50), default=states.PENDING) - result = mapped_column(sa.PickleType, nullable=True) - date_done = mapped_column( + id: Mapped[int] = mapped_column( + sa.Integer, sa.Sequence("task_id_sequence"), primary_key=True, autoincrement=True, init=False + ) + task_id: Mapped[str] = mapped_column(String(155), unique=True) + status: Mapped[str] = mapped_column(String(50), default=states.PENDING) + result: Mapped[bytes | None] = mapped_column(sa.PickleType, nullable=True, default=None) + date_done: Mapped[datetime | None] = mapped_column( DateTime, - default=lambda: naive_utc_now(), - onupdate=lambda: naive_utc_now(), + default=naive_utc_now, + onupdate=naive_utc_now, nullable=True, ) - traceback = mapped_column(sa.Text, nullable=True) - name = mapped_column(String(155), nullable=True) - args = mapped_column(sa.LargeBinary, nullable=True) - kwargs = mapped_column(sa.LargeBinary, nullable=True) - worker = mapped_column(String(155), nullable=True) - retries: Mapped[int | None] = mapped_column(sa.Integer, nullable=True) - queue = mapped_column(String(155), nullable=True) + traceback: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None) + name: Mapped[str | None] = mapped_column(String(155), nullable=True, default=None) + args: Mapped[bytes | None] = mapped_column(sa.LargeBinary, nullable=True, default=None) + kwargs: Mapped[bytes | None] = mapped_column(sa.LargeBinary, nullable=True, default=None) + worker: Mapped[str | None] = mapped_column(String(155), nullable=True, default=None) + retries: Mapped[int | None] = mapped_column(sa.Integer, nullable=True, default=None) + queue: Mapped[str | None] = mapped_column(String(155), nullable=True, default=None) -class CeleryTaskSet(Base): +class CeleryTaskSet(TypeBase): """TaskSet result.""" __tablename__ = "celery_tasksetmeta" id: Mapped[int] = mapped_column( - sa.Integer, sa.Sequence("taskset_id_sequence"), autoincrement=True, primary_key=True + sa.Integer, sa.Sequence("taskset_id_sequence"), autoincrement=True, primary_key=True, init=False ) - taskset_id = mapped_column(String(155), unique=True) - result = mapped_column(sa.PickleType, nullable=True) - date_done: Mapped[datetime | None] = mapped_column(DateTime, default=lambda: naive_utc_now(), nullable=True) + taskset_id: Mapped[str] = mapped_column(String(155), unique=True) + result: Mapped[bytes | None] = mapped_column(sa.PickleType, nullable=True, default=None) + date_done: Mapped[datetime | None] = mapped_column(DateTime, default=naive_utc_now, nullable=True) diff --git a/api/models/tools.py b/api/models/tools.py index d581d588a4..aec53da50c 100644 --- a/api/models/tools.py +++ b/api/models/tools.py @@ -1,6 +1,7 @@ import json from collections.abc import Mapping from datetime import datetime +from decimal import Decimal from typing import TYPE_CHECKING, Any, cast from urllib.parse import urlparse @@ -13,7 +14,7 @@ from core.helper import encrypter from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_bundle import ApiToolBundle from core.tools.entities.tool_entities import ApiProviderSchemaType, WorkflowToolParameterConfiguration -from models.base import Base, TypeBase +from models.base import TypeBase from .engine import db from .model import Account, App, Tenant @@ -42,28 +43,28 @@ class ToolOAuthSystemClient(TypeBase): # tenant level tool oauth client params (client_id, client_secret, etc.) -class ToolOAuthTenantClient(Base): +class ToolOAuthTenantClient(TypeBase): __tablename__ = "tool_oauth_tenant_clients" __table_args__ = ( sa.PrimaryKeyConstraint("id", name="tool_oauth_tenant_client_pkey"), sa.UniqueConstraint("tenant_id", "plugin_id", "provider", name="unique_tool_oauth_tenant_client"), ) - id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # tenant id tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) plugin_id: Mapped[str] = mapped_column(String(512), nullable=False) provider: Mapped[str] = mapped_column(String(255), nullable=False) - enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true")) + enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"), init=False) # oauth params of the tool provider - encrypted_oauth_params: Mapped[str] = mapped_column(sa.Text, nullable=False) + encrypted_oauth_params: Mapped[str] = mapped_column(sa.Text, nullable=False, init=False) @property def oauth_params(self) -> dict[str, Any]: return cast(dict[str, Any], json.loads(self.encrypted_oauth_params or "{}")) -class BuiltinToolProvider(Base): +class BuiltinToolProvider(TypeBase): """ This table stores the tool provider information for built-in tools for each tenant. """ @@ -75,37 +76,45 @@ class BuiltinToolProvider(Base): ) # id of the tool provider - id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) name: Mapped[str] = mapped_column( - String(256), nullable=False, server_default=sa.text("'API KEY 1'::character varying") + String(256), + nullable=False, + server_default=sa.text("'API KEY 1'::character varying"), ) # id of the tenant - tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=True) + tenant_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True) # who created this tool provider user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # name of the tool provider provider: Mapped[str] = mapped_column(String(256), nullable=False) # credential of the tool provider - encrypted_credentials: Mapped[str] = mapped_column(sa.Text, nullable=True) + encrypted_credentials: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None) created_at: Mapped[datetime] = mapped_column( - sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)") + sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False ) updated_at: Mapped[datetime] = mapped_column( - sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)") + sa.DateTime, + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP(0)"), + onupdate=func.current_timestamp(), + init=False, ) - is_default: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false")) + is_default: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"), default=False) # credential type, e.g., "api-key", "oauth2" credential_type: Mapped[str] = mapped_column( - String(32), nullable=False, server_default=sa.text("'api-key'::character varying") + String(32), nullable=False, server_default=sa.text("'api-key'::character varying"), default="api-key" ) - expires_at: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("-1")) + expires_at: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("-1"), default=-1) @property def credentials(self) -> dict[str, Any]: + if not self.encrypted_credentials: + return {} return cast(dict[str, Any], json.loads(self.encrypted_credentials)) -class ApiToolProvider(Base): +class ApiToolProvider(TypeBase): """ The table stores the api providers. """ @@ -116,31 +125,43 @@ class ApiToolProvider(Base): sa.UniqueConstraint("name", "tenant_id", name="unique_api_tool_provider"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # name of the api provider - name = mapped_column(String(255), nullable=False, server_default=sa.text("'API KEY 1'::character varying")) + name: Mapped[str] = mapped_column( + String(255), + nullable=False, + server_default=sa.text("'API KEY 1'::character varying"), + ) # icon icon: Mapped[str] = mapped_column(String(255), nullable=False) # original schema - schema = mapped_column(sa.Text, nullable=False) + schema: Mapped[str] = mapped_column(sa.Text, nullable=False) schema_type_str: Mapped[str] = mapped_column(String(40), nullable=False) # who created this tool - user_id = mapped_column(StringUUID, nullable=False) + user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # tenant id - tenant_id = mapped_column(StringUUID, nullable=False) + tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # description of the provider - description = mapped_column(sa.Text, nullable=False) + description: Mapped[str] = mapped_column(sa.Text, nullable=False) # json format tools - tools_str = mapped_column(sa.Text, nullable=False) + tools_str: Mapped[str] = mapped_column(sa.Text, nullable=False) # json format credentials - credentials_str = mapped_column(sa.Text, nullable=False) + credentials_str: Mapped[str] = mapped_column(sa.Text, nullable=False) # privacy policy - privacy_policy = mapped_column(String(255), nullable=True) + privacy_policy: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None) # custom_disclaimer custom_disclaimer: Mapped[str] = mapped_column(sa.TEXT, default="") - created_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) - updated_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) + created_at: Mapped[datetime] = mapped_column( + sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False + ) + updated_at: Mapped[datetime] = mapped_column( + sa.DateTime, + nullable=False, + server_default=func.current_timestamp(), + onupdate=func.current_timestamp(), + init=False, + ) @property def schema_type(self) -> "ApiProviderSchemaType": @@ -189,7 +210,7 @@ class ToolLabelBinding(TypeBase): label_name: Mapped[str] = mapped_column(String(40), nullable=False) -class WorkflowToolProvider(Base): +class WorkflowToolProvider(TypeBase): """ The table stores the workflow providers. """ @@ -201,7 +222,7 @@ class WorkflowToolProvider(Base): sa.UniqueConstraint("tenant_id", "app_id", name="unique_workflow_tool_provider_app_id"), ) - id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # name of the workflow provider name: Mapped[str] = mapped_column(String(255), nullable=False) # label of the workflow provider @@ -219,15 +240,19 @@ class WorkflowToolProvider(Base): # description of the provider description: Mapped[str] = mapped_column(sa.Text, nullable=False) # parameter configuration - parameter_configuration: Mapped[str] = mapped_column(sa.Text, nullable=False, server_default="[]") + parameter_configuration: Mapped[str] = mapped_column(sa.Text, nullable=False, server_default="[]", default="[]") # privacy policy - privacy_policy: Mapped[str] = mapped_column(String(255), nullable=True, server_default="") + privacy_policy: Mapped[str | None] = mapped_column(String(255), nullable=True, server_default="", default=None) created_at: Mapped[datetime] = mapped_column( - sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)") + sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False ) updated_at: Mapped[datetime] = mapped_column( - sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)") + sa.DateTime, + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP(0)"), + onupdate=func.current_timestamp(), + init=False, ) @property @@ -252,7 +277,7 @@ class WorkflowToolProvider(Base): return db.session.query(App).where(App.id == self.app_id).first() -class MCPToolProvider(Base): +class MCPToolProvider(TypeBase): """ The table stores the mcp providers. """ @@ -265,7 +290,7 @@ class MCPToolProvider(Base): sa.UniqueConstraint("tenant_id", "server_identifier", name="unique_mcp_provider_server_identifier"), ) - id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # name of the mcp provider name: Mapped[str] = mapped_column(String(40), nullable=False) # server identifier of the mcp provider @@ -275,27 +300,33 @@ class MCPToolProvider(Base): # hash of server_url for uniqueness check server_url_hash: Mapped[str] = mapped_column(String(64), nullable=False) # icon of the mcp provider - icon: Mapped[str] = mapped_column(String(255), nullable=True) + icon: Mapped[str | None] = mapped_column(String(255), nullable=True) # tenant id tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # who created this tool user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # encrypted credentials - encrypted_credentials: Mapped[str] = mapped_column(sa.Text, nullable=True) + encrypted_credentials: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None) # authed authed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=False) # tools tools: Mapped[str] = mapped_column(sa.Text, nullable=False, default="[]") created_at: Mapped[datetime] = mapped_column( - sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)") + sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False ) updated_at: Mapped[datetime] = mapped_column( - sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)") + sa.DateTime, + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP(0)"), + onupdate=func.current_timestamp(), + init=False, + ) + timeout: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default=sa.text("30"), default=30.0) + sse_read_timeout: Mapped[float] = mapped_column( + sa.Float, nullable=False, server_default=sa.text("300"), default=300.0 ) - timeout: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default=sa.text("30")) - sse_read_timeout: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default=sa.text("300")) # encrypted headers for MCP server requests - encrypted_headers: Mapped[str | None] = mapped_column(sa.Text, nullable=True) + encrypted_headers: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None) def load_user(self) -> Account | None: return db.session.query(Account).where(Account.id == self.user_id).first() @@ -306,9 +337,11 @@ class MCPToolProvider(Base): @property def credentials(self) -> dict[str, Any]: + if not self.encrypted_credentials: + return {} try: return cast(dict[str, Any], json.loads(self.encrypted_credentials)) or {} - except Exception: + except json.JSONDecodeError: return {} @property @@ -321,6 +354,7 @@ class MCPToolProvider(Base): def provider_icon(self) -> Mapping[str, str] | str: from core.file import helpers as file_helpers + assert self.icon try: return json.loads(self.icon) except json.JSONDecodeError: @@ -419,7 +453,7 @@ class MCPToolProvider(Base): return encrypter.decrypt(self.credentials) -class ToolModelInvoke(Base): +class ToolModelInvoke(TypeBase): """ store the invoke logs from tool invoke """ @@ -427,37 +461,47 @@ class ToolModelInvoke(Base): __tablename__ = "tool_model_invokes" __table_args__ = (sa.PrimaryKeyConstraint("id", name="tool_model_invoke_pkey"),) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # who invoke this tool - user_id = mapped_column(StringUUID, nullable=False) + user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # tenant id - tenant_id = mapped_column(StringUUID, nullable=False) + tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # provider provider: Mapped[str] = mapped_column(String(255), nullable=False) # type - tool_type = mapped_column(String(40), nullable=False) + tool_type: Mapped[str] = mapped_column(String(40), nullable=False) # tool name - tool_name = mapped_column(String(128), nullable=False) + tool_name: Mapped[str] = mapped_column(String(128), nullable=False) # invoke parameters - model_parameters = mapped_column(sa.Text, nullable=False) + model_parameters: Mapped[str] = mapped_column(sa.Text, nullable=False) # prompt messages - prompt_messages = mapped_column(sa.Text, nullable=False) + prompt_messages: Mapped[str] = mapped_column(sa.Text, nullable=False) # invoke response - model_response = mapped_column(sa.Text, nullable=False) + model_response: Mapped[str] = mapped_column(sa.Text, nullable=False) prompt_tokens: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0")) answer_tokens: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0")) - answer_unit_price = mapped_column(sa.Numeric(10, 4), nullable=False) - answer_price_unit = mapped_column(sa.Numeric(10, 7), nullable=False, server_default=sa.text("0.001")) - provider_response_latency = mapped_column(sa.Float, nullable=False, server_default=sa.text("0")) - total_price = mapped_column(sa.Numeric(10, 7)) + answer_unit_price: Mapped[Decimal] = mapped_column(sa.Numeric(10, 4), nullable=False) + answer_price_unit: Mapped[Decimal] = mapped_column( + sa.Numeric(10, 7), nullable=False, server_default=sa.text("0.001") + ) + provider_response_latency: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default=sa.text("0")) + total_price: Mapped[Decimal | None] = mapped_column(sa.Numeric(10, 7)) currency: Mapped[str] = mapped_column(String(255), nullable=False) - created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) - updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) + created_at: Mapped[datetime] = mapped_column( + sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False + ) + updated_at: Mapped[datetime] = mapped_column( + sa.DateTime, + nullable=False, + server_default=func.current_timestamp(), + onupdate=func.current_timestamp(), + init=False, + ) @deprecated -class ToolConversationVariables(Base): +class ToolConversationVariables(TypeBase): """ store the conversation variables from tool invoke """ @@ -470,18 +514,26 @@ class ToolConversationVariables(Base): sa.Index("conversation_id_idx", "conversation_id"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # conversation user id - user_id = mapped_column(StringUUID, nullable=False) + user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # tenant id - tenant_id = mapped_column(StringUUID, nullable=False) + tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # conversation id - conversation_id = mapped_column(StringUUID, nullable=False) + conversation_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # variables pool - variables_str = mapped_column(sa.Text, nullable=False) + variables_str: Mapped[str] = mapped_column(sa.Text, nullable=False) - created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) - updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp()) + created_at: Mapped[datetime] = mapped_column( + sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False + ) + updated_at: Mapped[datetime] = mapped_column( + sa.DateTime, + nullable=False, + server_default=func.current_timestamp(), + onupdate=func.current_timestamp(), + init=False, + ) @property def variables(self): @@ -519,7 +571,7 @@ class ToolFile(TypeBase): @deprecated -class DeprecatedPublishedAppTool(Base): +class DeprecatedPublishedAppTool(TypeBase): """ The table stores the apps published as a tool for each person. """ @@ -530,26 +582,34 @@ class DeprecatedPublishedAppTool(Base): sa.UniqueConstraint("app_id", "user_id", name="unique_published_app_tool"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) # id of the app - app_id = mapped_column(StringUUID, ForeignKey("apps.id"), nullable=False) + app_id: Mapped[str] = mapped_column(StringUUID, ForeignKey("apps.id"), nullable=False) user_id: Mapped[str] = mapped_column(StringUUID, nullable=False) # who published this tool - description = mapped_column(sa.Text, nullable=False) + description: Mapped[str] = mapped_column(sa.Text, nullable=False) # llm_description of the tool, for LLM - llm_description = mapped_column(sa.Text, nullable=False) + llm_description: Mapped[str] = mapped_column(sa.Text, nullable=False) # query description, query will be seem as a parameter of the tool, # to describe this parameter to llm, we need this field - query_description = mapped_column(sa.Text, nullable=False) + query_description: Mapped[str] = mapped_column(sa.Text, nullable=False) # query name, the name of the query parameter - query_name = mapped_column(String(40), nullable=False) + query_name: Mapped[str] = mapped_column(String(40), nullable=False) # name of the tool provider - tool_name = mapped_column(String(40), nullable=False) + tool_name: Mapped[str] = mapped_column(String(40), nullable=False) # author - author = mapped_column(String(40), nullable=False) - created_at = mapped_column(sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")) - updated_at = mapped_column(sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")) + author: Mapped[str] = mapped_column(String(40), nullable=False) + created_at: Mapped[datetime] = mapped_column( + sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False + ) + updated_at: Mapped[datetime] = mapped_column( + sa.DateTime, + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP(0)"), + onupdate=func.current_timestamp(), + init=False, + ) @property def description_i18n(self) -> "I18nObject": diff --git a/api/models/web.py b/api/models/web.py index 74f99e187b..7df5bd6e87 100644 --- a/api/models/web.py +++ b/api/models/web.py @@ -4,46 +4,58 @@ import sqlalchemy as sa from sqlalchemy import DateTime, String, func from sqlalchemy.orm import Mapped, mapped_column -from models.base import Base +from models.base import TypeBase from .engine import db from .model import Message from .types import StringUUID -class SavedMessage(Base): +class SavedMessage(TypeBase): __tablename__ = "saved_messages" __table_args__ = ( sa.PrimaryKeyConstraint("id", name="saved_message_pkey"), sa.Index("saved_message_message_idx", "app_id", "message_id", "created_by_role", "created_by"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) - app_id = mapped_column(StringUUID, nullable=False) - message_id = mapped_column(StringUUID, nullable=False) - created_by_role = mapped_column( + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) + app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) + message_id: Mapped[str] = mapped_column(StringUUID, nullable=False) + created_by_role: Mapped[str] = mapped_column( String(255), nullable=False, server_default=sa.text("'end_user'::character varying") ) - created_by = mapped_column(StringUUID, nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp()) + created_by: Mapped[str] = mapped_column(StringUUID, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime, + nullable=False, + server_default=func.current_timestamp(), + init=False, + ) @property def message(self): return db.session.query(Message).where(Message.id == self.message_id).first() -class PinnedConversation(Base): +class PinnedConversation(TypeBase): __tablename__ = "pinned_conversations" __table_args__ = ( sa.PrimaryKeyConstraint("id", name="pinned_conversation_pkey"), sa.Index("pinned_conversation_conversation_idx", "app_id", "conversation_id", "created_by_role", "created_by"), ) - id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()")) - app_id = mapped_column(StringUUID, nullable=False) + id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False) + app_id: Mapped[str] = mapped_column(StringUUID, nullable=False) conversation_id: Mapped[str] = mapped_column(StringUUID) - created_by_role = mapped_column( - String(255), nullable=False, server_default=sa.text("'end_user'::character varying") + created_by_role: Mapped[str] = mapped_column( + String(255), + nullable=False, + server_default=sa.text("'end_user'::character varying"), + ) + created_by: Mapped[str] = mapped_column(StringUUID, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime, + nullable=False, + server_default=func.current_timestamp(), + init=False, ) - created_by = mapped_column(StringUUID, nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp()) diff --git a/api/services/auth/api_key_auth_service.py b/api/services/auth/api_key_auth_service.py index 055cf65816..56aaf407ee 100644 --- a/api/services/auth/api_key_auth_service.py +++ b/api/services/auth/api_key_auth_service.py @@ -26,10 +26,9 @@ class ApiKeyAuthService: api_key = encrypter.encrypt_token(tenant_id, args["credentials"]["config"]["api_key"]) args["credentials"]["config"]["api_key"] = api_key - data_source_api_key_binding = DataSourceApiKeyAuthBinding() - data_source_api_key_binding.tenant_id = tenant_id - data_source_api_key_binding.category = args["category"] - data_source_api_key_binding.provider = args["provider"] + data_source_api_key_binding = DataSourceApiKeyAuthBinding( + tenant_id=tenant_id, category=args["category"], provider=args["provider"] + ) data_source_api_key_binding.credentials = json.dumps(args["credentials"], ensure_ascii=False) db.session.add(data_source_api_key_binding) db.session.commit() @@ -48,6 +47,8 @@ class ApiKeyAuthService: ) if not data_source_api_key_bindings: return None + if not data_source_api_key_bindings.credentials: + return None credentials = json.loads(data_source_api_key_bindings.credentials) return credentials diff --git a/api/services/tools/api_tools_manage_service.py b/api/services/tools/api_tools_manage_service.py index 2c0c63f634..bb024cc846 100644 --- a/api/services/tools/api_tools_manage_service.py +++ b/api/services/tools/api_tools_manage_service.py @@ -148,7 +148,7 @@ class ApiToolManageService: description=extra_info.get("description", ""), schema_type_str=schema_type, tools_str=json.dumps(jsonable_encoder(tool_bundles)), - credentials_str={}, + credentials_str="{}", privacy_policy=privacy_policy, custom_disclaimer=custom_disclaimer, ) diff --git a/api/services/tools/builtin_tools_manage_service.py b/api/services/tools/builtin_tools_manage_service.py index cab4a5c6ab..b5dcec17d0 100644 --- a/api/services/tools/builtin_tools_manage_service.py +++ b/api/services/tools/builtin_tools_manage_service.py @@ -683,7 +683,7 @@ class BuiltinToolManageService: cache=NoOpProviderCredentialCache(), ) original_params = encrypter.decrypt(custom_client_params.oauth_params) - new_params: dict = { + new_params = { key: value if value != HIDDEN_VALUE else original_params.get(key, UNKNOWN_VALUE) for key, value in client_params.items() } diff --git a/api/services/tools/mcp_tools_manage_service.py b/api/services/tools/mcp_tools_manage_service.py index 605ad8379b..54133d3801 100644 --- a/api/services/tools/mcp_tools_manage_service.py +++ b/api/services/tools/mcp_tools_manage_service.py @@ -188,6 +188,8 @@ class MCPToolManageService: raise user = mcp_provider.load_user() + if not mcp_provider.icon: + raise ValueError("MCP provider icon is required") return ToolProviderApiEntity( id=mcp_provider.id, name=mcp_provider.name, diff --git a/api/services/tools/tools_transform_service.py b/api/services/tools/tools_transform_service.py index 81b4d6993a..b7850ea150 100644 --- a/api/services/tools/tools_transform_service.py +++ b/api/services/tools/tools_transform_service.py @@ -152,7 +152,8 @@ class ToolTransformService: if decrypt_credentials: credentials = db_provider.credentials - + if not db_provider.tenant_id: + raise ValueError(f"Required tenant_id is missing for BuiltinToolProvider with id {db_provider.id}") # init tool configuration encrypter, _ = create_provider_encrypter( tenant_id=db_provider.tenant_id,