diff --git a/api/controllers/console/workspace/plugin.py b/api/controllers/console/workspace/plugin.py index 2edc0d6470..ebe706d65d 100644 --- a/api/controllers/console/workspace/plugin.py +++ b/api/controllers/console/workspace/plugin.py @@ -1,7 +1,6 @@ import io -import json -from flask import Response, request, send_file +from flask import request, send_file from flask_login import current_user from flask_restful import Resource, reqparse from werkzeug.exceptions import Forbidden @@ -11,7 +10,6 @@ from controllers.console import api from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required from core.model_runtime.utils.encoders import jsonable_encoder -from core.plugin.entities.plugin_daemon import InstallPluginMessage from libs.login import login_required from services.plugin.plugin_service import PluginService @@ -59,37 +57,63 @@ class PluginIconApi(Resource): return send_file(io.BytesIO(icon_bytes), mimetype=mimetype, max_age=icon_cache_max_age) -class PluginInstallCheckUniqueIdentifierApi(Resource): - @setup_required - @login_required - @account_initialization_required - def get(self): - req = reqparse.RequestParser() - req.add_argument("plugin_unique_identifier", type=str, required=True, location="args") - args = req.parse_args() - - user = current_user - tenant_id = user.current_tenant_id - - return {"installed": PluginService.check_plugin_unique_identifier(tenant_id, args["plugin_unique_identifier"])} - - -class PluginInstallFromUniqueIdentifierApi(Resource): +class PluginUploadPkgApi(Resource): @setup_required @login_required @account_initialization_required def post(self): - req = reqparse.RequestParser() - req.add_argument("plugin_unique_identifier", type=str, required=True, location="json") - args = req.parse_args() + user = current_user + if not user.is_admin_or_owner: + raise Forbidden() + tenant_id = user.current_tenant_id + file = request.files["pkg"] + content = file.read() + return {"plugin_unique_identifier": PluginService.upload_pkg(tenant_id, content)} + + +class PluginUploadFromPkgApi(Resource): + @setup_required + @login_required + @account_initialization_required + def post(self): user = current_user if not user.is_admin_or_owner: raise Forbidden() tenant_id = user.current_tenant_id - return {"success": PluginService.install_from_unique_identifier(tenant_id, args["plugin_unique_identifier"])} + file = request.files["pkg"] + content = file.read() + response = PluginService.upload_pkg(tenant_id, content) + + return { + "plugin_unique_identifier": response, + } + + +class PluginUploadFromGithubApi(Resource): + @setup_required + @login_required + @account_initialization_required + def post(self): + user = current_user + if not user.is_admin_or_owner: + raise Forbidden() + + tenant_id = user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument("repo", type=str, required=True, location="json") + parser.add_argument("version", type=str, required=True, location="json") + parser.add_argument("package", type=str, required=True, location="json") + args = parser.parse_args() + + response = PluginService.upload_pkg_from_github(tenant_id, args["repo"], args["version"], args["package"]) + + return { + "plugin_unique_identifier": response, + } class PluginInstallFromPkgApi(Resource): @@ -103,19 +127,15 @@ class PluginInstallFromPkgApi(Resource): tenant_id = user.current_tenant_id - file = request.files["pkg"] - content = file.read() + parser = reqparse.RequestParser() + parser.add_argument("plugin_unique_identifier", type=str, required=True, location="json") + args = parser.parse_args() - def generator(): - try: - response = PluginService.install_from_local_pkg(tenant_id, content) - for message in response: - yield f"data: {json.dumps(jsonable_encoder(message))}\n\n" - except ValueError as e: - error_message = InstallPluginMessage(event=InstallPluginMessage.Event.Error, data=str(e)) - yield f"data: {json.dumps(jsonable_encoder(error_message))}\n\n" + response = PluginService.install_from_local_pkg(tenant_id, args["plugin_unique_identifier"]) - return Response(generator(), mimetype="text/event-stream") + return { + "task_id": response, + } class PluginInstallFromGithubApi(Resource): @@ -133,20 +153,16 @@ class PluginInstallFromGithubApi(Resource): parser.add_argument("repo", type=str, required=True, location="json") parser.add_argument("version", type=str, required=True, location="json") parser.add_argument("package", type=str, required=True, location="json") + parser.add_argument("plugin_unique_identifier", type=str, required=True, location="json") args = parser.parse_args() - def generator(): - try: - response = PluginService.install_from_github_pkg( - tenant_id, args["repo"], args["version"], args["package"] - ) - for message in response: - yield f"data: {json.dumps(jsonable_encoder(message))}\n\n" - except ValueError as e: - error_message = InstallPluginMessage(event=InstallPluginMessage.Event.Error, data=str(e)) - yield f"data: {json.dumps(jsonable_encoder(error_message))}\n\n" + response = PluginService.install_from_github( + tenant_id, args["repo"], args["version"], args["package"], args["plugin_unique_identifier"] + ) - return Response(generator(), mimetype="text/event-stream") + return { + "task_id": response, + } class PluginInstallFromMarketplaceApi(Resource): @@ -164,16 +180,55 @@ class PluginInstallFromMarketplaceApi(Resource): parser.add_argument("plugin_unique_identifier", type=str, required=True, location="json") args = parser.parse_args() - def generator(): - try: - response = PluginService.install_from_marketplace_pkg(tenant_id, args["plugin_unique_identifier"]) - for message in response: - yield f"data: {json.dumps(jsonable_encoder(message))}\n\n" - except ValueError as e: - error_message = InstallPluginMessage(event=InstallPluginMessage.Event.Error, data=str(e)) - yield f"data: {json.dumps(jsonable_encoder(error_message))}\n\n" + response = PluginService.install_from_marketplace_pkg(tenant_id, args["plugin_unique_identifier"]) - return Response(generator(), mimetype="text/event-stream") + return { + "task_id": response, + } + + +class PluginFetchManifestApi(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self): + user = current_user + + parser = reqparse.RequestParser() + parser.add_argument("plugin_unique_identifier", type=str, required=True, location="args") + args = parser.parse_args() + + tenant_id = user.current_tenant_id + + return {"manifest": PluginService.fetch_plugin_manifest(tenant_id, args["plugin_unique_identifier"])} + + +class PluginFetchInstallTasksApi(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self): + user = current_user + if not user.is_admin_or_owner: + raise Forbidden() + + tenant_id = user.current_tenant_id + + return {"tasks": PluginService.fetch_install_tasks(tenant_id)} + + +class PluginFetchInstallTaskApi(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self, task_id: str): + user = current_user + if not user.is_admin_or_owner: + raise Forbidden() + + tenant_id = user.current_tenant_id + + return {"task": PluginService.fetch_install_task(tenant_id, task_id)} class PluginUninstallApi(Resource): @@ -197,9 +252,12 @@ class PluginUninstallApi(Resource): api.add_resource(PluginDebuggingKeyApi, "/workspaces/current/plugin/debugging-key") api.add_resource(PluginListApi, "/workspaces/current/plugin/list") api.add_resource(PluginIconApi, "/workspaces/current/plugin/icon") -api.add_resource(PluginInstallCheckUniqueIdentifierApi, "/workspaces/current/plugin/install/check_unique_identifier") -api.add_resource(PluginInstallFromUniqueIdentifierApi, "/workspaces/current/plugin/install/from_unique_identifier") -api.add_resource(PluginInstallFromPkgApi, "/workspaces/current/plugin/install/from_pkg") -api.add_resource(PluginInstallFromGithubApi, "/workspaces/current/plugin/install/from_github") -api.add_resource(PluginInstallFromMarketplaceApi, "/workspaces/current/plugin/install/from_marketplace") +api.add_resource(PluginUploadFromPkgApi, "/workspaces/current/plugin/upload/pkg") +api.add_resource(PluginUploadFromGithubApi, "/workspaces/current/plugin/upload/github") +api.add_resource(PluginInstallFromPkgApi, "/workspaces/current/plugin/install/pkg") +api.add_resource(PluginInstallFromGithubApi, "/workspaces/current/plugin/install/github") +api.add_resource(PluginInstallFromMarketplaceApi, "/workspaces/current/plugin/install/marketplace") +api.add_resource(PluginFetchManifestApi, "/workspaces/current/plugin/fetch-manifest") +api.add_resource(PluginFetchInstallTasksApi, "/workspaces/current/plugin/tasks") +api.add_resource(PluginFetchInstallTaskApi, "/workspaces/current/plugin/tasks/") api.add_resource(PluginUninstallApi, "/workspaces/current/plugin/uninstall") diff --git a/api/core/plugin/entities/plugin_daemon.py b/api/core/plugin/entities/plugin_daemon.py index 9596972def..d32dcf1adc 100644 --- a/api/core/plugin/entities/plugin_daemon.py +++ b/api/core/plugin/entities/plugin_daemon.py @@ -6,9 +6,10 @@ from pydantic import BaseModel, ConfigDict, Field from core.model_runtime.entities.model_entities import AIModelEntity from core.model_runtime.entities.provider_entities import ProviderEntity +from core.plugin.entities.base import BasePluginEntity from core.tools.entities.tool_entities import ToolProviderEntityWithPlugin -T = TypeVar("T", bound=(BaseModel | dict | list | bool)) +T = TypeVar("T", bound=(BaseModel | dict | list | bool | str)) class PluginDaemonBasicResponse(BaseModel, Generic[T]): @@ -106,3 +107,24 @@ class PluginDaemonInnerError(Exception): def __init__(self, code: int, message: str): self.code = code self.message = message + + +class PluginInstallTaskStatus(str, Enum): + Pending = "pending" + Running = "running" + Success = "success" + Failed = "failed" + + +class PluginInstallTaskPluginStatus(BaseModel): + plugin_unique_identifier: str = Field(description="The plugin unique identifier of the install task.") + plugin_id: str = Field(description="The plugin ID of the install task.") + status: PluginInstallTaskStatus = Field(description="The status of the install task.") + message: str = Field(description="The message of the install task.") + + +class PluginInstallTask(BasePluginEntity): + status: PluginInstallTaskStatus = Field(description="The status of the install task.") + total_plugins: int = Field(description="The total number of plugins to be installed.") + completed_plugins: int = Field(description="The number of plugins that have been installed.") + plugins: list[PluginInstallTaskPluginStatus] = Field(description="The status of the plugins.") diff --git a/api/core/plugin/manager/base.py b/api/core/plugin/manager/base.py index 3c46e8a3c3..9980f7c15d 100644 --- a/api/core/plugin/manager/base.py +++ b/api/core/plugin/manager/base.py @@ -19,7 +19,7 @@ from core.plugin.entities.plugin_daemon import PluginDaemonBasicResponse, Plugin plugin_daemon_inner_api_baseurl = dify_config.PLUGIN_API_URL plugin_daemon_inner_api_key = dify_config.PLUGIN_API_KEY -T = TypeVar("T", bound=(BaseModel | dict | list | bool)) +T = TypeVar("T", bound=(BaseModel | dict | list | bool | str)) class BasePluginManager: diff --git a/api/core/plugin/manager/plugin.py b/api/core/plugin/manager/plugin.py index 0e89c257e8..0912d473a5 100644 --- a/api/core/plugin/manager/plugin.py +++ b/api/core/plugin/manager/plugin.py @@ -1,16 +1,16 @@ -import json -from collections.abc import Generator, Mapping -from typing import Any +from collections.abc import Sequence -from core.plugin.entities.plugin import PluginEntity, PluginInstallationSource -from core.plugin.entities.plugin_daemon import InstallPluginMessage +from core.plugin.entities.plugin import PluginDeclaration, PluginEntity, PluginInstallationSource +from core.plugin.entities.plugin_daemon import PluginInstallTask from core.plugin.manager.base import BasePluginManager class PluginInstallationManager(BasePluginManager): - def fetch_plugin_by_identifier(self, tenant_id: str, identifier: str) -> bool: - # urlencode the identifier - + def fetch_plugin_by_identifier( + self, + tenant_id: str, + identifier: str, + ) -> bool: return self._request_with_plugin_daemon_response( "GET", f"plugin/{tenant_id}/management/fetch/identifier", @@ -26,37 +26,34 @@ class PluginInstallationManager(BasePluginManager): params={"page": 1, "page_size": 256}, ) - def install_from_pkg( + def upload_pkg( self, tenant_id: str, pkg: bytes, - source: PluginInstallationSource, - meta: Mapping[str, Any], verify_signature: bool = False, - ) -> Generator[InstallPluginMessage, None, None]: + ) -> str: """ - Install a plugin from a package. + Upload a plugin package and return the plugin unique identifier. """ - # using multipart/form-data to encode body body = { "dify_pkg": ("dify_pkg", pkg, "application/octet-stream"), } data = { "verify_signature": "true" if verify_signature else "false", - "source": source.value, - "meta": json.dumps(meta), } - return self._request_with_plugin_daemon_response_stream( + return self._request_with_plugin_daemon_response( "POST", - f"plugin/{tenant_id}/management/install/pkg", - InstallPluginMessage, + f"plugin/{tenant_id}/management/install/upload", + str, files=body, data=data, ) - def install_from_identifier(self, tenant_id: str, identifier: str) -> bool: + def install_from_identifiers( + self, tenant_id: str, identifiers: Sequence[str], source: PluginInstallationSource, meta: dict + ) -> str: """ Install a plugin from an identifier. """ @@ -64,13 +61,45 @@ class PluginInstallationManager(BasePluginManager): return self._request_with_plugin_daemon_response( "POST", f"plugin/{tenant_id}/management/install/identifier", - bool, + str, data={ - "plugin_unique_identifier": identifier, + "plugin_unique_identifiers": identifiers, + "source": source, + "meta": meta, }, headers={"Content-Type": "application/json"}, ) + def fetch_plugin_installation_tasks(self, tenant_id: str) -> Sequence[PluginInstallTask]: + """ + Fetch plugin installation tasks. + """ + return self._request_with_plugin_daemon_response( + "GET", + f"plugin/{tenant_id}/management/install/tasks", + list[PluginInstallTask], + ) + + def fetch_plugin_installation_task(self, tenant_id: str, task_id: str) -> PluginInstallTask: + """ + Fetch a plugin installation task. + """ + return self._request_with_plugin_daemon_response( + "GET", + f"plugin/{tenant_id}/management/install/tasks/{task_id}", + PluginInstallTask, + ) + + def fetch_plugin_manifest(self, tenant_id: str, plugin_unique_identifier: str) -> PluginDeclaration: + """ + Fetch a plugin manifest. + """ + return self._request_with_plugin_daemon_response( + "GET", + f"plugin/{tenant_id}/management/fetch/identifier", + PluginDeclaration, + ) + def uninstall(self, tenant_id: str, plugin_installation_id: str) -> bool: """ Uninstall a plugin. diff --git a/api/services/plugin/plugin_service.py b/api/services/plugin/plugin_service.py index 94d8858dc6..3b7d19aa0b 100644 --- a/api/services/plugin/plugin_service.py +++ b/api/services/plugin/plugin_service.py @@ -1,10 +1,10 @@ -from collections.abc import Generator +from collections.abc import Sequence from mimetypes import guess_type from core.helper.download import download_with_size_limit from core.helper.marketplace import download_plugin_pkg -from core.plugin.entities.plugin import PluginEntity, PluginInstallationSource -from core.plugin.entities.plugin_daemon import InstallPluginMessage, PluginDaemonInnerError +from core.plugin.entities.plugin import PluginDeclaration, PluginEntity, PluginInstallationSource +from core.plugin.entities.plugin_daemon import PluginInstallTask from core.plugin.manager.asset import PluginAssetManager from core.plugin.manager.debugging import PluginDebuggingManager from core.plugin.manager.plugin import PluginInstallationManager @@ -13,16 +13,25 @@ from core.plugin.manager.plugin import PluginInstallationManager class PluginService: @staticmethod def get_debugging_key(tenant_id: str) -> str: + """ + get the debugging key of the tenant + """ manager = PluginDebuggingManager() return manager.get_debugging_key(tenant_id) @staticmethod def list(tenant_id: str) -> list[PluginEntity]: + """ + list all plugins of the tenant + """ manager = PluginInstallationManager() return manager.list_plugins(tenant_id) @staticmethod def get_asset(tenant_id: str, asset_file: str) -> tuple[bytes, str]: + """ + get the asset file of the plugin + """ manager = PluginAssetManager() # guess mime type mime_type, _ = guess_type(asset_file) @@ -30,73 +39,104 @@ class PluginService: @staticmethod def check_plugin_unique_identifier(tenant_id: str, plugin_unique_identifier: str) -> bool: + """ + check if the plugin unique identifier is already installed by other tenant + """ manager = PluginInstallationManager() return manager.fetch_plugin_by_identifier(tenant_id, plugin_unique_identifier) @staticmethod - def install_from_unique_identifier(tenant_id: str, plugin_unique_identifier: str) -> bool: + def fetch_plugin_manifest(tenant_id: str, plugin_unique_identifier: str) -> PluginDeclaration: manager = PluginInstallationManager() - return manager.install_from_identifier(tenant_id, plugin_unique_identifier) + return manager.fetch_plugin_manifest(tenant_id, plugin_unique_identifier) @staticmethod - def install_from_local_pkg(tenant_id: str, pkg: bytes) -> Generator[InstallPluginMessage, None, None]: - """ - Install plugin from uploaded package files - """ + def fetch_install_tasks(tenant_id: str) -> Sequence[PluginInstallTask]: manager = PluginInstallationManager() - try: - yield from manager.install_from_pkg(tenant_id, pkg, PluginInstallationSource.Package, {}) - except PluginDaemonInnerError as e: - yield InstallPluginMessage(event=InstallPluginMessage.Event.Error, data=str(e.message)) + return manager.fetch_plugin_installation_tasks(tenant_id) @staticmethod - def install_from_github_pkg( - tenant_id: str, repo: str, version: str, package: str - ) -> Generator[InstallPluginMessage, None, None]: + def fetch_install_task(tenant_id: str, task_id: str) -> PluginInstallTask: + manager = PluginInstallationManager() + return manager.fetch_plugin_installation_task(tenant_id, task_id) + + @staticmethod + def upload_pkg(tenant_id: str, pkg: bytes) -> str: """ - Install plugin from github release package files + Upload plugin package files + + returns: plugin_unique_identifier + """ + manager = PluginInstallationManager() + return manager.upload_pkg(tenant_id, pkg) + + @staticmethod + def upload_pkg_from_github(tenant_id: str, repo: str, version: str, package: str) -> str: + """ + Install plugin from github release package files, + returns plugin_unique_identifier """ pkg = download_with_size_limit( f"https://github.com/{repo}/releases/download/{version}/{package}", 15 * 1024 * 1024 ) manager = PluginInstallationManager() - try: - yield from manager.install_from_pkg( - tenant_id, - pkg, - PluginInstallationSource.Github, - { - "repo": repo, - "version": version, - "package": package, - }, - ) - except PluginDaemonInnerError as e: - yield InstallPluginMessage(event=InstallPluginMessage.Event.Error, data=str(e.message)) + return manager.upload_pkg( + tenant_id, + pkg, + ) @staticmethod - def install_from_marketplace_pkg( - tenant_id: str, plugin_unique_identifier: str - ) -> Generator[InstallPluginMessage, None, None]: + def install_from_local_pkg(tenant_id: str, plugin_unique_identifier: str) -> str: + manager = PluginInstallationManager() + return manager.install_from_identifiers( + tenant_id, + [plugin_unique_identifier], + PluginInstallationSource.Package, + {}, + ) + + @staticmethod + def install_from_github( + tenant_id: str, plugin_unique_identifier: str, repo: str, version: str, package: str + ) -> str: """ - TODO: wait for marketplace api + Install plugin from github release package files, + returns plugin_unique_identifier + """ + manager = PluginInstallationManager() + return manager.install_from_identifiers( + tenant_id, + [plugin_unique_identifier], + PluginInstallationSource.Github, + { + "repo": repo, + "version": version, + "package": package, + }, + ) + + @staticmethod + def install_from_marketplace_pkg(tenant_id: str, plugin_unique_identifier: str) -> str: + """ + Install plugin from marketplace package files, + returns installation task id """ manager = PluginInstallationManager() pkg = download_plugin_pkg(plugin_unique_identifier) - try: - yield from manager.install_from_pkg( - tenant_id, - pkg, - PluginInstallationSource.Marketplace, - { - "plugin_unique_identifier": plugin_unique_identifier, - }, - ) - except PluginDaemonInnerError as e: - yield InstallPluginMessage(event=InstallPluginMessage.Event.Error, data=str(e.message)) + # upload pkg to plugin daemon + pkg_id = manager.upload_pkg(tenant_id, pkg) + + return manager.install_from_identifiers( + tenant_id, + [pkg_id], + PluginInstallationSource.Marketplace, + { + "plugin_unique_identifier": plugin_unique_identifier, + }, + ) @staticmethod def uninstall(tenant_id: str, plugin_installation_id: str) -> bool: