diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index bd6fc9e50c..96c4fc0c8e 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -677,26 +677,19 @@ class AppExportBundleApi(Resource): @account_initialization_required @edit_permission_required def get(self, app_model): - from io import BytesIO - - from flask import send_file - from services.app_bundle_service import AppBundleService args = AppExportBundleQuery.model_validate(request.args.to_dict(flat=True)) + current_user, _ = current_account_with_tenant() result = AppBundleService.export_bundle( app_model=app_model, + account_id=str(current_user.id), include_secret=args.include_secret, workflow_id=args.workflow_id, ) - return send_file( - BytesIO(result.zip_bytes), - mimetype="application/zip", - as_attachment=True, - download_name=result.filename, - ) + return result.model_dump(mode="json") @console_ns.route("/apps//name") diff --git a/api/controllers/console/app/workflow_draft_variable.py b/api/controllers/console/app/workflow_draft_variable.py index 1e40731439..50ce75ca8a 100644 --- a/api/controllers/console/app/workflow_draft_variable.py +++ b/api/controllers/console/app/workflow_draft_variable.py @@ -16,7 +16,6 @@ from controllers.console.app.wraps import get_app_model from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required from controllers.web.error import InvalidArgumentError, NotFoundError from core.file import helpers as file_helpers -from core.sandbox.manager import SandboxManager from core.variables.segment_group import SegmentGroup from core.variables.segments import ArrayFileSegment, ArrayPromptMessageSegment, FileSegment, Segment from core.variables.types import SegmentType @@ -27,6 +26,7 @@ from factories.file_factory import build_from_mapping, build_from_mappings from libs.login import current_account_with_tenant, login_required from models import App, AppMode from models.workflow import WorkflowDraftVariable +from services.sandbox.sandbox_service import SandboxService from services.workflow_draft_variable_service import WorkflowDraftVariableList, WorkflowDraftVariableService from services.workflow_service import WorkflowService @@ -268,9 +268,8 @@ class WorkflowVariableCollectionApi(Resource): @console_ns.response(204, "Workflow variables deleted successfully") @_api_prerequisite def delete(self, app_model: App): - # FIXME(Mairuis): move to SandboxArtifactService current_user, _ = current_account_with_tenant() - SandboxManager.delete_draft_storage(app_model.tenant_id, current_user.id) + SandboxService.delete_draft_storage(app_model.tenant_id, current_user.id) draft_var_srv = WorkflowDraftVariableService( session=db.session(), ) diff --git a/api/controllers/files/app_assets_download.py b/api/controllers/files/app_assets_download.py index 205b7353a9..1d829a5671 100644 --- a/api/controllers/files/app_assets_download.py +++ b/api/controllers/files/app_assets_download.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, Field from werkzeug.exceptions import Forbidden, NotFound from controllers.files import files_ns -from core.app_assets.storage import AppAssetSigner, AssetPath, app_asset_storage +from core.app_assets.storage import AppAssetSigner, AssetPath from extensions.ext_storage import storage DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" @@ -58,7 +58,7 @@ class AppAssetDownloadApi(Resource): ): raise Forbidden("Invalid or expired download link") - storage_key = app_asset_storage.get_storage_key(asset_path) + storage_key = asset_path.get_storage_key() try: generator = storage.load_stream(storage_key) diff --git a/api/controllers/files/app_assets_upload.py b/api/controllers/files/app_assets_upload.py index 92657d815a..c25bc5e100 100644 --- a/api/controllers/files/app_assets_upload.py +++ b/api/controllers/files/app_assets_upload.py @@ -4,7 +4,8 @@ from pydantic import BaseModel, Field from werkzeug.exceptions import Forbidden from controllers.files import files_ns -from core.app_assets.storage import AppAssetSigner, AssetPath, app_asset_storage +from core.app_assets.storage import AppAssetSigner, AssetPath +from services.app_asset_service import AppAssetService DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" @@ -56,5 +57,5 @@ class AppAssetUploadApi(Resource): raise Forbidden("Invalid or expired upload link") content = request.get_data() - app_asset_storage.save(asset_path, content) + AppAssetService.get_storage().save(asset_path, content) return Response(status=204) diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index 34fbeb104c..841c783861 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -35,7 +35,7 @@ from core.model_runtime.errors.invoke import InvokeAuthorizationError from core.ops.ops_trace_manager import TraceQueueManager from core.prompt.utils.get_thread_messages_length import get_thread_messages_length from core.repositories import DifyCoreRepositoryFactory -from core.sandbox import Sandbox, SandboxManager +from core.sandbox import Sandbox from core.workflow.repositories.draft_variable_repository import ( DraftVariableSaverFactory, ) @@ -50,6 +50,7 @@ from models.enums import WorkflowRunTriggeredFrom from models.workflow_features import WorkflowFeatures from services.conversation_service import ConversationService from services.sandbox.sandbox_provider_service import SandboxProviderService +from services.sandbox.sandbox_service import SandboxService from services.workflow_draft_variable_service import ( DraftVarLoader, WorkflowDraftVariableService, @@ -528,7 +529,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator): application_generate_entity.app_config.tenant_id ) if workflow.version == Workflow.VERSION_DRAFT: - sandbox = SandboxManager.create_draft( + sandbox = SandboxService.create_draft( tenant_id=application_generate_entity.app_config.tenant_id, app_id=application_generate_entity.app_config.app_id, user_id=application_generate_entity.user_id, @@ -537,7 +538,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator): else: if application_generate_entity.workflow_run_id is None: raise ValueError("workflow_run_id is required when sandbox is enabled") - sandbox = SandboxManager.create( + sandbox = SandboxService.create( tenant_id=application_generate_entity.app_config.tenant_id, app_id=application_generate_entity.app_config.app_id, user_id=application_generate_entity.user_id, diff --git a/api/core/app/apps/workflow/app_generator.py b/api/core/app/apps/workflow/app_generator.py index 2a7225fd4c..b1f197953e 100644 --- a/api/core/app/apps/workflow/app_generator.py +++ b/api/core/app/apps/workflow/app_generator.py @@ -31,7 +31,7 @@ from core.helper.trace_id_helper import extract_external_trace_id_from_args from core.model_runtime.errors.invoke import InvokeAuthorizationError from core.ops.ops_trace_manager import TraceQueueManager from core.repositories import DifyCoreRepositoryFactory -from core.sandbox import Sandbox, SandboxManager +from core.sandbox import Sandbox from core.workflow.graph_engine.layers.base import GraphEngineLayer from core.workflow.repositories.draft_variable_repository import DraftVariableSaverFactory from core.workflow.repositories.workflow_execution_repository import WorkflowExecutionRepository @@ -44,6 +44,7 @@ from models import Account, App, EndUser, Workflow, WorkflowNodeExecutionTrigger from models.enums import WorkflowRunTriggeredFrom from models.workflow_features import WorkflowFeatures from services.sandbox.sandbox_provider_service import SandboxProviderService +from services.sandbox.sandbox_service import SandboxService from services.workflow_draft_variable_service import DraftVarLoader, WorkflowDraftVariableService if TYPE_CHECKING: @@ -503,14 +504,14 @@ class WorkflowAppGenerator(BaseAppGenerator): application_generate_entity.app_config.tenant_id ) if workflow.version == Workflow.VERSION_DRAFT: - sandbox = SandboxManager.create_draft( + sandbox = SandboxService.create_draft( tenant_id=application_generate_entity.app_config.tenant_id, app_id=application_generate_entity.app_config.app_id, user_id=application_generate_entity.user_id, sandbox_provider=sandbox_provider, ) else: - sandbox = SandboxManager.create( + sandbox = SandboxService.create( tenant_id=application_generate_entity.app_config.tenant_id, app_id=application_generate_entity.app_config.app_id, user_id=application_generate_entity.user_id, diff --git a/api/core/app/entities/app_bundle_entities.py b/api/core/app/entities/app_bundle_entities.py index b81fec62ae..4ed7807346 100644 --- a/api/core/app/entities/app_bundle_entities.py +++ b/api/core/app/entities/app_bundle_entities.py @@ -24,7 +24,7 @@ class ZipSecurityError(Exception): # Entities class BundleExportResult(BaseModel): - zip_bytes: bytes = Field(description="ZIP file content as bytes") + download_url: str = Field(description="Temporary download URL for the ZIP") filename: str = Field(description="Suggested filename for the ZIP") diff --git a/api/core/app_assets/__init__.py b/api/core/app_assets/__init__.py index c81f5b2e55..7c11fedbcf 100644 --- a/api/core/app_assets/__init__.py +++ b/api/core/app_assets/__init__.py @@ -4,18 +4,10 @@ from .entities import ( FileAsset, SkillAsset, ) -from .packager import AssetPackager, AssetZipPackager -from .parser import AssetItemParser, AssetParser, FileAssetParser, SkillAssetParser __all__ = [ "AppAssetsAttrs", "AssetItem", - "AssetItemParser", - "AssetPackager", - "AssetParser", - "AssetZipPackager", "FileAsset", - "FileAssetParser", "SkillAsset", - "SkillAssetParser", ] diff --git a/api/core/app_assets/builder/file_builder.py b/api/core/app_assets/builder/file_builder.py index 617c68bfbb..8fcd29807a 100644 --- a/api/core/app_assets/builder/file_builder.py +++ b/api/core/app_assets/builder/file_builder.py @@ -1,17 +1,15 @@ from core.app.entities.app_asset_entities import AppAssetFileTree, AppAssetNode from core.app_assets.entities import AssetItem, FileAsset -from core.app_assets.storage import AppAssetStorage, AssetPath +from core.app_assets.storage import AssetPath from .base import BuildContext class FileBuilder: _nodes: list[tuple[AppAssetNode, str]] - _storage: AppAssetStorage - def __init__(self, storage: AppAssetStorage) -> None: + def __init__(self) -> None: self._nodes = [] - self._storage = storage def accept(self, node: AppAssetNode) -> bool: return True @@ -26,7 +24,7 @@ class FileBuilder: path=path, file_name=node.name, extension=node.extension or "", - storage_key=self._storage.get_storage_key(AssetPath.draft(ctx.tenant_id, ctx.app_id, node.id)), + storage_key=AssetPath.draft(ctx.tenant_id, ctx.app_id, node.id).get_storage_key(), ) for node, path in self._nodes ] diff --git a/api/core/app_assets/builder/skill_builder.py b/api/core/app_assets/builder/skill_builder.py index 9f75890b9c..702857e9e2 100644 --- a/api/core/app_assets/builder/skill_builder.py +++ b/api/core/app_assets/builder/skill_builder.py @@ -9,7 +9,6 @@ from core.app_assets.storage import AppAssetStorage, AssetPath, AssetPathBase from core.skill.entities.skill_bundle import SkillBundle from core.skill.entities.skill_document import SkillDocument from core.skill.skill_compiler import SkillCompiler -from core.skill.skill_manager import SkillManager from .base import BuildContext @@ -48,6 +47,8 @@ class SkillBuilder: self._nodes.append((node, path)) def build(self, tree: AppAssetFileTree, ctx: BuildContext) -> list[AssetItem]: + from core.skill.skill_manager import SkillManager + if not self._nodes: bundle = SkillBundle(assets_id=ctx.build_id) SkillManager.save_bundle(ctx.tenant_id, ctx.app_id, ctx.build_id, bundle) @@ -74,7 +75,7 @@ class SkillBuilder: node=skill.node, path=skill.path, ref=resolved_ref, - storage_key=self._storage.get_storage_key(resolved_ref), + storage_key=resolved_ref.get_storage_key(), content_bytes=artifact.content.encode("utf-8"), ) ) diff --git a/api/core/app_assets/converters.py b/api/core/app_assets/converters.py index d7740f2a97..c9bae8c564 100644 --- a/api/core/app_assets/converters.py +++ b/api/core/app_assets/converters.py @@ -3,14 +3,13 @@ from __future__ import annotations from core.app.entities.app_asset_entities import AppAssetFileTree, AssetNodeType from core.app_assets.entities import FileAsset from core.app_assets.entities.assets import AssetItem -from core.app_assets.storage import AppAssetStorage, AssetPath +from core.app_assets.storage import AssetPath def tree_to_asset_items( tree: AppAssetFileTree, tenant_id: str, app_id: str, - storage: AppAssetStorage, ) -> list[AssetItem]: """ Convert AppAssetFileTree to list of FileAsset for packaging. @@ -19,7 +18,6 @@ def tree_to_asset_items( tree: The asset file tree to convert tenant_id: Tenant ID for storage key generation app_id: App ID for storage key generation - storage: App asset storage for key mapping Returns: List of FileAsset items ready for packaging @@ -29,14 +27,13 @@ def tree_to_asset_items( if node.node_type == AssetNodeType.FILE: path = tree.get_path(node.id) asset_path = AssetPath.draft(tenant_id, app_id, node.id) - storage_key = storage.get_storage_key(asset_path) items.append( FileAsset( asset_id=node.id, path=path, file_name=node.name, extension=node.extension or "", - storage_key=storage_key, + storage_key=asset_path.get_storage_key(), ) ) return items diff --git a/api/core/app_assets/packager/__init__.py b/api/core/app_assets/packager/__init__.py deleted file mode 100644 index 2a52edcaa5..0000000000 --- a/api/core/app_assets/packager/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .asset_zip_packager import AssetZipPackager -from .base import AssetPackager - -__all__ = [ - "AssetPackager", - "AssetZipPackager", -] diff --git a/api/core/app_assets/packager/asset_zip_packager.py b/api/core/app_assets/packager/asset_zip_packager.py deleted file mode 100644 index 5b08398b00..0000000000 --- a/api/core/app_assets/packager/asset_zip_packager.py +++ /dev/null @@ -1,75 +0,0 @@ -from __future__ import annotations - -import io -import zipfile -from concurrent.futures import ThreadPoolExecutor -from threading import Lock - -from core.app_assets.entities import AssetItem -from extensions.storage.base_storage import BaseStorage - - -class AssetZipPackager: - """ - Unified ZIP packager for assets. - Automatically creates directory entries from asset paths. - """ - - def __init__(self, storage: BaseStorage, *, max_workers: int = 8) -> None: - self._storage = storage - self._max_workers = max_workers - - def package(self, assets: list[AssetItem], *, prefix: str = "") -> bytes: - """ - Package assets into a ZIP file. - - Args: - assets: List of assets to package - prefix: Optional prefix to add to all paths in the ZIP - - Returns: - ZIP file content as bytes - """ - zip_buffer = io.BytesIO() - - # Extract folder paths from asset paths - folder_paths = self._extract_folder_paths(assets, prefix) - - with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf: - # Create directory entries - for folder_path in sorted(folder_paths): - zf.writestr(zipfile.ZipInfo(folder_path + "/"), "") - - # Write files in parallel - if assets: - self._write_files_parallel(zf, assets, prefix) - - return zip_buffer.getvalue() - - def _extract_folder_paths(self, assets: list[AssetItem], prefix: str) -> set[str]: - """Extract all folder paths from asset paths.""" - folders: set[str] = set() - for asset in assets: - full_path = f"{prefix}/{asset.path}" if prefix else asset.path - parts = full_path.split("/")[:-1] # Remove filename - folders.update("/".join(parts[:i]) for i in range(1, len(parts) + 1)) - return folders - - def _write_files_parallel( - self, - zf: zipfile.ZipFile, - assets: list[AssetItem], - prefix: str, - ) -> None: - lock = Lock() - - def load_and_write(asset: AssetItem) -> None: - content = self._storage.load_once(asset.get_storage_key()) - full_path = f"{prefix}/{asset.path}" if prefix else asset.path - with lock: - zf.writestr(full_path, content) - - with ThreadPoolExecutor(max_workers=self._max_workers) as executor: - futures = [executor.submit(load_and_write, a) for a in assets] - for future in futures: - future.result() diff --git a/api/core/app_assets/packager/base.py b/api/core/app_assets/packager/base.py deleted file mode 100644 index 3094a45c76..0000000000 --- a/api/core/app_assets/packager/base.py +++ /dev/null @@ -1,9 +0,0 @@ -from abc import ABC, abstractmethod - -from core.app_assets.entities import AssetItem - - -class AssetPackager(ABC): - @abstractmethod - def package(self, assets: list[AssetItem]) -> bytes: - raise NotImplementedError diff --git a/api/core/app_assets/parser/__init__.py b/api/core/app_assets/parser/__init__.py deleted file mode 100644 index c2d7352c20..0000000000 --- a/api/core/app_assets/parser/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from .asset_parser import AssetParser -from .base import AssetItemParser, FileAssetParser -from .skill_parser import SkillAssetParser - -__all__ = [ - "AssetItemParser", - "AssetParser", - "FileAssetParser", - "SkillAssetParser", -] diff --git a/api/core/app_assets/parser/asset_parser.py b/api/core/app_assets/parser/asset_parser.py deleted file mode 100644 index a42fb2f879..0000000000 --- a/api/core/app_assets/parser/asset_parser.py +++ /dev/null @@ -1,36 +0,0 @@ -from core.app.entities.app_asset_entities import AppAssetFileTree -from core.app_assets.entities import AssetItem -from core.app_assets.storage import AssetPath, app_asset_storage - -from .base import AssetItemParser, FileAssetParser - - -class AssetParser: - def __init__( - self, - tree: AppAssetFileTree, - tenant_id: str, - app_id: str, - ) -> None: - self._tree = tree - self._tenant_id = tenant_id - self._app_id = app_id - self._parsers: dict[str, AssetItemParser] = {} - self._default_parser = FileAssetParser() - - def register(self, extension: str, parser: AssetItemParser) -> None: - self._parsers[extension] = parser - - def parse(self) -> list[AssetItem]: - assets: list[AssetItem] = [] - - for node in self._tree.walk_files(): - path = self._tree.get_path(node.id).lstrip("/") - storage_key = app_asset_storage.get_storage_key(AssetPath.draft(self._tenant_id, self._app_id, node.id)) - extension = node.extension or "" - - parser: AssetItemParser = self._parsers.get(extension, self._default_parser) - asset = parser.parse(node.id, path, node.name, extension, storage_key) - assets.append(asset) - - return assets diff --git a/api/core/app_assets/parser/base.py b/api/core/app_assets/parser/base.py deleted file mode 100644 index cd1807c79e..0000000000 --- a/api/core/app_assets/parser/base.py +++ /dev/null @@ -1,34 +0,0 @@ -from abc import ABC, abstractmethod - -from core.app_assets.entities import AssetItem, FileAsset - - -class AssetItemParser(ABC): - @abstractmethod - def parse( - self, - asset_id: str, - path: str, - file_name: str, - extension: str, - storage_key: str, - ) -> AssetItem: - raise NotImplementedError - - -class FileAssetParser(AssetItemParser): - def parse( - self, - asset_id: str, - path: str, - file_name: str, - extension: str, - storage_key: str, - ) -> FileAsset: - return FileAsset( - asset_id=asset_id, - path=path, - file_name=file_name, - extension=extension, - storage_key=storage_key, - ) diff --git a/api/core/app_assets/parser/skill_parser.py b/api/core/app_assets/parser/skill_parser.py deleted file mode 100644 index af7155c31a..0000000000 --- a/api/core/app_assets/parser/skill_parser.py +++ /dev/null @@ -1,57 +0,0 @@ -import json -import logging -from typing import Any - -from core.app_assets.entities import SkillAsset -from core.app_assets.entities.assets import AssetItem, FileAsset -from extensions.ext_storage import storage - -from .base import AssetItemParser - -logger = logging.getLogger(__name__) - - -class SkillAssetParser(AssetItemParser): - """ - Parser for skill assets. - - Responsibilities: - - Read file from storage - - Parse JSON structure - - Return SkillAsset with raw metadata (no parsing/resolution) - - Metadata parsing and content resolution are handled by SkillCompiler. - """ - - def parse( - self, - asset_id: str, - path: str, - file_name: str, - extension: str, - storage_key: str, - ) -> AssetItem: - try: - data = json.loads(storage.load_once(storage_key)) - if not isinstance(data, dict): - raise ValueError(f"Skill document {asset_id} must be a JSON object") - - metadata_raw: dict[str, Any] = data.get("metadata", {}) - - return SkillAsset( - asset_id=asset_id, - path=path, - file_name=file_name, - extension=extension, - storage_key=storage_key, - metadata=metadata_raw, - ) - except Exception: - logger.exception("Failed to parse skill asset %s", asset_id) - return FileAsset( - asset_id=asset_id, - path=path, - file_name=file_name, - extension=extension, - storage_key=storage_key, - ) diff --git a/api/core/app_assets/storage.py b/api/core/app_assets/storage.py index 2f656f1776..3ae9b4f04e 100644 --- a/api/core/app_assets/storage.py +++ b/api/core/app_assets/storage.py @@ -6,29 +6,20 @@ import hmac import os import time import urllib.parse +from abc import ABC, abstractmethod from collections.abc import Callable, Iterable from dataclasses import dataclass from typing import Any, ClassVar from uuid import UUID from configs import dify_config -from extensions.ext_redis import redis_client -from extensions.ext_storage import storage from extensions.storage.base_storage import BaseStorage from extensions.storage.cached_presign_storage import CachedPresignStorage -from extensions.storage.silent_storage import SilentStorage from libs import rsa _ASSET_BASE = "app_assets" _SILENT_STORAGE_NOT_FOUND = b"File Not Found" -_PATH_TEMPLATES: dict[str, str] = { - "draft": f"{_ASSET_BASE}/{{t}}/{{a}}/draft/{{r}}", - "build-zip": f"{_ASSET_BASE}/{{t}}/{{a}}/artifacts/{{r}}.zip", - "resolved": f"{_ASSET_BASE}/{{t}}/{{a}}/artifacts/{{r}}/resolved/{{s}}", - "skill-bundle": f"{_ASSET_BASE}/{{t}}/{{a}}/artifacts/{{r}}/skill_artifact_set.json", - "source-zip": f"{_ASSET_BASE}/{{t}}/{{a}}/sources/{{r}}.zip", -} -_ASSET_PATH_REGISTRY: dict[str, tuple[bool, Callable[..., AssetPathBase]]] = {} +_ASSET_PATH_REGISTRY: dict[str, tuple[bool, Callable[..., SignedAssetPath]]] = {} def _require_uuid(value: str, field_name: str) -> None: @@ -38,12 +29,12 @@ def _require_uuid(value: str, field_name: str) -> None: raise ValueError(f"{field_name} must be a UUID") from exc -def register_asset_path(asset_type: str, *, requires_node: bool, factory: Callable[..., AssetPathBase]) -> None: +def register_asset_path(asset_type: str, *, requires_node: bool, factory: Callable[..., SignedAssetPath]) -> None: _ASSET_PATH_REGISTRY[asset_type] = (requires_node, factory) @dataclass(frozen=True) -class AssetPathBase: +class AssetPathBase(ABC): asset_type: ClassVar[str] tenant_id: str app_id: str @@ -54,40 +45,54 @@ class AssetPathBase: _require_uuid(self.app_id, "app_id") _require_uuid(self.resource_id, "resource_id") + @abstractmethod def get_storage_key(self) -> str: - return _PATH_TEMPLATES[self.asset_type].format( - t=self.tenant_id, - a=self.app_id, - r=self.resource_id, - s=self.signature_sub_resource_id() or "", - ) + raise NotImplementedError - def signature_resource_id(self) -> str: - return self.resource_id - def signature_sub_resource_id(self) -> str: - return "" +class SignedAssetPath(AssetPathBase, ABC): + @abstractmethod + def signature_parts(self) -> tuple[str, str | None]: + """Return (resource_id, sub_resource_id) used for signing. + sub_resource_id should be None when not applicable. + """ + + @abstractmethod def proxy_path_parts(self) -> list[str]: - parts = [self.asset_type, self.tenant_id, self.app_id, self.signature_resource_id()] - sub_resource_id = self.signature_sub_resource_id() - if sub_resource_id: - parts.append(sub_resource_id) - return parts + raise NotImplementedError @dataclass(frozen=True) -class _DraftAssetPath(AssetPathBase): +class _DraftAssetPath(SignedAssetPath): asset_type: ClassVar[str] = "draft" + def get_storage_key(self) -> str: + return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/draft/{self.resource_id}" + + def signature_parts(self) -> tuple[str, str | None]: + return (self.resource_id, None) + + def proxy_path_parts(self) -> list[str]: + return [self.asset_type, self.tenant_id, self.app_id, self.resource_id] + @dataclass(frozen=True) -class _BuildZipAssetPath(AssetPathBase): +class _BuildZipAssetPath(SignedAssetPath): asset_type: ClassVar[str] = "build-zip" + def get_storage_key(self) -> str: + return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/artifacts/{self.resource_id}.zip" + + def signature_parts(self) -> tuple[str, str | None]: + return (self.resource_id, None) + + def proxy_path_parts(self) -> list[str]: + return [self.asset_type, self.tenant_id, self.app_id, self.resource_id] + @dataclass(frozen=True) -class _ResolvedAssetPath(AssetPathBase): +class _ResolvedAssetPath(SignedAssetPath): asset_type: ClassVar[str] = "resolved" node_id: str @@ -95,41 +100,86 @@ class _ResolvedAssetPath(AssetPathBase): super().__post_init__() _require_uuid(self.node_id, "node_id") - def signature_sub_resource_id(self) -> str: - return self.node_id + def get_storage_key(self) -> str: + return ( + f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/artifacts/" + f"{self.resource_id}/resolved/{self.node_id}" + ) + + def signature_parts(self) -> tuple[str, str | None]: + return (self.resource_id, self.node_id) + + def proxy_path_parts(self) -> list[str]: + return [self.asset_type, self.tenant_id, self.app_id, self.resource_id, self.node_id] @dataclass(frozen=True) -class _SkillBundleAssetPath(AssetPathBase): +class _SkillBundleAssetPath(SignedAssetPath): asset_type: ClassVar[str] = "skill-bundle" + def get_storage_key(self) -> str: + return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/artifacts/{self.resource_id}/skill_artifact_set.json" + + def signature_parts(self) -> tuple[str, str | None]: + return (self.resource_id, None) + + def proxy_path_parts(self) -> list[str]: + return [self.asset_type, self.tenant_id, self.app_id, self.resource_id] + @dataclass(frozen=True) -class _SourceZipAssetPath(AssetPathBase): +class _SourceZipAssetPath(SignedAssetPath): asset_type: ClassVar[str] = "source-zip" + def get_storage_key(self) -> str: + return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/sources/{self.resource_id}.zip" + + def signature_parts(self) -> tuple[str, str | None]: + return (self.resource_id, None) + + def proxy_path_parts(self) -> list[str]: + return [self.asset_type, self.tenant_id, self.app_id, self.resource_id] + + +@dataclass(frozen=True) +class _BundleExportZipAssetPath(SignedAssetPath): + asset_type: ClassVar[str] = "bundle-export-zip" + + def get_storage_key(self) -> str: + return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/bundle_exports/{self.resource_id}.zip" + + def signature_parts(self) -> tuple[str, str | None]: + return (self.resource_id, None) + + def proxy_path_parts(self) -> list[str]: + return [self.asset_type, self.tenant_id, self.app_id, self.resource_id] + class AssetPath: @staticmethod - def draft(tenant_id: str, app_id: str, node_id: str) -> AssetPathBase: + def draft(tenant_id: str, app_id: str, node_id: str) -> SignedAssetPath: return _DraftAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=node_id) @staticmethod - def build_zip(tenant_id: str, app_id: str, assets_id: str) -> AssetPathBase: + def build_zip(tenant_id: str, app_id: str, assets_id: str) -> SignedAssetPath: return _BuildZipAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=assets_id) @staticmethod - def resolved(tenant_id: str, app_id: str, assets_id: str, node_id: str) -> AssetPathBase: + def resolved(tenant_id: str, app_id: str, assets_id: str, node_id: str) -> SignedAssetPath: return _ResolvedAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=assets_id, node_id=node_id) @staticmethod - def skill_bundle(tenant_id: str, app_id: str, assets_id: str) -> AssetPathBase: + def skill_bundle(tenant_id: str, app_id: str, assets_id: str) -> SignedAssetPath: return _SkillBundleAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=assets_id) @staticmethod - def source_zip(tenant_id: str, app_id: str, workflow_id: str) -> AssetPathBase: + def source_zip(tenant_id: str, app_id: str, workflow_id: str) -> SignedAssetPath: return _SourceZipAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=workflow_id) + @staticmethod + def bundle_export_zip(tenant_id: str, app_id: str, export_id: str) -> SignedAssetPath: + return _BundleExportZipAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=export_id) + @staticmethod def from_components( asset_type: str, @@ -137,7 +187,7 @@ class AssetPath: app_id: str, resource_id: str, sub_resource_id: str | None = None, - ) -> AssetPathBase: + ) -> SignedAssetPath: entry = _ASSET_PATH_REGISTRY.get(asset_type) if not entry: raise ValueError(f"Unsupported asset type: {asset_type}") @@ -156,6 +206,7 @@ register_asset_path("build-zip", requires_node=False, factory=AssetPath.build_zi register_asset_path("resolved", requires_node=True, factory=AssetPath.resolved) register_asset_path("skill-bundle", requires_node=False, factory=AssetPath.skill_bundle) register_asset_path("source-zip", requires_node=False, factory=AssetPath.source_zip) +register_asset_path("bundle-export-zip", requires_node=False, factory=AssetPath.bundle_export_zip) class AppAssetSigner: @@ -165,7 +216,7 @@ class AppAssetSigner: OPERATION_UPLOAD = "upload" @classmethod - def create_download_signature(cls, asset_path: AssetPathBase, expires_at: int, nonce: str) -> str: + def create_download_signature(cls, asset_path: SignedAssetPath, expires_at: int, nonce: str) -> str: return cls._create_signature( asset_path=asset_path, operation=cls.OPERATION_DOWNLOAD, @@ -174,7 +225,7 @@ class AppAssetSigner: ) @classmethod - def create_upload_signature(cls, asset_path: AssetPathBase, expires_at: int, nonce: str) -> str: + def create_upload_signature(cls, asset_path: SignedAssetPath, expires_at: int, nonce: str) -> str: return cls._create_signature( asset_path=asset_path, operation=cls.OPERATION_UPLOAD, @@ -183,7 +234,7 @@ class AppAssetSigner: ) @classmethod - def verify_download_signature(cls, asset_path: AssetPathBase, expires_at: int, nonce: str, sign: str) -> bool: + def verify_download_signature(cls, asset_path: SignedAssetPath, expires_at: int, nonce: str, sign: str) -> bool: return cls._verify_signature( asset_path=asset_path, operation=cls.OPERATION_DOWNLOAD, @@ -193,7 +244,7 @@ class AppAssetSigner: ) @classmethod - def verify_upload_signature(cls, asset_path: AssetPathBase, expires_at: int, nonce: str, sign: str) -> bool: + def verify_upload_signature(cls, asset_path: SignedAssetPath, expires_at: int, nonce: str, sign: str) -> bool: return cls._verify_signature( asset_path=asset_path, operation=cls.OPERATION_UPLOAD, @@ -206,7 +257,7 @@ class AppAssetSigner: def _verify_signature( cls, *, - asset_path: AssetPathBase, + asset_path: SignedAssetPath, operation: str, expires_at: int, nonce: str, @@ -234,7 +285,7 @@ class AppAssetSigner: return True @classmethod - def _create_signature(cls, *, asset_path: AssetPathBase, operation: str, expires_at: int, nonce: str) -> str: + def _create_signature(cls, *, asset_path: SignedAssetPath, operation: str, expires_at: int, nonce: str) -> str: key = cls._tenant_key(asset_path.tenant_id) message = cls._signature_message( asset_path=asset_path, @@ -246,12 +297,12 @@ class AppAssetSigner: return base64.urlsafe_b64encode(sign).decode() @classmethod - def _signature_message(cls, *, asset_path: AssetPathBase, operation: str, expires_at: int, nonce: str) -> str: - sub_resource_id = asset_path.signature_sub_resource_id() + def _signature_message(cls, *, asset_path: SignedAssetPath, operation: str, expires_at: int, nonce: str) -> str: + resource_id, sub_resource_id = asset_path.signature_parts() return ( f"{cls.SIGNATURE_PREFIX}|{cls.SIGNATURE_VERSION}|{operation}|" f"{asset_path.asset_type}|{asset_path.tenant_id}|{asset_path.app_id}|" - f"{asset_path.signature_resource_id()}|{sub_resource_id}|{expires_at}|{nonce}" + f"{resource_id}|{sub_resource_id or ''}|{expires_at}|{nonce}" ) @classmethod @@ -301,7 +352,7 @@ class AppAssetStorage: def get_storage_key(self, asset_path: AssetPathBase) -> str: return asset_path.get_storage_key() - def get_download_url(self, asset_path: AssetPathBase, expires_in: int = 3600) -> str: + def get_download_url(self, asset_path: SignedAssetPath, expires_in: int = 3600) -> str: storage_key = self.get_storage_key(asset_path) try: return self._storage.get_download_url(storage_key, expires_in) @@ -312,7 +363,7 @@ class AppAssetStorage: def get_download_urls( self, - asset_paths: Iterable[AssetPathBase], + asset_paths: Iterable[SignedAssetPath], expires_in: int = 3600, ) -> list[str]: asset_paths_list = list(asset_paths) @@ -327,7 +378,7 @@ class AppAssetStorage: def get_upload_url( self, - asset_path: AssetPathBase, + asset_path: SignedAssetPath, expires_in: int = 3600, ) -> str: storage_key = self.get_storage_key(asset_path) @@ -338,7 +389,7 @@ class AppAssetStorage: return self._generate_signed_proxy_upload_url(asset_path, expires_in) - def _generate_signed_proxy_download_url(self, asset_path: AssetPathBase, expires_in: int) -> str: + def _generate_signed_proxy_download_url(self, asset_path: SignedAssetPath, expires_in: int) -> str: expires_in = min(expires_in, dify_config.FILES_ACCESS_TIMEOUT) expires_at = int(time.time()) + max(expires_in, 1) nonce = os.urandom(16).hex() @@ -349,7 +400,7 @@ class AppAssetStorage: query = urllib.parse.urlencode({"expires_at": expires_at, "nonce": nonce, "sign": sign}) return f"{url}?{query}" - def _generate_signed_proxy_upload_url(self, asset_path: AssetPathBase, expires_in: int) -> str: + def _generate_signed_proxy_upload_url(self, asset_path: SignedAssetPath, expires_in: int) -> str: expires_in = min(expires_in, dify_config.FILES_ACCESS_TIMEOUT) expires_at = int(time.time()) + max(expires_in, 1) nonce = os.urandom(16).hex() @@ -361,33 +412,7 @@ class AppAssetStorage: return f"{url}?{query}" @staticmethod - def _build_proxy_url(*, base_url: str, asset_path: AssetPathBase, action: str) -> str: + def _build_proxy_url(*, base_url: str, asset_path: SignedAssetPath, action: str) -> str: encoded_parts = [urllib.parse.quote(part, safe="") for part in asset_path.proxy_path_parts()] path = "/".join(encoded_parts) return f"{base_url}/files/app-assets/{path}/{action}" - - -class _LazyAppAssetStorage: - _instance: AppAssetStorage | None - _cache_key_prefix: str - - def __init__(self, *, cache_key_prefix: str) -> None: - self._instance = None - self._cache_key_prefix = cache_key_prefix - - def _get_instance(self) -> AppAssetStorage: - if self._instance is None: - if not hasattr(storage, "storage_runner"): - raise RuntimeError("Storage is not initialized; call storage.init_app before using app_asset_storage") - self._instance = AppAssetStorage( - storage=SilentStorage(storage.storage_runner), - redis_client=redis_client, - cache_key_prefix=self._cache_key_prefix, - ) - return self._instance - - def __getattr__(self, name: str): - return getattr(self._get_instance(), name) - - -app_asset_storage = _LazyAppAssetStorage(cache_key_prefix="app_assets") diff --git a/api/core/sandbox/__init__.py b/api/core/sandbox/__init__.py index 3d175c81cc..bf18a226c1 100644 --- a/api/core/sandbox/__init__.py +++ b/api/core/sandbox/__init__.py @@ -15,13 +15,13 @@ if TYPE_CHECKING: from .builder import SandboxBuilder, VMConfig from .entities import AppAssets, DifyCli, SandboxProviderApiEntity, SandboxType from .initializer import ( - AppAssetsInitializer, AsyncSandboxInitializer, - DifyCliInitializer, - DraftAppAssetsInitializer, SandboxInitializer, SyncSandboxInitializer, ) + from .initializer.app_assets_initializer import AppAssetsInitializer + from .initializer.dify_cli_initializer import DifyCliInitializer + from .initializer.draft_app_assets_initializer import DraftAppAssetsInitializer from .manager import SandboxManager from .sandbox import Sandbox from .storage import ArchiveSandboxStorage, SandboxStorage @@ -58,17 +58,17 @@ __all__ = [ _LAZY_IMPORTS = { "AppAssets": ("core.sandbox.entities", "AppAssets"), - "AppAssetsInitializer": ("core.sandbox.initializer", "AppAssetsInitializer"), + "AppAssetsInitializer": ("core.sandbox.initializer.app_assets_initializer", "AppAssetsInitializer"), "AsyncSandboxInitializer": ("core.sandbox.initializer", "AsyncSandboxInitializer"), "ArchiveSandboxStorage": ("core.sandbox.storage", "ArchiveSandboxStorage"), "DifyCli": ("core.sandbox.entities", "DifyCli"), "DifyCliBinary": ("core.sandbox.bash.dify_cli", "DifyCliBinary"), "DifyCliConfig": ("core.sandbox.bash.dify_cli", "DifyCliConfig"), "DifyCliEnvConfig": ("core.sandbox.bash.dify_cli", "DifyCliEnvConfig"), - "DifyCliInitializer": ("core.sandbox.initializer", "DifyCliInitializer"), + "DifyCliInitializer": ("core.sandbox.initializer.dify_cli_initializer", "DifyCliInitializer"), "DifyCliLocator": ("core.sandbox.bash.dify_cli", "DifyCliLocator"), "DifyCliToolConfig": ("core.sandbox.bash.dify_cli", "DifyCliToolConfig"), - "DraftAppAssetsInitializer": ("core.sandbox.initializer", "DraftAppAssetsInitializer"), + "DraftAppAssetsInitializer": ("core.sandbox.initializer.draft_app_assets_initializer", "DraftAppAssetsInitializer"), "Sandbox": ("core.sandbox.sandbox", "Sandbox"), "SandboxBashSession": ("core.sandbox.bash.session", "SandboxBashSession"), "SandboxBuilder": ("core.sandbox.builder", "SandboxBuilder"), diff --git a/api/core/sandbox/initializer/__init__.py b/api/core/sandbox/initializer/__init__.py index eca2b15a66..6933b9f385 100644 --- a/api/core/sandbox/initializer/__init__.py +++ b/api/core/sandbox/initializer/__init__.py @@ -1,13 +1,7 @@ -from .app_assets_initializer import AppAssetsInitializer from .base import AsyncSandboxInitializer, SandboxInitializer, SyncSandboxInitializer -from .dify_cli_initializer import DifyCliInitializer -from .draft_app_assets_initializer import DraftAppAssetsInitializer __all__ = [ - "AppAssetsInitializer", "AsyncSandboxInitializer", - "DifyCliInitializer", - "DraftAppAssetsInitializer", "SandboxInitializer", "SyncSandboxInitializer", ] diff --git a/api/core/sandbox/initializer/app_assets_initializer.py b/api/core/sandbox/initializer/app_assets_initializer.py index 34a55d9fc8..48eb8bf99c 100644 --- a/api/core/sandbox/initializer/app_assets_initializer.py +++ b/api/core/sandbox/initializer/app_assets_initializer.py @@ -1,10 +1,9 @@ import logging from core.app_assets.constants import AppAssetsAttrs -from core.app_assets.storage import AssetPath, app_asset_storage +from core.app_assets.storage import AssetPath from core.sandbox.sandbox import Sandbox from core.virtual_environment.__base.helpers import pipeline -from services.app_asset_service import AppAssetService from ..entities import AppAssets from .base import AsyncSandboxInitializer @@ -21,12 +20,15 @@ class AppAssetsInitializer(AsyncSandboxInitializer): self._assets_id = assets_id def initialize(self, sandbox: Sandbox) -> None: + from services.app_asset_package_service import AppAssetPackageService + from services.app_asset_service import AppAssetService + # Load published app assets and unzip the artifact bundle. - app_assets = AppAssetService.get_tenant_app_assets(self._tenant_id, self._assets_id) + app_assets = AppAssetPackageService.get_tenant_app_assets(self._tenant_id, self._assets_id) sandbox.attrs.set(AppAssetsAttrs.FILE_TREE, app_assets.asset_tree) sandbox.attrs.set(AppAssetsAttrs.APP_ASSETS_ID, self._assets_id) vm = sandbox.vm - asset_storage = app_asset_storage + asset_storage = AppAssetService.get_storage() zip_ref = AssetPath.build_zip(self._tenant_id, self._app_id, self._assets_id) download_url = asset_storage.get_download_url(zip_ref) diff --git a/api/core/sandbox/initializer/draft_app_assets_initializer.py b/api/core/sandbox/initializer/draft_app_assets_initializer.py index 0f30db02fa..d242f895e4 100644 --- a/api/core/sandbox/initializer/draft_app_assets_initializer.py +++ b/api/core/sandbox/initializer/draft_app_assets_initializer.py @@ -1,12 +1,13 @@ import logging from core.app_assets.constants import AppAssetsAttrs -from core.app_assets.storage import AssetPath, app_asset_storage +from core.app_assets.storage import AssetPath from core.sandbox.entities import AppAssets from core.sandbox.sandbox import Sandbox from core.sandbox.services import AssetDownloadService from core.sandbox.services.asset_download_service import AssetDownloadItem from core.virtual_environment.__base.helpers import pipeline +from services.app_asset_package_service import AppAssetPackageService from services.app_asset_service import AppAssetService from .base import AsyncSandboxInitializer @@ -25,14 +26,14 @@ class DraftAppAssetsInitializer(AsyncSandboxInitializer): def initialize(self, sandbox: Sandbox) -> None: # Load published app assets and unzip the artifact bundle. - app_assets = AppAssetService.get_tenant_app_assets(self._tenant_id, self._assets_id) + app_assets = AppAssetPackageService.get_tenant_app_assets(self._tenant_id, self._assets_id) sandbox.attrs.set(AppAssetsAttrs.FILE_TREE, app_assets.asset_tree) sandbox.attrs.set(AppAssetsAttrs.APP_ASSETS_ID, self._assets_id) vm = sandbox.vm build_id = self._assets_id tree = app_assets.asset_tree - storage = app_asset_storage + asset_storage = AppAssetService.get_storage() nodes = list(tree.walk_files()) if not nodes: return @@ -43,7 +44,7 @@ class DraftAppAssetsInitializer(AsyncSandboxInitializer): else AssetPath.draft(self._tenant_id, self._app_id, node.id) for node in nodes ] - urls = storage.get_download_urls(refs, DRAFT_ASSETS_EXPIRES_IN) + urls = asset_storage.get_download_urls(refs, DRAFT_ASSETS_EXPIRES_IN) items = [AssetDownloadItem(path=tree.get_path(node.id).lstrip("/"), url=url) for node, url in zip(nodes, urls)] script = AssetDownloadService.build_download_script(items, AppAssets.PATH) pipeline(vm).add( diff --git a/api/core/sandbox/manager.py b/api/core/sandbox/manager.py index 7c97d28a48..3fc3b7f8de 100644 --- a/api/core/sandbox/manager.py +++ b/api/core/sandbox/manager.py @@ -4,17 +4,7 @@ import logging import threading from typing import Final -from core.sandbox.builder import SandboxBuilder -from core.sandbox.entities import AppAssets, SandboxType -from core.sandbox.entities.providers import SandboxProviderEntity -from core.sandbox.initializer.app_assets_initializer import AppAssetsInitializer -from core.sandbox.initializer.dify_cli_initializer import DifyCliInitializer -from core.sandbox.initializer.draft_app_assets_initializer import DraftAppAssetsInitializer -from core.sandbox.initializer.skill_initializer import SkillInitializer -from core.sandbox.sandbox import Sandbox -from core.sandbox.storage.archive_storage import ArchiveSandboxStorage from core.virtual_environment.__base.virtual_environment import VirtualEnvironment -from services.app_asset_service import AppAssetService logger = logging.getLogger(__name__) @@ -103,99 +93,3 @@ class SandboxManager: @classmethod def count(cls) -> int: return sum(len(shard) for shard in cls._shards) - - @classmethod - def create( - cls, - tenant_id: str, - app_id: str, - user_id: str, - workflow_execution_id: str, - sandbox_provider: SandboxProviderEntity, - ) -> Sandbox: - assets = AppAssetService.get_assets(tenant_id, app_id, user_id, is_draft=False) - if not assets: - raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}") - - storage = ArchiveSandboxStorage(tenant_id, workflow_execution_id) - sandbox = ( - SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type)) - .options(sandbox_provider.config) - .user(user_id) - .app(app_id) - .initializer(AppAssetsInitializer(tenant_id, app_id, assets.id)) - .initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id)) - .initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id)) - .storage(storage, assets.id) - .build() - ) - - logger.info("Sandbox created: id=%s, assets=%s", sandbox.vm.metadata.id, sandbox.assets_id) - return sandbox - - @classmethod - def delete_draft_storage(cls, tenant_id: str, user_id: str) -> None: - storage = ArchiveSandboxStorage(tenant_id, SandboxBuilder.draft_id(user_id)) - storage.delete() - - @classmethod - def create_draft( - cls, - tenant_id: str, - app_id: str, - user_id: str, - sandbox_provider: SandboxProviderEntity, - ) -> Sandbox: - assets = AppAssetService.get_assets(tenant_id, app_id, user_id, is_draft=True) - if not assets: - raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}") - - AppAssetService.build_assets(tenant_id, app_id, assets) - sandbox_id = SandboxBuilder.draft_id(user_id) - storage = ArchiveSandboxStorage(tenant_id, sandbox_id, exclude_patterns=[AppAssets.PATH]) - - sandbox = ( - SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type)) - .options(sandbox_provider.config) - .user(user_id) - .app(app_id) - .initializer(DraftAppAssetsInitializer(tenant_id, app_id, assets.id)) - .initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id)) - .initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id)) - .storage(storage, assets.id) - .build() - ) - - logger.info("Draft sandbox created: id=%s, assets=%s", sandbox.vm.metadata.id, sandbox.assets_id) - return sandbox - - @classmethod - def create_for_single_step( - cls, - tenant_id: str, - app_id: str, - user_id: str, - sandbox_provider: SandboxProviderEntity, - ) -> Sandbox: - assets = AppAssetService.get_assets(tenant_id, app_id, user_id, is_draft=True) - if not assets: - raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}") - - AppAssetService.build_assets(tenant_id, app_id, assets) - sandbox_id = SandboxBuilder.draft_id(user_id) - storage = ArchiveSandboxStorage(tenant_id, sandbox_id, exclude_patterns=[AppAssets.PATH]) - - sandbox = ( - SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type)) - .options(sandbox_provider.config) - .user(user_id) - .app(app_id) - .initializer(DraftAppAssetsInitializer(tenant_id, app_id, assets.id)) - .initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id)) - .initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id)) - .storage(storage, assets.id) - .build() - ) - - logger.info("Single-step sandbox created: id=%s, assets=%s", sandbox.vm.metadata.id, sandbox.assets_id) - return sandbox diff --git a/api/core/skill/skill_manager.py b/api/core/skill/skill_manager.py index dd8c2c4fd4..29be138501 100644 --- a/api/core/skill/skill_manager.py +++ b/api/core/skill/skill_manager.py @@ -1,9 +1,8 @@ import logging -from core.app_assets.storage import AppAssetStorage, AssetPath +from core.app_assets.storage import AssetPath from core.skill.entities.skill_bundle import SkillBundle -from extensions.ext_redis import redis_client -from extensions.ext_storage import storage +from services.app_asset_service import AppAssetService logger = logging.getLogger(__name__) @@ -16,7 +15,7 @@ class SkillManager: assets_id: str, ) -> SkillBundle: asset_path = AssetPath.skill_bundle(tenant_id, app_id, assets_id) - data = AppAssetStorage(storage.storage_runner, redis_client=redis_client).load(asset_path) + data = AppAssetService.get_storage().load(asset_path) return SkillBundle.model_validate_json(data) @staticmethod @@ -27,7 +26,7 @@ class SkillManager: bundle: SkillBundle, ) -> None: asset_path = AssetPath.skill_bundle(tenant_id, app_id, assets_id) - AppAssetStorage(storage.storage_runner, redis_client=redis_client).save( + AppAssetService.get_storage().save( asset_path, bundle.model_dump_json(indent=2).encode("utf-8"), ) diff --git a/api/core/zip_sandbox/__init__.py b/api/core/zip_sandbox/__init__.py index 5d261b238a..746bb99a79 100644 --- a/api/core/zip_sandbox/__init__.py +++ b/api/core/zip_sandbox/__init__.py @@ -1,6 +1,7 @@ -from .session import SandboxArchiveFile, ZipSandbox +from .zip_sandbox import SandboxDownloadItem, SandboxFile, ZipSandbox __all__ = [ - "SandboxArchiveFile", + "SandboxDownloadItem", + "SandboxFile", "ZipSandbox", ] diff --git a/api/core/zip_sandbox/cli_strategy.py b/api/core/zip_sandbox/cli_strategy.py new file mode 100644 index 0000000000..e41365dc6a --- /dev/null +++ b/api/core/zip_sandbox/cli_strategy.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import posixpath +from typing import TYPE_CHECKING + +from core.virtual_environment.__base.exec import CommandExecutionError +from core.virtual_environment.__base.helpers import execute, try_execute + +from .strategy import ZipStrategy + +if TYPE_CHECKING: + from core.virtual_environment.__base.virtual_environment import VirtualEnvironment + + +class CliZipStrategy(ZipStrategy): + """Strategy using native zip/unzip CLI commands.""" + + def is_available(self, vm: VirtualEnvironment) -> bool: + result = try_execute(vm, ["which", "zip"], timeout=10) + has_zip = bool(result.stdout and result.stdout.strip()) + result = try_execute(vm, ["which", "unzip"], timeout=10) + has_unzip = bool(result.stdout and result.stdout.strip()) + return has_zip and has_unzip + + def zip( + self, + vm: VirtualEnvironment, + *, + src: str, + out_path: str, + cwd: str | None, + timeout: float, + ) -> None: + if src in (".", ""): + result = try_execute(vm, ["zip", "-qr", out_path, "."], timeout=timeout, cwd=cwd) + if not result.is_error: + return + # zip exits with 12 when there is nothing to do; create empty zip + if result.exit_code == 12: + self._write_empty_zip(vm, out_path) + return + raise CommandExecutionError("Failed to create zip archive", result) + + zip_cwd = posixpath.dirname(src) or "." + target = posixpath.basename(src) + result = try_execute(vm, ["zip", "-qr", out_path, target], timeout=timeout, cwd=zip_cwd) + if not result.is_error: + return + if result.exit_code == 12: + self._write_empty_zip(vm, out_path) + return + raise CommandExecutionError("Failed to create zip archive", result) + + def unzip( + self, + vm: VirtualEnvironment, + *, + archive_path: str, + dest_dir: str, + timeout: float, + ) -> None: + execute( + vm, + ["unzip", "-q", archive_path, "-d", dest_dir], + timeout=timeout, + error_message="Failed to unzip archive", + ) + + def _write_empty_zip(self, vm: VirtualEnvironment, out_path: str) -> None: + """Write an empty but valid zip file.""" + script = ( + 'printf "' + "\\x50\\x4b\\x05\\x06" + "\\x00\\x00\\x00\\x00" + "\\x00\\x00\\x00\\x00" + "\\x00\\x00\\x00\\x00" + "\\x00\\x00\\x00\\x00" + "\\x00\\x00\\x00\\x00" + '" > "$1"' + ) + execute(vm, ["sh", "-c", script, "sh", out_path], timeout=30, error_message="Failed to write empty zip") diff --git a/api/core/zip_sandbox/node_strategy.py b/api/core/zip_sandbox/node_strategy.py new file mode 100644 index 0000000000..d77a236a6d --- /dev/null +++ b/api/core/zip_sandbox/node_strategy.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from core.virtual_environment.__base.helpers import execute, try_execute + +from .strategy import ZipStrategy + +if TYPE_CHECKING: + from core.virtual_environment.__base.virtual_environment import VirtualEnvironment + + +ZIP_SCRIPT = r""" +const fs = require('fs'); +const path = require('path'); +const AdmZip = require('adm-zip'); + +const src = process.argv[2]; +const outPath = process.argv[3]; + +function walkAdd(zip, absPath, arcPrefix) { + const stat = fs.statSync(absPath); + if (stat.isDirectory()) { + const entries = fs.readdirSync(absPath); + if (entries.length === 0) { + zip.addFile(arcPrefix.replace(/\\/g, '/') + '/', Buffer.alloc(0)); + return; + } + for (const e of entries) { + walkAdd(zip, path.join(absPath, e), path.posix.join(arcPrefix, e)); + } + return; + } + if (stat.isFile()) { + const data = fs.readFileSync(absPath); + zip.addFile(arcPrefix.replace(/\\/g, '/'), data); + } +} + +const zip = new AdmZip(); +if (src === '.' || src === '') { + const entries = fs.readdirSync('.'); + for (const e of entries) { + walkAdd(zip, path.join('.', e), e); + } +} else { + const base = path.dirname(src) || '.'; + const prefix = path.basename(src.replace(/\/+$/, '')); + const root = path.join(base, prefix); + walkAdd(zip, root, prefix); +} + +zip.writeZip(outPath); +""" + +UNZIP_SCRIPT = r""" +const AdmZip = require('adm-zip'); +const archivePath = process.argv[2]; +const destDir = process.argv[3]; +const zip = new AdmZip(archivePath); +zip.extractAllTo(destDir, true); +""" + + +class NodeZipStrategy(ZipStrategy): + """Strategy using Node.js with adm-zip package.""" + + def is_available(self, vm: VirtualEnvironment) -> bool: + result = try_execute(vm, ["which", "node"], timeout=10) + if not (result.stdout and result.stdout.strip()): + return False + # Check if adm-zip module is available + result = try_execute(vm, ["node", "-e", "require('adm-zip')"], timeout=10) + return not result.is_error + + def zip( + self, + vm: VirtualEnvironment, + *, + src: str, + out_path: str, + cwd: str | None, + timeout: float, + ) -> None: + execute( + vm, + ["node", "-e", ZIP_SCRIPT, src, out_path], + timeout=timeout, + cwd=cwd, + error_message="Failed to create zip archive", + ) + + def unzip( + self, + vm: VirtualEnvironment, + *, + archive_path: str, + dest_dir: str, + timeout: float, + ) -> None: + execute( + vm, + ["node", "-e", UNZIP_SCRIPT, archive_path, dest_dir], + timeout=timeout, + error_message="Failed to unzip archive", + ) diff --git a/api/core/zip_sandbox/python_strategy.py b/api/core/zip_sandbox/python_strategy.py new file mode 100644 index 0000000000..5eae7efdac --- /dev/null +++ b/api/core/zip_sandbox/python_strategy.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from core.virtual_environment.__base.helpers import execute, try_execute + +from .strategy import ZipStrategy + +if TYPE_CHECKING: + from core.virtual_environment.__base.virtual_environment import VirtualEnvironment + + +ZIP_SCRIPT = r""" +import os +import sys +import zipfile + +src = sys.argv[1] +out_path = sys.argv[2] + +def is_cwd(p: str) -> bool: + return p in (".", "") + +src = src.rstrip("/") + +if is_cwd(src): + base = "." + root = "." + prefix = "" +else: + base = os.path.dirname(src) or "." + prefix = os.path.basename(src) + root = os.path.join(base, prefix) + +def add_empty_dir(zf: zipfile.ZipFile, arc_dir: str) -> None: + name = arc_dir.rstrip("/") + "/" + if name != "/": + zf.writestr(name, b"") + +with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: + if os.path.isfile(root): + zf.write(root, arcname=os.path.basename(root)) + else: + for dirpath, dirnames, filenames in os.walk(root): + rel_dir = os.path.relpath(dirpath, base) + rel_dir = "" if rel_dir == "." else rel_dir + if not dirnames and not filenames: + add_empty_dir(zf, rel_dir) + for fn in filenames: + fp = os.path.join(dirpath, fn) + arcname = os.path.join(rel_dir, fn) if rel_dir else fn + zf.write(fp, arcname=arcname) +""" + +UNZIP_SCRIPT = r""" +import sys +import zipfile + +archive_path = sys.argv[1] +dest_dir = sys.argv[2] + +with zipfile.ZipFile(archive_path, "r") as zf: + zf.extractall(dest_dir) +""" + + +class PythonZipStrategy(ZipStrategy): + """Strategy using Python's zipfile module.""" + + def __init__(self) -> None: + self._python_cmd: str | None = None + + def is_available(self, vm: VirtualEnvironment) -> bool: + for cmd in ("python3", "python"): + result = try_execute(vm, ["which", cmd], timeout=10) + if result.stdout and result.stdout.strip(): + self._python_cmd = cmd + return True + return False + + def zip( + self, + vm: VirtualEnvironment, + *, + src: str, + out_path: str, + cwd: str | None, + timeout: float, + ) -> None: + if self._python_cmd is None: + raise RuntimeError("Python not available") + + execute( + vm, + [self._python_cmd, "-c", ZIP_SCRIPT, src, out_path], + timeout=timeout, + cwd=cwd, + error_message="Failed to create zip archive", + ) + + def unzip( + self, + vm: VirtualEnvironment, + *, + archive_path: str, + dest_dir: str, + timeout: float, + ) -> None: + if self._python_cmd is None: + raise RuntimeError("Python not available") + + execute( + vm, + [self._python_cmd, "-c", UNZIP_SCRIPT, archive_path, dest_dir], + timeout=timeout, + error_message="Failed to unzip archive", + ) diff --git a/api/core/zip_sandbox/strategy.py b/api/core/zip_sandbox/strategy.py new file mode 100644 index 0000000000..f7356c4f79 --- /dev/null +++ b/api/core/zip_sandbox/strategy.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from core.virtual_environment.__base.virtual_environment import VirtualEnvironment + + +class ZipStrategy(ABC): + """Abstract base class for zip/unzip strategies.""" + + @abstractmethod + def is_available(self, vm: VirtualEnvironment) -> bool: + """Check if this strategy is available in the given VM.""" + ... + + @abstractmethod + def zip( + self, + vm: VirtualEnvironment, + *, + src: str, + out_path: str, + cwd: str | None, + timeout: float, + ) -> None: + """Create a zip archive.""" + ... + + @abstractmethod + def unzip( + self, + vm: VirtualEnvironment, + *, + archive_path: str, + dest_dir: str, + timeout: float, + ) -> None: + """Extract a zip archive.""" + ... diff --git a/api/core/zip_sandbox/session.py b/api/core/zip_sandbox/zip_sandbox.py similarity index 64% rename from api/core/zip_sandbox/session.py rename to api/core/zip_sandbox/zip_sandbox.py index cb10b5aef3..ea2b541719 100644 --- a/api/core/zip_sandbox/session.py +++ b/api/core/zip_sandbox/zip_sandbox.py @@ -1,6 +1,5 @@ from __future__ import annotations -import hashlib import posixpath from dataclasses import dataclass from io import BytesIO @@ -20,26 +19,38 @@ from core.virtual_environment.__base.helpers import execute, pipeline from core.virtual_environment.__base.virtual_environment import VirtualEnvironment from services.sandbox.sandbox_provider_service import SandboxProviderService +from .cli_strategy import CliZipStrategy +from .node_strategy import NodeZipStrategy +from .python_strategy import PythonZipStrategy +from .strategy import ZipStrategy + @dataclass(frozen=True) -class SandboxArchiveFile: - file_path: str - size_bytes: int - sha256: str +class SandboxDownloadItem: + url: str + path: str + + +@dataclass(frozen=True) +class SandboxFile: + """A handle to a file in the sandbox.""" + + path: str class ZipSandbox: - """A sandbox specifically for archive (tar) operations. + """A sandbox for archive (zip) operations. Usage: with ZipSandbox(tenant_id=..., user_id=...) as zs: - zs.write_file("a.txt", b"hello") - archive = zs.tar() - zs.upload(path=archive.file_path, target_url=url) + zs.download_items(items) + archive = zs.zip() + zs.upload(archive, upload_url) # VM automatically released on exit """ _DEFAULT_TIMEOUT_SECONDS = 60 * 5 + _STRATEGIES: list[ZipStrategy] = [CliZipStrategy(), PythonZipStrategy(), NodeZipStrategy()] def __init__( self, @@ -49,7 +60,6 @@ class ZipSandbox: app_id: str = "zip-sandbox", sandbox_provider_type: str | None = None, sandbox_provider_options: dict[str, Any] | None = None, - # For testing: allow injecting a VM directly _vm: VirtualEnvironment | None = None, ) -> None: self._tenant_id = tenant_id @@ -62,6 +72,7 @@ class ZipSandbox: self._sandbox: Sandbox | None = None self._sandbox_id: str | None = None self._vm: VirtualEnvironment | None = None + self._strategy: ZipStrategy | None = None def __enter__(self) -> ZipSandbox: self._start() @@ -79,7 +90,6 @@ class ZipSandbox: if self._vm is not None: raise RuntimeError("ZipSandbox already started") - # If VM is injected (for testing), use it directly if self._injected_vm is not None: self._vm = self._injected_vm self._sandbox_id = uuid4().hex @@ -127,6 +137,7 @@ class ZipSandbox: self._vm = None self._sandbox = None self._sandbox_id = None + self._strategy = None @property def vm(self) -> VirtualEnvironment: @@ -134,10 +145,21 @@ class ZipSandbox: raise RuntimeError("ZipSandbox not started. Use 'with ZipSandbox(...) as zs:'") return self._vm + def _get_strategy(self) -> ZipStrategy: + if self._strategy is not None: + return self._strategy + + for strategy in self._STRATEGIES: + if strategy.is_available(self.vm): + self._strategy = strategy + return strategy + + raise RuntimeError("No available zip backend (zip/python/node+adm-zip)") + # ========== Path utilities ========== @staticmethod - def _normalize_workspace_path(path: str | None) -> str: + def _normalize_path(path: str | None) -> str: raw = (path or ".").strip() if raw == "": raw = "." @@ -163,7 +185,7 @@ class ZipSandbox: # ========== File operations ========== def write_file(self, path: str, data: bytes) -> None: - path = self._normalize_workspace_path(path) + path = self._normalize_path(path) if path in ("", "."): raise ValueError("path must point to a file") @@ -173,7 +195,7 @@ class ZipSandbox: raise RuntimeError(f"Failed to write file to sandbox: {exc}") from exc def read_file(self, path: str, *, max_bytes: int = 10 * 1024 * 1024) -> bytes: - path = self._normalize_workspace_path(path) + path = self._normalize_path(path) if max_bytes <= 0: raise ValueError("max_bytes must be positive") @@ -192,9 +214,9 @@ class ZipSandbox: if not urls: return [] - dest_dir = self._normalize_workspace_path(dest_dir) - + dest_dir = self._normalize_path(dest_dir) paths = [self._dest_path_for_url(dest_dir, u) for u in urls] + p = pipeline(self.vm) p.add(["mkdir", "-p", dest_dir], error_message="Failed to create download directory") for url, out_path in zip(urls, paths, strict=True): @@ -207,14 +229,42 @@ class ZipSandbox: return paths + def download_items(self, items: list[SandboxDownloadItem], *, dest_dir: str = ".") -> list[str]: + if not items: + return [] + + dest_dir = self._normalize_path(dest_dir) + p = pipeline(self.vm) + p.add(["mkdir", "-p", dest_dir], error_message="Failed to create download directory") + + out_paths: list[str] = [] + for item in items: + rel = self._normalize_path(item.path) + if rel in ("", "."): + raise ValueError("Download item path must point to a file") + out_path = posixpath.join(dest_dir, rel) + out_paths.append(out_path) + out_dir = posixpath.dirname(out_path) + if out_dir not in ("", "."): + p.add(["mkdir", "-p", out_dir], error_message="Failed to create download directory") + p.add(["curl", "-fsSL", item.url, "-o", out_path], error_message="Failed to download file") + + try: + p.execute(timeout=self._DEFAULT_TIMEOUT_SECONDS, raise_on_error=True) + except Exception as exc: + raise RuntimeError(str(exc)) from exc + + return out_paths + def download_archive(self, archive_url: str, *, path: str = "input.tar.gz") -> str: - path = self._normalize_workspace_path(path) + path = self._normalize_path(path) dir_path = posixpath.dirname(path) p = pipeline(self.vm) if dir_path not in ("", "."): - p.add(["mkdir", "-p", dir_path], error_message=f"Failed to create archive download directory {dir_path}") + p.add(["mkdir", "-p", dir_path], error_message=f"Failed to create directory {dir_path}") p.add(["curl", "-fsSL", archive_url, "-o", path], error_message=f"Failed to download archive to {path}") + try: p.execute(timeout=self._DEFAULT_TIMEOUT_SECONDS, raise_on_error=True) except Exception as exc: @@ -224,15 +274,12 @@ class ZipSandbox: # ========== Upload operations ========== - def upload(self, *, path: str, target_url: str) -> None: - path = self._normalize_workspace_path(path) - if path in ("", "."): - raise ValueError("path must point to a file") - + def upload(self, file: SandboxFile, target_url: str) -> None: + """Upload a sandbox file to the given URL.""" try: execute( self.vm, - ["curl", "-fsSL", "-X", "PUT", "-T", path, target_url], + ["curl", "-fsSL", "-X", "PUT", "-T", file.path, target_url], timeout=self._DEFAULT_TIMEOUT_SECONDS, error_message="Failed to upload file from sandbox", ) @@ -241,55 +288,58 @@ class ZipSandbox: # ========== Archive operations ========== - def tar(self, src: str = ".", *, out_path: str | None = None) -> SandboxArchiveFile: - src = self._normalize_workspace_path(src) - if out_path is None: - out_path = f"{uuid4().hex}.tar" - out_path = self._normalize_workspace_path(out_path) - lower_out = out_path.lower() - if not (lower_out.endswith(".tar") or lower_out.endswith(".tar.gz") or lower_out.endswith(".tgz")): - raise ValueError("out_path must end with .tar/.tar.gz/.tgz") + def zip(self, src: str = ".", *, include_base: bool = True) -> SandboxFile: + """Create a zip archive and return a handle to it.""" + src = self._normalize_path(src) + out_path = f"/tmp/{uuid4().hex}.zip" - out_dir = posixpath.dirname(out_path) - is_gz = lower_out.endswith(".tar.gz") or lower_out.endswith(".tgz") - tar_flag = "-czf" if is_gz else "-cf" - is_cwd = src in (".", "") - - # Avoid "archive cannot contain itself" when archiving the current directory. - # Create the archive outside the workspace tree and move it into place. - tmp_archive = f"/tmp/{uuid4().hex}{'.tar.gz' if is_gz else '.tar'}" + cwd = None + src_for_strategy = src + if src not in (".", "") and not include_base: + cwd = src + src_for_strategy = "." try: - ( - pipeline(self.vm) - .add( - ["mkdir", "-p", out_dir], - error_message="Failed to create archive output directory", - on=out_dir not in ("", "."), - ) - .add( - ["tar", tar_flag, tmp_archive, "-C", ".", "."], - error_message="Failed to create tar archive", - on=is_cwd, - ) - .add(["tar", tar_flag, tmp_archive, src], error_message="Failed to create tar archive", on=not is_cwd) - .add(["mv", "-f", tmp_archive, out_path], error_message="Failed to move tar archive into place") - .execute(timeout=self._DEFAULT_TIMEOUT_SECONDS, raise_on_error=True) + self._get_strategy().zip( + self.vm, + src=src_for_strategy, + out_path=out_path, + cwd=cwd, + timeout=self._DEFAULT_TIMEOUT_SECONDS, ) - except PipelineExecutionError as exc: + except (PipelineExecutionError, CommandExecutionError) as exc: raise RuntimeError(str(exc)) from exc - # Compute size + sha256 on host side (avoid requiring sha256sum in sandbox). - try: - data = self.vm.download_file(out_path).getvalue() - except Exception as exc: - raise RuntimeError(f"Failed to read tar result from sandbox: {exc}") from exc + return SandboxFile(path=out_path) - return SandboxArchiveFile(file_path=out_path, size_bytes=len(data), sha256=hashlib.sha256(data).hexdigest()) + def unzip(self, *, archive_path: str, dest_dir: str = "unpacked") -> str: + """Extract a zip archive to the destination directory.""" + archive_path = self._normalize_path(archive_path) + dest_dir = self._normalize_path(dest_dir) + + if not archive_path.lower().endswith(".zip"): + raise ValueError("archive_path must end with .zip") + + try: + pipeline(self.vm).add( + ["mkdir", "-p", dest_dir], error_message="Failed to create destination directory" + ).execute(timeout=self._DEFAULT_TIMEOUT_SECONDS, raise_on_error=True) + + self._get_strategy().unzip( + self.vm, + archive_path=archive_path, + dest_dir=dest_dir, + timeout=self._DEFAULT_TIMEOUT_SECONDS, + ) + except (PipelineExecutionError, CommandExecutionError) as exc: + raise RuntimeError(str(exc)) from exc + + return dest_dir def untar(self, *, archive_path: str, dest_dir: str = "unpacked") -> str: - archive_path = self._normalize_workspace_path(archive_path) - dest_dir = self._normalize_workspace_path(dest_dir) + """Extract a tar archive to the destination directory.""" + archive_path = self._normalize_path(archive_path) + dest_dir = self._normalize_path(dest_dir) lower = archive_path.lower() is_gz = lower.endswith(".tar.gz") or lower.endswith(".tgz") @@ -298,7 +348,7 @@ class ZipSandbox: try: ( pipeline(self.vm) - .add(["mkdir", "-p", dest_dir], error_message="Failed to create untar destination directory") + .add(["mkdir", "-p", dest_dir], error_message="Failed to create destination directory") .add(["tar", extract_flag, archive_path, "-C", dest_dir], error_message="Failed to extract tar archive") .execute(timeout=self._DEFAULT_TIMEOUT_SECONDS, raise_on_error=True) ) diff --git a/api/services/app_asset_package_service.py b/api/services/app_asset_package_service.py new file mode 100644 index 0000000000..582a42d36a --- /dev/null +++ b/api/services/app_asset_package_service.py @@ -0,0 +1,185 @@ +"""Service for packaging and publishing app assets. + +This service handles operations that require core.zip_sandbox, +separated from AppAssetService to avoid circular imports. + +Dependency flow: + core/* -> AppAssetPackageService -> AppAssetService + (core modules can import this service without circular dependency) +""" + +import logging +from uuid import uuid4 + +from sqlalchemy.orm import Session + +from core.app.entities.app_asset_entities import AppAssetFileTree +from core.app_assets.builder import AssetBuildPipeline, BuildContext +from core.app_assets.builder.file_builder import FileBuilder +from core.app_assets.builder.skill_builder import SkillBuilder +from core.app_assets.entities.assets import AssetItem, FileAsset +from core.app_assets.storage import AssetPath +from core.zip_sandbox import SandboxDownloadItem, ZipSandbox +from models.app_asset import AppAssets +from models.model import App + +logger = logging.getLogger(__name__) + + +class AppAssetPackageService: + """Service for packaging and publishing app assets. + + This service is designed to be imported by core/* modules without + causing circular imports. It depends on AppAssetService for basic + asset operations but provides the packaging/publishing functionality + that requires core.zip_sandbox. + """ + + @staticmethod + def get_tenant_app_assets(tenant_id: str, assets_id: str) -> AppAssets: + """Get app assets by tenant_id and assets_id. + + This is a read-only operation that doesn't require AppAssetService. + """ + from extensions.ext_database import db + + with Session(db.engine, expire_on_commit=False) as session: + app_assets = ( + session.query(AppAssets) + .filter( + AppAssets.tenant_id == tenant_id, + AppAssets.id == assets_id, + ) + .first() + ) + if not app_assets: + raise ValueError(f"App assets not found for tenant_id={tenant_id}, assets_id={assets_id}") + + return app_assets + + @staticmethod + def get_draft_asset_items(tenant_id: str, app_id: str, file_tree: AppAssetFileTree) -> list[AssetItem]: + """Convert file tree to asset items for packaging.""" + files = file_tree.walk_files() + return [ + FileAsset( + asset_id=f.id, + path=file_tree.get_path(f.id), + file_name=f.name, + extension=f.extension, + storage_key=AssetPath.draft(tenant_id, app_id, f.id).get_storage_key(), + ) + for f in files + ] + + @staticmethod + def package_and_upload( + *, + assets: list[AssetItem], + upload_url: str, + tenant_id: str, + user_id: str, + ) -> None: + """Package assets into a ZIP and upload directly to the given URL.""" + from services.app_asset_service import AppAssetService + + if not assets: + import io + import zipfile + + import requests + + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w"): + pass + buf.seek(0) + requests.put(upload_url, data=buf.getvalue(), timeout=30) + return + + asset_storage = AppAssetService.get_storage() + storage_keys = [a.get_storage_key() for a in assets] + download_urls = asset_storage.storage.get_download_urls(storage_keys, 10 * 60) + + download_items = [ + SandboxDownloadItem(url=url, path=a.path) for a, url in zip(assets, download_urls, strict=True) + ] + + with ZipSandbox(tenant_id=tenant_id, user_id=user_id, app_id="asset-packager") as zs: + zs.download_items(download_items) + archive = zs.zip() + zs.upload(archive, upload_url) + + @staticmethod + def publish(session: Session, app_model: App, account_id: str, workflow_id: str) -> AppAssets: + """Publish app assets for a workflow. + + Creates a versioned copy of draft assets and packages them for runtime use. + """ + from services.app_asset_service import AppAssetService + + tenant_id = app_model.tenant_id + app_id = app_model.id + + assets = AppAssetService.get_or_create_assets(session, app_model, account_id) + tree = assets.asset_tree + + publish_id = str(uuid4()) + + published = AppAssets( + id=publish_id, + tenant_id=tenant_id, + app_id=app_id, + version=workflow_id, + created_by=account_id, + ) + published.asset_tree = tree + session.add(published) + session.flush() + + asset_storage = AppAssetService.get_storage() + ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=publish_id) + built_assets = AssetBuildPipeline([SkillBuilder(storage=asset_storage), FileBuilder()]).build_all(tree, ctx) + + runtime_zip_path = AssetPath.build_zip(tenant_id, app_id, publish_id) + runtime_upload_url = asset_storage.get_upload_url(runtime_zip_path) + AppAssetPackageService.package_and_upload( + assets=built_assets, + upload_url=runtime_upload_url, + tenant_id=tenant_id, + user_id=account_id, + ) + + source_items = AppAssetService.get_draft_assets(tenant_id, app_id) + source_zip_path = AssetPath.source_zip(tenant_id, app_id, workflow_id) + source_upload_url = asset_storage.get_upload_url(source_zip_path) + AppAssetPackageService.package_and_upload( + assets=source_items, + upload_url=source_upload_url, + tenant_id=tenant_id, + user_id=account_id, + ) + + return published + + @staticmethod + def build_assets(tenant_id: str, app_id: str, assets: AppAssets) -> None: + """Build resolved draft assets without packaging into a zip.""" + from services.app_asset_service import AppAssetService + + tree = assets.asset_tree + + asset_storage = AppAssetService.get_storage() + ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=assets.id) + built_assets: list[AssetItem] = AssetBuildPipeline( + [SkillBuilder(storage=asset_storage), FileBuilder()] + ).build_all(tree, ctx) + + user_id = getattr(assets, "updated_by", None) or getattr(assets, "created_by", None) or "system" + zip_path = AssetPath.build_zip(tenant_id, app_id, assets.id) + upload_url = asset_storage.get_upload_url(zip_path) + AppAssetPackageService.package_and_upload( + assets=built_assets, + upload_url=upload_url, + tenant_id=tenant_id, + user_id=user_id, + ) diff --git a/api/services/app_asset_service.py b/api/services/app_asset_service.py index 932df8e076..c8aeca8605 100644 --- a/api/services/app_asset_service.py +++ b/api/services/app_asset_service.py @@ -13,15 +13,12 @@ from core.app.entities.app_asset_entities import ( TreeParentNotFoundError, TreePathConflictError, ) -from core.app_assets.builder import AssetBuildPipeline, BuildContext -from core.app_assets.builder.file_builder import FileBuilder -from core.app_assets.builder.skill_builder import SkillBuilder -from core.app_assets.converters import tree_to_asset_items -from core.app_assets.entities.assets import AssetItem -from core.app_assets.packager import AssetZipPackager -from core.app_assets.storage import AssetPath, app_asset_storage +from core.app_assets.entities.assets import AssetItem, FileAsset +from core.app_assets.storage import AppAssetStorage, AssetPath from extensions.ext_database import db from extensions.ext_redis import redis_client +from extensions.ext_storage import storage +from extensions.storage.silent_storage import SilentStorage from models.app_asset import AppAssets from models.model import App @@ -40,10 +37,53 @@ class AppAssetService: _LOCK_TIMEOUT_SECONDS = 60 _DRAFT_CACHE_KEY_PREFIX = "app_asset:draft_download" + @staticmethod + def get_storage() -> AppAssetStorage: + """Get a lazily-initialized AppAssetStorage instance. + + This method creates an AppAssetStorage each time it's called, + ensuring storage.storage_runner is only accessed after init_app. + """ + return AppAssetStorage( + storage=SilentStorage(storage.storage_runner), + redis_client=redis_client, + cache_key_prefix="app_assets", + ) + @staticmethod def _lock(app_id: str): return redis_client.lock(f"app_asset:lock:{app_id}", timeout=AppAssetService._LOCK_TIMEOUT_SECONDS) + @staticmethod + def get_draft_assets(tenant_id: str, app_id: str) -> list[AssetItem]: + with Session(db.engine) as session: + assets = ( + session.query(AppAssets) + .filter( + AppAssets.tenant_id == tenant_id, + AppAssets.app_id == app_id, + AppAssets.version == AppAssets.VERSION_DRAFT, + ) + .first() + ) + if not assets: + return [] + return AppAssetService.get_draft_asset_items(assets.tenant_id, assets.app_id, assets.asset_tree) + + @staticmethod + def get_draft_asset_items(tenant_id: str, app_id: str, file_tree: AppAssetFileTree) -> list[AssetItem]: + files = file_tree.walk_files() + return [ + FileAsset( + asset_id=f.id, + path=file_tree.get_path(f.id), + file_name=f.name, + extension=f.extension, + storage_key=AssetPath.draft(tenant_id, app_id, f.id).get_storage_key(), + ) + for f in files + ] + @staticmethod def get_or_create_assets(session: Session, app_model: App, account_id: str) -> AppAssets: assets = ( @@ -161,7 +201,7 @@ class AppAssetService: max_size_mb = AppAssetService.MAX_PREVIEW_CONTENT_SIZE / 1024 / 1024 raise AppAssetNodeTooLargeError(f"File node {node_id} size exceeded the limit: {max_size_mb} MB") - asset_storage = app_asset_storage + asset_storage = AppAssetService.get_storage() asset_path = AssetPath.draft(app_model.tenant_id, app_model.id, node_id) return asset_storage.load(asset_path) @@ -182,7 +222,7 @@ class AppAssetService: except TreeNodeNotFoundError as e: raise AppAssetNodeNotFoundError(str(e)) from e - asset_storage = app_asset_storage + asset_storage = AppAssetService.get_storage() asset_path = AssetPath.draft(app_model.tenant_id, app_model.id, node_id) asset_storage.save(asset_path, content) @@ -282,7 +322,7 @@ class AppAssetService: # FIXME(Mairuis): sync deletion queue, failed is fine def _delete_file_from_storage(tenant_id: str, app_id: str, node_ids: list[str]) -> None: - asset_storage = app_asset_storage + asset_storage = AppAssetService.get_storage() for nid in node_ids: asset_path = AssetPath.draft(tenant_id, app_id, nid) try: @@ -290,7 +330,7 @@ class AppAssetService: except Exception: logger.warning( "Failed to delete storage file %s", - asset_storage.get_storage_key(asset_path), + asset_path.get_storage_key(), exc_info=True, ) @@ -298,62 +338,6 @@ class AppAssetService: target=lambda: _delete_file_from_storage(app_model.tenant_id, app_model.id, removed_ids) ).start() - @staticmethod - def publish(session: Session, app_model: App, account_id: str, workflow_id: str) -> AppAssets: - tenant_id = app_model.tenant_id - app_id = app_model.id - - assets = AppAssetService.get_or_create_assets(session, app_model, account_id) - tree = assets.asset_tree - - publish_id = str(uuid4()) - - published = AppAssets( - id=publish_id, - tenant_id=tenant_id, - app_id=app_id, - version=workflow_id, - created_by=account_id, - ) - published.asset_tree = tree - session.add(published) - session.flush() - - asset_storage = app_asset_storage - ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=publish_id) - built_assets = AssetBuildPipeline( - [SkillBuilder(storage=asset_storage), FileBuilder(storage=asset_storage)] - ).build_all(tree, ctx) - - packager = AssetZipPackager(asset_storage.storage) - - runtime_zip_bytes = packager.package(built_assets) - runtime_zip_path = AssetPath.build_zip(tenant_id, app_id, publish_id) - asset_storage.save(runtime_zip_path, runtime_zip_bytes) - - source_items = tree_to_asset_items(tree, tenant_id, app_id, asset_storage) - source_zip_bytes = packager.package(source_items) - source_zip_path = AssetPath.source_zip(tenant_id, app_id, workflow_id) - asset_storage.save(source_zip_path, source_zip_bytes) - - return published - - @staticmethod - def build_assets(tenant_id: str, app_id: str, assets: AppAssets) -> None: - # Build resolved draft assets without packaging into a zip. - tree = assets.asset_tree - - asset_storage = app_asset_storage - ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=assets.id) - built_assets: list[AssetItem] = AssetBuildPipeline( - [SkillBuilder(storage=asset_storage), FileBuilder(storage=asset_storage)] - ).build_all(tree, ctx) - - packager = AssetZipPackager(storage=asset_storage.storage) - zip_bytes = packager.package(built_assets) - zip_path = AssetPath.build_zip(tenant_id, app_id, assets.id) - asset_storage.save(zip_path, zip_bytes) - @staticmethod def get_file_download_url( app_model: App, @@ -369,17 +353,17 @@ class AppAssetService: if not node or node.node_type != AssetNodeType.FILE: raise AppAssetNodeNotFoundError(f"File node {node_id} not found") - asset_storage = app_asset_storage + asset_storage = AppAssetService.get_storage() asset_path = AssetPath.draft(app_model.tenant_id, app_model.id, node_id) return asset_storage.get_download_url(asset_path, expires_in) @staticmethod def get_source_zip_bytes(tenant_id: str, app_id: str, workflow_id: str) -> bytes | None: - asset_storage = app_asset_storage + asset_storage = AppAssetService.get_storage() asset_path = AssetPath.source_zip(tenant_id, app_id, workflow_id) source_zip = asset_storage.load_or_none(asset_path) if source_zip is None: - logger.warning("Source zip not found: %s", asset_storage.get_storage_key(asset_path)) + logger.warning("Source zip not found: %s", asset_path.get_storage_key()) return source_zip @staticmethod @@ -435,7 +419,7 @@ class AppAssetService: session.commit() asset_path = AssetPath.draft(app_model.tenant_id, app_model.id, node_id) - asset_storage = app_asset_storage + asset_storage = AppAssetService.get_storage() # put empty content to create the file record # which avoids file not found error when uploading via presigned URL is never touched @@ -477,7 +461,7 @@ class AppAssetService: assets.updated_by = account_id session.commit() - asset_storage = app_asset_storage + asset_storage = AppAssetService.get_storage() def fill_urls(node: BatchUploadNode) -> None: if node.node_type == AssetNodeType.FILE and node.id: diff --git a/api/services/app_bundle_service.py b/api/services/app_bundle_service.py index ab48762135..1a1102a66d 100644 --- a/api/services/app_bundle_service.py +++ b/api/services/app_bundle_service.py @@ -4,6 +4,7 @@ import io import logging import re import zipfile +from uuid import uuid4 import yaml from sqlalchemy.orm import Session @@ -15,13 +16,13 @@ from core.app.entities.app_bundle_entities import ( BundleFormatError, ZipSecurityError, ) -from core.app_assets.converters import tree_to_asset_items -from core.app_assets.packager import AssetZipPackager -from core.app_assets.storage import app_asset_storage +from core.app_assets.storage import AssetPath from core.app_bundle import SourceZipExtractor +from core.zip_sandbox import SandboxDownloadItem, ZipSandbox from extensions.ext_database import db from models import Account, App +from .app_asset_package_service import AppAssetPackageService from .app_asset_service import AppAssetService from .app_dsl_service import AppDslService, Import @@ -54,7 +55,7 @@ class AppBundleService: ) # 2. Publish assets (bound to workflow_id) - AppAssetService.publish( + AppAssetPackageService.publish( session=session, app_model=app_model, account_id=account.id, @@ -65,31 +66,61 @@ class AppBundleService: @staticmethod def export_bundle( + *, app_model: App, + account_id: str, include_secret: bool = False, workflow_id: str | None = None, + expires_in: int = 10 * 60, ) -> BundleExportResult: + """Export bundle and return a temporary download URL. + + Uses sandbox VM to build the ZIP, avoiding memory pressure in API process. + """ + tenant_id = app_model.tenant_id + app_id = app_model.id + safe_name = AppBundleService._sanitize_filename(app_model.name) + filename = f"{safe_name}.zip" + + export_id = uuid4().hex + export_path = AssetPath.bundle_export_zip(tenant_id, app_id, export_id) + asset_storage = AppAssetService.get_storage() + upload_url = asset_storage.get_upload_url(export_path, expires_in) + dsl_content = AppDslService.export_dsl( app_model=app_model, include_secret=include_secret, workflow_id=workflow_id, ) - safe_name = AppBundleService._sanitize_filename(app_model.name) - assets_prefix = safe_name + with ZipSandbox(tenant_id=tenant_id, user_id=account_id, app_id="app-bundle-export") as zs: + zs.write_file(f"bundle_root/{safe_name}.yml", dsl_content.encode("utf-8")) - zip_buffer = io.BytesIO() - with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf: - zf.writestr(f"{safe_name}.yml", dsl_content.encode("utf-8")) + # Published assets: use stored source zip and unzip into /... + if workflow_id is not None: + source_zip_path = AssetPath.source_zip(tenant_id, app_id, workflow_id) + source_url = asset_storage.get_download_url(source_zip_path, expires_in) + zs.download_archive(source_url, path="tmp/source_assets.zip") + zs.unzip(archive_path="tmp/source_assets.zip", dest_dir=f"bundle_root/{safe_name}") + else: + # Draft assets: download individual files and place under /... + asset_items = AppAssetService.get_draft_assets(tenant_id, app_id) + asset_urls = asset_storage.get_download_urls( + [AssetPath.draft(tenant_id, app_id, a.asset_id) for a in asset_items], expires_in + ) + zs.download_items( + [ + SandboxDownloadItem(url=url, path=f"{safe_name}/{a.path}") + for a, url in zip(asset_items, asset_urls, strict=True) + ], + dest_dir="bundle_root", + ) - assets_zip_bytes = AppBundleService._get_assets_zip_bytes(app_model, workflow_id) - if assets_zip_bytes: - AppBundleService._merge_assets_into_bundle(zf, assets_zip_bytes, assets_prefix) + archive = zs.zip(src="bundle_root", include_base=False) + zs.upload(archive, upload_url) - return BundleExportResult( - zip_bytes=zip_buffer.getvalue(), - filename=f"{safe_name}.zip", - ) + download_url = asset_storage.get_download_url(export_path, expires_in) + return BundleExportResult(download_url=download_url, filename=filename) @staticmethod def import_bundle( @@ -131,51 +162,6 @@ class AppBundleService: return import_result - @staticmethod - def _get_assets_zip_bytes(app_model: App, workflow_id: str | None) -> bytes | None: - tenant_id = app_model.tenant_id - app_id = app_model.id - - if workflow_id is None: - return AppBundleService._package_draft_assets(app_model) - else: - return AppAssetService.get_source_zip_bytes(tenant_id, app_id, workflow_id) - - @staticmethod - def _package_draft_assets(app_model: App) -> bytes | None: - assets = AppAssetService.get_assets( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - user_id="", - is_draft=True, - ) - if not assets: - return None - - tree = assets.asset_tree - if not tree.nodes: - return None - - asset_storage = app_asset_storage - items = tree_to_asset_items(tree, app_model.tenant_id, app_model.id, asset_storage) - packager = AssetZipPackager(asset_storage.storage) - return packager.package(items) - - @staticmethod - def _merge_assets_into_bundle( - bundle_zf: zipfile.ZipFile, - assets_zip_bytes: bytes, - prefix: str, - ) -> None: - with zipfile.ZipFile(io.BytesIO(assets_zip_bytes), "r") as assets_zf: - for info in assets_zf.infolist(): - content = assets_zf.read(info) - new_path = f"{prefix}/{info.filename}" - if info.is_dir(): - bundle_zf.writestr(zipfile.ZipInfo(new_path), "") - else: - bundle_zf.writestr(new_path, content) - @staticmethod def _extract_dsl_from_bundle(zip_bytes: bytes) -> tuple[str, str | None]: dsl_content: str | None = None @@ -221,7 +207,7 @@ class AppBundleService: logger.warning("App not found for asset import: %s", app_id) return - asset_storage = app_asset_storage + asset_storage = AppAssetService.get_storage() extractor = SourceZipExtractor(asset_storage) try: folders, files = extractor.extract_entries( diff --git a/api/services/sandbox/sandbox_service.py b/api/services/sandbox/sandbox_service.py new file mode 100644 index 0000000000..9bb97f0ed2 --- /dev/null +++ b/api/services/sandbox/sandbox_service.py @@ -0,0 +1,113 @@ +import logging + +from core.sandbox.builder import SandboxBuilder +from core.sandbox.entities import AppAssets, SandboxType +from core.sandbox.entities.providers import SandboxProviderEntity +from core.sandbox.initializer.app_assets_initializer import AppAssetsInitializer +from core.sandbox.initializer.dify_cli_initializer import DifyCliInitializer +from core.sandbox.initializer.draft_app_assets_initializer import DraftAppAssetsInitializer +from core.sandbox.initializer.skill_initializer import SkillInitializer +from core.sandbox.sandbox import Sandbox +from core.sandbox.storage.archive_storage import ArchiveSandboxStorage +from services.app_asset_package_service import AppAssetPackageService +from services.app_asset_service import AppAssetService + +logger = logging.getLogger(__name__) + + +class SandboxService: + @classmethod + def create( + cls, + tenant_id: str, + app_id: str, + user_id: str, + workflow_execution_id: str, + sandbox_provider: SandboxProviderEntity, + ) -> Sandbox: + assets = AppAssetService.get_assets(tenant_id, app_id, user_id, is_draft=False) + if not assets: + raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}") + + storage = ArchiveSandboxStorage(tenant_id, workflow_execution_id) + sandbox = ( + SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type)) + .options(sandbox_provider.config) + .user(user_id) + .app(app_id) + .initializer(AppAssetsInitializer(tenant_id, app_id, assets.id)) + .initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id)) + .initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id)) + .storage(storage, assets.id) + .build() + ) + + logger.info("Sandbox created: id=%s, assets=%s", sandbox.vm.metadata.id, sandbox.assets_id) + return sandbox + + @classmethod + def delete_draft_storage(cls, tenant_id: str, user_id: str) -> None: + storage = ArchiveSandboxStorage(tenant_id, SandboxBuilder.draft_id(user_id)) + storage.delete() + + @classmethod + def create_draft( + cls, + tenant_id: str, + app_id: str, + user_id: str, + sandbox_provider: SandboxProviderEntity, + ) -> Sandbox: + assets = AppAssetService.get_assets(tenant_id, app_id, user_id, is_draft=True) + if not assets: + raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}") + + AppAssetPackageService.build_assets(tenant_id, app_id, assets) + sandbox_id = SandboxBuilder.draft_id(user_id) + storage = ArchiveSandboxStorage(tenant_id, sandbox_id, exclude_patterns=[AppAssets.PATH]) + + sandbox = ( + SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type)) + .options(sandbox_provider.config) + .user(user_id) + .app(app_id) + .initializer(DraftAppAssetsInitializer(tenant_id, app_id, assets.id)) + .initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id)) + .initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id)) + .storage(storage, assets.id) + .build() + ) + + logger.info("Draft sandbox created: id=%s, assets=%s", sandbox.vm.metadata.id, sandbox.assets_id) + return sandbox + + @classmethod + def create_for_single_step( + cls, + tenant_id: str, + app_id: str, + user_id: str, + sandbox_provider: SandboxProviderEntity, + ) -> Sandbox: + assets = AppAssetService.get_assets(tenant_id, app_id, user_id, is_draft=True) + if not assets: + raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}") + + AppAssetPackageService.build_assets(tenant_id, app_id, assets) + sandbox_id = SandboxBuilder.draft_id(user_id) + storage = ArchiveSandboxStorage(tenant_id, sandbox_id, exclude_patterns=[AppAssets.PATH]) + + sandbox = ( + SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type)) + .options(sandbox_provider.config) + .user(user_id) + .app(app_id) + .initializer(DraftAppAssetsInitializer(tenant_id, app_id, assets.id)) + .initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id)) + .initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id)) + .storage(storage, assets.id) + .build() + ) + + logger.info("Single-step sandbox created: id=%s, assets=%s", sandbox.vm.metadata.id, sandbox.assets_id) + return sandbox diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index 4ab91be0ce..fa86772dfc 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -14,7 +14,6 @@ from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfig from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager from core.file import File from core.repositories import DifyCoreRepositoryFactory -from core.sandbox.manager import SandboxManager from core.variables import Variable, VariableBase from core.workflow.entities import WorkflowNodeExecution from core.workflow.enums import ErrorStrategy, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus @@ -44,6 +43,7 @@ from services.billing_service import BillingService from services.enterprise.plugin_manager_service import PluginCredentialType from services.errors.app import IsDraftWorkflowError, TriggerNodeLimitExceededError, WorkflowHashNotEqualError from services.sandbox.sandbox_provider_service import SandboxProviderService +from services.sandbox.sandbox_service import SandboxService from services.workflow.workflow_converter import WorkflowConverter from .errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError @@ -773,7 +773,7 @@ class WorkflowService: sandbox = None if draft_workflow.get_feature(WorkflowFeatures.SANDBOX).enabled: sandbox_provider = SandboxProviderService.get_sandbox_provider(draft_workflow.tenant_id) - sandbox = SandboxManager.create_for_single_step( + sandbox = SandboxService.create_for_single_step( tenant_id=draft_workflow.tenant_id, app_id=app_model.id, user_id=account.id, diff --git a/web/service/apps.ts b/web/service/apps.ts index 284bc7bcf4..3ac5395a11 100644 --- a/web/service/apps.ts +++ b/web/service/apps.ts @@ -141,21 +141,12 @@ export const exportAppBundle = async ({ appID, include = false, workflowID }: { if (!response.ok) throw new Error('Export bundle failed') - const blob = await response.blob() - const contentDisposition = response.headers.get('content-disposition') - let filename = `app-${appID}.zip` - if (contentDisposition) { - const filenameMatch = contentDisposition.match(/filename="?([^";\n]+)"?/) - if (filenameMatch) - filename = filenameMatch[1] - } + const result: { download_url: string, filename: string } = await response.json() - const downloadUrl = URL.createObjectURL(blob) const a = document.createElement('a') - a.href = downloadUrl - a.download = filename + a.href = result.download_url + a.download = result.filename a.click() - URL.revokeObjectURL(downloadUrl) } export const importDSL = ({ mode, yaml_content, yaml_url, app_id, name, description, icon_type, icon, icon_background }: { mode: DSLImportMode, yaml_content?: string, yaml_url?: string, app_id?: string, name?: string, description?: string, icon_type?: AppIconType, icon?: string, icon_background?: string }): Promise => {