opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-05-10 05:56:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(sandbox-zip-service): using sandbox to zip files

Browse Source 
- refactor allllllllll!!!!!!
This commit is contained in:
Harry 2026-01-27 13:09:16 +08:00
parent 9094f9d313
commit 64b6a5dd31
37 changed files with 1025 additions and 700 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
api/controllers/console/app/app.py
View File

@ -677,26 +677,19 @@ class AppExportBundleApi(Resource):
@account_initialization_required
@edit_permission_required
def get(self, app_model):
from io import BytesIO
from flask import send_file
from services.app_bundle_service import AppBundleService
args = AppExportBundleQuery.model_validate(request.args.to_dict(flat=True))
current_user, _ = current_account_with_tenant()
result = AppBundleService.export_bundle(
app_model=app_model,
account_id=str(current_user.id),
include_secret=args.include_secret,
workflow_id=args.workflow_id,
)
return send_file(
BytesIO(result.zip_bytes),
mimetype="application/zip",
as_attachment=True,
download_name=result.filename,
)
return result.model_dump(mode="json")
@console_ns.route("/apps/<uuid:app_id>/name")

5
api/controllers/console/app/workflow_draft_variable.py
View File

@ -16,7 +16,6 @@ from controllers.console.app.wraps import get_app_model
from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required
from controllers.web.error import InvalidArgumentError, NotFoundError
from core.file import helpers as file_helpers
from core.sandbox.manager import SandboxManager
from core.variables.segment_group import SegmentGroup
from core.variables.segments import ArrayFileSegment, ArrayPromptMessageSegment, FileSegment, Segment
from core.variables.types import SegmentType
@ -27,6 +26,7 @@ from factories.file_factory import build_from_mapping, build_from_mappings
from libs.login import current_account_with_tenant, login_required
from models import App, AppMode
from models.workflow import WorkflowDraftVariable
from services.sandbox.sandbox_service import SandboxService
from services.workflow_draft_variable_service import WorkflowDraftVariableList, WorkflowDraftVariableService
from services.workflow_service import WorkflowService
@ -268,9 +268,8 @@ class WorkflowVariableCollectionApi(Resource):
@console_ns.response(204, "Workflow variables deleted successfully")
@_api_prerequisite
def delete(self, app_model: App):
# FIXME(Mairuis): move to SandboxArtifactService
current_user, _ = current_account_with_tenant()
SandboxManager.delete_draft_storage(app_model.tenant_id, current_user.id)
SandboxService.delete_draft_storage(app_model.tenant_id, current_user.id)
draft_var_srv = WorkflowDraftVariableService(
session=db.session(),
)

4
api/controllers/files/app_assets_download.py
View File

@ -6,7 +6,7 @@ from pydantic import BaseModel, Field
from werkzeug.exceptions import Forbidden, NotFound
from controllers.files import files_ns
from core.app_assets.storage import AppAssetSigner, AssetPath, app_asset_storage
from core.app_assets.storage import AppAssetSigner, AssetPath
from extensions.ext_storage import storage
DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
@ -58,7 +58,7 @@ class AppAssetDownloadApi(Resource):
):
raise Forbidden("Invalid or expired download link")
storage_key = app_asset_storage.get_storage_key(asset_path)
storage_key = asset_path.get_storage_key()
try:
generator = storage.load_stream(storage_key)

5
api/controllers/files/app_assets_upload.py
View File

@ -4,7 +4,8 @@ from pydantic import BaseModel, Field
from werkzeug.exceptions import Forbidden
from controllers.files import files_ns
from core.app_assets.storage import AppAssetSigner, AssetPath, app_asset_storage
from core.app_assets.storage import AppAssetSigner, AssetPath
from services.app_asset_service import AppAssetService
DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
@ -56,5 +57,5 @@ class AppAssetUploadApi(Resource):
raise Forbidden("Invalid or expired upload link")
content = request.get_data()
app_asset_storage.save(asset_path, content)
AppAssetService.get_storage().save(asset_path, content)
return Response(status=204)

7
api/core/app/apps/advanced_chat/app_generator.py
View File

@ -35,7 +35,7 @@ from core.model_runtime.errors.invoke import InvokeAuthorizationError
from core.ops.ops_trace_manager import TraceQueueManager
from core.prompt.utils.get_thread_messages_length import get_thread_messages_length
from core.repositories import DifyCoreRepositoryFactory
from core.sandbox import Sandbox, SandboxManager
from core.sandbox import Sandbox
from core.workflow.repositories.draft_variable_repository import (
DraftVariableSaverFactory,
)
@ -50,6 +50,7 @@ from models.enums import WorkflowRunTriggeredFrom
from models.workflow_features import WorkflowFeatures
from services.conversation_service import ConversationService
from services.sandbox.sandbox_provider_service import SandboxProviderService
from services.sandbox.sandbox_service import SandboxService
from services.workflow_draft_variable_service import (
DraftVarLoader,
WorkflowDraftVariableService,
@ -528,7 +529,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
application_generate_entity.app_config.tenant_id
)
if workflow.version == Workflow.VERSION_DRAFT:
sandbox = SandboxManager.create_draft(
sandbox = SandboxService.create_draft(
tenant_id=application_generate_entity.app_config.tenant_id,
app_id=application_generate_entity.app_config.app_id,
user_id=application_generate_entity.user_id,
@ -537,7 +538,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
else:
if application_generate_entity.workflow_run_id is None:
raise ValueError("workflow_run_id is required when sandbox is enabled")
sandbox = SandboxManager.create(
sandbox = SandboxService.create(
tenant_id=application_generate_entity.app_config.tenant_id,
app_id=application_generate_entity.app_config.app_id,
user_id=application_generate_entity.user_id,

7
api/core/app/apps/workflow/app_generator.py
View File

@ -31,7 +31,7 @@ from core.helper.trace_id_helper import extract_external_trace_id_from_args
from core.model_runtime.errors.invoke import InvokeAuthorizationError
from core.ops.ops_trace_manager import TraceQueueManager
from core.repositories import DifyCoreRepositoryFactory
from core.sandbox import Sandbox, SandboxManager
from core.sandbox import Sandbox
from core.workflow.graph_engine.layers.base import GraphEngineLayer
from core.workflow.repositories.draft_variable_repository import DraftVariableSaverFactory
from core.workflow.repositories.workflow_execution_repository import WorkflowExecutionRepository
@ -44,6 +44,7 @@ from models import Account, App, EndUser, Workflow, WorkflowNodeExecutionTrigger
from models.enums import WorkflowRunTriggeredFrom
from models.workflow_features import WorkflowFeatures
from services.sandbox.sandbox_provider_service import SandboxProviderService
from services.sandbox.sandbox_service import SandboxService
from services.workflow_draft_variable_service import DraftVarLoader, WorkflowDraftVariableService
if TYPE_CHECKING:
@ -503,14 +504,14 @@ class WorkflowAppGenerator(BaseAppGenerator):
application_generate_entity.app_config.tenant_id
)
if workflow.version == Workflow.VERSION_DRAFT:
sandbox = SandboxManager.create_draft(
sandbox = SandboxService.create_draft(
tenant_id=application_generate_entity.app_config.tenant_id,
app_id=application_generate_entity.app_config.app_id,
user_id=application_generate_entity.user_id,
sandbox_provider=sandbox_provider,
)
else:
sandbox = SandboxManager.create(
sandbox = SandboxService.create(
tenant_id=application_generate_entity.app_config.tenant_id,
app_id=application_generate_entity.app_config.app_id,
user_id=application_generate_entity.user_id,

2
api/core/app/entities/app_bundle_entities.py
View File

@ -24,7 +24,7 @@ class ZipSecurityError(Exception):
# Entities
class BundleExportResult(BaseModel):
zip_bytes: bytes = Field(description="ZIP file content as bytes")
download_url: str = Field(description="Temporary download URL for the ZIP")
filename: str = Field(description="Suggested filename for the ZIP")

8
api/core/app_assets/__init__.py
View File

@ -4,18 +4,10 @@ from .entities import (
FileAsset,
SkillAsset,
)
from .packager import AssetPackager, AssetZipPackager
from .parser import AssetItemParser, AssetParser, FileAssetParser, SkillAssetParser
__all__ = [
"AppAssetsAttrs",
"AssetItem",
"AssetItemParser",
"AssetPackager",
"AssetParser",
"AssetZipPackager",
"FileAsset",
"FileAssetParser",
"SkillAsset",
"SkillAssetParser",
]

8
api/core/app_assets/builder/file_builder.py
View File

@ -1,17 +1,15 @@
from core.app.entities.app_asset_entities import AppAssetFileTree, AppAssetNode
from core.app_assets.entities import AssetItem, FileAsset
from core.app_assets.storage import AppAssetStorage, AssetPath
from core.app_assets.storage import AssetPath
from .base import BuildContext
class FileBuilder:
_nodes: list[tuple[AppAssetNode, str]]
_storage: AppAssetStorage
def __init__(self, storage: AppAssetStorage) -> None:
def __init__(self) -> None:
self._nodes = []
self._storage = storage
def accept(self, node: AppAssetNode) -> bool:
return True
@ -26,7 +24,7 @@ class FileBuilder:
path=path,
file_name=node.name,
extension=node.extension or "",
storage_key=self._storage.get_storage_key(AssetPath.draft(ctx.tenant_id, ctx.app_id, node.id)),
storage_key=AssetPath.draft(ctx.tenant_id, ctx.app_id, node.id).get_storage_key(),
)
for node, path in self._nodes
]

5
api/core/app_assets/builder/skill_builder.py
View File

@ -9,7 +9,6 @@ from core.app_assets.storage import AppAssetStorage, AssetPath, AssetPathBase
from core.skill.entities.skill_bundle import SkillBundle
from core.skill.entities.skill_document import SkillDocument
from core.skill.skill_compiler import SkillCompiler
from core.skill.skill_manager import SkillManager
from .base import BuildContext
@ -48,6 +47,8 @@ class SkillBuilder:
self._nodes.append((node, path))
def build(self, tree: AppAssetFileTree, ctx: BuildContext) -> list[AssetItem]:
from core.skill.skill_manager import SkillManager
if not self._nodes:
bundle = SkillBundle(assets_id=ctx.build_id)
SkillManager.save_bundle(ctx.tenant_id, ctx.app_id, ctx.build_id, bundle)
@ -74,7 +75,7 @@ class SkillBuilder:
node=skill.node,
path=skill.path,
ref=resolved_ref,
storage_key=self._storage.get_storage_key(resolved_ref),
storage_key=resolved_ref.get_storage_key(),
content_bytes=artifact.content.encode("utf-8"),
)
)

7
api/core/app_assets/converters.py
View File

@ -3,14 +3,13 @@ from __future__ import annotations
from core.app.entities.app_asset_entities import AppAssetFileTree, AssetNodeType
from core.app_assets.entities import FileAsset
from core.app_assets.entities.assets import AssetItem
from core.app_assets.storage import AppAssetStorage, AssetPath
from core.app_assets.storage import AssetPath
def tree_to_asset_items(
tree: AppAssetFileTree,
tenant_id: str,
app_id: str,
storage: AppAssetStorage,
) -> list[AssetItem]:
"""
Convert AppAssetFileTree to list of FileAsset for packaging.
@ -19,7 +18,6 @@ def tree_to_asset_items(
tree: The asset file tree to convert
tenant_id: Tenant ID for storage key generation
app_id: App ID for storage key generation
storage: App asset storage for key mapping
Returns:
List of FileAsset items ready for packaging
@ -29,14 +27,13 @@ def tree_to_asset_items(
if node.node_type == AssetNodeType.FILE:
path = tree.get_path(node.id)
asset_path = AssetPath.draft(tenant_id, app_id, node.id)
storage_key = storage.get_storage_key(asset_path)
items.append(
FileAsset(
asset_id=node.id,
path=path,
file_name=node.name,
extension=node.extension or "",
storage_key=storage_key,
storage_key=asset_path.get_storage_key(),
)
)
return items

7
api/core/app_assets/packager/__init__.py
View File

@ -1,7 +0,0 @@
from .asset_zip_packager import AssetZipPackager
from .base import AssetPackager
__all__ = [
"AssetPackager",
"AssetZipPackager",
]

75
api/core/app_assets/packager/asset_zip_packager.py
View File

@ -1,75 +0,0 @@
from __future__ import annotations
import io
import zipfile
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
from core.app_assets.entities import AssetItem
from extensions.storage.base_storage import BaseStorage
class AssetZipPackager:
"""
Unified ZIP packager for assets.
Automatically creates directory entries from asset paths.
"""
def __init__(self, storage: BaseStorage, *, max_workers: int = 8) -> None:
self._storage = storage
self._max_workers = max_workers
def package(self, assets: list[AssetItem], *, prefix: str = "") -> bytes:
"""
Package assets into a ZIP file.
Args:
assets: List of assets to package
prefix: Optional prefix to add to all paths in the ZIP
Returns:
ZIP file content as bytes
"""
zip_buffer = io.BytesIO()
# Extract folder paths from asset paths
folder_paths = self._extract_folder_paths(assets, prefix)
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
# Create directory entries
for folder_path in sorted(folder_paths):
zf.writestr(zipfile.ZipInfo(folder_path + "/"), "")
# Write files in parallel
if assets:
self._write_files_parallel(zf, assets, prefix)
return zip_buffer.getvalue()
def _extract_folder_paths(self, assets: list[AssetItem], prefix: str) -> set[str]:
"""Extract all folder paths from asset paths."""
folders: set[str] = set()
for asset in assets:
full_path = f"{prefix}/{asset.path}" if prefix else asset.path
parts = full_path.split("/")[:-1] # Remove filename
folders.update("/".join(parts[:i]) for i in range(1, len(parts) + 1))
return folders
def _write_files_parallel(
self,
zf: zipfile.ZipFile,
assets: list[AssetItem],
prefix: str,
) -> None:
lock = Lock()
def load_and_write(asset: AssetItem) -> None:
content = self._storage.load_once(asset.get_storage_key())
full_path = f"{prefix}/{asset.path}" if prefix else asset.path
with lock:
zf.writestr(full_path, content)
with ThreadPoolExecutor(max_workers=self._max_workers) as executor:
futures = [executor.submit(load_and_write, a) for a in assets]
for future in futures:
future.result()

9
api/core/app_assets/packager/base.py
View File

@ -1,9 +0,0 @@
from abc import ABC, abstractmethod
from core.app_assets.entities import AssetItem
class AssetPackager(ABC):
@abstractmethod
def package(self, assets: list[AssetItem]) -> bytes:
raise NotImplementedError

10
api/core/app_assets/parser/__init__.py
View File

@ -1,10 +0,0 @@
from .asset_parser import AssetParser
from .base import AssetItemParser, FileAssetParser
from .skill_parser import SkillAssetParser
__all__ = [
"AssetItemParser",
"AssetParser",
"FileAssetParser",
"SkillAssetParser",
]

36
api/core/app_assets/parser/asset_parser.py
View File

@ -1,36 +0,0 @@
from core.app.entities.app_asset_entities import AppAssetFileTree
from core.app_assets.entities import AssetItem
from core.app_assets.storage import AssetPath, app_asset_storage
from .base import AssetItemParser, FileAssetParser
class AssetParser:
def __init__(
self,
tree: AppAssetFileTree,
tenant_id: str,
app_id: str,
) -> None:
self._tree = tree
self._tenant_id = tenant_id
self._app_id = app_id
self._parsers: dict[str, AssetItemParser] = {}
self._default_parser = FileAssetParser()
def register(self, extension: str, parser: AssetItemParser) -> None:
self._parsers[extension] = parser
def parse(self) -> list[AssetItem]:
assets: list[AssetItem] = []
for node in self._tree.walk_files():
path = self._tree.get_path(node.id).lstrip("/")
storage_key = app_asset_storage.get_storage_key(AssetPath.draft(self._tenant_id, self._app_id, node.id))
extension = node.extension or ""
parser: AssetItemParser = self._parsers.get(extension, self._default_parser)
asset = parser.parse(node.id, path, node.name, extension, storage_key)
assets.append(asset)
return assets

34
api/core/app_assets/parser/base.py
View File

@ -1,34 +0,0 @@
from abc import ABC, abstractmethod
from core.app_assets.entities import AssetItem, FileAsset
class AssetItemParser(ABC):
@abstractmethod
def parse(
self,
asset_id: str,
path: str,
file_name: str,
extension: str,
storage_key: str,
) -> AssetItem:
raise NotImplementedError
class FileAssetParser(AssetItemParser):
def parse(
self,
asset_id: str,
path: str,
file_name: str,
extension: str,
storage_key: str,
) -> FileAsset:
return FileAsset(
asset_id=asset_id,
path=path,
file_name=file_name,
extension=extension,
storage_key=storage_key,
)

57
api/core/app_assets/parser/skill_parser.py
View File

@ -1,57 +0,0 @@
import json
import logging
from typing import Any
from core.app_assets.entities import SkillAsset
from core.app_assets.entities.assets import AssetItem, FileAsset
from extensions.ext_storage import storage
from .base import AssetItemParser
logger = logging.getLogger(__name__)
class SkillAssetParser(AssetItemParser):
"""
Parser for skill assets.
Responsibilities:
- Read file from storage
- Parse JSON structure
- Return SkillAsset with raw metadata (no parsing/resolution)
Metadata parsing and content resolution are handled by SkillCompiler.
"""
def parse(
self,
asset_id: str,
path: str,
file_name: str,
extension: str,
storage_key: str,
) -> AssetItem:
try:
data = json.loads(storage.load_once(storage_key))
if not isinstance(data, dict):
raise ValueError(f"Skill document {asset_id} must be a JSON object")
metadata_raw: dict[str, Any] = data.get("metadata", {})
return SkillAsset(
asset_id=asset_id,
path=path,
file_name=file_name,
extension=extension,
storage_key=storage_key,
metadata=metadata_raw,
)
except Exception:
logger.exception("Failed to parse skill asset %s", asset_id)
return FileAsset(
asset_id=asset_id,
path=path,
file_name=file_name,
extension=extension,
storage_key=storage_key,
)

189
api/core/app_assets/storage.py
View File

@ -6,29 +6,20 @@ import hmac
import os
import time
import urllib.parse
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable
from dataclasses import dataclass
from typing import Any, ClassVar
from uuid import UUID
from configs import dify_config
from extensions.ext_redis import redis_client
from extensions.ext_storage import storage
from extensions.storage.base_storage import BaseStorage
from extensions.storage.cached_presign_storage import CachedPresignStorage
from extensions.storage.silent_storage import SilentStorage
from libs import rsa
_ASSET_BASE = "app_assets"
_SILENT_STORAGE_NOT_FOUND = b"File Not Found"
_PATH_TEMPLATES: dict[str, str] = {
"draft": f"{_ASSET_BASE}/{{t}}/{{a}}/draft/{{r}}",
"build-zip": f"{_ASSET_BASE}/{{t}}/{{a}}/artifacts/{{r}}.zip",
"resolved": f"{_ASSET_BASE}/{{t}}/{{a}}/artifacts/{{r}}/resolved/{{s}}",
"skill-bundle": f"{_ASSET_BASE}/{{t}}/{{a}}/artifacts/{{r}}/skill_artifact_set.json",
"source-zip": f"{_ASSET_BASE}/{{t}}/{{a}}/sources/{{r}}.zip",
}
_ASSET_PATH_REGISTRY: dict[str, tuple[bool, Callable[..., AssetPathBase]]] = {}
_ASSET_PATH_REGISTRY: dict[str, tuple[bool, Callable[..., SignedAssetPath]]] = {}
def _require_uuid(value: str, field_name: str) -> None:
@ -38,12 +29,12 @@ def _require_uuid(value: str, field_name: str) -> None:
raise ValueError(f"{field_name} must be a UUID") from exc
def register_asset_path(asset_type: str, *, requires_node: bool, factory: Callable[..., AssetPathBase]) -> None:
def register_asset_path(asset_type: str, *, requires_node: bool, factory: Callable[..., SignedAssetPath]) -> None:
_ASSET_PATH_REGISTRY[asset_type] = (requires_node, factory)
@dataclass(frozen=True)
class AssetPathBase:
class AssetPathBase(ABC):
asset_type: ClassVar[str]
tenant_id: str
app_id: str
@ -54,40 +45,54 @@ class AssetPathBase:
_require_uuid(self.app_id, "app_id")
_require_uuid(self.resource_id, "resource_id")
@abstractmethod
def get_storage_key(self) -> str:
return _PATH_TEMPLATES[self.asset_type].format(
t=self.tenant_id,
a=self.app_id,
r=self.resource_id,
s=self.signature_sub_resource_id() or "",
)
raise NotImplementedError
def signature_resource_id(self) -> str:
return self.resource_id
def signature_sub_resource_id(self) -> str:
return ""
class SignedAssetPath(AssetPathBase, ABC):
@abstractmethod
def signature_parts(self) -> tuple[str, str | None]:
"""Return (resource_id, sub_resource_id) used for signing.
sub_resource_id should be None when not applicable.
"""
@abstractmethod
def proxy_path_parts(self) -> list[str]:
parts = [self.asset_type, self.tenant_id, self.app_id, self.signature_resource_id()]
sub_resource_id = self.signature_sub_resource_id()
if sub_resource_id:
parts.append(sub_resource_id)
return parts
raise NotImplementedError
@dataclass(frozen=True)
class _DraftAssetPath(AssetPathBase):
class _DraftAssetPath(SignedAssetPath):
asset_type: ClassVar[str] = "draft"
def get_storage_key(self) -> str:
return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/draft/{self.resource_id}"
def signature_parts(self) -> tuple[str, str | None]:
return (self.resource_id, None)
def proxy_path_parts(self) -> list[str]:
return [self.asset_type, self.tenant_id, self.app_id, self.resource_id]
@dataclass(frozen=True)
class _BuildZipAssetPath(AssetPathBase):
class _BuildZipAssetPath(SignedAssetPath):
asset_type: ClassVar[str] = "build-zip"
def get_storage_key(self) -> str:
return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/artifacts/{self.resource_id}.zip"
def signature_parts(self) -> tuple[str, str | None]:
return (self.resource_id, None)
def proxy_path_parts(self) -> list[str]:
return [self.asset_type, self.tenant_id, self.app_id, self.resource_id]
@dataclass(frozen=True)
class _ResolvedAssetPath(AssetPathBase):
class _ResolvedAssetPath(SignedAssetPath):
asset_type: ClassVar[str] = "resolved"
node_id: str
@ -95,41 +100,86 @@ class _ResolvedAssetPath(AssetPathBase):
super().__post_init__()
_require_uuid(self.node_id, "node_id")
def signature_sub_resource_id(self) -> str:
return self.node_id
def get_storage_key(self) -> str:
return (
f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/artifacts/"
f"{self.resource_id}/resolved/{self.node_id}"
)
def signature_parts(self) -> tuple[str, str | None]:
return (self.resource_id, self.node_id)
def proxy_path_parts(self) -> list[str]:
return [self.asset_type, self.tenant_id, self.app_id, self.resource_id, self.node_id]
@dataclass(frozen=True)
class _SkillBundleAssetPath(AssetPathBase):
class _SkillBundleAssetPath(SignedAssetPath):
asset_type: ClassVar[str] = "skill-bundle"
def get_storage_key(self) -> str:
return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/artifacts/{self.resource_id}/skill_artifact_set.json"
def signature_parts(self) -> tuple[str, str | None]:
return (self.resource_id, None)
def proxy_path_parts(self) -> list[str]:
return [self.asset_type, self.tenant_id, self.app_id, self.resource_id]
@dataclass(frozen=True)
class _SourceZipAssetPath(AssetPathBase):
class _SourceZipAssetPath(SignedAssetPath):
asset_type: ClassVar[str] = "source-zip"
def get_storage_key(self) -> str:
return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/sources/{self.resource_id}.zip"
def signature_parts(self) -> tuple[str, str | None]:
return (self.resource_id, None)
def proxy_path_parts(self) -> list[str]:
return [self.asset_type, self.tenant_id, self.app_id, self.resource_id]
@dataclass(frozen=True)
class _BundleExportZipAssetPath(SignedAssetPath):
asset_type: ClassVar[str] = "bundle-export-zip"
def get_storage_key(self) -> str:
return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/bundle_exports/{self.resource_id}.zip"
def signature_parts(self) -> tuple[str, str | None]:
return (self.resource_id, None)
def proxy_path_parts(self) -> list[str]:
return [self.asset_type, self.tenant_id, self.app_id, self.resource_id]
class AssetPath:
@staticmethod
def draft(tenant_id: str, app_id: str, node_id: str) -> AssetPathBase:
def draft(tenant_id: str, app_id: str, node_id: str) -> SignedAssetPath:
return _DraftAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=node_id)
@staticmethod
def build_zip(tenant_id: str, app_id: str, assets_id: str) -> AssetPathBase:
def build_zip(tenant_id: str, app_id: str, assets_id: str) -> SignedAssetPath:
return _BuildZipAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=assets_id)
@staticmethod
def resolved(tenant_id: str, app_id: str, assets_id: str, node_id: str) -> AssetPathBase:
def resolved(tenant_id: str, app_id: str, assets_id: str, node_id: str) -> SignedAssetPath:
return _ResolvedAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=assets_id, node_id=node_id)
@staticmethod
def skill_bundle(tenant_id: str, app_id: str, assets_id: str) -> AssetPathBase:
def skill_bundle(tenant_id: str, app_id: str, assets_id: str) -> SignedAssetPath:
return _SkillBundleAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=assets_id)
@staticmethod
def source_zip(tenant_id: str, app_id: str, workflow_id: str) -> AssetPathBase:
def source_zip(tenant_id: str, app_id: str, workflow_id: str) -> SignedAssetPath:
return _SourceZipAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=workflow_id)
@staticmethod
def bundle_export_zip(tenant_id: str, app_id: str, export_id: str) -> SignedAssetPath:
return _BundleExportZipAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=export_id)
@staticmethod
def from_components(
asset_type: str,
@ -137,7 +187,7 @@ class AssetPath:
app_id: str,
resource_id: str,
sub_resource_id: str | None = None,
) -> AssetPathBase:
) -> SignedAssetPath:
entry = _ASSET_PATH_REGISTRY.get(asset_type)
if not entry:
raise ValueError(f"Unsupported asset type: {asset_type}")
@ -156,6 +206,7 @@ register_asset_path("build-zip", requires_node=False, factory=AssetPath.build_zi
register_asset_path("resolved", requires_node=True, factory=AssetPath.resolved)
register_asset_path("skill-bundle", requires_node=False, factory=AssetPath.skill_bundle)
register_asset_path("source-zip", requires_node=False, factory=AssetPath.source_zip)
register_asset_path("bundle-export-zip", requires_node=False, factory=AssetPath.bundle_export_zip)
class AppAssetSigner:
@ -165,7 +216,7 @@ class AppAssetSigner:
OPERATION_UPLOAD = "upload"
@classmethod
def create_download_signature(cls, asset_path: AssetPathBase, expires_at: int, nonce: str) -> str:
def create_download_signature(cls, asset_path: SignedAssetPath, expires_at: int, nonce: str) -> str:
return cls._create_signature(
asset_path=asset_path,
operation=cls.OPERATION_DOWNLOAD,
@ -174,7 +225,7 @@ class AppAssetSigner:
)
@classmethod
def create_upload_signature(cls, asset_path: AssetPathBase, expires_at: int, nonce: str) -> str:
def create_upload_signature(cls, asset_path: SignedAssetPath, expires_at: int, nonce: str) -> str:
return cls._create_signature(
asset_path=asset_path,
operation=cls.OPERATION_UPLOAD,
@ -183,7 +234,7 @@ class AppAssetSigner:
)
@classmethod
def verify_download_signature(cls, asset_path: AssetPathBase, expires_at: int, nonce: str, sign: str) -> bool:
def verify_download_signature(cls, asset_path: SignedAssetPath, expires_at: int, nonce: str, sign: str) -> bool:
return cls._verify_signature(
asset_path=asset_path,
operation=cls.OPERATION_DOWNLOAD,
@ -193,7 +244,7 @@ class AppAssetSigner:
)
@classmethod
def verify_upload_signature(cls, asset_path: AssetPathBase, expires_at: int, nonce: str, sign: str) -> bool:
def verify_upload_signature(cls, asset_path: SignedAssetPath, expires_at: int, nonce: str, sign: str) -> bool:
return cls._verify_signature(
asset_path=asset_path,
operation=cls.OPERATION_UPLOAD,
@ -206,7 +257,7 @@ class AppAssetSigner:
def _verify_signature(
cls,
*,
asset_path: AssetPathBase,
asset_path: SignedAssetPath,
operation: str,
expires_at: int,
nonce: str,
@ -234,7 +285,7 @@ class AppAssetSigner:
return True
@classmethod
def _create_signature(cls, *, asset_path: AssetPathBase, operation: str, expires_at: int, nonce: str) -> str:
def _create_signature(cls, *, asset_path: SignedAssetPath, operation: str, expires_at: int, nonce: str) -> str:
key = cls._tenant_key(asset_path.tenant_id)
message = cls._signature_message(
asset_path=asset_path,
@ -246,12 +297,12 @@ class AppAssetSigner:
return base64.urlsafe_b64encode(sign).decode()
@classmethod
def _signature_message(cls, *, asset_path: AssetPathBase, operation: str, expires_at: int, nonce: str) -> str:
sub_resource_id = asset_path.signature_sub_resource_id()
def _signature_message(cls, *, asset_path: SignedAssetPath, operation: str, expires_at: int, nonce: str) -> str:
resource_id, sub_resource_id = asset_path.signature_parts()
return (
f"{cls.SIGNATURE_PREFIX}|{cls.SIGNATURE_VERSION}|{operation}|"
f"{asset_path.asset_type}|{asset_path.tenant_id}|{asset_path.app_id}|"
f"{asset_path.signature_resource_id()}|{sub_resource_id}|{expires_at}|{nonce}"
f"{resource_id}|{sub_resource_id or ''}|{expires_at}|{nonce}"
)
@classmethod
@ -301,7 +352,7 @@ class AppAssetStorage:
def get_storage_key(self, asset_path: AssetPathBase) -> str:
return asset_path.get_storage_key()
def get_download_url(self, asset_path: AssetPathBase, expires_in: int = 3600) -> str:
def get_download_url(self, asset_path: SignedAssetPath, expires_in: int = 3600) -> str:
storage_key = self.get_storage_key(asset_path)
try:
return self._storage.get_download_url(storage_key, expires_in)
@ -312,7 +363,7 @@ class AppAssetStorage:
def get_download_urls(
self,
asset_paths: Iterable[AssetPathBase],
asset_paths: Iterable[SignedAssetPath],
expires_in: int = 3600,
) -> list[str]:
asset_paths_list = list(asset_paths)
@ -327,7 +378,7 @@ class AppAssetStorage:
def get_upload_url(
self,
asset_path: AssetPathBase,
asset_path: SignedAssetPath,
expires_in: int = 3600,
) -> str:
storage_key = self.get_storage_key(asset_path)
@ -338,7 +389,7 @@ class AppAssetStorage:
return self._generate_signed_proxy_upload_url(asset_path, expires_in)
def _generate_signed_proxy_download_url(self, asset_path: AssetPathBase, expires_in: int) -> str:
def _generate_signed_proxy_download_url(self, asset_path: SignedAssetPath, expires_in: int) -> str:
expires_in = min(expires_in, dify_config.FILES_ACCESS_TIMEOUT)
expires_at = int(time.time()) + max(expires_in, 1)
nonce = os.urandom(16).hex()
@ -349,7 +400,7 @@ class AppAssetStorage:
query = urllib.parse.urlencode({"expires_at": expires_at, "nonce": nonce, "sign": sign})
return f"{url}?{query}"
def _generate_signed_proxy_upload_url(self, asset_path: AssetPathBase, expires_in: int) -> str:
def _generate_signed_proxy_upload_url(self, asset_path: SignedAssetPath, expires_in: int) -> str:
expires_in = min(expires_in, dify_config.FILES_ACCESS_TIMEOUT)
expires_at = int(time.time()) + max(expires_in, 1)
nonce = os.urandom(16).hex()
@ -361,33 +412,7 @@ class AppAssetStorage:
return f"{url}?{query}"
@staticmethod
def _build_proxy_url(*, base_url: str, asset_path: AssetPathBase, action: str) -> str:
def _build_proxy_url(*, base_url: str, asset_path: SignedAssetPath, action: str) -> str:
encoded_parts = [urllib.parse.quote(part, safe="") for part in asset_path.proxy_path_parts()]
path = "/".join(encoded_parts)
return f"{base_url}/files/app-assets/{path}/{action}"
class _LazyAppAssetStorage:
_instance: AppAssetStorage | None
_cache_key_prefix: str
def __init__(self, *, cache_key_prefix: str) -> None:
self._instance = None
self._cache_key_prefix = cache_key_prefix
def _get_instance(self) -> AppAssetStorage:
if self._instance is None:
if not hasattr(storage, "storage_runner"):
raise RuntimeError("Storage is not initialized; call storage.init_app before using app_asset_storage")
self._instance = AppAssetStorage(
storage=SilentStorage(storage.storage_runner),
redis_client=redis_client,
cache_key_prefix=self._cache_key_prefix,
)
return self._instance
def __getattr__(self, name: str):
return getattr(self._get_instance(), name)
app_asset_storage = _LazyAppAssetStorage(cache_key_prefix="app_assets")

12
api/core/sandbox/__init__.py
View File

@ -15,13 +15,13 @@ if TYPE_CHECKING:
from .builder import SandboxBuilder, VMConfig
from .entities import AppAssets, DifyCli, SandboxProviderApiEntity, SandboxType
from .initializer import (
AppAssetsInitializer,
AsyncSandboxInitializer,
DifyCliInitializer,
DraftAppAssetsInitializer,
SandboxInitializer,
SyncSandboxInitializer,
)
from .initializer.app_assets_initializer import AppAssetsInitializer
from .initializer.dify_cli_initializer import DifyCliInitializer
from .initializer.draft_app_assets_initializer import DraftAppAssetsInitializer
from .manager import SandboxManager
from .sandbox import Sandbox
from .storage import ArchiveSandboxStorage, SandboxStorage
@ -58,17 +58,17 @@ __all__ = [
_LAZY_IMPORTS = {
"AppAssets": ("core.sandbox.entities", "AppAssets"),
"AppAssetsInitializer": ("core.sandbox.initializer", "AppAssetsInitializer"),
"AppAssetsInitializer": ("core.sandbox.initializer.app_assets_initializer", "AppAssetsInitializer"),
"AsyncSandboxInitializer": ("core.sandbox.initializer", "AsyncSandboxInitializer"),
"ArchiveSandboxStorage": ("core.sandbox.storage", "ArchiveSandboxStorage"),
"DifyCli": ("core.sandbox.entities", "DifyCli"),
"DifyCliBinary": ("core.sandbox.bash.dify_cli", "DifyCliBinary"),
"DifyCliConfig": ("core.sandbox.bash.dify_cli", "DifyCliConfig"),
"DifyCliEnvConfig": ("core.sandbox.bash.dify_cli", "DifyCliEnvConfig"),
"DifyCliInitializer": ("core.sandbox.initializer", "DifyCliInitializer"),
"DifyCliInitializer": ("core.sandbox.initializer.dify_cli_initializer", "DifyCliInitializer"),
"DifyCliLocator": ("core.sandbox.bash.dify_cli", "DifyCliLocator"),
"DifyCliToolConfig": ("core.sandbox.bash.dify_cli", "DifyCliToolConfig"),
"DraftAppAssetsInitializer": ("core.sandbox.initializer", "DraftAppAssetsInitializer"),
"DraftAppAssetsInitializer": ("core.sandbox.initializer.draft_app_assets_initializer", "DraftAppAssetsInitializer"),
"Sandbox": ("core.sandbox.sandbox", "Sandbox"),
"SandboxBashSession": ("core.sandbox.bash.session", "SandboxBashSession"),
"SandboxBuilder": ("core.sandbox.builder", "SandboxBuilder"),

6
api/core/sandbox/initializer/__init__.py
View File

@ -1,13 +1,7 @@
from .app_assets_initializer import AppAssetsInitializer
from .base import AsyncSandboxInitializer, SandboxInitializer, SyncSandboxInitializer
from .dify_cli_initializer import DifyCliInitializer
from .draft_app_assets_initializer import DraftAppAssetsInitializer
__all__ = [
"AppAssetsInitializer",
"AsyncSandboxInitializer",
"DifyCliInitializer",
"DraftAppAssetsInitializer",
"SandboxInitializer",
"SyncSandboxInitializer",
]

10
api/core/sandbox/initializer/app_assets_initializer.py
View File

@ -1,10 +1,9 @@
import logging
from core.app_assets.constants import AppAssetsAttrs
from core.app_assets.storage import AssetPath, app_asset_storage
from core.app_assets.storage import AssetPath
from core.sandbox.sandbox import Sandbox
from core.virtual_environment.__base.helpers import pipeline
from services.app_asset_service import AppAssetService
from ..entities import AppAssets
from .base import AsyncSandboxInitializer
@ -21,12 +20,15 @@ class AppAssetsInitializer(AsyncSandboxInitializer):
self._assets_id = assets_id
def initialize(self, sandbox: Sandbox) -> None:
from services.app_asset_package_service import AppAssetPackageService
from services.app_asset_service import AppAssetService
# Load published app assets and unzip the artifact bundle.
app_assets = AppAssetService.get_tenant_app_assets(self._tenant_id, self._assets_id)
app_assets = AppAssetPackageService.get_tenant_app_assets(self._tenant_id, self._assets_id)
sandbox.attrs.set(AppAssetsAttrs.FILE_TREE, app_assets.asset_tree)
sandbox.attrs.set(AppAssetsAttrs.APP_ASSETS_ID, self._assets_id)
vm = sandbox.vm
asset_storage = app_asset_storage
asset_storage = AppAssetService.get_storage()
zip_ref = AssetPath.build_zip(self._tenant_id, self._app_id, self._assets_id)
download_url = asset_storage.get_download_url(zip_ref)

9
api/core/sandbox/initializer/draft_app_assets_initializer.py
View File

@ -1,12 +1,13 @@
import logging
from core.app_assets.constants import AppAssetsAttrs
from core.app_assets.storage import AssetPath, app_asset_storage
from core.app_assets.storage import AssetPath
from core.sandbox.entities import AppAssets
from core.sandbox.sandbox import Sandbox
from core.sandbox.services import AssetDownloadService
from core.sandbox.services.asset_download_service import AssetDownloadItem
from core.virtual_environment.__base.helpers import pipeline
from services.app_asset_package_service import AppAssetPackageService
from services.app_asset_service import AppAssetService
from .base import AsyncSandboxInitializer
@ -25,14 +26,14 @@ class DraftAppAssetsInitializer(AsyncSandboxInitializer):
def initialize(self, sandbox: Sandbox) -> None:
# Load published app assets and unzip the artifact bundle.
app_assets = AppAssetService.get_tenant_app_assets(self._tenant_id, self._assets_id)
app_assets = AppAssetPackageService.get_tenant_app_assets(self._tenant_id, self._assets_id)
sandbox.attrs.set(AppAssetsAttrs.FILE_TREE, app_assets.asset_tree)
sandbox.attrs.set(AppAssetsAttrs.APP_ASSETS_ID, self._assets_id)
vm = sandbox.vm
build_id = self._assets_id
tree = app_assets.asset_tree
storage = app_asset_storage
asset_storage = AppAssetService.get_storage()
nodes = list(tree.walk_files())
if not nodes:
return
@ -43,7 +44,7 @@ class DraftAppAssetsInitializer(AsyncSandboxInitializer):
else AssetPath.draft(self._tenant_id, self._app_id, node.id)
for node in nodes
]
urls = storage.get_download_urls(refs, DRAFT_ASSETS_EXPIRES_IN)
urls = asset_storage.get_download_urls(refs, DRAFT_ASSETS_EXPIRES_IN)
items = [AssetDownloadItem(path=tree.get_path(node.id).lstrip("/"), url=url) for node, url in zip(nodes, urls)]
script = AssetDownloadService.build_download_script(items, AppAssets.PATH)
pipeline(vm).add(

106
api/core/sandbox/manager.py
View File

@ -4,17 +4,7 @@ import logging
import threading
from typing import Final
from core.sandbox.builder import SandboxBuilder
from core.sandbox.entities import AppAssets, SandboxType
from core.sandbox.entities.providers import SandboxProviderEntity
from core.sandbox.initializer.app_assets_initializer import AppAssetsInitializer
from core.sandbox.initializer.dify_cli_initializer import DifyCliInitializer
from core.sandbox.initializer.draft_app_assets_initializer import DraftAppAssetsInitializer
from core.sandbox.initializer.skill_initializer import SkillInitializer
from core.sandbox.sandbox import Sandbox
from core.sandbox.storage.archive_storage import ArchiveSandboxStorage
from core.virtual_environment.__base.virtual_environment import VirtualEnvironment
from services.app_asset_service import AppAssetService
logger = logging.getLogger(__name__)
@ -103,99 +93,3 @@ class SandboxManager:
@classmethod
def count(cls) -> int:
return sum(len(shard) for shard in cls._shards)
@classmethod
def create(
cls,
tenant_id: str,
app_id: str,
user_id: str,
workflow_execution_id: str,
sandbox_provider: SandboxProviderEntity,
) -> Sandbox:
assets = AppAssetService.get_assets(tenant_id, app_id, user_id, is_draft=False)
if not assets:
raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}")
storage = ArchiveSandboxStorage(tenant_id, workflow_execution_id)
sandbox = (
SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type))
.options(sandbox_provider.config)
.user(user_id)
.app(app_id)
.initializer(AppAssetsInitializer(tenant_id, app_id, assets.id))
.initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id))
.initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id))
.storage(storage, assets.id)
.build()
)
logger.info("Sandbox created: id=%s, assets=%s", sandbox.vm.metadata.id, sandbox.assets_id)
return sandbox
@classmethod
def delete_draft_storage(cls, tenant_id: str, user_id: str) -> None:
storage = ArchiveSandboxStorage(tenant_id, SandboxBuilder.draft_id(user_id))
storage.delete()
@classmethod
def create_draft(
cls,
tenant_id: str,
app_id: str,
user_id: str,
sandbox_provider: SandboxProviderEntity,
) -> Sandbox:
assets = AppAssetService.get_assets(tenant_id, app_id, user_id, is_draft=True)
if not assets:
raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}")
AppAssetService.build_assets(tenant_id, app_id, assets)
sandbox_id = SandboxBuilder.draft_id(user_id)
storage = ArchiveSandboxStorage(tenant_id, sandbox_id, exclude_patterns=[AppAssets.PATH])
sandbox = (
SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type))
.options(sandbox_provider.config)
.user(user_id)
.app(app_id)
.initializer(DraftAppAssetsInitializer(tenant_id, app_id, assets.id))
.initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id))
.initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id))
.storage(storage, assets.id)
.build()
)
logger.info("Draft sandbox created: id=%s, assets=%s", sandbox.vm.metadata.id, sandbox.assets_id)
return sandbox
@classmethod
def create_for_single_step(
cls,
tenant_id: str,
app_id: str,
user_id: str,
sandbox_provider: SandboxProviderEntity,
) -> Sandbox:
assets = AppAssetService.get_assets(tenant_id, app_id, user_id, is_draft=True)
if not assets:
raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}")
AppAssetService.build_assets(tenant_id, app_id, assets)
sandbox_id = SandboxBuilder.draft_id(user_id)
storage = ArchiveSandboxStorage(tenant_id, sandbox_id, exclude_patterns=[AppAssets.PATH])
sandbox = (
SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type))
.options(sandbox_provider.config)
.user(user_id)
.app(app_id)
.initializer(DraftAppAssetsInitializer(tenant_id, app_id, assets.id))
.initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id))
.initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id))
.storage(storage, assets.id)
.build()
)
logger.info("Single-step sandbox created: id=%s, assets=%s", sandbox.vm.metadata.id, sandbox.assets_id)
return sandbox

9
api/core/skill/skill_manager.py
View File

@ -1,9 +1,8 @@
import logging
from core.app_assets.storage import AppAssetStorage, AssetPath
from core.app_assets.storage import AssetPath
from core.skill.entities.skill_bundle import SkillBundle
from extensions.ext_redis import redis_client
from extensions.ext_storage import storage
from services.app_asset_service import AppAssetService
logger = logging.getLogger(__name__)
@ -16,7 +15,7 @@ class SkillManager:
assets_id: str,
) -> SkillBundle:
asset_path = AssetPath.skill_bundle(tenant_id, app_id, assets_id)
data = AppAssetStorage(storage.storage_runner, redis_client=redis_client).load(asset_path)
data = AppAssetService.get_storage().load(asset_path)
return SkillBundle.model_validate_json(data)
@staticmethod
@ -27,7 +26,7 @@ class SkillManager:
bundle: SkillBundle,
) -> None:
asset_path = AssetPath.skill_bundle(tenant_id, app_id, assets_id)
AppAssetStorage(storage.storage_runner, redis_client=redis_client).save(
AppAssetService.get_storage().save(
asset_path,
bundle.model_dump_json(indent=2).encode("utf-8"),
)

5
api/core/zip_sandbox/__init__.py
View File

@ -1,6 +1,7 @@
from .session import SandboxArchiveFile, ZipSandbox
from .zip_sandbox import SandboxDownloadItem, SandboxFile, ZipSandbox
__all__ = [
"SandboxArchiveFile",
"SandboxDownloadItem",
"SandboxFile",
"ZipSandbox",
]

81
api/core/zip_sandbox/cli_strategy.py Normal file
View File

@ -0,0 +1,81 @@
from __future__ import annotations
import posixpath
from typing import TYPE_CHECKING
from core.virtual_environment.__base.exec import CommandExecutionError
from core.virtual_environment.__base.helpers import execute, try_execute
from .strategy import ZipStrategy
if TYPE_CHECKING:
from core.virtual_environment.__base.virtual_environment import VirtualEnvironment
class CliZipStrategy(ZipStrategy):
"""Strategy using native zip/unzip CLI commands."""
def is_available(self, vm: VirtualEnvironment) -> bool:
result = try_execute(vm, ["which", "zip"], timeout=10)
has_zip = bool(result.stdout and result.stdout.strip())
result = try_execute(vm, ["which", "unzip"], timeout=10)
has_unzip = bool(result.stdout and result.stdout.strip())
return has_zip and has_unzip
def zip(
self,
vm: VirtualEnvironment,
*,
src: str,
out_path: str,
cwd: str | None,
timeout: float,
) -> None:
if src in (".", ""):
result = try_execute(vm, ["zip", "-qr", out_path, "."], timeout=timeout, cwd=cwd)
if not result.is_error:
return
# zip exits with 12 when there is nothing to do; create empty zip
if result.exit_code == 12:
self._write_empty_zip(vm, out_path)
return
raise CommandExecutionError("Failed to create zip archive", result)
zip_cwd = posixpath.dirname(src) or "."
target = posixpath.basename(src)
result = try_execute(vm, ["zip", "-qr", out_path, target], timeout=timeout, cwd=zip_cwd)
if not result.is_error:
return
if result.exit_code == 12:
self._write_empty_zip(vm, out_path)
return
raise CommandExecutionError("Failed to create zip archive", result)
def unzip(
self,
vm: VirtualEnvironment,
*,
archive_path: str,
dest_dir: str,
timeout: float,
) -> None:
execute(
vm,
["unzip", "-q", archive_path, "-d", dest_dir],
timeout=timeout,
error_message="Failed to unzip archive",
)
def _write_empty_zip(self, vm: VirtualEnvironment, out_path: str) -> None:
"""Write an empty but valid zip file."""
script = (
'printf "'
"\\x50\\x4b\\x05\\x06"
"\\x00\\x00\\x00\\x00"
"\\x00\\x00\\x00\\x00"
"\\x00\\x00\\x00\\x00"
"\\x00\\x00\\x00\\x00"
"\\x00\\x00\\x00\\x00"
'" > "$1"'
)
execute(vm, ["sh", "-c", script, "sh", out_path], timeout=30, error_message="Failed to write empty zip")

106
api/core/zip_sandbox/node_strategy.py Normal file
View File

@ -0,0 +1,106 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from core.virtual_environment.__base.helpers import execute, try_execute
from .strategy import ZipStrategy
if TYPE_CHECKING:
from core.virtual_environment.__base.virtual_environment import VirtualEnvironment
ZIP_SCRIPT = r"""
const fs = require('fs');
const path = require('path');
const AdmZip = require('adm-zip');
const src = process.argv[2];
const outPath = process.argv[3];
function walkAdd(zip, absPath, arcPrefix) {
const stat = fs.statSync(absPath);
if (stat.isDirectory()) {
const entries = fs.readdirSync(absPath);
if (entries.length === 0) {
zip.addFile(arcPrefix.replace(/\\/g, '/') + '/', Buffer.alloc(0));
return;
}
for (const e of entries) {
walkAdd(zip, path.join(absPath, e), path.posix.join(arcPrefix, e));
}
return;
}
if (stat.isFile()) {
const data = fs.readFileSync(absPath);
zip.addFile(arcPrefix.replace(/\\/g, '/'), data);
}
}
const zip = new AdmZip();
if (src === '.' || src === '') {
const entries = fs.readdirSync('.');
for (const e of entries) {
walkAdd(zip, path.join('.', e), e);
}
} else {
const base = path.dirname(src) || '.';
const prefix = path.basename(src.replace(/\/+$/, ''));
const root = path.join(base, prefix);
walkAdd(zip, root, prefix);
}
zip.writeZip(outPath);
"""
UNZIP_SCRIPT = r"""
const AdmZip = require('adm-zip');
const archivePath = process.argv[2];
const destDir = process.argv[3];
const zip = new AdmZip(archivePath);
zip.extractAllTo(destDir, true);
"""
class NodeZipStrategy(ZipStrategy):
"""Strategy using Node.js with adm-zip package."""
def is_available(self, vm: VirtualEnvironment) -> bool:
result = try_execute(vm, ["which", "node"], timeout=10)
if not (result.stdout and result.stdout.strip()):
return False
# Check if adm-zip module is available
result = try_execute(vm, ["node", "-e", "require('adm-zip')"], timeout=10)
return not result.is_error
def zip(
self,
vm: VirtualEnvironment,
*,
src: str,
out_path: str,
cwd: str | None,
timeout: float,
) -> None:
execute(
vm,
["node", "-e", ZIP_SCRIPT, src, out_path],
timeout=timeout,
cwd=cwd,
error_message="Failed to create zip archive",
)
def unzip(
self,
vm: VirtualEnvironment,
*,
archive_path: str,
dest_dir: str,
timeout: float,
) -> None:
execute(
vm,
["node", "-e", UNZIP_SCRIPT, archive_path, dest_dir],
timeout=timeout,
error_message="Failed to unzip archive",
)

117
api/core/zip_sandbox/python_strategy.py Normal file
View File

@ -0,0 +1,117 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from core.virtual_environment.__base.helpers import execute, try_execute
from .strategy import ZipStrategy
if TYPE_CHECKING:
from core.virtual_environment.__base.virtual_environment import VirtualEnvironment
ZIP_SCRIPT = r"""
import os
import sys
import zipfile
src = sys.argv[1]
out_path = sys.argv[2]
def is_cwd(p: str) -> bool:
return p in (".", "")
src = src.rstrip("/")
if is_cwd(src):
base = "."
root = "."
prefix = ""
else:
base = os.path.dirname(src) or "."
prefix = os.path.basename(src)
root = os.path.join(base, prefix)
def add_empty_dir(zf: zipfile.ZipFile, arc_dir: str) -> None:
name = arc_dir.rstrip("/") + "/"
if name != "/":
zf.writestr(name, b"")
with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
if os.path.isfile(root):
zf.write(root, arcname=os.path.basename(root))
else:
for dirpath, dirnames, filenames in os.walk(root):
rel_dir = os.path.relpath(dirpath, base)
rel_dir = "" if rel_dir == "." else rel_dir
if not dirnames and not filenames:
add_empty_dir(zf, rel_dir)
for fn in filenames:
fp = os.path.join(dirpath, fn)
arcname = os.path.join(rel_dir, fn) if rel_dir else fn
zf.write(fp, arcname=arcname)
"""
UNZIP_SCRIPT = r"""
import sys
import zipfile
archive_path = sys.argv[1]
dest_dir = sys.argv[2]
with zipfile.ZipFile(archive_path, "r") as zf:
zf.extractall(dest_dir)
"""
class PythonZipStrategy(ZipStrategy):
"""Strategy using Python's zipfile module."""
def __init__(self) -> None:
self._python_cmd: str | None = None
def is_available(self, vm: VirtualEnvironment) -> bool:
for cmd in ("python3", "python"):
result = try_execute(vm, ["which", cmd], timeout=10)
if result.stdout and result.stdout.strip():
self._python_cmd = cmd
return True
return False
def zip(
self,
vm: VirtualEnvironment,
*,
src: str,
out_path: str,
cwd: str | None,
timeout: float,
) -> None:
if self._python_cmd is None:
raise RuntimeError("Python not available")
execute(
vm,
[self._python_cmd, "-c", ZIP_SCRIPT, src, out_path],
timeout=timeout,
cwd=cwd,
error_message="Failed to create zip archive",
)
def unzip(
self,
vm: VirtualEnvironment,
*,
archive_path: str,
dest_dir: str,
timeout: float,
) -> None:
if self._python_cmd is None:
raise RuntimeError("Python not available")
execute(
vm,
[self._python_cmd, "-c", UNZIP_SCRIPT, archive_path, dest_dir],
timeout=timeout,
error_message="Failed to unzip archive",
)

41
api/core/zip_sandbox/strategy.py Normal file
View File

@ -0,0 +1,41 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from core.virtual_environment.__base.virtual_environment import VirtualEnvironment
class ZipStrategy(ABC):
"""Abstract base class for zip/unzip strategies."""
@abstractmethod
def is_available(self, vm: VirtualEnvironment) -> bool:
"""Check if this strategy is available in the given VM."""
...
@abstractmethod
def zip(
self,
vm: VirtualEnvironment,
*,
src: str,
out_path: str,
cwd: str | None,
timeout: float,
) -> None:
"""Create a zip archive."""
...
@abstractmethod
def unzip(
self,
vm: VirtualEnvironment,
*,
archive_path: str,
dest_dir: str,
timeout: float,
) -> None:
"""Extract a zip archive."""
...

180
api/core/zip_sandbox/session.py → api/core/zip_sandbox/zip_sandbox.py
View File

@ -1,6 +1,5 @@
from __future__ import annotations
import hashlib
import posixpath
from dataclasses import dataclass
from io import BytesIO
@ -20,26 +19,38 @@ from core.virtual_environment.__base.helpers import execute, pipeline
from core.virtual_environment.__base.virtual_environment import VirtualEnvironment
from services.sandbox.sandbox_provider_service import SandboxProviderService
from .cli_strategy import CliZipStrategy
from .node_strategy import NodeZipStrategy
from .python_strategy import PythonZipStrategy
from .strategy import ZipStrategy
@dataclass(frozen=True)
class SandboxArchiveFile:
file_path: str
size_bytes: int
sha256: str
class SandboxDownloadItem:
url: str
path: str
@dataclass(frozen=True)
class SandboxFile:
"""A handle to a file in the sandbox."""
path: str
class ZipSandbox:
"""A sandbox specifically for archive (tar) operations.
"""A sandbox for archive (zip) operations.
Usage:
with ZipSandbox(tenant_id=..., user_id=...) as zs:
zs.write_file("a.txt", b"hello")
archive = zs.tar()
zs.upload(path=archive.file_path, target_url=url)
zs.download_items(items)
archive = zs.zip()
zs.upload(archive, upload_url)
# VM automatically released on exit
"""
_DEFAULT_TIMEOUT_SECONDS = 60 * 5
_STRATEGIES: list[ZipStrategy] = [CliZipStrategy(), PythonZipStrategy(), NodeZipStrategy()]
def __init__(
self,
@ -49,7 +60,6 @@ class ZipSandbox:
app_id: str = "zip-sandbox",
sandbox_provider_type: str | None = None,
sandbox_provider_options: dict[str, Any] | None = None,
# For testing: allow injecting a VM directly
_vm: VirtualEnvironment | None = None,
) -> None:
self._tenant_id = tenant_id
@ -62,6 +72,7 @@ class ZipSandbox:
self._sandbox: Sandbox | None = None
self._sandbox_id: str | None = None
self._vm: VirtualEnvironment | None = None
self._strategy: ZipStrategy | None = None
def __enter__(self) -> ZipSandbox:
self._start()
@ -79,7 +90,6 @@ class ZipSandbox:
if self._vm is not None:
raise RuntimeError("ZipSandbox already started")
# If VM is injected (for testing), use it directly
if self._injected_vm is not None:
self._vm = self._injected_vm
self._sandbox_id = uuid4().hex
@ -127,6 +137,7 @@ class ZipSandbox:
self._vm = None
self._sandbox = None
self._sandbox_id = None
self._strategy = None
@property
def vm(self) -> VirtualEnvironment:
@ -134,10 +145,21 @@ class ZipSandbox:
raise RuntimeError("ZipSandbox not started. Use 'with ZipSandbox(...) as zs:'")
return self._vm
def _get_strategy(self) -> ZipStrategy:
if self._strategy is not None:
return self._strategy
for strategy in self._STRATEGIES:
if strategy.is_available(self.vm):
self._strategy = strategy
return strategy
raise RuntimeError("No available zip backend (zip/python/node+adm-zip)")
# ========== Path utilities ==========
@staticmethod
def _normalize_workspace_path(path: str | None) -> str:
def _normalize_path(path: str | None) -> str:
raw = (path or ".").strip()
if raw == "":
raw = "."
@ -163,7 +185,7 @@ class ZipSandbox:
# ========== File operations ==========
def write_file(self, path: str, data: bytes) -> None:
path = self._normalize_workspace_path(path)
path = self._normalize_path(path)
if path in ("", "."):
raise ValueError("path must point to a file")
@ -173,7 +195,7 @@ class ZipSandbox:
raise RuntimeError(f"Failed to write file to sandbox: {exc}") from exc
def read_file(self, path: str, *, max_bytes: int = 10 * 1024 * 1024) -> bytes:
path = self._normalize_workspace_path(path)
path = self._normalize_path(path)
if max_bytes <= 0:
raise ValueError("max_bytes must be positive")
@ -192,9 +214,9 @@ class ZipSandbox:
if not urls:
return []
dest_dir = self._normalize_workspace_path(dest_dir)
dest_dir = self._normalize_path(dest_dir)
paths = [self._dest_path_for_url(dest_dir, u) for u in urls]
p = pipeline(self.vm)
p.add(["mkdir", "-p", dest_dir], error_message="Failed to create download directory")
for url, out_path in zip(urls, paths, strict=True):
@ -207,14 +229,42 @@ class ZipSandbox:
return paths
def download_items(self, items: list[SandboxDownloadItem], *, dest_dir: str = ".") -> list[str]:
if not items:
return []
dest_dir = self._normalize_path(dest_dir)
p = pipeline(self.vm)
p.add(["mkdir", "-p", dest_dir], error_message="Failed to create download directory")
out_paths: list[str] = []
for item in items:
rel = self._normalize_path(item.path)
if rel in ("", "."):
raise ValueError("Download item path must point to a file")
out_path = posixpath.join(dest_dir, rel)
out_paths.append(out_path)
out_dir = posixpath.dirname(out_path)
if out_dir not in ("", "."):
p.add(["mkdir", "-p", out_dir], error_message="Failed to create download directory")
p.add(["curl", "-fsSL", item.url, "-o", out_path], error_message="Failed to download file")
try:
p.execute(timeout=self._DEFAULT_TIMEOUT_SECONDS, raise_on_error=True)
except Exception as exc:
raise RuntimeError(str(exc)) from exc
return out_paths
def download_archive(self, archive_url: str, *, path: str = "input.tar.gz") -> str:
path = self._normalize_workspace_path(path)
path = self._normalize_path(path)
dir_path = posixpath.dirname(path)
p = pipeline(self.vm)
if dir_path not in ("", "."):
p.add(["mkdir", "-p", dir_path], error_message=f"Failed to create archive download directory {dir_path}")
p.add(["mkdir", "-p", dir_path], error_message=f"Failed to create directory {dir_path}")
p.add(["curl", "-fsSL", archive_url, "-o", path], error_message=f"Failed to download archive to {path}")
try:
p.execute(timeout=self._DEFAULT_TIMEOUT_SECONDS, raise_on_error=True)
except Exception as exc:
@ -224,15 +274,12 @@ class ZipSandbox:
# ========== Upload operations ==========
def upload(self, *, path: str, target_url: str) -> None:
path = self._normalize_workspace_path(path)
if path in ("", "."):
raise ValueError("path must point to a file")
def upload(self, file: SandboxFile, target_url: str) -> None:
"""Upload a sandbox file to the given URL."""
try:
execute(
self.vm,
["curl", "-fsSL", "-X", "PUT", "-T", path, target_url],
["curl", "-fsSL", "-X", "PUT", "-T", file.path, target_url],
timeout=self._DEFAULT_TIMEOUT_SECONDS,
error_message="Failed to upload file from sandbox",
)
@ -241,55 +288,58 @@ class ZipSandbox:
# ========== Archive operations ==========
def tar(self, src: str = ".", *, out_path: str | None = None) -> SandboxArchiveFile:
src = self._normalize_workspace_path(src)
if out_path is None:
out_path = f"{uuid4().hex}.tar"
out_path = self._normalize_workspace_path(out_path)
lower_out = out_path.lower()
if not (lower_out.endswith(".tar") or lower_out.endswith(".tar.gz") or lower_out.endswith(".tgz")):
raise ValueError("out_path must end with .tar/.tar.gz/.tgz")
def zip(self, src: str = ".", *, include_base: bool = True) -> SandboxFile:
"""Create a zip archive and return a handle to it."""
src = self._normalize_path(src)
out_path = f"/tmp/{uuid4().hex}.zip"
out_dir = posixpath.dirname(out_path)
is_gz = lower_out.endswith(".tar.gz") or lower_out.endswith(".tgz")
tar_flag = "-czf" if is_gz else "-cf"
is_cwd = src in (".", "")
# Avoid "archive cannot contain itself" when archiving the current directory.
# Create the archive outside the workspace tree and move it into place.
tmp_archive = f"/tmp/{uuid4().hex}{'.tar.gz' if is_gz else '.tar'}"
cwd = None
src_for_strategy = src
if src not in (".", "") and not include_base:
cwd = src
src_for_strategy = "."
try:
(
pipeline(self.vm)
.add(
["mkdir", "-p", out_dir],
error_message="Failed to create archive output directory",
on=out_dir not in ("", "."),
)
.add(
["tar", tar_flag, tmp_archive, "-C", ".", "."],
error_message="Failed to create tar archive",
on=is_cwd,
)
.add(["tar", tar_flag, tmp_archive, src], error_message="Failed to create tar archive", on=not is_cwd)
.add(["mv", "-f", tmp_archive, out_path], error_message="Failed to move tar archive into place")
.execute(timeout=self._DEFAULT_TIMEOUT_SECONDS, raise_on_error=True)
self._get_strategy().zip(
self.vm,
src=src_for_strategy,
out_path=out_path,
cwd=cwd,
timeout=self._DEFAULT_TIMEOUT_SECONDS,
)
except PipelineExecutionError as exc:
except (PipelineExecutionError, CommandExecutionError) as exc:
raise RuntimeError(str(exc)) from exc
# Compute size + sha256 on host side (avoid requiring sha256sum in sandbox).
try:
data = self.vm.download_file(out_path).getvalue()
except Exception as exc:
raise RuntimeError(f"Failed to read tar result from sandbox: {exc}") from exc
return SandboxFile(path=out_path)
return SandboxArchiveFile(file_path=out_path, size_bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
def unzip(self, *, archive_path: str, dest_dir: str = "unpacked") -> str:
"""Extract a zip archive to the destination directory."""
archive_path = self._normalize_path(archive_path)
dest_dir = self._normalize_path(dest_dir)
if not archive_path.lower().endswith(".zip"):
raise ValueError("archive_path must end with .zip")
try:
pipeline(self.vm).add(
["mkdir", "-p", dest_dir], error_message="Failed to create destination directory"
).execute(timeout=self._DEFAULT_TIMEOUT_SECONDS, raise_on_error=True)
self._get_strategy().unzip(
self.vm,
archive_path=archive_path,
dest_dir=dest_dir,
timeout=self._DEFAULT_TIMEOUT_SECONDS,
)
except (PipelineExecutionError, CommandExecutionError) as exc:
raise RuntimeError(str(exc)) from exc
return dest_dir
def untar(self, *, archive_path: str, dest_dir: str = "unpacked") -> str:
archive_path = self._normalize_workspace_path(archive_path)
dest_dir = self._normalize_workspace_path(dest_dir)
"""Extract a tar archive to the destination directory."""
archive_path = self._normalize_path(archive_path)
dest_dir = self._normalize_path(dest_dir)
lower = archive_path.lower()
is_gz = lower.endswith(".tar.gz") or lower.endswith(".tgz")
@ -298,7 +348,7 @@ class ZipSandbox:
try:
(
pipeline(self.vm)
.add(["mkdir", "-p", dest_dir], error_message="Failed to create untar destination directory")
.add(["mkdir", "-p", dest_dir], error_message="Failed to create destination directory")
.add(["tar", extract_flag, archive_path, "-C", dest_dir], error_message="Failed to extract tar archive")
.execute(timeout=self._DEFAULT_TIMEOUT_SECONDS, raise_on_error=True)
)

185
api/services/app_asset_package_service.py Normal file
View File

@ -0,0 +1,185 @@
"""Service for packaging and publishing app assets.
This service handles operations that require core.zip_sandbox,
separated from AppAssetService to avoid circular imports.
Dependency flow:
core/* -> AppAssetPackageService -> AppAssetService
(core modules can import this service without circular dependency)
"""
import logging
from uuid import uuid4
from sqlalchemy.orm import Session
from core.app.entities.app_asset_entities import AppAssetFileTree
from core.app_assets.builder import AssetBuildPipeline, BuildContext
from core.app_assets.builder.file_builder import FileBuilder
from core.app_assets.builder.skill_builder import SkillBuilder
from core.app_assets.entities.assets import AssetItem, FileAsset
from core.app_assets.storage import AssetPath
from core.zip_sandbox import SandboxDownloadItem, ZipSandbox
from models.app_asset import AppAssets
from models.model import App
logger = logging.getLogger(__name__)
class AppAssetPackageService:
"""Service for packaging and publishing app assets.
This service is designed to be imported by core/* modules without
causing circular imports. It depends on AppAssetService for basic
asset operations but provides the packaging/publishing functionality
that requires core.zip_sandbox.
"""
@staticmethod
def get_tenant_app_assets(tenant_id: str, assets_id: str) -> AppAssets:
"""Get app assets by tenant_id and assets_id.
This is a read-only operation that doesn't require AppAssetService.
"""
from extensions.ext_database import db
with Session(db.engine, expire_on_commit=False) as session:
app_assets = (
session.query(AppAssets)
.filter(
AppAssets.tenant_id == tenant_id,
AppAssets.id == assets_id,
)
.first()
)
if not app_assets:
raise ValueError(f"App assets not found for tenant_id={tenant_id}, assets_id={assets_id}")
return app_assets
@staticmethod
def get_draft_asset_items(tenant_id: str, app_id: str, file_tree: AppAssetFileTree) -> list[AssetItem]:
"""Convert file tree to asset items for packaging."""
files = file_tree.walk_files()
return [
FileAsset(
asset_id=f.id,
path=file_tree.get_path(f.id),
file_name=f.name,
extension=f.extension,
storage_key=AssetPath.draft(tenant_id, app_id, f.id).get_storage_key(),
)
for f in files
]
@staticmethod
def package_and_upload(
*,
assets: list[AssetItem],
upload_url: str,
tenant_id: str,
user_id: str,
) -> None:
"""Package assets into a ZIP and upload directly to the given URL."""
from services.app_asset_service import AppAssetService
if not assets:
import io
import zipfile
import requests
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w"):
pass
buf.seek(0)
requests.put(upload_url, data=buf.getvalue(), timeout=30)
return
asset_storage = AppAssetService.get_storage()
storage_keys = [a.get_storage_key() for a in assets]
download_urls = asset_storage.storage.get_download_urls(storage_keys, 10 * 60)
download_items = [
SandboxDownloadItem(url=url, path=a.path) for a, url in zip(assets, download_urls, strict=True)
]
with ZipSandbox(tenant_id=tenant_id, user_id=user_id, app_id="asset-packager") as zs:
zs.download_items(download_items)
archive = zs.zip()
zs.upload(archive, upload_url)
@staticmethod
def publish(session: Session, app_model: App, account_id: str, workflow_id: str) -> AppAssets:
"""Publish app assets for a workflow.
Creates a versioned copy of draft assets and packages them for runtime use.
"""
from services.app_asset_service import AppAssetService
tenant_id = app_model.tenant_id
app_id = app_model.id
assets = AppAssetService.get_or_create_assets(session, app_model, account_id)
tree = assets.asset_tree
publish_id = str(uuid4())
published = AppAssets(
id=publish_id,
tenant_id=tenant_id,
app_id=app_id,
version=workflow_id,
created_by=account_id,
)
published.asset_tree = tree
session.add(published)
session.flush()
asset_storage = AppAssetService.get_storage()
ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=publish_id)
built_assets = AssetBuildPipeline([SkillBuilder(storage=asset_storage), FileBuilder()]).build_all(tree, ctx)
runtime_zip_path = AssetPath.build_zip(tenant_id, app_id, publish_id)
runtime_upload_url = asset_storage.get_upload_url(runtime_zip_path)
AppAssetPackageService.package_and_upload(
assets=built_assets,
upload_url=runtime_upload_url,
tenant_id=tenant_id,
user_id=account_id,
)
source_items = AppAssetService.get_draft_assets(tenant_id, app_id)
source_zip_path = AssetPath.source_zip(tenant_id, app_id, workflow_id)
source_upload_url = asset_storage.get_upload_url(source_zip_path)
AppAssetPackageService.package_and_upload(
assets=source_items,
upload_url=source_upload_url,
tenant_id=tenant_id,
user_id=account_id,
)
return published
@staticmethod
def build_assets(tenant_id: str, app_id: str, assets: AppAssets) -> None:
"""Build resolved draft assets without packaging into a zip."""
from services.app_asset_service import AppAssetService
tree = assets.asset_tree
asset_storage = AppAssetService.get_storage()
ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=assets.id)
built_assets: list[AssetItem] = AssetBuildPipeline(
[SkillBuilder(storage=asset_storage), FileBuilder()]
).build_all(tree, ctx)
user_id = getattr(assets, "updated_by", None) or getattr(assets, "created_by", None) or "system"
zip_path = AssetPath.build_zip(tenant_id, app_id, assets.id)
upload_url = asset_storage.get_upload_url(zip_path)
AppAssetPackageService.package_and_upload(
assets=built_assets,
upload_url=upload_url,
tenant_id=tenant_id,
user_id=user_id,
)

128
api/services/app_asset_service.py
View File

@ -13,15 +13,12 @@ from core.app.entities.app_asset_entities import (
TreeParentNotFoundError,
TreePathConflictError,
)
from core.app_assets.builder import AssetBuildPipeline, BuildContext
from core.app_assets.builder.file_builder import FileBuilder
from core.app_assets.builder.skill_builder import SkillBuilder
from core.app_assets.converters import tree_to_asset_items
from core.app_assets.entities.assets import AssetItem
from core.app_assets.packager import AssetZipPackager
from core.app_assets.storage import AssetPath, app_asset_storage
from core.app_assets.entities.assets import AssetItem, FileAsset
from core.app_assets.storage import AppAssetStorage, AssetPath
from extensions.ext_database import db
from extensions.ext_redis import redis_client
from extensions.ext_storage import storage
from extensions.storage.silent_storage import SilentStorage
from models.app_asset import AppAssets
from models.model import App
@ -40,10 +37,53 @@ class AppAssetService:
_LOCK_TIMEOUT_SECONDS = 60
_DRAFT_CACHE_KEY_PREFIX = "app_asset:draft_download"
@staticmethod
def get_storage() -> AppAssetStorage:
"""Get a lazily-initialized AppAssetStorage instance.
This method creates an AppAssetStorage each time it's called,
ensuring storage.storage_runner is only accessed after init_app.
"""
return AppAssetStorage(
storage=SilentStorage(storage.storage_runner),
redis_client=redis_client,
cache_key_prefix="app_assets",
)
@staticmethod
def _lock(app_id: str):
return redis_client.lock(f"app_asset:lock:{app_id}", timeout=AppAssetService._LOCK_TIMEOUT_SECONDS)
@staticmethod
def get_draft_assets(tenant_id: str, app_id: str) -> list[AssetItem]:
with Session(db.engine) as session:
assets = (
session.query(AppAssets)
.filter(
AppAssets.tenant_id == tenant_id,
AppAssets.app_id == app_id,
AppAssets.version == AppAssets.VERSION_DRAFT,
)
.first()
)
if not assets:
return []
return AppAssetService.get_draft_asset_items(assets.tenant_id, assets.app_id, assets.asset_tree)
@staticmethod
def get_draft_asset_items(tenant_id: str, app_id: str, file_tree: AppAssetFileTree) -> list[AssetItem]:
files = file_tree.walk_files()
return [
FileAsset(
asset_id=f.id,
path=file_tree.get_path(f.id),
file_name=f.name,
extension=f.extension,
storage_key=AssetPath.draft(tenant_id, app_id, f.id).get_storage_key(),
)
for f in files
]
@staticmethod
def get_or_create_assets(session: Session, app_model: App, account_id: str) -> AppAssets:
assets = (
@ -161,7 +201,7 @@ class AppAssetService:
max_size_mb = AppAssetService.MAX_PREVIEW_CONTENT_SIZE / 1024 / 1024
raise AppAssetNodeTooLargeError(f"File node {node_id} size exceeded the limit: {max_size_mb} MB")
asset_storage = app_asset_storage
asset_storage = AppAssetService.get_storage()
asset_path = AssetPath.draft(app_model.tenant_id, app_model.id, node_id)
return asset_storage.load(asset_path)
@ -182,7 +222,7 @@ class AppAssetService:
except TreeNodeNotFoundError as e:
raise AppAssetNodeNotFoundError(str(e)) from e
asset_storage = app_asset_storage
asset_storage = AppAssetService.get_storage()
asset_path = AssetPath.draft(app_model.tenant_id, app_model.id, node_id)
asset_storage.save(asset_path, content)
@ -282,7 +322,7 @@ class AppAssetService:
# FIXME(Mairuis): sync deletion queue, failed is fine
def _delete_file_from_storage(tenant_id: str, app_id: str, node_ids: list[str]) -> None:
asset_storage = app_asset_storage
asset_storage = AppAssetService.get_storage()
for nid in node_ids:
asset_path = AssetPath.draft(tenant_id, app_id, nid)
try:
@ -290,7 +330,7 @@ class AppAssetService:
except Exception:
logger.warning(
"Failed to delete storage file %s",
asset_storage.get_storage_key(asset_path),
asset_path.get_storage_key(),
exc_info=True,
)
@ -298,62 +338,6 @@ class AppAssetService:
target=lambda: _delete_file_from_storage(app_model.tenant_id, app_model.id, removed_ids)
).start()
@staticmethod
def publish(session: Session, app_model: App, account_id: str, workflow_id: str) -> AppAssets:
tenant_id = app_model.tenant_id
app_id = app_model.id
assets = AppAssetService.get_or_create_assets(session, app_model, account_id)
tree = assets.asset_tree
publish_id = str(uuid4())
published = AppAssets(
id=publish_id,
tenant_id=tenant_id,
app_id=app_id,
version=workflow_id,
created_by=account_id,
)
published.asset_tree = tree
session.add(published)
session.flush()
asset_storage = app_asset_storage
ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=publish_id)
built_assets = AssetBuildPipeline(
[SkillBuilder(storage=asset_storage), FileBuilder(storage=asset_storage)]
).build_all(tree, ctx)
packager = AssetZipPackager(asset_storage.storage)
runtime_zip_bytes = packager.package(built_assets)
runtime_zip_path = AssetPath.build_zip(tenant_id, app_id, publish_id)
asset_storage.save(runtime_zip_path, runtime_zip_bytes)
source_items = tree_to_asset_items(tree, tenant_id, app_id, asset_storage)
source_zip_bytes = packager.package(source_items)
source_zip_path = AssetPath.source_zip(tenant_id, app_id, workflow_id)
asset_storage.save(source_zip_path, source_zip_bytes)
return published
@staticmethod
def build_assets(tenant_id: str, app_id: str, assets: AppAssets) -> None:
# Build resolved draft assets without packaging into a zip.
tree = assets.asset_tree
asset_storage = app_asset_storage
ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=assets.id)
built_assets: list[AssetItem] = AssetBuildPipeline(
[SkillBuilder(storage=asset_storage), FileBuilder(storage=asset_storage)]
).build_all(tree, ctx)
packager = AssetZipPackager(storage=asset_storage.storage)
zip_bytes = packager.package(built_assets)
zip_path = AssetPath.build_zip(tenant_id, app_id, assets.id)
asset_storage.save(zip_path, zip_bytes)
@staticmethod
def get_file_download_url(
app_model: App,
@ -369,17 +353,17 @@ class AppAssetService:
if not node or node.node_type != AssetNodeType.FILE:
raise AppAssetNodeNotFoundError(f"File node {node_id} not found")
asset_storage = app_asset_storage
asset_storage = AppAssetService.get_storage()
asset_path = AssetPath.draft(app_model.tenant_id, app_model.id, node_id)
return asset_storage.get_download_url(asset_path, expires_in)
@staticmethod
def get_source_zip_bytes(tenant_id: str, app_id: str, workflow_id: str) -> bytes | None:
asset_storage = app_asset_storage
asset_storage = AppAssetService.get_storage()
asset_path = AssetPath.source_zip(tenant_id, app_id, workflow_id)
source_zip = asset_storage.load_or_none(asset_path)
if source_zip is None:
logger.warning("Source zip not found: %s", asset_storage.get_storage_key(asset_path))
logger.warning("Source zip not found: %s", asset_path.get_storage_key())
return source_zip
@staticmethod
@ -435,7 +419,7 @@ class AppAssetService:
session.commit()
asset_path = AssetPath.draft(app_model.tenant_id, app_model.id, node_id)
asset_storage = app_asset_storage
asset_storage = AppAssetService.get_storage()
# put empty content to create the file record
# which avoids file not found error when uploading via presigned URL is never touched
@ -477,7 +461,7 @@ class AppAssetService:
assets.updated_by = account_id
session.commit()
asset_storage = app_asset_storage
asset_storage = AppAssetService.get_storage()
def fill_urls(node: BatchUploadNode) -> None:
if node.node_type == AssetNodeType.FILE and node.id:

110
api/services/app_bundle_service.py
View File

@ -4,6 +4,7 @@ import io
import logging
import re
import zipfile
from uuid import uuid4
import yaml
from sqlalchemy.orm import Session
@ -15,13 +16,13 @@ from core.app.entities.app_bundle_entities import (
BundleFormatError,
ZipSecurityError,
)
from core.app_assets.converters import tree_to_asset_items
from core.app_assets.packager import AssetZipPackager
from core.app_assets.storage import app_asset_storage
from core.app_assets.storage import AssetPath
from core.app_bundle import SourceZipExtractor
from core.zip_sandbox import SandboxDownloadItem, ZipSandbox
from extensions.ext_database import db
from models import Account, App
from .app_asset_package_service import AppAssetPackageService
from .app_asset_service import AppAssetService
from .app_dsl_service import AppDslService, Import
@ -54,7 +55,7 @@ class AppBundleService:
)
# 2. Publish assets (bound to workflow_id)
AppAssetService.publish(
AppAssetPackageService.publish(
session=session,
app_model=app_model,
account_id=account.id,
@ -65,31 +66,61 @@ class AppBundleService:
@staticmethod
def export_bundle(
*,
app_model: App,
account_id: str,
include_secret: bool = False,
workflow_id: str | None = None,
expires_in: int = 10 * 60,
) -> BundleExportResult:
"""Export bundle and return a temporary download URL.
Uses sandbox VM to build the ZIP, avoiding memory pressure in API process.
"""
tenant_id = app_model.tenant_id
app_id = app_model.id
safe_name = AppBundleService._sanitize_filename(app_model.name)
filename = f"{safe_name}.zip"
export_id = uuid4().hex
export_path = AssetPath.bundle_export_zip(tenant_id, app_id, export_id)
asset_storage = AppAssetService.get_storage()
upload_url = asset_storage.get_upload_url(export_path, expires_in)
dsl_content = AppDslService.export_dsl(
app_model=app_model,
include_secret=include_secret,
workflow_id=workflow_id,
)
safe_name = AppBundleService._sanitize_filename(app_model.name)
assets_prefix = safe_name
with ZipSandbox(tenant_id=tenant_id, user_id=account_id, app_id="app-bundle-export") as zs:
zs.write_file(f"bundle_root/{safe_name}.yml", dsl_content.encode("utf-8"))
zip_buffer = io.BytesIO()
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
zf.writestr(f"{safe_name}.yml", dsl_content.encode("utf-8"))
# Published assets: use stored source zip and unzip into <safe_name>/...
if workflow_id is not None:
source_zip_path = AssetPath.source_zip(tenant_id, app_id, workflow_id)
source_url = asset_storage.get_download_url(source_zip_path, expires_in)
zs.download_archive(source_url, path="tmp/source_assets.zip")
zs.unzip(archive_path="tmp/source_assets.zip", dest_dir=f"bundle_root/{safe_name}")
else:
# Draft assets: download individual files and place under <safe_name>/...
asset_items = AppAssetService.get_draft_assets(tenant_id, app_id)
asset_urls = asset_storage.get_download_urls(
[AssetPath.draft(tenant_id, app_id, a.asset_id) for a in asset_items], expires_in
)
zs.download_items(
[
SandboxDownloadItem(url=url, path=f"{safe_name}/{a.path}")
for a, url in zip(asset_items, asset_urls, strict=True)
],
dest_dir="bundle_root",
)
assets_zip_bytes = AppBundleService._get_assets_zip_bytes(app_model, workflow_id)
if assets_zip_bytes:
AppBundleService._merge_assets_into_bundle(zf, assets_zip_bytes, assets_prefix)
archive = zs.zip(src="bundle_root", include_base=False)
zs.upload(archive, upload_url)
return BundleExportResult(
zip_bytes=zip_buffer.getvalue(),
filename=f"{safe_name}.zip",
)
download_url = asset_storage.get_download_url(export_path, expires_in)
return BundleExportResult(download_url=download_url, filename=filename)
@staticmethod
def import_bundle(
@ -131,51 +162,6 @@ class AppBundleService:
return import_result
@staticmethod
def _get_assets_zip_bytes(app_model: App, workflow_id: str | None) -> bytes | None:
tenant_id = app_model.tenant_id
app_id = app_model.id
if workflow_id is None:
return AppBundleService._package_draft_assets(app_model)
else:
return AppAssetService.get_source_zip_bytes(tenant_id, app_id, workflow_id)
@staticmethod
def _package_draft_assets(app_model: App) -> bytes | None:
assets = AppAssetService.get_assets(
tenant_id=app_model.tenant_id,
app_id=app_model.id,
user_id="",
is_draft=True,
)
if not assets:
return None
tree = assets.asset_tree
if not tree.nodes:
return None
asset_storage = app_asset_storage
items = tree_to_asset_items(tree, app_model.tenant_id, app_model.id, asset_storage)
packager = AssetZipPackager(asset_storage.storage)
return packager.package(items)
@staticmethod
def _merge_assets_into_bundle(
bundle_zf: zipfile.ZipFile,
assets_zip_bytes: bytes,
prefix: str,
) -> None:
with zipfile.ZipFile(io.BytesIO(assets_zip_bytes), "r") as assets_zf:
for info in assets_zf.infolist():
content = assets_zf.read(info)
new_path = f"{prefix}/{info.filename}"
if info.is_dir():
bundle_zf.writestr(zipfile.ZipInfo(new_path), "")
else:
bundle_zf.writestr(new_path, content)
@staticmethod
def _extract_dsl_from_bundle(zip_bytes: bytes) -> tuple[str, str | None]:
dsl_content: str | None = None
@ -221,7 +207,7 @@ class AppBundleService:
logger.warning("App not found for asset import: %s", app_id)
return
asset_storage = app_asset_storage
asset_storage = AppAssetService.get_storage()
extractor = SourceZipExtractor(asset_storage)
try:
folders, files = extractor.extract_entries(

113
api/services/sandbox/sandbox_service.py Normal file
View File

@ -0,0 +1,113 @@
import logging
from core.sandbox.builder import SandboxBuilder
from core.sandbox.entities import AppAssets, SandboxType
from core.sandbox.entities.providers import SandboxProviderEntity
from core.sandbox.initializer.app_assets_initializer import AppAssetsInitializer
from core.sandbox.initializer.dify_cli_initializer import DifyCliInitializer
from core.sandbox.initializer.draft_app_assets_initializer import DraftAppAssetsInitializer
from core.sandbox.initializer.skill_initializer import SkillInitializer
from core.sandbox.sandbox import Sandbox
from core.sandbox.storage.archive_storage import ArchiveSandboxStorage
from services.app_asset_package_service import AppAssetPackageService
from services.app_asset_service import AppAssetService
logger = logging.getLogger(__name__)
class SandboxService:
@classmethod
def create(
cls,
tenant_id: str,
app_id: str,
user_id: str,
workflow_execution_id: str,
sandbox_provider: SandboxProviderEntity,
) -> Sandbox:
assets = AppAssetService.get_assets(tenant_id, app_id, user_id, is_draft=False)
if not assets:
raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}")
storage = ArchiveSandboxStorage(tenant_id, workflow_execution_id)
sandbox = (
SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type))
.options(sandbox_provider.config)
.user(user_id)
.app(app_id)
.initializer(AppAssetsInitializer(tenant_id, app_id, assets.id))
.initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id))
.initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id))
.storage(storage, assets.id)
.build()
)
logger.info("Sandbox created: id=%s, assets=%s", sandbox.vm.metadata.id, sandbox.assets_id)
return sandbox
@classmethod
def delete_draft_storage(cls, tenant_id: str, user_id: str) -> None:
storage = ArchiveSandboxStorage(tenant_id, SandboxBuilder.draft_id(user_id))
storage.delete()
@classmethod
def create_draft(
cls,
tenant_id: str,
app_id: str,
user_id: str,
sandbox_provider: SandboxProviderEntity,
) -> Sandbox:
assets = AppAssetService.get_assets(tenant_id, app_id, user_id, is_draft=True)
if not assets:
raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}")
AppAssetPackageService.build_assets(tenant_id, app_id, assets)
sandbox_id = SandboxBuilder.draft_id(user_id)
storage = ArchiveSandboxStorage(tenant_id, sandbox_id, exclude_patterns=[AppAssets.PATH])
sandbox = (
SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type))
.options(sandbox_provider.config)
.user(user_id)
.app(app_id)
.initializer(DraftAppAssetsInitializer(tenant_id, app_id, assets.id))
.initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id))
.initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id))
.storage(storage, assets.id)
.build()
)
logger.info("Draft sandbox created: id=%s, assets=%s", sandbox.vm.metadata.id, sandbox.assets_id)
return sandbox
@classmethod
def create_for_single_step(
cls,
tenant_id: str,
app_id: str,
user_id: str,
sandbox_provider: SandboxProviderEntity,
) -> Sandbox:
assets = AppAssetService.get_assets(tenant_id, app_id, user_id, is_draft=True)
if not assets:
raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}")
AppAssetPackageService.build_assets(tenant_id, app_id, assets)
sandbox_id = SandboxBuilder.draft_id(user_id)
storage = ArchiveSandboxStorage(tenant_id, sandbox_id, exclude_patterns=[AppAssets.PATH])
sandbox = (
SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type))
.options(sandbox_provider.config)
.user(user_id)
.app(app_id)
.initializer(DraftAppAssetsInitializer(tenant_id, app_id, assets.id))
.initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id))
.initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id))
.storage(storage, assets.id)
.build()
)
logger.info("Single-step sandbox created: id=%s, assets=%s", sandbox.vm.metadata.id, sandbox.assets_id)
return sandbox

4
api/services/workflow_service.py
View File

@ -14,7 +14,6 @@ from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfig
from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager
from core.file import File
from core.repositories import DifyCoreRepositoryFactory
from core.sandbox.manager import SandboxManager
from core.variables import Variable, VariableBase
from core.workflow.entities import WorkflowNodeExecution
from core.workflow.enums import ErrorStrategy, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus
@ -44,6 +43,7 @@ from services.billing_service import BillingService
from services.enterprise.plugin_manager_service import PluginCredentialType
from services.errors.app import IsDraftWorkflowError, TriggerNodeLimitExceededError, WorkflowHashNotEqualError
from services.sandbox.sandbox_provider_service import SandboxProviderService
from services.sandbox.sandbox_service import SandboxService
from services.workflow.workflow_converter import WorkflowConverter
from .errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError
@ -773,7 +773,7 @@ class WorkflowService:
sandbox = None
if draft_workflow.get_feature(WorkflowFeatures.SANDBOX).enabled:
sandbox_provider = SandboxProviderService.get_sandbox_provider(draft_workflow.tenant_id)
sandbox = SandboxManager.create_for_single_step(
sandbox = SandboxService.create_for_single_step(
tenant_id=draft_workflow.tenant_id,
app_id=app_model.id,
user_id=account.id,

15
web/service/apps.ts
View File

@ -141,21 +141,12 @@ export const exportAppBundle = async ({ appID, include = false, workflowID }: {
if (!response.ok)
throw new Error('Export bundle failed')
const blob = await response.blob()
const contentDisposition = response.headers.get('content-disposition')
let filename = `app-${appID}.zip`
if (contentDisposition) {
const filenameMatch = contentDisposition.match(/filename="?([^";\n]+)"?/)
if (filenameMatch)
filename = filenameMatch[1]
}
const result: { download_url: string, filename: string } = await response.json()
const downloadUrl = URL.createObjectURL(blob)
const a = document.createElement('a')
a.href = downloadUrl
a.download = filename
a.href = result.download_url
a.download = result.filename
a.click()
URL.revokeObjectURL(downloadUrl)
}
export const importDSL = ({ mode, yaml_content, yaml_url, app_id, name, description, icon_type, icon, icon_background }: { mode: DSLImportMode, yaml_content?: string, yaml_url?: string, app_id?: string, name?: string, description?: string, icon_type?: AppIconType, icon?: string, icon_background?: string }): Promise<DSLImportResponse> => {