feat(app-bundle): implement app bundle import/export functionality

- Introduced AppBundleService for managing app bundle publishing and importing, integrating workflow and asset services.
- Added methods for exporting app bundles as ZIP files, including DSL and asset management.
- Implemented source zip extraction and validation to enhance asset import processes.
- Refactored asset packaging to utilize AssetZipPackager for improved performance and organization.
- Enhanced error handling for bundle format and security during import operations.
This commit is contained in:
Harry 2026-01-22 22:40:42 +08:00
parent a43efef9f0
commit 521b66c488
19 changed files with 1364 additions and 1258 deletions

View File

@ -99,6 +99,11 @@ class AppExportQuery(BaseModel):
workflow_id: str | None = Field(default=None, description="Specific workflow ID to export")
class AppExportBundleQuery(BaseModel):
include_secret: bool = Field(default=False, description="Include secrets in export")
workflow_id: str | None = Field(default=None, description="Specific workflow ID to export")
class AppNamePayload(BaseModel):
name: str = Field(..., min_length=1, description="Name to check")
@ -650,6 +655,36 @@ class AppExportApi(Resource):
return payload.model_dump(mode="json")
@console_ns.route("/apps/<uuid:app_id>/export-bundle")
class AppExportBundleApi(Resource):
@get_app_model
@setup_required
@login_required
@account_initialization_required
@edit_permission_required
def get(self, app_model):
from io import BytesIO
from flask import send_file
from services.app_bundle_service import AppBundleService
args = AppExportBundleQuery.model_validate(request.args.to_dict(flat=True))
result = AppBundleService.export_bundle(
app_model=app_model,
include_secret=args.include_secret,
workflow_id=args.workflow_id,
)
return send_file(
BytesIO(result.zip_bytes),
mimetype="application/zip",
as_attachment=True,
download_name=result.filename,
)
@console_ns.route("/apps/<uuid:app_id>/name")
class AppNameApi(Resource):
@console_ns.doc("check_app_name")

View File

@ -243,22 +243,6 @@ class AppAssetNodeReorderResource(Resource):
raise AppAssetNodeNotFoundError()
@console_ns.route("/apps/<string:app_id>/assets/publish")
class AppAssetPublishResource(Resource):
@setup_required
@login_required
@account_initialization_required
@get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
def post(self, app_model: App):
current_user, _ = current_account_with_tenant()
published = AppAssetService.publish(app_model, current_user.id)
return {
"id": published.id,
"version": published.version,
"asset_tree": published.asset_tree.model_dump(),
}, 201
@console_ns.route("/apps/<string:app_id>/assets/files/<string:node_id>/download-url")
class AppAssetFileDownloadUrlResource(Resource):
@setup_required

View File

@ -51,6 +51,14 @@ class AppImportPayload(BaseModel):
app_id: str | None = None
class AppImportBundlePayload(BaseModel):
name: str | None = None
description: str | None = None
icon_type: str | None = None
icon: str | None = None
icon_background: str | None = None
console_ns.schema_model(
AppImportPayload.__name__, AppImportPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0)
)
@ -139,3 +147,55 @@ class AppImportCheckDependenciesApi(Resource):
result = import_service.check_dependencies(app_model=app_model)
return result.model_dump(mode="json"), 200
@console_ns.route("/apps/imports-bundle")
class AppImportBundleApi(Resource):
@setup_required
@login_required
@account_initialization_required
@marshal_with(app_import_model)
@cloud_edition_billing_resource_check("apps")
@edit_permission_required
def post(self):
from flask import request
from core.app.entities.app_bundle_entities import BundleFormatError
from services.app_bundle_service import AppBundleService
current_user, _ = current_account_with_tenant()
if "file" not in request.files:
return {"error": "No file provided"}, 400
file = request.files["file"]
if not file.filename or not file.filename.endswith(".zip"):
return {"error": "Invalid file format, expected .zip"}, 400
zip_bytes = file.read()
form_data = request.form.to_dict()
args = AppImportBundlePayload.model_validate(form_data)
try:
result = AppBundleService.import_bundle(
account=current_user,
zip_bytes=zip_bytes,
name=args.name,
description=args.description,
icon_type=args.icon_type,
icon=args.icon,
icon_background=args.icon_background,
)
except BundleFormatError as e:
return {"error": str(e)}, 400
if result.app_id and FeatureService.get_system_features().webapp_auth.enabled:
EnterpriseService.WebAppAuth.update_app_access_mode(result.app_id, "private")
status = result.status
if status == ImportStatus.FAILED:
return result.model_dump(mode="json"), 400
elif status == ImportStatus.PENDING:
return result.model_dump(mode="json"), 202
return result.model_dump(mode="json"), 200

View File

@ -686,13 +686,14 @@ class PublishedWorkflowApi(Resource):
"""
Publish workflow
"""
from services.app_bundle_service import AppBundleService
current_user, _ = current_account_with_tenant()
args = PublishWorkflowPayload.model_validate(console_ns.payload or {})
workflow_service = WorkflowService()
with Session(db.engine) as session:
workflow = workflow_service.publish_workflow(
workflow = AppBundleService.publish(
session=session,
app_model=app_model,
account=current_user,

View File

@ -0,0 +1,42 @@
from __future__ import annotations
import re
from pydantic import BaseModel, Field
# Constants
BUNDLE_DSL_FILENAME_PATTERN = re.compile(r"^[^/]+\.ya?ml$")
BUNDLE_MAX_SIZE = 50 * 1024 * 1024 # 50MB
# Exceptions
class BundleFormatError(Exception):
"""Raised when bundle format is invalid."""
pass
class ZipSecurityError(Exception):
"""Raised when zip file contains security violations."""
pass
# Entities
class BundleExportResult(BaseModel):
zip_bytes: bytes = Field(description="ZIP file content as bytes")
filename: str = Field(description="Suggested filename for the ZIP")
class SourceFileEntry(BaseModel):
path: str = Field(description="File path within the ZIP")
node_id: str = Field(description="Node ID in the asset tree")
class ExtractedFile(BaseModel):
path: str = Field(description="Relative path of the extracted file")
content: bytes = Field(description="File content as bytes")
class ExtractedFolder(BaseModel):
path: str = Field(description="Relative path of the extracted folder")

View File

@ -4,7 +4,7 @@ from .entities import (
FileAsset,
SkillAsset,
)
from .packager import AssetPackager, ZipPackager
from .packager import AssetPackager, AssetZipPackager
from .parser import AssetItemParser, AssetParser, FileAssetParser, SkillAssetParser
from .paths import AssetPaths
@ -15,9 +15,9 @@ __all__ = [
"AssetPackager",
"AssetParser",
"AssetPaths",
"AssetZipPackager",
"FileAsset",
"FileAssetParser",
"SkillAsset",
"SkillAssetParser",
"ZipPackager",
]

View File

@ -0,0 +1,38 @@
from __future__ import annotations
from core.app.entities.app_asset_entities import AppAssetFileTree, AssetNodeType
from core.app_assets.entities import FileAsset
from core.app_assets.paths import AssetPaths
def tree_to_asset_items(
tree: AppAssetFileTree,
tenant_id: str,
app_id: str,
) -> list[FileAsset]:
"""
Convert AppAssetFileTree to list of FileAsset for packaging.
Args:
tree: The asset file tree to convert
tenant_id: Tenant ID for storage key generation
app_id: App ID for storage key generation
Returns:
List of FileAsset items ready for packaging
"""
items: list[FileAsset] = []
for node in tree.nodes:
if node.node_type == AssetNodeType.FILE:
path = tree.get_path(node.id)
storage_key = AssetPaths.draft_file(tenant_id, app_id, node.id)
items.append(
FileAsset(
asset_id=node.id,
path=path,
file_name=node.name,
extension=node.extension or "",
storage_key=storage_key,
)
)
return items

View File

@ -1,7 +1,7 @@
from .asset_zip_packager import AssetZipPackager
from .base import AssetPackager
from .zip_packager import ZipPackager
__all__ = [
"AssetPackager",
"ZipPackager",
"AssetZipPackager",
]

View File

@ -0,0 +1,78 @@
from __future__ import annotations
import io
import zipfile
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
from typing import TYPE_CHECKING
from core.app_assets.entities import AssetItem
if TYPE_CHECKING:
from extensions.ext_storage import Storage
class AssetZipPackager:
"""
Unified ZIP packager for assets.
Automatically creates directory entries from asset paths.
"""
def __init__(self, storage: Storage, *, max_workers: int = 8) -> None:
self._storage = storage
self._max_workers = max_workers
def package(self, assets: list[AssetItem], *, prefix: str = "") -> bytes:
"""
Package assets into a ZIP file.
Args:
assets: List of assets to package
prefix: Optional prefix to add to all paths in the ZIP
Returns:
ZIP file content as bytes
"""
zip_buffer = io.BytesIO()
# Extract folder paths from asset paths
folder_paths = self._extract_folder_paths(assets, prefix)
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
# Create directory entries
for folder_path in sorted(folder_paths):
zf.writestr(zipfile.ZipInfo(folder_path + "/"), "")
# Write files in parallel
if assets:
self._write_files_parallel(zf, assets, prefix)
return zip_buffer.getvalue()
def _extract_folder_paths(self, assets: list[AssetItem], prefix: str) -> set[str]:
"""Extract all folder paths from asset paths."""
folders: set[str] = set()
for asset in assets:
full_path = f"{prefix}/{asset.path}" if prefix else asset.path
parts = full_path.split("/")[:-1] # Remove filename
folders.update("/".join(parts[:i]) for i in range(1, len(parts) + 1))
return folders
def _write_files_parallel(
self,
zf: zipfile.ZipFile,
assets: list[AssetItem],
prefix: str,
) -> None:
lock = Lock()
def load_and_write(asset: AssetItem) -> None:
content = self._storage.load_once(asset.get_storage_key())
full_path = f"{prefix}/{asset.path}" if prefix else asset.path
with lock:
zf.writestr(full_path, content)
with ThreadPoolExecutor(max_workers=self._max_workers) as executor:
futures = [executor.submit(load_and_write, a) for a in assets]
for future in futures:
future.result()

View File

@ -1,42 +0,0 @@
import io
import zipfile
from concurrent.futures import Future, ThreadPoolExecutor
from threading import Lock
from typing import TYPE_CHECKING
from core.app_assets.entities import AssetItem
from .base import AssetPackager
if TYPE_CHECKING:
from extensions.ext_storage import Storage
class ZipPackager(AssetPackager):
_storage: "Storage"
def __init__(self, storage: "Storage") -> None:
self._storage = storage
def package(self, assets: list[AssetItem]) -> bytes:
zip_buffer = io.BytesIO()
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
lock = Lock()
# FOR DELVELPMENT AND TESTING ONLY, TODO: optimize
with ThreadPoolExecutor(max_workers=8) as executor:
futures: list[Future[None]] = []
for asset in assets:
def _write_asset(a: AssetItem) -> None:
content = self._storage.load_once(a.get_storage_key())
with lock:
zf.writestr(a.path, content)
futures.append(executor.submit(_write_asset, asset))
# Wait for all futures to complete
for future in futures:
future.result()
return zip_buffer.getvalue()

View File

@ -16,3 +16,8 @@ class AssetPaths:
@staticmethod
def build_skill_artifact_set(tenant_id: str, app_id: str, assets_id: str) -> str:
return f"{AssetPaths._BASE}/{tenant_id}/{app_id}/artifacts/{assets_id}/skill_artifact_set.json"
@staticmethod
def build_source_zip(tenant_id: str, app_id: str, workflow_id: str) -> str:
"""Storage key for source assets zip (editable files snapshot at publish time)."""
return f"{AssetPaths._BASE}/{tenant_id}/{app_id}/sources/{workflow_id}.zip"

View File

@ -0,0 +1,5 @@
from .source_zip_extractor import SourceZipExtractor
__all__ = [
"SourceZipExtractor",
]

View File

@ -0,0 +1,101 @@
from __future__ import annotations
import hashlib
import io
import zipfile
from collections.abc import Callable
from typing import TYPE_CHECKING
from uuid import uuid4
from core.app.entities.app_asset_entities import AppAssetFileTree, AppAssetNode
from core.app.entities.app_bundle_entities import ExtractedFile, ExtractedFolder, ZipSecurityError
if TYPE_CHECKING:
from extensions.ext_storage import Storage
class SourceZipExtractor:
def __init__(self, storage: Storage) -> None:
self._storage = storage
def extract_entries(
self, zip_bytes: bytes, *, expected_prefix: str
) -> tuple[list[ExtractedFolder], list[ExtractedFile]]:
folders: list[ExtractedFolder] = []
files: list[ExtractedFile] = []
with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as zf:
for info in zf.infolist():
name = info.filename
self._validate_path(name)
if not name.startswith(expected_prefix):
continue
relative_path = name[len(expected_prefix) :].lstrip("/")
if not relative_path:
continue
if info.is_dir():
folders.append(ExtractedFolder(path=relative_path.rstrip("/")))
else:
content = zf.read(info)
files.append(ExtractedFile(path=relative_path, content=content))
return folders, files
def build_tree_and_save(
self,
folders: list[ExtractedFolder],
files: list[ExtractedFile],
tenant_id: str,
app_id: str,
storage_key_fn: Callable[[str, str, str], str],
) -> AppAssetFileTree:
tree = AppAssetFileTree()
path_to_node_id: dict[str, str] = {}
all_folder_paths = {f.path for f in folders}
for file in files:
self._ensure_parent_folders(file.path, all_folder_paths)
sorted_folders = sorted(all_folder_paths, key=lambda p: p.count("/"))
for folder_path in sorted_folders:
node_id = str(uuid4())
name = folder_path.rsplit("/", 1)[-1]
parent_path = folder_path.rsplit("/", 1)[0] if "/" in folder_path else None
parent_id = path_to_node_id.get(parent_path) if parent_path else None
node = AppAssetNode.create_folder(node_id, name, parent_id)
tree.add(node)
path_to_node_id[folder_path] = node_id
sorted_files = sorted(files, key=lambda f: f.path)
for file in sorted_files:
node_id = str(uuid4())
name = file.path.rsplit("/", 1)[-1]
parent_path = file.path.rsplit("/", 1)[0] if "/" in file.path else None
parent_id = path_to_node_id.get(parent_path) if parent_path else None
checksum = hashlib.sha256(file.content).hexdigest()
node = AppAssetNode.create_file(node_id, name, parent_id, len(file.content), checksum)
tree.add(node)
storage_key = storage_key_fn(tenant_id, app_id, node_id)
self._storage.save(storage_key, file.content)
return tree
def _validate_path(self, path: str) -> None:
if ".." in path:
raise ZipSecurityError(f"Path traversal detected: {path}")
if path.startswith("/"):
raise ZipSecurityError(f"Absolute path detected: {path}")
if "\\" in path:
raise ZipSecurityError(f"Backslash in path: {path}")
def _ensure_parent_folders(self, file_path: str, folder_set: set[str]) -> None:
parts = file_path.split("/")[:-1]
for i in range(1, len(parts) + 1):
parent = "/".join(parts[:i])
folder_set.add(parent)

View File

@ -37,6 +37,11 @@ class AppAssetsInitializer(AsyncSandboxInitializer):
["sh", "-c", f"unzip {AppAssets.ZIP_PATH} -d {AppAssets.PATH} 2>/dev/null || [ $? -eq 1 ]"],
error_message="Failed to unzip assets",
)
# Ensure directories have execute permission for traversal and files are readable
.add(
["sh", "-c", f"chmod -R u+rwX,go+rX {AppAssets.PATH}"],
error_message="Failed to set permissions on assets",
)
.execute(timeout=APP_ASSETS_DOWNLOAD_TIMEOUT, raise_on_error=True)
)

View File

@ -1,248 +1,390 @@
import hashlib
import logging
import re
from collections.abc import Mapping
from datetime import UTC, datetime
from typing import Any
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from typing import Any, Protocol, cast
from core.app.entities.app_asset_entities import AppAssetFileTree
from core.skill.entities.asset_references import AssetReferences
from core.skill.entities.skill_bundle import SkillBundle
from core.skill.entities.skill_bundle_entry import SkillBundleEntry, SourceInfo
from core.skill.entities.skill_document import SkillDocument
from core.skill.entities.skill_metadata import (
FileReference,
SkillMetadata,
ToolConfiguration,
ToolReference,
)
from core.skill.entities.skill_metadata import FileReference, SkillMetadata, ToolConfiguration, ToolReference
from core.skill.entities.tool_dependencies import ToolDependencies, ToolDependency
from core.tools.entities.tool_entities import ToolProviderType
logger = logging.getLogger(__name__)
TOOL_REFERENCE_PATTERN = re.compile(r"§\[tool\]\.\[([^\]]+)\]\.\[([^\]]+)\]\.\[([^\]]+)\")
FILE_REFERENCE_PATTERN = re.compile(r"§\[file\]\.\[([^\]]+)\]\.\[([^\]]+)\")
class PathResolver(Protocol):
def resolve(self, source_id: str, target_id: str) -> str: ...
class ToolResolver(Protocol):
def resolve(self, tool_ref: ToolReference) -> str: ...
@dataclass(frozen=True)
class CompilerConfig:
tool_pattern: re.Pattern[str] = re.compile(r"§\[tool\]\.\[.*?\]\.\[.*?\]\.\[(.*?)\")
file_pattern: re.Pattern[str] = re.compile(r"§\[file\]\.\[.*?\]\.\[(.*?)\")
class FileTreePathResolver:
def __init__(self, tree: AppAssetFileTree, base_path: str = ""):
self._tree = tree
self._base_path = base_path.rstrip("/")
def resolve(self, source_id: str, target_id: str) -> str:
source_node = self._tree.get(source_id)
target_node = self._tree.get(target_id)
if target_node is None:
return "[File not found]"
if source_node is not None:
return self._tree.relative_path(source_node, target_node)
full_path = self._tree.get_path(target_node.id)
if self._base_path:
return f"{self._base_path}/{full_path}"
return full_path
class DefaultToolResolver:
def resolve(self, tool_ref: ToolReference) -> str:
return f"[Executable: {tool_ref.tool_name}_{tool_ref.uuid} --help command]"
class SkillCompiler:
def _parse_metadata(self, content: str, raw_metadata: Mapping[str, Any]) -> SkillMetadata:
tools_raw: dict[str, Any] = dict(raw_metadata.get("tools", {}))
tools: dict[str, ToolReference] = {}
files: list[FileReference] = []
for match in TOOL_REFERENCE_PATTERN.finditer(content):
tool_id = match.group(3)
tool_name = match.group(2)
tool_provider = match.group(1)
tool_meta = tools_raw.get(tool_id)
if tool_meta is None:
continue
config_raw = tool_meta.get("configuration", {})
configuration = ToolConfiguration.model_validate(config_raw) if config_raw else None
tools[tool_id] = ToolReference(
uuid=tool_id,
type=ToolProviderType.value_of(tool_meta.get("type")),
provider=tool_provider,
tool_name=tool_name,
credential_id=tool_meta.get("credential_id"),
configuration=configuration,
)
for match in FILE_REFERENCE_PATTERN.finditer(content):
files.append(
FileReference(
source=match.group(1),
asset_id=match.group(2),
)
)
return SkillMetadata(tools=tools, files=files)
def __init__(
self,
path_resolver: PathResolver | None = None,
tool_resolver: ToolResolver | None = None,
config: CompilerConfig | None = None,
):
self._path_resolver = path_resolver
self._tool_resolver = tool_resolver or DefaultToolResolver()
self._config = config or CompilerConfig()
def compile_all(
self,
documents: list[SkillDocument],
documents: Iterable[SkillDocument],
file_tree: AppAssetFileTree,
assets_id: str,
) -> SkillBundle:
bundle = SkillBundle(
assets_id=assets_id,
built_at=datetime.now(UTC),
)
doc_map: dict[str, SkillDocument] = {doc.skill_id: doc for doc in documents}
parsed_metadata: dict[str, SkillMetadata] = {}
for doc in documents:
metadata = self._parse_metadata(doc.content, doc.metadata)
parsed_metadata[doc.skill_id] = metadata
direct_skill_refs = self._extract_skill_refs(metadata, doc_map)
bundle.dependency_graph[doc.skill_id] = list(direct_skill_refs)
for ref_id in direct_skill_refs:
if ref_id not in bundle.reverse_graph:
bundle.reverse_graph[ref_id] = []
bundle.reverse_graph[ref_id].append(doc.skill_id)
for doc in documents:
metadata = parsed_metadata[doc.skill_id]
entry = self._compile_single(doc, metadata, bundle, parsed_metadata, file_tree)
bundle.upsert(entry)
return bundle
path_resolver = self._path_resolver or FileTreePathResolver(file_tree)
return self._compile_batch_internal(documents, assets_id, path_resolver)
def compile_one(
self,
bundle: SkillBundle,
document: SkillDocument,
file_tree: AppAssetFileTree,
all_documents: dict[str, SkillDocument] | None = None,
base_path: str = "",
) -> SkillBundleEntry:
doc_map = all_documents or {}
if document.skill_id not in doc_map:
doc_map[document.skill_id] = document
parsed_metadata: dict[str, SkillMetadata] = {}
for skill_id, doc in doc_map.items():
parsed_metadata[skill_id] = self._parse_metadata(doc.content, doc.metadata)
metadata = parsed_metadata[document.skill_id]
direct_skill_refs = self._extract_skill_refs(metadata, doc_map)
bundle.dependency_graph[document.skill_id] = list(direct_skill_refs)
for ref_id in direct_skill_refs:
if ref_id not in bundle.reverse_graph:
bundle.reverse_graph[ref_id] = []
if document.skill_id not in bundle.reverse_graph[ref_id]:
bundle.reverse_graph[ref_id].append(document.skill_id)
return self._compile_single(document, metadata, bundle, parsed_metadata, file_tree)
def _compile_single(
self,
document: SkillDocument,
metadata: SkillMetadata,
bundle: SkillBundle,
parsed_metadata: dict[str, SkillMetadata],
file_tree: AppAssetFileTree,
) -> SkillBundleEntry:
all_tools, all_files = self._compute_transitive_closure(
document.skill_id, bundle, parsed_metadata
path_resolver = self._path_resolver or FileTreePathResolver(file_tree, base_path)
resolved_content, tool_dependencies = self._compile_template_internal(
document.content, document.metadata, bundle, path_resolver
)
current_node = file_tree.get(document.skill_id)
resolved_content = self._resolve_content(
document.content, metadata, current_node, file_tree
)
content_digest = hashlib.sha256(document.content.encode("utf-8")).hexdigest()
metadata = self._parse_metadata(document.content, document.metadata)
final_files: dict[str, FileReference] = {f.asset_id: f for f in metadata.files}
return SkillBundleEntry(
skill_id=document.skill_id,
source=SourceInfo(
asset_id=document.skill_id,
content_digest=content_digest,
content_digest=hashlib.sha256(document.content.encode("utf-8")).hexdigest(),
),
tools=tool_dependencies,
files=AssetReferences(references=list(final_files.values())),
content=resolved_content,
)
def _compile_batch_internal(
self,
documents: Iterable[SkillDocument],
assets_id: str,
path_resolver: PathResolver,
) -> SkillBundle:
doc_map = {doc.skill_id: doc for doc in documents}
graph: dict[str, set[str]] = {}
metadata_cache: dict[str, SkillMetadata] = {}
# Phase 1: Parse metadata and build dependency graph
for doc in doc_map.values():
metadata = self._parse_metadata(doc.content, doc.metadata)
metadata_cache[doc.skill_id] = metadata
deps: set[str] = set()
for file_ref in metadata.files:
if file_ref.asset_id in doc_map:
deps.add(file_ref.asset_id)
graph[doc.skill_id] = deps
bundle = SkillBundle(assets_id=assets_id)
bundle.dependency_graph = {k: list(v) for k, v in graph.items()}
# Build reverse graph for propagation
reverse_graph: dict[str, set[str]] = {skill_id: set() for skill_id in doc_map}
for skill_id, deps in graph.items():
for dep_id in deps:
if dep_id in reverse_graph:
reverse_graph[dep_id].add(skill_id)
bundle.reverse_graph = {k: list(v) for k, v in reverse_graph.items()}
# Phase 2: Compile each skill independently (content + direct dependencies only)
for skill_id, doc in doc_map.items():
metadata = metadata_cache[skill_id]
entry = self._compile_node_direct(doc, metadata, path_resolver)
bundle.upsert(entry)
# Phase 3: Propagate transitive dependencies until fixed-point
self._propagate_transitive_dependencies(bundle, graph)
return bundle
def _compile_node_direct(
self,
doc: SkillDocument,
metadata: SkillMetadata,
path_resolver: PathResolver,
) -> SkillBundleEntry:
"""Compile a single skill with only its direct dependencies (no transitive)."""
direct_tools: dict[str, ToolDependency] = {}
direct_refs: dict[str, ToolReference] = {}
for tool_ref in metadata.tools.values():
key = f"{tool_ref.provider}.{tool_ref.tool_name}"
if key not in direct_tools:
direct_tools[key] = ToolDependency(
type=tool_ref.type,
provider=tool_ref.provider,
tool_name=tool_ref.tool_name,
)
direct_refs[tool_ref.uuid] = tool_ref
direct_files: dict[str, FileReference] = {f.asset_id: f for f in metadata.files}
resolved_content = self._resolve_content(doc.content, metadata, path_resolver, doc.skill_id)
return SkillBundleEntry(
skill_id=doc.skill_id,
source=SourceInfo(
asset_id=doc.skill_id,
content_digest=hashlib.sha256(doc.content.encode("utf-8")).hexdigest(),
),
tools=ToolDependencies(
dependencies=list(all_tools.values()),
references=list(metadata.tools.values()),
dependencies=list(direct_tools.values()),
references=list(direct_refs.values()),
),
files=AssetReferences(
references=list(all_files.values()),
references=list(direct_files.values()),
),
content=resolved_content,
)
def _extract_skill_refs(
def _propagate_transitive_dependencies(
self,
metadata: SkillMetadata,
doc_map: dict[str, SkillDocument],
) -> set[str]:
skill_refs: set[str] = set()
for file_ref in metadata.files:
if file_ref.asset_id in doc_map:
skill_refs.add(file_ref.asset_id)
return skill_refs
def _compute_transitive_closure(
self,
skill_id: str,
bundle: SkillBundle,
parsed_metadata: dict[str, SkillMetadata],
) -> tuple[dict[str, ToolDependency], dict[str, FileReference]]:
all_tools: dict[str, ToolDependency] = {}
all_files: dict[str, FileReference] = {}
graph: dict[str, set[str]],
) -> None:
"""Iteratively propagate transitive dependencies until no changes occur."""
changed = True
while changed:
changed = False
for skill_id, dep_ids in graph.items():
entry = bundle.get(skill_id)
if not entry:
continue
visited: set[str] = set()
queue = [skill_id]
# Collect current tools and files
current_tools: dict[str, ToolDependency] = {
f"{d.provider}.{d.tool_name}": d for d in entry.tools.dependencies
}
current_refs: dict[str, ToolReference] = {r.uuid: r for r in entry.tools.references}
current_files: dict[str, FileReference] = {f.asset_id: f for f in entry.files.references}
while queue:
current_id = queue.pop(0)
if current_id in visited:
continue
visited.add(current_id)
original_tool_count = len(current_tools)
original_ref_count = len(current_refs)
original_file_count = len(current_files)
metadata = parsed_metadata.get(current_id)
if metadata is None:
existing_entry = bundle.get(current_id)
if existing_entry:
for dep in existing_entry.tools.dependencies:
key = f"{dep.provider}.{dep.tool_name}"
if key not in all_tools:
all_tools[key] = dep
for file_ref in existing_entry.files.references:
if file_ref.asset_id not in all_files:
all_files[file_ref.asset_id] = file_ref
continue
# Merge from dependencies
for dep_id in dep_ids:
dep_entry = bundle.get(dep_id)
if not dep_entry:
continue
for tool_ref in metadata.tools.values():
key = f"{tool_ref.provider}.{tool_ref.tool_name}"
if key not in all_tools:
all_tools[key] = ToolDependency(
type=tool_ref.type,
provider=tool_ref.provider,
tool_name=tool_ref.tool_name,
for tool_dep in dep_entry.tools.dependencies:
key = f"{tool_dep.provider}.{tool_dep.tool_name}"
if key not in current_tools:
current_tools[key] = tool_dep
for tool_ref in dep_entry.tools.references:
if tool_ref.uuid not in current_refs:
current_refs[tool_ref.uuid] = tool_ref
for file_ref in dep_entry.files.references:
if file_ref.asset_id not in current_files:
current_files[file_ref.asset_id] = file_ref
# Check if anything changed
if (
len(current_tools) != original_tool_count
or len(current_refs) != original_ref_count
or len(current_files) != original_file_count
):
changed = True
# Update the entry with new transitive dependencies
updated_entry = SkillBundleEntry(
skill_id=entry.skill_id,
source=entry.source,
tools=ToolDependencies(
dependencies=list(current_tools.values()),
references=list(current_refs.values()),
),
files=AssetReferences(
references=list(current_files.values()),
),
content=entry.content,
)
bundle.upsert(updated_entry)
for file_ref in metadata.files:
if file_ref.asset_id not in all_files:
all_files[file_ref.asset_id] = file_ref
for dep_id in bundle.dependency_graph.get(current_id, []):
if dep_id not in visited:
queue.append(dep_id)
return all_tools, all_files
def _resolve_content(
def _compile_template_internal(
self,
content: str,
metadata_dict: Mapping[str, Any],
context: SkillBundle,
path_resolver: PathResolver,
) -> tuple[str, ToolDependencies]:
metadata = self._parse_metadata(content, metadata_dict)
direct_deps: list[SkillBundleEntry] = []
for file_ref in metadata.files:
artifact = context.get(file_ref.asset_id)
if artifact:
direct_deps.append(artifact)
final_tools, final_refs = self._aggregate_dependencies(metadata, direct_deps)
resolved_content = self._resolve_content(content, metadata, path_resolver, current_id="<template>")
return resolved_content, ToolDependencies(
dependencies=list(final_tools.values()), references=list(final_refs.values())
)
def _compile_node(
self,
doc: SkillDocument,
metadata: SkillMetadata,
current_node: Any,
file_tree: AppAssetFileTree,
direct_deps: Sequence[SkillBundleEntry],
path_resolver: PathResolver,
) -> SkillBundleEntry:
final_tools, final_refs = self._aggregate_dependencies(metadata, direct_deps)
final_files: dict[str, FileReference] = {}
for f in metadata.files:
final_files[f.asset_id] = f
for dep in direct_deps:
for f in dep.files.references:
if f.asset_id not in final_files:
final_files[f.asset_id] = f
resolved_content = self._resolve_content(doc.content, metadata, path_resolver, doc.skill_id)
return SkillBundleEntry(
skill_id=doc.skill_id,
source=SourceInfo(
asset_id=doc.skill_id,
content_digest=hashlib.sha256(doc.content.encode("utf-8")).hexdigest(),
),
tools=ToolDependencies(
dependencies=list(final_tools.values()),
references=list(final_refs.values()),
),
files=AssetReferences(
references=list(final_files.values()),
),
content=resolved_content,
)
def _aggregate_dependencies(
self, metadata: SkillMetadata, direct_deps: Sequence[SkillBundleEntry]
) -> tuple[dict[str, ToolDependency], dict[str, ToolReference]]:
all_tools: dict[str, ToolDependency] = {}
all_refs: dict[str, ToolReference] = {}
for tool_ref in metadata.tools.values():
key = f"{tool_ref.provider}.{tool_ref.tool_name}"
if key not in all_tools:
all_tools[key] = ToolDependency(
type=tool_ref.type,
provider=tool_ref.provider,
tool_name=tool_ref.tool_name,
)
all_refs[tool_ref.uuid] = tool_ref
for dep in direct_deps:
for tool_dep in dep.tools.dependencies:
key = f"{tool_dep.provider}.{tool_dep.tool_name}"
if key not in all_tools:
all_tools[key] = tool_dep
for tool_ref in dep.tools.references:
if tool_ref.uuid not in all_refs:
all_refs[tool_ref.uuid] = tool_ref
return all_tools, all_refs
def _resolve_content(
self, content: str, metadata: SkillMetadata, path_resolver: PathResolver, current_id: str
) -> str:
if not content:
return content
def replace_file(match: re.Match[str]) -> str:
target_id = match.group(1)
try:
return path_resolver.resolve(current_id, target_id)
except Exception:
return match.group(0)
for match in FILE_REFERENCE_PATTERN.finditer(content):
file_id = match.group(2)
file_node = file_tree.get(file_id)
if file_node is None:
logger.warning("File not found for id=%s, skipping", file_id)
content = content.replace(match.group(0), "[File not found]")
continue
if current_node is not None:
content = content.replace(match.group(0), file_tree.relative_path(current_node, file_node))
else:
content = content.replace(match.group(0), f"[{file_node.name}]")
for match in TOOL_REFERENCE_PATTERN.finditer(content):
tool_id = match.group(3)
tool = metadata.tools.get(tool_id)
if tool is None:
logger.warning("Tool not found for id=%s, skipping", tool_id)
content = content.replace(match.group(0), f"[Tool not found: {tool_id}]")
continue
content = content.replace(match.group(0), f"[Bash Command: {tool.tool_name}_{tool_id}]")
def replace_tool(match: re.Match[str]) -> str:
tool_id = match.group(1)
tool_ref = metadata.tools.get(tool_id)
if not tool_ref:
return f"[Tool not found: {tool_id}]"
return self._tool_resolver.resolve(tool_ref)
content = self._config.file_pattern.sub(replace_file, content)
content = self._config.tool_pattern.sub(replace_tool, content)
return content
def _parse_metadata(self, content: str, raw_metadata: Mapping[str, Any]) -> SkillMetadata:
tools_raw = dict(raw_metadata.get("tools", {}))
tools: dict[str, ToolReference] = {}
tool_iter = re.finditer(r"§\[tool\]\.\[([^\]]+)\]\.\[([^\]]+)\]\.\[([^\]]+)\", content)
for match in tool_iter:
provider, name, uuid = match.group(1), match.group(2), match.group(3)
if uuid in tools_raw:
meta = tools_raw[uuid]
if isinstance(meta, ToolReference):
tools[uuid] = meta
elif isinstance(meta, dict):
tool_type_str = cast(str | None, meta.get("type"))
if tool_type_str:
tools[uuid] = ToolReference(
uuid=uuid,
type=ToolProviderType.value_of(tool_type_str),
provider=provider,
tool_name=name,
credential_id=cast(str | None, meta.get("credential_id")),
configuration=ToolConfiguration.model_validate(meta.get("configuration", {}))
if meta.get("configuration")
else None,
)
parsed_files: list[FileReference] = []
file_iter = re.finditer(r"§\[file\]\.\[([^\]]+)\]\.\[([^\]]+)\", content)
for match in file_iter:
source, asset_id = match.group(1), match.group(2)
parsed_files.append(FileReference(source=source, asset_id=asset_id))
return SkillMetadata(tools=tools, files=parsed_files)

View File

@ -13,13 +13,13 @@ from core.app.entities.app_asset_entities import (
TreePathConflictError,
)
from core.app_assets.builder import AssetBuildPipeline, BuildContext
from core.app_assets.packager.zip_packager import ZipPackager
from core.app_assets.converters import tree_to_asset_items
from core.app_assets.packager import AssetZipPackager
from core.app_assets.paths import AssetPaths
from extensions.ext_database import db
from extensions.ext_redis import redis_client
from extensions.ext_storage import storage
from extensions.storage.file_presign_storage import FilePresignStorage
from libs.datetime_utils import naive_utc_now
from models.app_asset import AppAssets
from models.model import App
@ -445,35 +445,39 @@ class AppAssetService:
AppAssetService._clear_draft_download_cache(cache_keys)
@staticmethod
def publish(app_model: App, account_id: str) -> AppAssets:
def publish(session: Session, app_model: App, account_id: str, workflow_id: str) -> AppAssets:
tenant_id = app_model.tenant_id
app_id = app_model.id
with Session(db.engine, expire_on_commit=False) as session:
assets = AppAssetService.get_or_create_assets(session, app_model, account_id)
tree = assets.asset_tree
publish_id = str(uuid4())
assets = AppAssetService.get_or_create_assets(session, app_model, account_id)
tree = assets.asset_tree
published = AppAssets(
id=publish_id,
tenant_id=tenant_id,
app_id=app_id,
version=str(naive_utc_now()),
created_by=account_id,
)
published.asset_tree = tree
session.add(published)
session.flush()
publish_id = str(uuid4())
ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=publish_id)
built_assets = AssetBuildPipeline().build_all(tree, ctx)
published = AppAssets(
id=publish_id,
tenant_id=tenant_id,
app_id=app_id,
version=workflow_id,
created_by=account_id,
)
published.asset_tree = tree
session.add(published)
session.flush()
packager = ZipPackager(storage)
zip_bytes = packager.package(built_assets)
zip_key = AssetPaths.build_zip(tenant_id, app_id, publish_id)
storage.save(zip_key, zip_bytes)
ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=publish_id)
built_assets = AssetBuildPipeline().build_all(tree, ctx)
session.commit()
packager = AssetZipPackager(storage)
runtime_zip_bytes = packager.package(built_assets)
runtime_zip_key = AssetPaths.build_zip(tenant_id, app_id, publish_id)
storage.save(runtime_zip_key, runtime_zip_bytes)
source_items = tree_to_asset_items(tree, tenant_id, app_id)
source_zip_bytes = packager.package(source_items)
source_zip_key = AssetPaths.build_source_zip(tenant_id, app_id, workflow_id)
storage.save(source_zip_key, source_zip_bytes)
return published
@ -483,7 +487,12 @@ class AppAssetService:
tree = assets.asset_tree
ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=assets.id)
AssetBuildPipeline().build_all(tree, ctx)
built_assets = AssetBuildPipeline().build_all(tree, ctx)
packager = AssetZipPackager(storage)
zip_bytes = packager.package(built_assets)
zip_key = AssetPaths.build_zip(tenant_id, app_id, assets.id)
storage.save(zip_key, zip_bytes)
@staticmethod
def get_file_download_url(
@ -503,3 +512,39 @@ class AppAssetService:
storage_key = AssetPaths.draft_file(app_model.tenant_id, app_model.id, node_id)
presign_storage = FilePresignStorage(storage.storage_runner)
return presign_storage.get_download_url(storage_key, expires_in)
@staticmethod
def get_published_assets_by_workflow_id(tenant_id: str, app_id: str, workflow_id: str) -> AppAssets | None:
with Session(db.engine, expire_on_commit=False) as session:
return (
session.query(AppAssets)
.filter(
AppAssets.tenant_id == tenant_id,
AppAssets.app_id == app_id,
AppAssets.version == workflow_id,
)
.first()
)
@staticmethod
def get_source_zip_bytes(tenant_id: str, app_id: str, workflow_id: str) -> bytes | None:
source_zip_key = AssetPaths.build_source_zip(tenant_id, app_id, workflow_id)
try:
return storage.load_once(source_zip_key)
except Exception:
logger.warning("Source zip not found: %s", source_zip_key)
return None
@staticmethod
def set_draft_assets(
app_model: App,
account_id: str,
new_tree: AppAssetFileTree,
) -> AppAssets:
with Session(db.engine, expire_on_commit=False) as session:
assets = AppAssetService.get_or_create_assets(session, app_model, account_id)
assets.asset_tree = new_tree
assets.updated_by = account_id
session.commit()
return assets

View File

@ -0,0 +1,255 @@
from __future__ import annotations
import io
import logging
import re
import zipfile
import yaml
from sqlalchemy.orm import Session
from core.app.entities.app_bundle_entities import (
BUNDLE_DSL_FILENAME_PATTERN,
BUNDLE_MAX_SIZE,
BundleExportResult,
BundleFormatError,
ZipSecurityError,
)
from core.app_assets.converters import tree_to_asset_items
from core.app_assets.packager import AssetZipPackager
from core.app_assets.paths import AssetPaths
from core.app_bundle import SourceZipExtractor
from extensions.ext_database import db
from extensions.ext_storage import storage
from models import Account, App
from .app_asset_service import AppAssetService
from .app_dsl_service import AppDslService, Import
logger = logging.getLogger(__name__)
class AppBundleService:
@staticmethod
def publish(
session: Session,
app_model: App,
account: Account,
marked_name: str = "",
marked_comment: str = "",
):
"""
Publish App Bundle (workflow + assets).
Coordinates WorkflowService and AppAssetService publishing in a single transaction.
"""
from models.workflow import Workflow
from services.workflow_service import WorkflowService
# 1. Publish workflow
workflow: Workflow = WorkflowService().publish_workflow(
session=session,
app_model=app_model,
account=account,
marked_name=marked_name,
marked_comment=marked_comment,
)
# 2. Publish assets (bound to workflow_id)
AppAssetService.publish(
session=session,
app_model=app_model,
account_id=account.id,
workflow_id=workflow.id,
)
return workflow
@staticmethod
def export_bundle(
app_model: App,
include_secret: bool = False,
workflow_id: str | None = None,
) -> BundleExportResult:
dsl_content = AppDslService.export_dsl(
app_model=app_model,
include_secret=include_secret,
workflow_id=workflow_id,
)
safe_name = AppBundleService._sanitize_filename(app_model.name)
assets_prefix = safe_name
zip_buffer = io.BytesIO()
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
zf.writestr(f"{safe_name}.yml", dsl_content.encode("utf-8"))
assets_zip_bytes = AppBundleService._get_assets_zip_bytes(app_model, workflow_id)
if assets_zip_bytes:
AppBundleService._merge_assets_into_bundle(zf, assets_zip_bytes, assets_prefix)
return BundleExportResult(
zip_bytes=zip_buffer.getvalue(),
filename=f"{safe_name}.zip",
)
@staticmethod
def import_bundle(
account: Account,
zip_bytes: bytes,
name: str | None = None,
description: str | None = None,
icon_type: str | None = None,
icon: str | None = None,
icon_background: str | None = None,
) -> Import:
if len(zip_bytes) > BUNDLE_MAX_SIZE:
raise BundleFormatError(f"Bundle size exceeds limit: {BUNDLE_MAX_SIZE} bytes")
dsl_content, assets_prefix = AppBundleService._extract_dsl_from_bundle(zip_bytes)
with Session(db.engine) as session:
dsl_service = AppDslService(session)
import_result = dsl_service.import_app(
account=account,
import_mode="yaml-content",
yaml_content=dsl_content,
name=name,
description=description,
icon_type=icon_type,
icon=icon,
icon_background=icon_background,
app_id=None,
)
session.commit()
if import_result.app_id and assets_prefix:
AppBundleService._import_assets_from_bundle(
zip_bytes=zip_bytes,
assets_prefix=assets_prefix,
app_id=import_result.app_id,
account_id=account.id,
)
return import_result
@staticmethod
def _get_assets_zip_bytes(app_model: App, workflow_id: str | None) -> bytes | None:
tenant_id = app_model.tenant_id
app_id = app_model.id
if workflow_id is None:
return AppBundleService._package_draft_assets(app_model)
else:
return AppAssetService.get_source_zip_bytes(tenant_id, app_id, workflow_id)
@staticmethod
def _package_draft_assets(app_model: App) -> bytes | None:
assets = AppAssetService.get_assets(
tenant_id=app_model.tenant_id,
app_id=app_model.id,
user_id="",
is_draft=True,
)
if not assets:
return None
tree = assets.asset_tree
if not tree.nodes:
return None
items = tree_to_asset_items(tree, app_model.tenant_id, app_model.id)
packager = AssetZipPackager(storage)
return packager.package(items)
@staticmethod
def _merge_assets_into_bundle(
bundle_zf: zipfile.ZipFile,
assets_zip_bytes: bytes,
prefix: str,
) -> None:
with zipfile.ZipFile(io.BytesIO(assets_zip_bytes), "r") as assets_zf:
for info in assets_zf.infolist():
content = assets_zf.read(info)
new_path = f"{prefix}/{info.filename}"
if info.is_dir():
bundle_zf.writestr(zipfile.ZipInfo(new_path), "")
else:
bundle_zf.writestr(new_path, content)
@staticmethod
def _extract_dsl_from_bundle(zip_bytes: bytes) -> tuple[str, str | None]:
dsl_content: str | None = None
dsl_filename: str | None = None
with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as zf:
for info in zf.infolist():
if info.is_dir():
continue
if BUNDLE_DSL_FILENAME_PATTERN.match(info.filename):
if dsl_content is not None:
raise BundleFormatError("Multiple DSL files found in bundle")
dsl_content = zf.read(info).decode("utf-8")
dsl_filename = info.filename
if dsl_content is None or dsl_filename is None:
raise BundleFormatError("No DSL file (*.yml or *.yaml) found in bundle root")
yaml.safe_load(dsl_content)
assets_prefix = dsl_filename.rsplit(".", 1)[0]
has_assets = AppBundleService._check_assets_prefix_exists(zip_bytes, assets_prefix)
return dsl_content, assets_prefix if has_assets else None
@staticmethod
def _check_assets_prefix_exists(zip_bytes: bytes, prefix: str) -> bool:
with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as zf:
for info in zf.infolist():
if info.filename.startswith(f"{prefix}/"):
return True
return False
@staticmethod
def _import_assets_from_bundle(
zip_bytes: bytes,
assets_prefix: str,
app_id: str,
account_id: str,
) -> None:
app_model = db.session.query(App).filter(App.id == app_id).first()
if not app_model:
logger.warning("App not found for asset import: %s", app_id)
return
extractor = SourceZipExtractor(storage)
try:
folders, files = extractor.extract_entries(
zip_bytes,
expected_prefix=f"{assets_prefix}/",
)
except ZipSecurityError as e:
logger.warning("Zip security error during asset import: %s", e)
return
if not folders and not files:
return
new_tree = extractor.build_tree_and_save(
folders=folders,
files=files,
tenant_id=app_model.tenant_id,
app_id=app_model.id,
storage_key_fn=AssetPaths.draft_file,
)
AppAssetService.set_draft_assets(
app_model=app_model,
account_id=account_id,
new_tree=new_tree,
)
@staticmethod
def _sanitize_filename(name: str) -> str:
safe = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "_", name)
safe = safe.strip(". ")
return safe[:100] if safe else "app"

View File

@ -40,7 +40,6 @@ from models.tools import WorkflowToolProvider
from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowType
from models.workflow_features import WorkflowFeatures
from repositories.factory import DifyAPIRepositoryFactory
from services.app_asset_service import AppAssetService
from services.billing_service import BillingService
from services.enterprise.plugin_manager_service import PluginCredentialType
from services.errors.app import IsDraftWorkflowError, TriggerNodeLimitExceededError, WorkflowHashNotEqualError
@ -156,8 +155,6 @@ class WorkflowService:
.first()
)
AppAssetService.publish(app_model=app_model, account_id=app_model.created_by)
return workflow
def get_all_published_workflow(

File diff suppressed because it is too large Load Diff