mirror of
https://github.com/langgenius/dify.git
synced 2026-05-10 05:56:31 +08:00
feat(app-bundle): implement app bundle import/export functionality
- Introduced AppBundleService for managing app bundle publishing and importing, integrating workflow and asset services. - Added methods for exporting app bundles as ZIP files, including DSL and asset management. - Implemented source zip extraction and validation to enhance asset import processes. - Refactored asset packaging to utilize AssetZipPackager for improved performance and organization. - Enhanced error handling for bundle format and security during import operations.
This commit is contained in:
parent
a43efef9f0
commit
521b66c488
@ -99,6 +99,11 @@ class AppExportQuery(BaseModel):
|
||||
workflow_id: str | None = Field(default=None, description="Specific workflow ID to export")
|
||||
|
||||
|
||||
class AppExportBundleQuery(BaseModel):
|
||||
include_secret: bool = Field(default=False, description="Include secrets in export")
|
||||
workflow_id: str | None = Field(default=None, description="Specific workflow ID to export")
|
||||
|
||||
|
||||
class AppNamePayload(BaseModel):
|
||||
name: str = Field(..., min_length=1, description="Name to check")
|
||||
|
||||
@ -650,6 +655,36 @@ class AppExportApi(Resource):
|
||||
return payload.model_dump(mode="json")
|
||||
|
||||
|
||||
@console_ns.route("/apps/<uuid:app_id>/export-bundle")
|
||||
class AppExportBundleApi(Resource):
|
||||
@get_app_model
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
def get(self, app_model):
|
||||
from io import BytesIO
|
||||
|
||||
from flask import send_file
|
||||
|
||||
from services.app_bundle_service import AppBundleService
|
||||
|
||||
args = AppExportBundleQuery.model_validate(request.args.to_dict(flat=True))
|
||||
|
||||
result = AppBundleService.export_bundle(
|
||||
app_model=app_model,
|
||||
include_secret=args.include_secret,
|
||||
workflow_id=args.workflow_id,
|
||||
)
|
||||
|
||||
return send_file(
|
||||
BytesIO(result.zip_bytes),
|
||||
mimetype="application/zip",
|
||||
as_attachment=True,
|
||||
download_name=result.filename,
|
||||
)
|
||||
|
||||
|
||||
@console_ns.route("/apps/<uuid:app_id>/name")
|
||||
class AppNameApi(Resource):
|
||||
@console_ns.doc("check_app_name")
|
||||
|
||||
@ -243,22 +243,6 @@ class AppAssetNodeReorderResource(Resource):
|
||||
raise AppAssetNodeNotFoundError()
|
||||
|
||||
|
||||
@console_ns.route("/apps/<string:app_id>/assets/publish")
|
||||
class AppAssetPublishResource(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
|
||||
def post(self, app_model: App):
|
||||
current_user, _ = current_account_with_tenant()
|
||||
published = AppAssetService.publish(app_model, current_user.id)
|
||||
return {
|
||||
"id": published.id,
|
||||
"version": published.version,
|
||||
"asset_tree": published.asset_tree.model_dump(),
|
||||
}, 201
|
||||
|
||||
|
||||
@console_ns.route("/apps/<string:app_id>/assets/files/<string:node_id>/download-url")
|
||||
class AppAssetFileDownloadUrlResource(Resource):
|
||||
@setup_required
|
||||
|
||||
@ -51,6 +51,14 @@ class AppImportPayload(BaseModel):
|
||||
app_id: str | None = None
|
||||
|
||||
|
||||
class AppImportBundlePayload(BaseModel):
|
||||
name: str | None = None
|
||||
description: str | None = None
|
||||
icon_type: str | None = None
|
||||
icon: str | None = None
|
||||
icon_background: str | None = None
|
||||
|
||||
|
||||
console_ns.schema_model(
|
||||
AppImportPayload.__name__, AppImportPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0)
|
||||
)
|
||||
@ -139,3 +147,55 @@ class AppImportCheckDependenciesApi(Resource):
|
||||
result = import_service.check_dependencies(app_model=app_model)
|
||||
|
||||
return result.model_dump(mode="json"), 200
|
||||
|
||||
|
||||
@console_ns.route("/apps/imports-bundle")
|
||||
class AppImportBundleApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@marshal_with(app_import_model)
|
||||
@cloud_edition_billing_resource_check("apps")
|
||||
@edit_permission_required
|
||||
def post(self):
|
||||
from flask import request
|
||||
|
||||
from core.app.entities.app_bundle_entities import BundleFormatError
|
||||
from services.app_bundle_service import AppBundleService
|
||||
|
||||
current_user, _ = current_account_with_tenant()
|
||||
|
||||
if "file" not in request.files:
|
||||
return {"error": "No file provided"}, 400
|
||||
|
||||
file = request.files["file"]
|
||||
if not file.filename or not file.filename.endswith(".zip"):
|
||||
return {"error": "Invalid file format, expected .zip"}, 400
|
||||
|
||||
zip_bytes = file.read()
|
||||
|
||||
form_data = request.form.to_dict()
|
||||
args = AppImportBundlePayload.model_validate(form_data)
|
||||
|
||||
try:
|
||||
result = AppBundleService.import_bundle(
|
||||
account=current_user,
|
||||
zip_bytes=zip_bytes,
|
||||
name=args.name,
|
||||
description=args.description,
|
||||
icon_type=args.icon_type,
|
||||
icon=args.icon,
|
||||
icon_background=args.icon_background,
|
||||
)
|
||||
except BundleFormatError as e:
|
||||
return {"error": str(e)}, 400
|
||||
|
||||
if result.app_id and FeatureService.get_system_features().webapp_auth.enabled:
|
||||
EnterpriseService.WebAppAuth.update_app_access_mode(result.app_id, "private")
|
||||
|
||||
status = result.status
|
||||
if status == ImportStatus.FAILED:
|
||||
return result.model_dump(mode="json"), 400
|
||||
elif status == ImportStatus.PENDING:
|
||||
return result.model_dump(mode="json"), 202
|
||||
return result.model_dump(mode="json"), 200
|
||||
|
||||
@ -686,13 +686,14 @@ class PublishedWorkflowApi(Resource):
|
||||
"""
|
||||
Publish workflow
|
||||
"""
|
||||
from services.app_bundle_service import AppBundleService
|
||||
|
||||
current_user, _ = current_account_with_tenant()
|
||||
|
||||
args = PublishWorkflowPayload.model_validate(console_ns.payload or {})
|
||||
|
||||
workflow_service = WorkflowService()
|
||||
with Session(db.engine) as session:
|
||||
workflow = workflow_service.publish_workflow(
|
||||
workflow = AppBundleService.publish(
|
||||
session=session,
|
||||
app_model=app_model,
|
||||
account=current_user,
|
||||
|
||||
42
api/core/app/entities/app_bundle_entities.py
Normal file
42
api/core/app/entities/app_bundle_entities.py
Normal file
@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Constants
|
||||
BUNDLE_DSL_FILENAME_PATTERN = re.compile(r"^[^/]+\.ya?ml$")
|
||||
BUNDLE_MAX_SIZE = 50 * 1024 * 1024 # 50MB
|
||||
|
||||
|
||||
# Exceptions
|
||||
class BundleFormatError(Exception):
|
||||
"""Raised when bundle format is invalid."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ZipSecurityError(Exception):
|
||||
"""Raised when zip file contains security violations."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
# Entities
|
||||
class BundleExportResult(BaseModel):
|
||||
zip_bytes: bytes = Field(description="ZIP file content as bytes")
|
||||
filename: str = Field(description="Suggested filename for the ZIP")
|
||||
|
||||
|
||||
class SourceFileEntry(BaseModel):
|
||||
path: str = Field(description="File path within the ZIP")
|
||||
node_id: str = Field(description="Node ID in the asset tree")
|
||||
|
||||
|
||||
class ExtractedFile(BaseModel):
|
||||
path: str = Field(description="Relative path of the extracted file")
|
||||
content: bytes = Field(description="File content as bytes")
|
||||
|
||||
|
||||
class ExtractedFolder(BaseModel):
|
||||
path: str = Field(description="Relative path of the extracted folder")
|
||||
@ -4,7 +4,7 @@ from .entities import (
|
||||
FileAsset,
|
||||
SkillAsset,
|
||||
)
|
||||
from .packager import AssetPackager, ZipPackager
|
||||
from .packager import AssetPackager, AssetZipPackager
|
||||
from .parser import AssetItemParser, AssetParser, FileAssetParser, SkillAssetParser
|
||||
from .paths import AssetPaths
|
||||
|
||||
@ -15,9 +15,9 @@ __all__ = [
|
||||
"AssetPackager",
|
||||
"AssetParser",
|
||||
"AssetPaths",
|
||||
"AssetZipPackager",
|
||||
"FileAsset",
|
||||
"FileAssetParser",
|
||||
"SkillAsset",
|
||||
"SkillAssetParser",
|
||||
"ZipPackager",
|
||||
]
|
||||
|
||||
38
api/core/app_assets/converters.py
Normal file
38
api/core/app_assets/converters.py
Normal file
@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from core.app.entities.app_asset_entities import AppAssetFileTree, AssetNodeType
|
||||
from core.app_assets.entities import FileAsset
|
||||
from core.app_assets.paths import AssetPaths
|
||||
|
||||
|
||||
def tree_to_asset_items(
|
||||
tree: AppAssetFileTree,
|
||||
tenant_id: str,
|
||||
app_id: str,
|
||||
) -> list[FileAsset]:
|
||||
"""
|
||||
Convert AppAssetFileTree to list of FileAsset for packaging.
|
||||
|
||||
Args:
|
||||
tree: The asset file tree to convert
|
||||
tenant_id: Tenant ID for storage key generation
|
||||
app_id: App ID for storage key generation
|
||||
|
||||
Returns:
|
||||
List of FileAsset items ready for packaging
|
||||
"""
|
||||
items: list[FileAsset] = []
|
||||
for node in tree.nodes:
|
||||
if node.node_type == AssetNodeType.FILE:
|
||||
path = tree.get_path(node.id)
|
||||
storage_key = AssetPaths.draft_file(tenant_id, app_id, node.id)
|
||||
items.append(
|
||||
FileAsset(
|
||||
asset_id=node.id,
|
||||
path=path,
|
||||
file_name=node.name,
|
||||
extension=node.extension or "",
|
||||
storage_key=storage_key,
|
||||
)
|
||||
)
|
||||
return items
|
||||
@ -1,7 +1,7 @@
|
||||
from .asset_zip_packager import AssetZipPackager
|
||||
from .base import AssetPackager
|
||||
from .zip_packager import ZipPackager
|
||||
|
||||
__all__ = [
|
||||
"AssetPackager",
|
||||
"ZipPackager",
|
||||
"AssetZipPackager",
|
||||
]
|
||||
|
||||
78
api/core/app_assets/packager/asset_zip_packager.py
Normal file
78
api/core/app_assets/packager/asset_zip_packager.py
Normal file
@ -0,0 +1,78 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from core.app_assets.entities import AssetItem
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from extensions.ext_storage import Storage
|
||||
|
||||
|
||||
class AssetZipPackager:
|
||||
"""
|
||||
Unified ZIP packager for assets.
|
||||
Automatically creates directory entries from asset paths.
|
||||
"""
|
||||
|
||||
def __init__(self, storage: Storage, *, max_workers: int = 8) -> None:
|
||||
self._storage = storage
|
||||
self._max_workers = max_workers
|
||||
|
||||
def package(self, assets: list[AssetItem], *, prefix: str = "") -> bytes:
|
||||
"""
|
||||
Package assets into a ZIP file.
|
||||
|
||||
Args:
|
||||
assets: List of assets to package
|
||||
prefix: Optional prefix to add to all paths in the ZIP
|
||||
|
||||
Returns:
|
||||
ZIP file content as bytes
|
||||
"""
|
||||
zip_buffer = io.BytesIO()
|
||||
|
||||
# Extract folder paths from asset paths
|
||||
folder_paths = self._extract_folder_paths(assets, prefix)
|
||||
|
||||
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
# Create directory entries
|
||||
for folder_path in sorted(folder_paths):
|
||||
zf.writestr(zipfile.ZipInfo(folder_path + "/"), "")
|
||||
|
||||
# Write files in parallel
|
||||
if assets:
|
||||
self._write_files_parallel(zf, assets, prefix)
|
||||
|
||||
return zip_buffer.getvalue()
|
||||
|
||||
def _extract_folder_paths(self, assets: list[AssetItem], prefix: str) -> set[str]:
|
||||
"""Extract all folder paths from asset paths."""
|
||||
folders: set[str] = set()
|
||||
for asset in assets:
|
||||
full_path = f"{prefix}/{asset.path}" if prefix else asset.path
|
||||
parts = full_path.split("/")[:-1] # Remove filename
|
||||
folders.update("/".join(parts[:i]) for i in range(1, len(parts) + 1))
|
||||
return folders
|
||||
|
||||
def _write_files_parallel(
|
||||
self,
|
||||
zf: zipfile.ZipFile,
|
||||
assets: list[AssetItem],
|
||||
prefix: str,
|
||||
) -> None:
|
||||
lock = Lock()
|
||||
|
||||
def load_and_write(asset: AssetItem) -> None:
|
||||
content = self._storage.load_once(asset.get_storage_key())
|
||||
full_path = f"{prefix}/{asset.path}" if prefix else asset.path
|
||||
with lock:
|
||||
zf.writestr(full_path, content)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=self._max_workers) as executor:
|
||||
futures = [executor.submit(load_and_write, a) for a in assets]
|
||||
for future in futures:
|
||||
future.result()
|
||||
@ -1,42 +0,0 @@
|
||||
import io
|
||||
import zipfile
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from core.app_assets.entities import AssetItem
|
||||
|
||||
from .base import AssetPackager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from extensions.ext_storage import Storage
|
||||
|
||||
|
||||
class ZipPackager(AssetPackager):
|
||||
_storage: "Storage"
|
||||
|
||||
def __init__(self, storage: "Storage") -> None:
|
||||
self._storage = storage
|
||||
|
||||
def package(self, assets: list[AssetItem]) -> bytes:
|
||||
zip_buffer = io.BytesIO()
|
||||
|
||||
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
lock = Lock()
|
||||
# FOR DELVELPMENT AND TESTING ONLY, TODO: optimize
|
||||
with ThreadPoolExecutor(max_workers=8) as executor:
|
||||
futures: list[Future[None]] = []
|
||||
for asset in assets:
|
||||
|
||||
def _write_asset(a: AssetItem) -> None:
|
||||
content = self._storage.load_once(a.get_storage_key())
|
||||
with lock:
|
||||
zf.writestr(a.path, content)
|
||||
|
||||
futures.append(executor.submit(_write_asset, asset))
|
||||
|
||||
# Wait for all futures to complete
|
||||
for future in futures:
|
||||
future.result()
|
||||
|
||||
return zip_buffer.getvalue()
|
||||
@ -16,3 +16,8 @@ class AssetPaths:
|
||||
@staticmethod
|
||||
def build_skill_artifact_set(tenant_id: str, app_id: str, assets_id: str) -> str:
|
||||
return f"{AssetPaths._BASE}/{tenant_id}/{app_id}/artifacts/{assets_id}/skill_artifact_set.json"
|
||||
|
||||
@staticmethod
|
||||
def build_source_zip(tenant_id: str, app_id: str, workflow_id: str) -> str:
|
||||
"""Storage key for source assets zip (editable files snapshot at publish time)."""
|
||||
return f"{AssetPaths._BASE}/{tenant_id}/{app_id}/sources/{workflow_id}.zip"
|
||||
|
||||
5
api/core/app_bundle/__init__.py
Normal file
5
api/core/app_bundle/__init__.py
Normal file
@ -0,0 +1,5 @@
|
||||
from .source_zip_extractor import SourceZipExtractor
|
||||
|
||||
__all__ = [
|
||||
"SourceZipExtractor",
|
||||
]
|
||||
101
api/core/app_bundle/source_zip_extractor.py
Normal file
101
api/core/app_bundle/source_zip_extractor.py
Normal file
@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import zipfile
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING
|
||||
from uuid import uuid4
|
||||
|
||||
from core.app.entities.app_asset_entities import AppAssetFileTree, AppAssetNode
|
||||
from core.app.entities.app_bundle_entities import ExtractedFile, ExtractedFolder, ZipSecurityError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from extensions.ext_storage import Storage
|
||||
|
||||
|
||||
class SourceZipExtractor:
|
||||
def __init__(self, storage: Storage) -> None:
|
||||
self._storage = storage
|
||||
|
||||
def extract_entries(
|
||||
self, zip_bytes: bytes, *, expected_prefix: str
|
||||
) -> tuple[list[ExtractedFolder], list[ExtractedFile]]:
|
||||
folders: list[ExtractedFolder] = []
|
||||
files: list[ExtractedFile] = []
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as zf:
|
||||
for info in zf.infolist():
|
||||
name = info.filename
|
||||
self._validate_path(name)
|
||||
|
||||
if not name.startswith(expected_prefix):
|
||||
continue
|
||||
|
||||
relative_path = name[len(expected_prefix) :].lstrip("/")
|
||||
if not relative_path:
|
||||
continue
|
||||
|
||||
if info.is_dir():
|
||||
folders.append(ExtractedFolder(path=relative_path.rstrip("/")))
|
||||
else:
|
||||
content = zf.read(info)
|
||||
files.append(ExtractedFile(path=relative_path, content=content))
|
||||
|
||||
return folders, files
|
||||
|
||||
def build_tree_and_save(
|
||||
self,
|
||||
folders: list[ExtractedFolder],
|
||||
files: list[ExtractedFile],
|
||||
tenant_id: str,
|
||||
app_id: str,
|
||||
storage_key_fn: Callable[[str, str, str], str],
|
||||
) -> AppAssetFileTree:
|
||||
tree = AppAssetFileTree()
|
||||
path_to_node_id: dict[str, str] = {}
|
||||
|
||||
all_folder_paths = {f.path for f in folders}
|
||||
for file in files:
|
||||
self._ensure_parent_folders(file.path, all_folder_paths)
|
||||
|
||||
sorted_folders = sorted(all_folder_paths, key=lambda p: p.count("/"))
|
||||
for folder_path in sorted_folders:
|
||||
node_id = str(uuid4())
|
||||
name = folder_path.rsplit("/", 1)[-1]
|
||||
parent_path = folder_path.rsplit("/", 1)[0] if "/" in folder_path else None
|
||||
parent_id = path_to_node_id.get(parent_path) if parent_path else None
|
||||
|
||||
node = AppAssetNode.create_folder(node_id, name, parent_id)
|
||||
tree.add(node)
|
||||
path_to_node_id[folder_path] = node_id
|
||||
|
||||
sorted_files = sorted(files, key=lambda f: f.path)
|
||||
for file in sorted_files:
|
||||
node_id = str(uuid4())
|
||||
name = file.path.rsplit("/", 1)[-1]
|
||||
parent_path = file.path.rsplit("/", 1)[0] if "/" in file.path else None
|
||||
parent_id = path_to_node_id.get(parent_path) if parent_path else None
|
||||
|
||||
checksum = hashlib.sha256(file.content).hexdigest()
|
||||
node = AppAssetNode.create_file(node_id, name, parent_id, len(file.content), checksum)
|
||||
tree.add(node)
|
||||
|
||||
storage_key = storage_key_fn(tenant_id, app_id, node_id)
|
||||
self._storage.save(storage_key, file.content)
|
||||
|
||||
return tree
|
||||
|
||||
def _validate_path(self, path: str) -> None:
|
||||
if ".." in path:
|
||||
raise ZipSecurityError(f"Path traversal detected: {path}")
|
||||
if path.startswith("/"):
|
||||
raise ZipSecurityError(f"Absolute path detected: {path}")
|
||||
if "\\" in path:
|
||||
raise ZipSecurityError(f"Backslash in path: {path}")
|
||||
|
||||
def _ensure_parent_folders(self, file_path: str, folder_set: set[str]) -> None:
|
||||
parts = file_path.split("/")[:-1]
|
||||
for i in range(1, len(parts) + 1):
|
||||
parent = "/".join(parts[:i])
|
||||
folder_set.add(parent)
|
||||
@ -37,6 +37,11 @@ class AppAssetsInitializer(AsyncSandboxInitializer):
|
||||
["sh", "-c", f"unzip {AppAssets.ZIP_PATH} -d {AppAssets.PATH} 2>/dev/null || [ $? -eq 1 ]"],
|
||||
error_message="Failed to unzip assets",
|
||||
)
|
||||
# Ensure directories have execute permission for traversal and files are readable
|
||||
.add(
|
||||
["sh", "-c", f"chmod -R u+rwX,go+rX {AppAssets.PATH}"],
|
||||
error_message="Failed to set permissions on assets",
|
||||
)
|
||||
.execute(timeout=APP_ASSETS_DOWNLOAD_TIMEOUT, raise_on_error=True)
|
||||
)
|
||||
|
||||
|
||||
@ -1,248 +1,390 @@
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Protocol, cast
|
||||
|
||||
from core.app.entities.app_asset_entities import AppAssetFileTree
|
||||
from core.skill.entities.asset_references import AssetReferences
|
||||
from core.skill.entities.skill_bundle import SkillBundle
|
||||
from core.skill.entities.skill_bundle_entry import SkillBundleEntry, SourceInfo
|
||||
from core.skill.entities.skill_document import SkillDocument
|
||||
from core.skill.entities.skill_metadata import (
|
||||
FileReference,
|
||||
SkillMetadata,
|
||||
ToolConfiguration,
|
||||
ToolReference,
|
||||
)
|
||||
from core.skill.entities.skill_metadata import FileReference, SkillMetadata, ToolConfiguration, ToolReference
|
||||
from core.skill.entities.tool_dependencies import ToolDependencies, ToolDependency
|
||||
from core.tools.entities.tool_entities import ToolProviderType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TOOL_REFERENCE_PATTERN = re.compile(r"§\[tool\]\.\[([^\]]+)\]\.\[([^\]]+)\]\.\[([^\]]+)\]§")
|
||||
FILE_REFERENCE_PATTERN = re.compile(r"§\[file\]\.\[([^\]]+)\]\.\[([^\]]+)\]§")
|
||||
class PathResolver(Protocol):
|
||||
def resolve(self, source_id: str, target_id: str) -> str: ...
|
||||
|
||||
|
||||
class ToolResolver(Protocol):
|
||||
def resolve(self, tool_ref: ToolReference) -> str: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompilerConfig:
|
||||
tool_pattern: re.Pattern[str] = re.compile(r"§\[tool\]\.\[.*?\]\.\[.*?\]\.\[(.*?)\]§")
|
||||
file_pattern: re.Pattern[str] = re.compile(r"§\[file\]\.\[.*?\]\.\[(.*?)\]§")
|
||||
|
||||
|
||||
class FileTreePathResolver:
|
||||
def __init__(self, tree: AppAssetFileTree, base_path: str = ""):
|
||||
self._tree = tree
|
||||
self._base_path = base_path.rstrip("/")
|
||||
|
||||
def resolve(self, source_id: str, target_id: str) -> str:
|
||||
source_node = self._tree.get(source_id)
|
||||
target_node = self._tree.get(target_id)
|
||||
|
||||
if target_node is None:
|
||||
return "[File not found]"
|
||||
|
||||
if source_node is not None:
|
||||
return self._tree.relative_path(source_node, target_node)
|
||||
|
||||
full_path = self._tree.get_path(target_node.id)
|
||||
if self._base_path:
|
||||
return f"{self._base_path}/{full_path}"
|
||||
return full_path
|
||||
|
||||
|
||||
class DefaultToolResolver:
|
||||
def resolve(self, tool_ref: ToolReference) -> str:
|
||||
return f"[Executable: {tool_ref.tool_name}_{tool_ref.uuid} --help command]"
|
||||
|
||||
|
||||
class SkillCompiler:
|
||||
def _parse_metadata(self, content: str, raw_metadata: Mapping[str, Any]) -> SkillMetadata:
|
||||
tools_raw: dict[str, Any] = dict(raw_metadata.get("tools", {}))
|
||||
tools: dict[str, ToolReference] = {}
|
||||
files: list[FileReference] = []
|
||||
|
||||
for match in TOOL_REFERENCE_PATTERN.finditer(content):
|
||||
tool_id = match.group(3)
|
||||
tool_name = match.group(2)
|
||||
tool_provider = match.group(1)
|
||||
tool_meta = tools_raw.get(tool_id)
|
||||
if tool_meta is None:
|
||||
continue
|
||||
|
||||
config_raw = tool_meta.get("configuration", {})
|
||||
configuration = ToolConfiguration.model_validate(config_raw) if config_raw else None
|
||||
tools[tool_id] = ToolReference(
|
||||
uuid=tool_id,
|
||||
type=ToolProviderType.value_of(tool_meta.get("type")),
|
||||
provider=tool_provider,
|
||||
tool_name=tool_name,
|
||||
credential_id=tool_meta.get("credential_id"),
|
||||
configuration=configuration,
|
||||
)
|
||||
|
||||
for match in FILE_REFERENCE_PATTERN.finditer(content):
|
||||
files.append(
|
||||
FileReference(
|
||||
source=match.group(1),
|
||||
asset_id=match.group(2),
|
||||
)
|
||||
)
|
||||
|
||||
return SkillMetadata(tools=tools, files=files)
|
||||
def __init__(
|
||||
self,
|
||||
path_resolver: PathResolver | None = None,
|
||||
tool_resolver: ToolResolver | None = None,
|
||||
config: CompilerConfig | None = None,
|
||||
):
|
||||
self._path_resolver = path_resolver
|
||||
self._tool_resolver = tool_resolver or DefaultToolResolver()
|
||||
self._config = config or CompilerConfig()
|
||||
|
||||
def compile_all(
|
||||
self,
|
||||
documents: list[SkillDocument],
|
||||
documents: Iterable[SkillDocument],
|
||||
file_tree: AppAssetFileTree,
|
||||
assets_id: str,
|
||||
) -> SkillBundle:
|
||||
bundle = SkillBundle(
|
||||
assets_id=assets_id,
|
||||
built_at=datetime.now(UTC),
|
||||
)
|
||||
|
||||
doc_map: dict[str, SkillDocument] = {doc.skill_id: doc for doc in documents}
|
||||
parsed_metadata: dict[str, SkillMetadata] = {}
|
||||
|
||||
for doc in documents:
|
||||
metadata = self._parse_metadata(doc.content, doc.metadata)
|
||||
parsed_metadata[doc.skill_id] = metadata
|
||||
direct_skill_refs = self._extract_skill_refs(metadata, doc_map)
|
||||
bundle.dependency_graph[doc.skill_id] = list(direct_skill_refs)
|
||||
for ref_id in direct_skill_refs:
|
||||
if ref_id not in bundle.reverse_graph:
|
||||
bundle.reverse_graph[ref_id] = []
|
||||
bundle.reverse_graph[ref_id].append(doc.skill_id)
|
||||
|
||||
for doc in documents:
|
||||
metadata = parsed_metadata[doc.skill_id]
|
||||
entry = self._compile_single(doc, metadata, bundle, parsed_metadata, file_tree)
|
||||
bundle.upsert(entry)
|
||||
|
||||
return bundle
|
||||
path_resolver = self._path_resolver or FileTreePathResolver(file_tree)
|
||||
return self._compile_batch_internal(documents, assets_id, path_resolver)
|
||||
|
||||
def compile_one(
|
||||
self,
|
||||
bundle: SkillBundle,
|
||||
document: SkillDocument,
|
||||
file_tree: AppAssetFileTree,
|
||||
all_documents: dict[str, SkillDocument] | None = None,
|
||||
base_path: str = "",
|
||||
) -> SkillBundleEntry:
|
||||
doc_map = all_documents or {}
|
||||
if document.skill_id not in doc_map:
|
||||
doc_map[document.skill_id] = document
|
||||
|
||||
parsed_metadata: dict[str, SkillMetadata] = {}
|
||||
for skill_id, doc in doc_map.items():
|
||||
parsed_metadata[skill_id] = self._parse_metadata(doc.content, doc.metadata)
|
||||
|
||||
metadata = parsed_metadata[document.skill_id]
|
||||
direct_skill_refs = self._extract_skill_refs(metadata, doc_map)
|
||||
bundle.dependency_graph[document.skill_id] = list(direct_skill_refs)
|
||||
for ref_id in direct_skill_refs:
|
||||
if ref_id not in bundle.reverse_graph:
|
||||
bundle.reverse_graph[ref_id] = []
|
||||
if document.skill_id not in bundle.reverse_graph[ref_id]:
|
||||
bundle.reverse_graph[ref_id].append(document.skill_id)
|
||||
|
||||
return self._compile_single(document, metadata, bundle, parsed_metadata, file_tree)
|
||||
|
||||
def _compile_single(
|
||||
self,
|
||||
document: SkillDocument,
|
||||
metadata: SkillMetadata,
|
||||
bundle: SkillBundle,
|
||||
parsed_metadata: dict[str, SkillMetadata],
|
||||
file_tree: AppAssetFileTree,
|
||||
) -> SkillBundleEntry:
|
||||
all_tools, all_files = self._compute_transitive_closure(
|
||||
document.skill_id, bundle, parsed_metadata
|
||||
path_resolver = self._path_resolver or FileTreePathResolver(file_tree, base_path)
|
||||
resolved_content, tool_dependencies = self._compile_template_internal(
|
||||
document.content, document.metadata, bundle, path_resolver
|
||||
)
|
||||
|
||||
current_node = file_tree.get(document.skill_id)
|
||||
|
||||
resolved_content = self._resolve_content(
|
||||
document.content, metadata, current_node, file_tree
|
||||
)
|
||||
|
||||
content_digest = hashlib.sha256(document.content.encode("utf-8")).hexdigest()
|
||||
metadata = self._parse_metadata(document.content, document.metadata)
|
||||
final_files: dict[str, FileReference] = {f.asset_id: f for f in metadata.files}
|
||||
|
||||
return SkillBundleEntry(
|
||||
skill_id=document.skill_id,
|
||||
source=SourceInfo(
|
||||
asset_id=document.skill_id,
|
||||
content_digest=content_digest,
|
||||
content_digest=hashlib.sha256(document.content.encode("utf-8")).hexdigest(),
|
||||
),
|
||||
tools=tool_dependencies,
|
||||
files=AssetReferences(references=list(final_files.values())),
|
||||
content=resolved_content,
|
||||
)
|
||||
|
||||
def _compile_batch_internal(
|
||||
self,
|
||||
documents: Iterable[SkillDocument],
|
||||
assets_id: str,
|
||||
path_resolver: PathResolver,
|
||||
) -> SkillBundle:
|
||||
doc_map = {doc.skill_id: doc for doc in documents}
|
||||
graph: dict[str, set[str]] = {}
|
||||
metadata_cache: dict[str, SkillMetadata] = {}
|
||||
|
||||
# Phase 1: Parse metadata and build dependency graph
|
||||
for doc in doc_map.values():
|
||||
metadata = self._parse_metadata(doc.content, doc.metadata)
|
||||
metadata_cache[doc.skill_id] = metadata
|
||||
|
||||
deps: set[str] = set()
|
||||
for file_ref in metadata.files:
|
||||
if file_ref.asset_id in doc_map:
|
||||
deps.add(file_ref.asset_id)
|
||||
graph[doc.skill_id] = deps
|
||||
|
||||
bundle = SkillBundle(assets_id=assets_id)
|
||||
bundle.dependency_graph = {k: list(v) for k, v in graph.items()}
|
||||
|
||||
# Build reverse graph for propagation
|
||||
reverse_graph: dict[str, set[str]] = {skill_id: set() for skill_id in doc_map}
|
||||
for skill_id, deps in graph.items():
|
||||
for dep_id in deps:
|
||||
if dep_id in reverse_graph:
|
||||
reverse_graph[dep_id].add(skill_id)
|
||||
bundle.reverse_graph = {k: list(v) for k, v in reverse_graph.items()}
|
||||
|
||||
# Phase 2: Compile each skill independently (content + direct dependencies only)
|
||||
for skill_id, doc in doc_map.items():
|
||||
metadata = metadata_cache[skill_id]
|
||||
entry = self._compile_node_direct(doc, metadata, path_resolver)
|
||||
bundle.upsert(entry)
|
||||
|
||||
# Phase 3: Propagate transitive dependencies until fixed-point
|
||||
self._propagate_transitive_dependencies(bundle, graph)
|
||||
|
||||
return bundle
|
||||
|
||||
def _compile_node_direct(
|
||||
self,
|
||||
doc: SkillDocument,
|
||||
metadata: SkillMetadata,
|
||||
path_resolver: PathResolver,
|
||||
) -> SkillBundleEntry:
|
||||
"""Compile a single skill with only its direct dependencies (no transitive)."""
|
||||
direct_tools: dict[str, ToolDependency] = {}
|
||||
direct_refs: dict[str, ToolReference] = {}
|
||||
|
||||
for tool_ref in metadata.tools.values():
|
||||
key = f"{tool_ref.provider}.{tool_ref.tool_name}"
|
||||
if key not in direct_tools:
|
||||
direct_tools[key] = ToolDependency(
|
||||
type=tool_ref.type,
|
||||
provider=tool_ref.provider,
|
||||
tool_name=tool_ref.tool_name,
|
||||
)
|
||||
direct_refs[tool_ref.uuid] = tool_ref
|
||||
|
||||
direct_files: dict[str, FileReference] = {f.asset_id: f for f in metadata.files}
|
||||
resolved_content = self._resolve_content(doc.content, metadata, path_resolver, doc.skill_id)
|
||||
|
||||
return SkillBundleEntry(
|
||||
skill_id=doc.skill_id,
|
||||
source=SourceInfo(
|
||||
asset_id=doc.skill_id,
|
||||
content_digest=hashlib.sha256(doc.content.encode("utf-8")).hexdigest(),
|
||||
),
|
||||
tools=ToolDependencies(
|
||||
dependencies=list(all_tools.values()),
|
||||
references=list(metadata.tools.values()),
|
||||
dependencies=list(direct_tools.values()),
|
||||
references=list(direct_refs.values()),
|
||||
),
|
||||
files=AssetReferences(
|
||||
references=list(all_files.values()),
|
||||
references=list(direct_files.values()),
|
||||
),
|
||||
content=resolved_content,
|
||||
)
|
||||
|
||||
def _extract_skill_refs(
|
||||
def _propagate_transitive_dependencies(
|
||||
self,
|
||||
metadata: SkillMetadata,
|
||||
doc_map: dict[str, SkillDocument],
|
||||
) -> set[str]:
|
||||
skill_refs: set[str] = set()
|
||||
for file_ref in metadata.files:
|
||||
if file_ref.asset_id in doc_map:
|
||||
skill_refs.add(file_ref.asset_id)
|
||||
return skill_refs
|
||||
|
||||
def _compute_transitive_closure(
|
||||
self,
|
||||
skill_id: str,
|
||||
bundle: SkillBundle,
|
||||
parsed_metadata: dict[str, SkillMetadata],
|
||||
) -> tuple[dict[str, ToolDependency], dict[str, FileReference]]:
|
||||
all_tools: dict[str, ToolDependency] = {}
|
||||
all_files: dict[str, FileReference] = {}
|
||||
graph: dict[str, set[str]],
|
||||
) -> None:
|
||||
"""Iteratively propagate transitive dependencies until no changes occur."""
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for skill_id, dep_ids in graph.items():
|
||||
entry = bundle.get(skill_id)
|
||||
if not entry:
|
||||
continue
|
||||
|
||||
visited: set[str] = set()
|
||||
queue = [skill_id]
|
||||
# Collect current tools and files
|
||||
current_tools: dict[str, ToolDependency] = {
|
||||
f"{d.provider}.{d.tool_name}": d for d in entry.tools.dependencies
|
||||
}
|
||||
current_refs: dict[str, ToolReference] = {r.uuid: r for r in entry.tools.references}
|
||||
current_files: dict[str, FileReference] = {f.asset_id: f for f in entry.files.references}
|
||||
|
||||
while queue:
|
||||
current_id = queue.pop(0)
|
||||
if current_id in visited:
|
||||
continue
|
||||
visited.add(current_id)
|
||||
original_tool_count = len(current_tools)
|
||||
original_ref_count = len(current_refs)
|
||||
original_file_count = len(current_files)
|
||||
|
||||
metadata = parsed_metadata.get(current_id)
|
||||
if metadata is None:
|
||||
existing_entry = bundle.get(current_id)
|
||||
if existing_entry:
|
||||
for dep in existing_entry.tools.dependencies:
|
||||
key = f"{dep.provider}.{dep.tool_name}"
|
||||
if key not in all_tools:
|
||||
all_tools[key] = dep
|
||||
for file_ref in existing_entry.files.references:
|
||||
if file_ref.asset_id not in all_files:
|
||||
all_files[file_ref.asset_id] = file_ref
|
||||
continue
|
||||
# Merge from dependencies
|
||||
for dep_id in dep_ids:
|
||||
dep_entry = bundle.get(dep_id)
|
||||
if not dep_entry:
|
||||
continue
|
||||
|
||||
for tool_ref in metadata.tools.values():
|
||||
key = f"{tool_ref.provider}.{tool_ref.tool_name}"
|
||||
if key not in all_tools:
|
||||
all_tools[key] = ToolDependency(
|
||||
type=tool_ref.type,
|
||||
provider=tool_ref.provider,
|
||||
tool_name=tool_ref.tool_name,
|
||||
for tool_dep in dep_entry.tools.dependencies:
|
||||
key = f"{tool_dep.provider}.{tool_dep.tool_name}"
|
||||
if key not in current_tools:
|
||||
current_tools[key] = tool_dep
|
||||
|
||||
for tool_ref in dep_entry.tools.references:
|
||||
if tool_ref.uuid not in current_refs:
|
||||
current_refs[tool_ref.uuid] = tool_ref
|
||||
|
||||
for file_ref in dep_entry.files.references:
|
||||
if file_ref.asset_id not in current_files:
|
||||
current_files[file_ref.asset_id] = file_ref
|
||||
|
||||
# Check if anything changed
|
||||
if (
|
||||
len(current_tools) != original_tool_count
|
||||
or len(current_refs) != original_ref_count
|
||||
or len(current_files) != original_file_count
|
||||
):
|
||||
changed = True
|
||||
# Update the entry with new transitive dependencies
|
||||
updated_entry = SkillBundleEntry(
|
||||
skill_id=entry.skill_id,
|
||||
source=entry.source,
|
||||
tools=ToolDependencies(
|
||||
dependencies=list(current_tools.values()),
|
||||
references=list(current_refs.values()),
|
||||
),
|
||||
files=AssetReferences(
|
||||
references=list(current_files.values()),
|
||||
),
|
||||
content=entry.content,
|
||||
)
|
||||
bundle.upsert(updated_entry)
|
||||
|
||||
for file_ref in metadata.files:
|
||||
if file_ref.asset_id not in all_files:
|
||||
all_files[file_ref.asset_id] = file_ref
|
||||
|
||||
for dep_id in bundle.dependency_graph.get(current_id, []):
|
||||
if dep_id not in visited:
|
||||
queue.append(dep_id)
|
||||
|
||||
return all_tools, all_files
|
||||
|
||||
def _resolve_content(
|
||||
def _compile_template_internal(
|
||||
self,
|
||||
content: str,
|
||||
metadata_dict: Mapping[str, Any],
|
||||
context: SkillBundle,
|
||||
path_resolver: PathResolver,
|
||||
) -> tuple[str, ToolDependencies]:
|
||||
metadata = self._parse_metadata(content, metadata_dict)
|
||||
|
||||
direct_deps: list[SkillBundleEntry] = []
|
||||
for file_ref in metadata.files:
|
||||
artifact = context.get(file_ref.asset_id)
|
||||
if artifact:
|
||||
direct_deps.append(artifact)
|
||||
|
||||
final_tools, final_refs = self._aggregate_dependencies(metadata, direct_deps)
|
||||
|
||||
resolved_content = self._resolve_content(content, metadata, path_resolver, current_id="<template>")
|
||||
|
||||
return resolved_content, ToolDependencies(
|
||||
dependencies=list(final_tools.values()), references=list(final_refs.values())
|
||||
)
|
||||
|
||||
def _compile_node(
|
||||
self,
|
||||
doc: SkillDocument,
|
||||
metadata: SkillMetadata,
|
||||
current_node: Any,
|
||||
file_tree: AppAssetFileTree,
|
||||
direct_deps: Sequence[SkillBundleEntry],
|
||||
path_resolver: PathResolver,
|
||||
) -> SkillBundleEntry:
|
||||
final_tools, final_refs = self._aggregate_dependencies(metadata, direct_deps)
|
||||
|
||||
final_files: dict[str, FileReference] = {}
|
||||
for f in metadata.files:
|
||||
final_files[f.asset_id] = f
|
||||
|
||||
for dep in direct_deps:
|
||||
for f in dep.files.references:
|
||||
if f.asset_id not in final_files:
|
||||
final_files[f.asset_id] = f
|
||||
|
||||
resolved_content = self._resolve_content(doc.content, metadata, path_resolver, doc.skill_id)
|
||||
|
||||
return SkillBundleEntry(
|
||||
skill_id=doc.skill_id,
|
||||
source=SourceInfo(
|
||||
asset_id=doc.skill_id,
|
||||
content_digest=hashlib.sha256(doc.content.encode("utf-8")).hexdigest(),
|
||||
),
|
||||
tools=ToolDependencies(
|
||||
dependencies=list(final_tools.values()),
|
||||
references=list(final_refs.values()),
|
||||
),
|
||||
files=AssetReferences(
|
||||
references=list(final_files.values()),
|
||||
),
|
||||
content=resolved_content,
|
||||
)
|
||||
|
||||
def _aggregate_dependencies(
|
||||
self, metadata: SkillMetadata, direct_deps: Sequence[SkillBundleEntry]
|
||||
) -> tuple[dict[str, ToolDependency], dict[str, ToolReference]]:
|
||||
all_tools: dict[str, ToolDependency] = {}
|
||||
all_refs: dict[str, ToolReference] = {}
|
||||
|
||||
for tool_ref in metadata.tools.values():
|
||||
key = f"{tool_ref.provider}.{tool_ref.tool_name}"
|
||||
if key not in all_tools:
|
||||
all_tools[key] = ToolDependency(
|
||||
type=tool_ref.type,
|
||||
provider=tool_ref.provider,
|
||||
tool_name=tool_ref.tool_name,
|
||||
)
|
||||
all_refs[tool_ref.uuid] = tool_ref
|
||||
|
||||
for dep in direct_deps:
|
||||
for tool_dep in dep.tools.dependencies:
|
||||
key = f"{tool_dep.provider}.{tool_dep.tool_name}"
|
||||
if key not in all_tools:
|
||||
all_tools[key] = tool_dep
|
||||
|
||||
for tool_ref in dep.tools.references:
|
||||
if tool_ref.uuid not in all_refs:
|
||||
all_refs[tool_ref.uuid] = tool_ref
|
||||
|
||||
return all_tools, all_refs
|
||||
|
||||
def _resolve_content(
|
||||
self, content: str, metadata: SkillMetadata, path_resolver: PathResolver, current_id: str
|
||||
) -> str:
|
||||
if not content:
|
||||
return content
|
||||
def replace_file(match: re.Match[str]) -> str:
|
||||
target_id = match.group(1)
|
||||
try:
|
||||
return path_resolver.resolve(current_id, target_id)
|
||||
except Exception:
|
||||
return match.group(0)
|
||||
|
||||
for match in FILE_REFERENCE_PATTERN.finditer(content):
|
||||
file_id = match.group(2)
|
||||
file_node = file_tree.get(file_id)
|
||||
if file_node is None:
|
||||
logger.warning("File not found for id=%s, skipping", file_id)
|
||||
content = content.replace(match.group(0), "[File not found]")
|
||||
continue
|
||||
if current_node is not None:
|
||||
content = content.replace(match.group(0), file_tree.relative_path(current_node, file_node))
|
||||
else:
|
||||
content = content.replace(match.group(0), f"[{file_node.name}]")
|
||||
|
||||
for match in TOOL_REFERENCE_PATTERN.finditer(content):
|
||||
tool_id = match.group(3)
|
||||
tool = metadata.tools.get(tool_id)
|
||||
if tool is None:
|
||||
logger.warning("Tool not found for id=%s, skipping", tool_id)
|
||||
content = content.replace(match.group(0), f"[Tool not found: {tool_id}]")
|
||||
continue
|
||||
content = content.replace(match.group(0), f"[Bash Command: {tool.tool_name}_{tool_id}]")
|
||||
def replace_tool(match: re.Match[str]) -> str:
|
||||
tool_id = match.group(1)
|
||||
tool_ref = metadata.tools.get(tool_id)
|
||||
if not tool_ref:
|
||||
return f"[Tool not found: {tool_id}]"
|
||||
return self._tool_resolver.resolve(tool_ref)
|
||||
|
||||
content = self._config.file_pattern.sub(replace_file, content)
|
||||
content = self._config.tool_pattern.sub(replace_tool, content)
|
||||
return content
|
||||
|
||||
def _parse_metadata(self, content: str, raw_metadata: Mapping[str, Any]) -> SkillMetadata:
|
||||
tools_raw = dict(raw_metadata.get("tools", {}))
|
||||
tools: dict[str, ToolReference] = {}
|
||||
|
||||
tool_iter = re.finditer(r"§\[tool\]\.\[([^\]]+)\]\.\[([^\]]+)\]\.\[([^\]]+)\]§", content)
|
||||
for match in tool_iter:
|
||||
provider, name, uuid = match.group(1), match.group(2), match.group(3)
|
||||
if uuid in tools_raw:
|
||||
meta = tools_raw[uuid]
|
||||
if isinstance(meta, ToolReference):
|
||||
tools[uuid] = meta
|
||||
elif isinstance(meta, dict):
|
||||
tool_type_str = cast(str | None, meta.get("type"))
|
||||
if tool_type_str:
|
||||
tools[uuid] = ToolReference(
|
||||
uuid=uuid,
|
||||
type=ToolProviderType.value_of(tool_type_str),
|
||||
provider=provider,
|
||||
tool_name=name,
|
||||
credential_id=cast(str | None, meta.get("credential_id")),
|
||||
configuration=ToolConfiguration.model_validate(meta.get("configuration", {}))
|
||||
if meta.get("configuration")
|
||||
else None,
|
||||
)
|
||||
|
||||
parsed_files: list[FileReference] = []
|
||||
file_iter = re.finditer(r"§\[file\]\.\[([^\]]+)\]\.\[([^\]]+)\]§", content)
|
||||
for match in file_iter:
|
||||
source, asset_id = match.group(1), match.group(2)
|
||||
parsed_files.append(FileReference(source=source, asset_id=asset_id))
|
||||
|
||||
return SkillMetadata(tools=tools, files=parsed_files)
|
||||
|
||||
@ -13,13 +13,13 @@ from core.app.entities.app_asset_entities import (
|
||||
TreePathConflictError,
|
||||
)
|
||||
from core.app_assets.builder import AssetBuildPipeline, BuildContext
|
||||
from core.app_assets.packager.zip_packager import ZipPackager
|
||||
from core.app_assets.converters import tree_to_asset_items
|
||||
from core.app_assets.packager import AssetZipPackager
|
||||
from core.app_assets.paths import AssetPaths
|
||||
from extensions.ext_database import db
|
||||
from extensions.ext_redis import redis_client
|
||||
from extensions.ext_storage import storage
|
||||
from extensions.storage.file_presign_storage import FilePresignStorage
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models.app_asset import AppAssets
|
||||
from models.model import App
|
||||
|
||||
@ -445,35 +445,39 @@ class AppAssetService:
|
||||
AppAssetService._clear_draft_download_cache(cache_keys)
|
||||
|
||||
@staticmethod
|
||||
def publish(app_model: App, account_id: str) -> AppAssets:
|
||||
def publish(session: Session, app_model: App, account_id: str, workflow_id: str) -> AppAssets:
|
||||
tenant_id = app_model.tenant_id
|
||||
app_id = app_model.id
|
||||
with Session(db.engine, expire_on_commit=False) as session:
|
||||
assets = AppAssetService.get_or_create_assets(session, app_model, account_id)
|
||||
tree = assets.asset_tree
|
||||
|
||||
publish_id = str(uuid4())
|
||||
assets = AppAssetService.get_or_create_assets(session, app_model, account_id)
|
||||
tree = assets.asset_tree
|
||||
|
||||
published = AppAssets(
|
||||
id=publish_id,
|
||||
tenant_id=tenant_id,
|
||||
app_id=app_id,
|
||||
version=str(naive_utc_now()),
|
||||
created_by=account_id,
|
||||
)
|
||||
published.asset_tree = tree
|
||||
session.add(published)
|
||||
session.flush()
|
||||
publish_id = str(uuid4())
|
||||
|
||||
ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=publish_id)
|
||||
built_assets = AssetBuildPipeline().build_all(tree, ctx)
|
||||
published = AppAssets(
|
||||
id=publish_id,
|
||||
tenant_id=tenant_id,
|
||||
app_id=app_id,
|
||||
version=workflow_id,
|
||||
created_by=account_id,
|
||||
)
|
||||
published.asset_tree = tree
|
||||
session.add(published)
|
||||
session.flush()
|
||||
|
||||
packager = ZipPackager(storage)
|
||||
zip_bytes = packager.package(built_assets)
|
||||
zip_key = AssetPaths.build_zip(tenant_id, app_id, publish_id)
|
||||
storage.save(zip_key, zip_bytes)
|
||||
ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=publish_id)
|
||||
built_assets = AssetBuildPipeline().build_all(tree, ctx)
|
||||
|
||||
session.commit()
|
||||
packager = AssetZipPackager(storage)
|
||||
|
||||
runtime_zip_bytes = packager.package(built_assets)
|
||||
runtime_zip_key = AssetPaths.build_zip(tenant_id, app_id, publish_id)
|
||||
storage.save(runtime_zip_key, runtime_zip_bytes)
|
||||
|
||||
source_items = tree_to_asset_items(tree, tenant_id, app_id)
|
||||
source_zip_bytes = packager.package(source_items)
|
||||
source_zip_key = AssetPaths.build_source_zip(tenant_id, app_id, workflow_id)
|
||||
storage.save(source_zip_key, source_zip_bytes)
|
||||
|
||||
return published
|
||||
|
||||
@ -483,7 +487,12 @@ class AppAssetService:
|
||||
tree = assets.asset_tree
|
||||
|
||||
ctx = BuildContext(tenant_id=tenant_id, app_id=app_id, build_id=assets.id)
|
||||
AssetBuildPipeline().build_all(tree, ctx)
|
||||
built_assets = AssetBuildPipeline().build_all(tree, ctx)
|
||||
|
||||
packager = AssetZipPackager(storage)
|
||||
zip_bytes = packager.package(built_assets)
|
||||
zip_key = AssetPaths.build_zip(tenant_id, app_id, assets.id)
|
||||
storage.save(zip_key, zip_bytes)
|
||||
|
||||
@staticmethod
|
||||
def get_file_download_url(
|
||||
@ -503,3 +512,39 @@ class AppAssetService:
|
||||
storage_key = AssetPaths.draft_file(app_model.tenant_id, app_model.id, node_id)
|
||||
presign_storage = FilePresignStorage(storage.storage_runner)
|
||||
return presign_storage.get_download_url(storage_key, expires_in)
|
||||
|
||||
@staticmethod
|
||||
def get_published_assets_by_workflow_id(tenant_id: str, app_id: str, workflow_id: str) -> AppAssets | None:
|
||||
with Session(db.engine, expire_on_commit=False) as session:
|
||||
return (
|
||||
session.query(AppAssets)
|
||||
.filter(
|
||||
AppAssets.tenant_id == tenant_id,
|
||||
AppAssets.app_id == app_id,
|
||||
AppAssets.version == workflow_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_source_zip_bytes(tenant_id: str, app_id: str, workflow_id: str) -> bytes | None:
|
||||
source_zip_key = AssetPaths.build_source_zip(tenant_id, app_id, workflow_id)
|
||||
try:
|
||||
return storage.load_once(source_zip_key)
|
||||
except Exception:
|
||||
logger.warning("Source zip not found: %s", source_zip_key)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def set_draft_assets(
|
||||
app_model: App,
|
||||
account_id: str,
|
||||
new_tree: AppAssetFileTree,
|
||||
) -> AppAssets:
|
||||
with Session(db.engine, expire_on_commit=False) as session:
|
||||
assets = AppAssetService.get_or_create_assets(session, app_model, account_id)
|
||||
assets.asset_tree = new_tree
|
||||
assets.updated_by = account_id
|
||||
session.commit()
|
||||
|
||||
return assets
|
||||
|
||||
255
api/services/app_bundle_service.py
Normal file
255
api/services/app_bundle_service.py
Normal file
@ -0,0 +1,255 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
import zipfile
|
||||
|
||||
import yaml
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.app.entities.app_bundle_entities import (
|
||||
BUNDLE_DSL_FILENAME_PATTERN,
|
||||
BUNDLE_MAX_SIZE,
|
||||
BundleExportResult,
|
||||
BundleFormatError,
|
||||
ZipSecurityError,
|
||||
)
|
||||
from core.app_assets.converters import tree_to_asset_items
|
||||
from core.app_assets.packager import AssetZipPackager
|
||||
from core.app_assets.paths import AssetPaths
|
||||
from core.app_bundle import SourceZipExtractor
|
||||
from extensions.ext_database import db
|
||||
from extensions.ext_storage import storage
|
||||
from models import Account, App
|
||||
|
||||
from .app_asset_service import AppAssetService
|
||||
from .app_dsl_service import AppDslService, Import
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AppBundleService:
|
||||
@staticmethod
|
||||
def publish(
|
||||
session: Session,
|
||||
app_model: App,
|
||||
account: Account,
|
||||
marked_name: str = "",
|
||||
marked_comment: str = "",
|
||||
):
|
||||
"""
|
||||
Publish App Bundle (workflow + assets).
|
||||
Coordinates WorkflowService and AppAssetService publishing in a single transaction.
|
||||
"""
|
||||
from models.workflow import Workflow
|
||||
from services.workflow_service import WorkflowService
|
||||
|
||||
# 1. Publish workflow
|
||||
workflow: Workflow = WorkflowService().publish_workflow(
|
||||
session=session,
|
||||
app_model=app_model,
|
||||
account=account,
|
||||
marked_name=marked_name,
|
||||
marked_comment=marked_comment,
|
||||
)
|
||||
|
||||
# 2. Publish assets (bound to workflow_id)
|
||||
AppAssetService.publish(
|
||||
session=session,
|
||||
app_model=app_model,
|
||||
account_id=account.id,
|
||||
workflow_id=workflow.id,
|
||||
)
|
||||
|
||||
return workflow
|
||||
|
||||
@staticmethod
|
||||
def export_bundle(
|
||||
app_model: App,
|
||||
include_secret: bool = False,
|
||||
workflow_id: str | None = None,
|
||||
) -> BundleExportResult:
|
||||
dsl_content = AppDslService.export_dsl(
|
||||
app_model=app_model,
|
||||
include_secret=include_secret,
|
||||
workflow_id=workflow_id,
|
||||
)
|
||||
|
||||
safe_name = AppBundleService._sanitize_filename(app_model.name)
|
||||
assets_prefix = safe_name
|
||||
|
||||
zip_buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
zf.writestr(f"{safe_name}.yml", dsl_content.encode("utf-8"))
|
||||
|
||||
assets_zip_bytes = AppBundleService._get_assets_zip_bytes(app_model, workflow_id)
|
||||
if assets_zip_bytes:
|
||||
AppBundleService._merge_assets_into_bundle(zf, assets_zip_bytes, assets_prefix)
|
||||
|
||||
return BundleExportResult(
|
||||
zip_bytes=zip_buffer.getvalue(),
|
||||
filename=f"{safe_name}.zip",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def import_bundle(
|
||||
account: Account,
|
||||
zip_bytes: bytes,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
icon_type: str | None = None,
|
||||
icon: str | None = None,
|
||||
icon_background: str | None = None,
|
||||
) -> Import:
|
||||
if len(zip_bytes) > BUNDLE_MAX_SIZE:
|
||||
raise BundleFormatError(f"Bundle size exceeds limit: {BUNDLE_MAX_SIZE} bytes")
|
||||
|
||||
dsl_content, assets_prefix = AppBundleService._extract_dsl_from_bundle(zip_bytes)
|
||||
|
||||
with Session(db.engine) as session:
|
||||
dsl_service = AppDslService(session)
|
||||
import_result = dsl_service.import_app(
|
||||
account=account,
|
||||
import_mode="yaml-content",
|
||||
yaml_content=dsl_content,
|
||||
name=name,
|
||||
description=description,
|
||||
icon_type=icon_type,
|
||||
icon=icon,
|
||||
icon_background=icon_background,
|
||||
app_id=None,
|
||||
)
|
||||
session.commit()
|
||||
|
||||
if import_result.app_id and assets_prefix:
|
||||
AppBundleService._import_assets_from_bundle(
|
||||
zip_bytes=zip_bytes,
|
||||
assets_prefix=assets_prefix,
|
||||
app_id=import_result.app_id,
|
||||
account_id=account.id,
|
||||
)
|
||||
|
||||
return import_result
|
||||
|
||||
@staticmethod
|
||||
def _get_assets_zip_bytes(app_model: App, workflow_id: str | None) -> bytes | None:
|
||||
tenant_id = app_model.tenant_id
|
||||
app_id = app_model.id
|
||||
|
||||
if workflow_id is None:
|
||||
return AppBundleService._package_draft_assets(app_model)
|
||||
else:
|
||||
return AppAssetService.get_source_zip_bytes(tenant_id, app_id, workflow_id)
|
||||
|
||||
@staticmethod
|
||||
def _package_draft_assets(app_model: App) -> bytes | None:
|
||||
assets = AppAssetService.get_assets(
|
||||
tenant_id=app_model.tenant_id,
|
||||
app_id=app_model.id,
|
||||
user_id="",
|
||||
is_draft=True,
|
||||
)
|
||||
if not assets:
|
||||
return None
|
||||
|
||||
tree = assets.asset_tree
|
||||
if not tree.nodes:
|
||||
return None
|
||||
|
||||
items = tree_to_asset_items(tree, app_model.tenant_id, app_model.id)
|
||||
packager = AssetZipPackager(storage)
|
||||
return packager.package(items)
|
||||
|
||||
@staticmethod
|
||||
def _merge_assets_into_bundle(
|
||||
bundle_zf: zipfile.ZipFile,
|
||||
assets_zip_bytes: bytes,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
with zipfile.ZipFile(io.BytesIO(assets_zip_bytes), "r") as assets_zf:
|
||||
for info in assets_zf.infolist():
|
||||
content = assets_zf.read(info)
|
||||
new_path = f"{prefix}/{info.filename}"
|
||||
if info.is_dir():
|
||||
bundle_zf.writestr(zipfile.ZipInfo(new_path), "")
|
||||
else:
|
||||
bundle_zf.writestr(new_path, content)
|
||||
|
||||
@staticmethod
|
||||
def _extract_dsl_from_bundle(zip_bytes: bytes) -> tuple[str, str | None]:
|
||||
dsl_content: str | None = None
|
||||
dsl_filename: str | None = None
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as zf:
|
||||
for info in zf.infolist():
|
||||
if info.is_dir():
|
||||
continue
|
||||
if BUNDLE_DSL_FILENAME_PATTERN.match(info.filename):
|
||||
if dsl_content is not None:
|
||||
raise BundleFormatError("Multiple DSL files found in bundle")
|
||||
dsl_content = zf.read(info).decode("utf-8")
|
||||
dsl_filename = info.filename
|
||||
|
||||
if dsl_content is None or dsl_filename is None:
|
||||
raise BundleFormatError("No DSL file (*.yml or *.yaml) found in bundle root")
|
||||
|
||||
yaml.safe_load(dsl_content)
|
||||
|
||||
assets_prefix = dsl_filename.rsplit(".", 1)[0]
|
||||
has_assets = AppBundleService._check_assets_prefix_exists(zip_bytes, assets_prefix)
|
||||
|
||||
return dsl_content, assets_prefix if has_assets else None
|
||||
|
||||
@staticmethod
|
||||
def _check_assets_prefix_exists(zip_bytes: bytes, prefix: str) -> bool:
|
||||
with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as zf:
|
||||
for info in zf.infolist():
|
||||
if info.filename.startswith(f"{prefix}/"):
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _import_assets_from_bundle(
|
||||
zip_bytes: bytes,
|
||||
assets_prefix: str,
|
||||
app_id: str,
|
||||
account_id: str,
|
||||
) -> None:
|
||||
app_model = db.session.query(App).filter(App.id == app_id).first()
|
||||
if not app_model:
|
||||
logger.warning("App not found for asset import: %s", app_id)
|
||||
return
|
||||
|
||||
extractor = SourceZipExtractor(storage)
|
||||
try:
|
||||
folders, files = extractor.extract_entries(
|
||||
zip_bytes,
|
||||
expected_prefix=f"{assets_prefix}/",
|
||||
)
|
||||
except ZipSecurityError as e:
|
||||
logger.warning("Zip security error during asset import: %s", e)
|
||||
return
|
||||
|
||||
if not folders and not files:
|
||||
return
|
||||
|
||||
new_tree = extractor.build_tree_and_save(
|
||||
folders=folders,
|
||||
files=files,
|
||||
tenant_id=app_model.tenant_id,
|
||||
app_id=app_model.id,
|
||||
storage_key_fn=AssetPaths.draft_file,
|
||||
)
|
||||
|
||||
AppAssetService.set_draft_assets(
|
||||
app_model=app_model,
|
||||
account_id=account_id,
|
||||
new_tree=new_tree,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_filename(name: str) -> str:
|
||||
safe = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "_", name)
|
||||
safe = safe.strip(". ")
|
||||
return safe[:100] if safe else "app"
|
||||
@ -40,7 +40,6 @@ from models.tools import WorkflowToolProvider
|
||||
from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowType
|
||||
from models.workflow_features import WorkflowFeatures
|
||||
from repositories.factory import DifyAPIRepositoryFactory
|
||||
from services.app_asset_service import AppAssetService
|
||||
from services.billing_service import BillingService
|
||||
from services.enterprise.plugin_manager_service import PluginCredentialType
|
||||
from services.errors.app import IsDraftWorkflowError, TriggerNodeLimitExceededError, WorkflowHashNotEqualError
|
||||
@ -156,8 +155,6 @@ class WorkflowService:
|
||||
.first()
|
||||
)
|
||||
|
||||
AppAssetService.publish(app_model=app_model, account_id=app_model.created_by)
|
||||
|
||||
return workflow
|
||||
|
||||
def get_all_published_workflow(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user