mirror of
https://github.com/langgenius/dify.git
synced 2026-05-10 05:56:31 +08:00
feat: parallelize asset packing
This commit is contained in:
parent
c4943ff4f5
commit
b5e31c0f25
14
api/agent-notes/core/app_assets/packager/zip_packager.py.md
Normal file
14
api/agent-notes/core/app_assets/packager/zip_packager.py.md
Normal file
@ -0,0 +1,14 @@
|
||||
# Zip Packager Notes
|
||||
|
||||
## Purpose
|
||||
- Builds a ZIP archive of asset contents stored via the configured storage backend.
|
||||
|
||||
## Key Decisions
|
||||
- Packaging writes assets into an in-memory zip buffer returned as bytes.
|
||||
- Asset fetch + zip writing are executed via a thread pool with a lock guarding `ZipFile` writes.
|
||||
|
||||
## Edge Cases
|
||||
- ZIP writes are serialized by the lock; storage reads still run in parallel.
|
||||
|
||||
## Tests/Verification
|
||||
- None yet.
|
||||
@ -1,5 +1,7 @@
|
||||
import io
|
||||
import zipfile
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from core.app_assets.entities import AssetItem
|
||||
@ -20,8 +22,21 @@ class ZipPackager(AssetPackager):
|
||||
zip_buffer = io.BytesIO()
|
||||
|
||||
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
for asset in assets:
|
||||
content = self._storage.load_once(asset.get_storage_key())
|
||||
zf.writestr(asset.path, content)
|
||||
lock = Lock()
|
||||
# FOR DELVELPMENT AND TESTING ONLY, TODO: optimize
|
||||
with ThreadPoolExecutor(max_workers=8) as executor:
|
||||
futures: list[Future[None]] = []
|
||||
for asset in assets:
|
||||
|
||||
def _write_asset(a: AssetItem) -> None:
|
||||
content = self._storage.load_once(a.get_storage_key())
|
||||
with lock:
|
||||
zf.writestr(a.path, content)
|
||||
|
||||
futures.append(executor.submit(_write_asset, asset))
|
||||
|
||||
# Wait for all futures to complete
|
||||
for future in futures:
|
||||
future.result()
|
||||
|
||||
return zip_buffer.getvalue()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user