feat: parallelize asset packing

This commit is contained in:
Yeuoly 2026-01-21 16:23:44 +08:00
parent c4943ff4f5
commit b5e31c0f25
2 changed files with 32 additions and 3 deletions

View File

@ -0,0 +1,14 @@
# Zip Packager Notes
## Purpose
- Builds a ZIP archive of asset contents stored via the configured storage backend.
## Key Decisions
- Packaging writes assets into an in-memory zip buffer returned as bytes.
- Asset fetch + zip writing are executed via a thread pool with a lock guarding `ZipFile` writes.
## Edge Cases
- ZIP writes are serialized by the lock; storage reads still run in parallel.
## Tests/Verification
- None yet.

View File

@ -1,5 +1,7 @@
import io
import zipfile
from concurrent.futures import Future, ThreadPoolExecutor
from threading import Lock
from typing import TYPE_CHECKING
from core.app_assets.entities import AssetItem
@ -20,8 +22,21 @@ class ZipPackager(AssetPackager):
zip_buffer = io.BytesIO()
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
for asset in assets:
content = self._storage.load_once(asset.get_storage_key())
zf.writestr(asset.path, content)
lock = Lock()
# FOR DELVELPMENT AND TESTING ONLY, TODO: optimize
with ThreadPoolExecutor(max_workers=8) as executor:
futures: list[Future[None]] = []
for asset in assets:
def _write_asset(a: AssetItem) -> None:
content = self._storage.load_once(a.get_storage_key())
with lock:
zf.writestr(a.path, content)
futures.append(executor.submit(_write_asset, asset))
# Wait for all futures to complete
for future in futures:
future.result()
return zip_buffer.getvalue()