diff --git a/api/core/plugin/plugin_service.py b/api/core/plugin/plugin_service.py index a6d83e85dbc..805c52329b3 100644 --- a/api/core/plugin/plugin_service.py +++ b/api/core/plugin/plugin_service.py @@ -4,7 +4,7 @@ This module owns plugin daemon management calls that are shared by API services and core runtimes. Plugin model provider discovery is cached here, alongside plugin install, uninstall, and upgrade invalidation, so all cache mutations for plugin-owned provider metadata stay tenant-scoped and in one place. -Provider cache payloads may be stored as prefixed zlib bytes; readers also +Provider cache payloads may be stored as prefixed zstd bytes; readers also accept legacy plain JSON payloads for rolling upgrades and existing Redis keys. The console plugin list also normalizes endpoint setup counters against live @@ -16,12 +16,12 @@ metadata. import logging import time -import zlib from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager from mimetypes import guess_type from typing import Literal, Protocol +import zstandard from pydantic import BaseModel, TypeAdapter, ValidationError from redis import RedisError from redis.exceptions import LockError @@ -95,7 +95,7 @@ class PluginService: PLUGIN_MODEL_PROVIDERS_LOCK_TTL = 30 PLUGIN_MODEL_PROVIDERS_LOCK_WAIT_TIMEOUT = 2.0 PLUGIN_MODEL_PROVIDERS_LOCK_WAIT_INTERVAL = 0.05 - PLUGIN_MODEL_PROVIDERS_CACHE_COMPRESSION_PREFIX = b"\x00dify-plugin-model-providers-zlib-v1:" + PLUGIN_MODEL_PROVIDERS_CACHE_COMPRESSION_PREFIX = b"\x00dify-plugin-model-providers-zstd-v1:" PLUGIN_MODEL_PROVIDERS_CACHE_COMPRESSION_MIN_BYTES = 64 * 1024 PLUGIN_INSTALL_TASK_TERMINAL_STATUSES = (PluginInstallTaskStatus.Success, PluginInstallTaskStatus.Failed) # Mirror the detail-panel endpoint query size so list reconciliation and @@ -152,7 +152,7 @@ class PluginService: if len(payload) < cls.PLUGIN_MODEL_PROVIDERS_CACHE_COMPRESSION_MIN_BYTES: return payload - return cls.PLUGIN_MODEL_PROVIDERS_CACHE_COMPRESSION_PREFIX + zlib.compress(payload, level=1) + return cls.PLUGIN_MODEL_PROVIDERS_CACHE_COMPRESSION_PREFIX + zstandard.compress(payload, level=1) @classmethod def _decode_plugin_model_providers_cache_payload(cls, payload: bytes | bytearray | str) -> bytes | bytearray | str: @@ -164,8 +164,8 @@ class PluginService: return payload try: - return zlib.decompress(payload[len(prefix) :]) - except zlib.error as exc: + return zstandard.decompress(payload[len(prefix) :]) + except zstandard.ZstdError as exc: raise ValueError("Invalid compressed plugin model providers cache payload.") from exc @classmethod diff --git a/api/pyproject.toml b/api/pyproject.toml index 9978dad738f..d3fcb59d694 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ "opentelemetry-propagator-b3>=1.41.1,<2.0.0", "readabilipy==0.3.0", "resend>=2.27.0,<3.0.0", + "zstandard==0.25.0", # Emerging: newer and fast-moving, use compatible pins "fastopenapi[flask]==0.7.0", "graphon==0.6.0", diff --git a/api/tests/unit_tests/services/plugin/test_plugin_service.py b/api/tests/unit_tests/services/plugin/test_plugin_service.py index 11230b25b40..bd8d84834fb 100644 --- a/api/tests/unit_tests/services/plugin/test_plugin_service.py +++ b/api/tests/unit_tests/services/plugin/test_plugin_service.py @@ -1,10 +1,10 @@ import datetime import uuid -import zlib from types import SimpleNamespace from unittest.mock import MagicMock, Mock, call, patch import pytest +import zstandard from pydantic import TypeAdapter from redis import RedisError @@ -154,7 +154,7 @@ class TestPluginModelProviderCache: prefix = PluginService.PLUGIN_MODEL_PROVIDERS_CACHE_COMPRESSION_PREFIX assert stored_payload.startswith(prefix) assert len(stored_payload) < len(raw_payload) - assert zlib.decompress(stored_payload[len(prefix) :]) == raw_payload + assert zstandard.decompress(stored_payload[len(prefix) :]) == raw_payload def test_fetch_plugin_model_providers_reads_compressed_cached_provider_without_calling_daemon(self) -> None: """Compressed tenant cache entries are decoded before provider schema validation.""" @@ -166,7 +166,7 @@ class TestPluginModelProviderCache: from core.plugin.plugin_service import PluginService - compressed_payload = PluginService.PLUGIN_MODEL_PROVIDERS_CACHE_COMPRESSION_PREFIX + zlib.compress( + compressed_payload = PluginService.PLUGIN_MODEL_PROVIDERS_CACHE_COMPRESSION_PREFIX + zstandard.compress( cached_payload, level=1 ) diff --git a/api/uv.lock b/api/uv.lock index df9effefdf5..6e7e1811064 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -1375,6 +1375,7 @@ dependencies = [ { name = "resend" }, { name = "sendgrid" }, { name = "sseclient-py" }, + { name = "zstandard" }, ] [package.dev-dependencies] @@ -1658,6 +1659,7 @@ requires-dist = [ { name = "resend", specifier = ">=2.27.0,<3.0.0" }, { name = "sendgrid", specifier = ">=6.12.5,<7.0.0" }, { name = "sseclient-py", specifier = ">=1.8.0,<2.0.0" }, + { name = "zstandard", specifier = "==0.25.0" }, ] [package.metadata.requires-dev]