feat(dify-agent): expand and enforce agent run limits (#40641)

This commit is contained in:
盐粒 Yanli 2026-08-14 08:27:19 +00:00 committed by GitHub
parent 9ddbc23b16
commit 3acc9470d4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 355 additions and 66 deletions

View File

@ -707,7 +707,6 @@ AGENT_BACKEND_BASE_URL=http://localhost:5050
AGENT_BACKEND_API_TOKEN=dify-agent-run-token-for-dev-only
AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS=30
AGENT_BACKEND_STREAM_MAX_RECONNECTS=3
AGENT_BACKEND_RUN_TIMEOUT_SECONDS=1200
# KnowledgeFS (Dataset 2.0)
KNOWLEDGE_FS_ENABLED=false

View File

@ -92,11 +92,9 @@ class DifyAgentBackendRunClient:
client: _DifyAgentSyncClient,
*,
stream_max_reconnects: int = 3,
stream_timeout_seconds: float = 1200,
) -> None:
self.client = client
self._stream_max_reconnects = stream_max_reconnects
self._stream_timeout_seconds = stream_timeout_seconds
def create_run(self, request: CreateRunRequest) -> CreateRunResponse:
"""Create one run through ``POST /runs`` and normalize client exceptions."""
@ -125,7 +123,6 @@ class DifyAgentBackendRunClient:
run_id,
after=after,
max_reconnects=self._stream_max_reconnects,
timeout_seconds=self._stream_timeout_seconds,
should_stop=should_stop,
)
except Exception as exc:

View File

@ -22,7 +22,6 @@ def create_agent_backend_run_client(
fake_scenario: str | FakeAgentBackendScenario = FakeAgentBackendScenario.SUCCESS,
stream_read_timeout_seconds: float = 30,
stream_max_reconnects: int = 3,
stream_run_timeout_seconds: float = 1200,
) -> AgentBackendRunClient:
"""Create the API-side run client without hiding the ``dify-agent`` protocol."""
if use_fake:
@ -36,5 +35,4 @@ def create_agent_backend_run_client(
stream_timeout=stream_read_timeout_seconds,
),
stream_max_reconnects=stream_max_reconnects,
stream_timeout_seconds=stream_run_timeout_seconds,
)

View File

@ -37,11 +37,6 @@ class AgentBackendConfig(BaseSettings):
default=3,
)
AGENT_BACKEND_RUN_TIMEOUT_SECONDS: PositiveFloat = Field(
description="Total deadline for one Agent backend run event stream.",
default=1200,
)
AGENT_SHELL_ENABLED: bool = Field(
description=(
"Inject the Home, Workspace, Sandbox, and Shell runtime layers into Agent runs. "

View File

@ -553,7 +553,6 @@ class AgentAppGenerator(MessageBasedAppGenerator):
fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
stream_read_timeout_seconds=dify_config.AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS,
stream_max_reconnects=dify_config.AGENT_BACKEND_STREAM_MAX_RECONNECTS,
stream_run_timeout_seconds=dify_config.AGENT_BACKEND_RUN_TIMEOUT_SECONDS,
),
event_adapter=AgentBackendRunEventAdapter(),
session_store=AgentAppWorkspaceStore(),

View File

@ -539,7 +539,6 @@ class DifyNodeFactory(NodeFactory):
fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
stream_read_timeout_seconds=dify_config.AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS,
stream_max_reconnects=dify_config.AGENT_BACKEND_STREAM_MAX_RECONNECTS,
stream_run_timeout_seconds=dify_config.AGENT_BACKEND_RUN_TIMEOUT_SECONDS,
),
"event_adapter": AgentBackendRunEventAdapter(),
# Agent Files §4.6: reback file outputs from the ToolFile row so

View File

@ -1,5 +1,5 @@
from collections.abc import Callable, Iterator
from typing import override
from typing import cast, override
import pytest
from dify_agent.client import DifyAgentHTTPError, DifyAgentStreamError, DifyAgentTimeoutError, DifyAgentValidationError
@ -25,8 +25,10 @@ from clients.agent_backend import (
DifyAgentBackendRunClient,
)
_STREAM_TIMEOUT_UNSET = object()
def _request():
def _request() -> CreateRunRequest:
return AgentBackendRunRequestBuilder().build_for_workflow_node(
AgentBackendWorkflowNodeRunInput(
model=AgentBackendModelConfig(
@ -48,7 +50,7 @@ def _request():
class _SuccessfulClient:
stream_options: tuple[int | None, float | None, Callable[[], bool] | None] | None = None
stream_options: tuple[int | None, object, Callable[[], bool] | None] | None = None
def create_run_sync(self, request: CreateRunRequest) -> CreateRunResponse:
assert isinstance(request, CreateRunRequest)
@ -64,7 +66,7 @@ class _SuccessfulClient:
*,
after: str | None = None,
max_reconnects: int | None = None,
timeout_seconds: float | None = None,
timeout_seconds: float | None = cast(float | None, _STREAM_TIMEOUT_UNSET),
should_stop: Callable[[], bool] | None = None,
) -> Iterator[RunEvent]:
del after
@ -83,9 +85,9 @@ class _SuccessfulClient:
)
def test_dify_agent_backend_run_client_delegates_sync_methods():
def test_dify_agent_backend_run_client_delegates_sync_methods() -> None:
wrapped = _SuccessfulClient()
client = DifyAgentBackendRunClient(wrapped, stream_max_reconnects=2, stream_timeout_seconds=45)
client = DifyAgentBackendRunClient(wrapped, stream_max_reconnects=2)
def should_stop() -> bool:
return False
@ -99,10 +101,10 @@ def test_dify_agent_backend_run_client_delegates_sync_methods():
assert cancelled.status == "cancelled"
assert events[0].type == "run_started"
assert status.status == "succeeded"
assert wrapped.stream_options == (2, 45, should_stop)
assert wrapped.stream_options == (2, _STREAM_TIMEOUT_UNSET, should_stop)
def test_dify_agent_backend_run_client_maps_validation_error():
def test_dify_agent_backend_run_client_maps_validation_error() -> None:
class InvalidClient(_SuccessfulClient):
@override
def create_run_sync(self, request: CreateRunRequest) -> CreateRunResponse:
@ -114,7 +116,7 @@ def test_dify_agent_backend_run_client_maps_validation_error():
assert exc_info.value.detail == {"field": "bad"}
def test_dify_agent_backend_run_client_maps_http_error():
def test_dify_agent_backend_run_client_maps_http_error() -> None:
class HTTPErrorClient(_SuccessfulClient):
@override
def create_run_sync(self, request: CreateRunRequest) -> CreateRunResponse:
@ -127,7 +129,7 @@ def test_dify_agent_backend_run_client_maps_http_error():
assert exc_info.value.detail == "unavailable"
def test_dify_agent_backend_run_client_maps_timeout_error():
def test_dify_agent_backend_run_client_maps_timeout_error() -> None:
class TimeoutClient(_SuccessfulClient):
@override
def wait_run_sync(self, run_id: str, *, timeout_seconds: float | None = None) -> RunStatusResponse:
@ -139,7 +141,7 @@ def test_dify_agent_backend_run_client_maps_timeout_error():
assert str(exc_info.value) == "timeout"
def test_dify_agent_backend_run_client_maps_stream_error():
def test_dify_agent_backend_run_client_maps_stream_error() -> None:
class StreamClient(_SuccessfulClient):
@override
def stream_events_sync(

View File

@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
import pytest
from dify_agent.client import Client
from clients.agent_backend.factory import create_agent_backend_client
from clients.agent_backend.factory import create_agent_backend_client, create_agent_backend_run_client
from configs import dify_config
from services import agent_app_sandbox_service
from services.agent import home_snapshot_service, workspace_service
@ -35,6 +35,21 @@ def test_create_agent_backend_client_forwards_authentication(
)
@patch("clients.agent_backend.factory.create_agent_backend_client")
def test_create_agent_backend_run_client_forwards_stream_read_timeout(create_client: MagicMock) -> None:
create_agent_backend_run_client(
base_url="http://agent-backend",
api_token="secret-token",
stream_read_timeout_seconds=17.5,
)
create_client.assert_called_once_with(
base_url="http://agent-backend",
api_token="secret-token",
stream_timeout=17.5,
)
@pytest.mark.parametrize(
("factory", "module"),
[

View File

@ -196,9 +196,11 @@ Neither path creates a fallback Workspace or switches backends.
`DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS` limits continuous active time for an E2B
resource to one hour. The limit covers the complete Agent run held by one
RuntimeLease rather than an individual tool call. Runtime resources pause on
timeout. It is not a retention TTL and does not delete paused resources or
immutable snapshots.
RuntimeLease rather than an individual tool call. Its 3600-second default is
intentionally the same as `DIFY_AGENT_RUN_TIMEOUT_SECONDS`, but the two settings
remain independently configurable. Runtime resources pause on timeout, but this
resource setting does not own the Agent run terminal state. It is not a retention
TTL and does not delete paused resources or immutable snapshots.
See the [Shell layer](../../user-manual/shell-layer/index.md) for request
composition and the [Operations Guide](../../guide/index.md) for Local and E2B

View File

@ -35,6 +35,7 @@ also reads `.env` and `dify-agent/.env` when present.
| `DIFY_AGENT_REDIS_PREFIX` | `dify-agent` | Prefix for Redis record and event keys. |
| `DIFY_AGENT_SHUTDOWN_GRACE_SECONDS` | `30` | Seconds to wait for active local runs during graceful shutdown before cancellation. |
| `DIFY_AGENT_RUN_RETENTION_SECONDS` | `259200` | Seconds to retain Redis run records and per-run event streams; defaults to 3 days. |
| `DIFY_AGENT_RUN_TIMEOUT_SECONDS` | `3600` | Wall-clock deadline in seconds for the Pydantic AI `agent.run(...)` model/tool loop. Deadline failures use `agent_run_limit_exceeded`. Its default intentionally matches `DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS`, but the settings are independently configurable. |
| `DIFY_AGENT_API_TOKEN` | empty | Optional Bearer token required by private run, Execution Binding, Home Snapshot, and Binding file control-plane routes. Must match Dify API `AGENT_BACKEND_API_TOKEN`. |
| `DIFY_AGENT_PLUGIN_DAEMON_URL` | `http://localhost:5002` | Base URL for the Dify plugin daemon. |
| `DIFY_AGENT_PLUGIN_DAEMON_API_KEY` | empty | API key sent to the Dify plugin daemon. |
@ -52,7 +53,7 @@ also reads `.env` and `dify-agent/.env` when present.
| `DIFY_AGENT_ENTERPRISE_SANDBOX_PROXY_TIMEOUT` | `60` | Enterprise shellctl-proxy timeout in seconds. |
| `DIFY_AGENT_E2B_API_KEY` | empty | E2B API key; required for E2B. |
| `DIFY_AGENT_E2B_TEMPLATE` | `difys-default-team/dify-agent-local-sandbox` | Prepared E2B template containing shellctl and the deployment-default Home environment. |
| `DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS` | `3600` | Maximum continuous active time for the RuntimeLease spanning one complete Agent run. Binding resources pause on timeout. This is not a retention TTL. |
| `DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS` | `3600` | Maximum continuous active time for the RuntimeLease spanning one complete Agent run. Its default intentionally matches `DIFY_AGENT_RUN_TIMEOUT_SECONDS`, but the settings are independently configurable. Binding resources pause on timeout; this setting does not own the run terminal state and is not a retention TTL. |
| `DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN` | empty | Optional bearer token expected by shellctl inside the E2B template. |
| `DIFY_AGENT_E2B_SHELLCTL_PORT` | `5004` | shellctl port exposed by the E2B template. |
| `DIFY_AGENT_SHELL_REDACT_PATTERNS` | empty | JSON array of additional regex patterns redacted from Shell output. |
@ -74,6 +75,7 @@ DIFY_AGENT_REDIS_URL=redis://localhost:6379/0
DIFY_AGENT_REDIS_PREFIX=dify-agent-dev
DIFY_AGENT_SHUTDOWN_GRACE_SECONDS=30
DIFY_AGENT_RUN_RETENTION_SECONDS=259200
DIFY_AGENT_RUN_TIMEOUT_SECONDS=3600
DIFY_AGENT_API_TOKEN=replace-with-agent-backend-token
DIFY_AGENT_PLUGIN_DAEMON_URL=http://localhost:5002
DIFY_AGENT_PLUGIN_DAEMON_API_KEY=replace-with-daemon-key
@ -206,8 +208,16 @@ docker compose \
`DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS` controls continuous active E2B time.
The physical resource behind a Binding pauses when that timeout fires, preserving
the current Workspace. The setting is not a resource-age TTL and does not delete
paused resources or immutable snapshots.
the current Workspace. The setting is not a resource-age TTL, does not delete
paused resources or immutable snapshots, and does not authoritatively finalize
the Agent run. If the paused Sandbox is first observed by a Shell tool, that
provider failure is returned to Pydantic AI as a tool error observation.
The run and E2B defaults both equal 3600 seconds, but independent clocks and
asynchronous E2B pause propagation make their ordering nondeterministic. A Shell
provider `RuntimeError` observed first becomes a tool observation. In contrast,
run-deadline cancellation propagates through the Shell boundary; only the Dify
Agent run deadline owns the terminal `agent_run_limit_exceeded` failure.
## Run runtime-backend integration contracts
@ -257,10 +267,17 @@ automatic retry layer. Request-shaped runtime failures such as bad composition,
prompt, output, or snapshot inputs are reported later as failed runs rather than
rejected synchronously once the request DTO itself is accepted.
Each run explicitly limits Pydantic AI to 100 model-request steps. Tool calls do
Each run explicitly limits Pydantic AI to 500 model-request steps. Tool calls do
not have a separate count limit, but every model request used to continue the
tool loop consumes one of those steps.
`DIFY_AGENT_RUN_TIMEOUT_SECONDS` additionally applies a wall-clock deadline only
around Pydantic AI's `agent.run(...)`, including its model/tool loop and event
handler. It does not include compositor entry, RuntimeLease acquisition, tool
preparation, session snapshot generation, or resource exit. Expiry cancels the
active run task, allows the compositor to release resources, and finalizes the
run as failed with `error_type: "agent_run_limit_exceeded"`.
During FastAPI shutdown the scheduler rejects new runs, waits up to
`DIFY_AGENT_SHUTDOWN_GRACE_SECONDS` for active tasks, then cancels remaining tasks
and attempts to finalize them as failed. Success, failure, cancellation, and this
@ -358,13 +375,12 @@ Failed event payloads contain the diagnostic `error`, optional source-specific
`reason`, and optional stable `error_type`. Pydantic AI request/step budget
exhaustion enforced by Dify Agent is reported as
`error_type: "agent_run_limit_exceeded"`; consumers should branch on that value
rather than parsing the error text. This type does not classify wall-clock run
timeouts, whose classification is not implemented in this release, or provider
and connection timeouts. The matching failed run record and terminal event are
committed atomically with the same error type. For independently deployed Agent
backend and API services, deploy consumers that accept the optional field before
producers begin emitting it because the public protocol models reject unknown
fields.
rather than parsing the error text. The Dify Agent-owned wall-clock run deadline
uses the same error type; provider and connection timeouts do not. The matching
failed run record and terminal event are committed atomically with the same error
type. For independently deployed Agent backend and API services, deploy consumers
that accept the optional field before producers begin emitting it because the
public protocol models reject unknown fields.
## Examples

View File

@ -24,7 +24,7 @@ from agenton.compositor import LayerProviderInput
from dify_agent.protocol.schemas import CancelRunRequest, CancelRunResponse, CreateRunRequest
from dify_agent.runtime.compositor_factory import create_default_layer_providers
from dify_agent.runtime.event_sink import RunEventSink, emit_run_cancelled, emit_run_failed
from dify_agent.runtime.runner import AgentRunRunner
from dify_agent.runtime.runner import DEFAULT_AGENT_RUN_TIMEOUT_SECONDS, AgentRunRunner
from dify_agent.server.schemas import RunRecord
logger = logging.getLogger(__name__)
@ -75,6 +75,7 @@ class RunScheduler:
store: RunStore
shutdown_grace_seconds: float
run_timeout_seconds: float
active_tasks: dict[str, asyncio.Task[None]]
stopping: bool
runner_factory: RunRunnerFactory | None
@ -90,11 +91,13 @@ class RunScheduler:
plugin_daemon_http_client: httpx.AsyncClient,
dify_api_http_client: httpx.AsyncClient,
shutdown_grace_seconds: float = 30,
run_timeout_seconds: float = DEFAULT_AGENT_RUN_TIMEOUT_SECONDS,
layer_providers: tuple[LayerProviderInput, ...] | None = None,
runner_factory: RunRunnerFactory | None = None,
) -> None:
self.store = store
self.shutdown_grace_seconds = shutdown_grace_seconds
self.run_timeout_seconds = run_timeout_seconds
self.active_tasks = {}
self.stopping = False
self.plugin_daemon_http_client = plugin_daemon_http_client
@ -224,6 +227,7 @@ class RunScheduler:
dify_api_http_client=self.dify_api_http_client,
layer_providers=self.layer_providers,
is_cancelled=is_cancelled,
run_timeout_seconds=self.run_timeout_seconds,
)
def _discard_active_run(self, run_id: str) -> None:

View File

@ -81,7 +81,8 @@ from dify_agent.runtime.user_prompt_validation import EMPTY_USER_PROMPTS_ERROR,
_AGENT_OUTPUT_ADAPTER = TypeAdapter(object)
_MAX_AGENT_STEPS_PER_RUN = 100
_MAX_AGENT_STEPS_PER_RUN = 500
DEFAULT_AGENT_RUN_TIMEOUT_SECONDS = 60 * 60
@runtime_checkable
@ -181,6 +182,7 @@ class AgentRunRunner:
plugin_daemon_http_client: httpx.AsyncClient
dify_api_http_client: httpx.AsyncClient
is_cancelled: Callable[[], bool]
run_timeout_seconds: float
def __init__(
self,
@ -192,6 +194,7 @@ class AgentRunRunner:
dify_api_http_client: httpx.AsyncClient,
layer_providers: tuple[LayerProviderInput, ...] | None = None,
is_cancelled: Callable[[], bool] | None = None,
run_timeout_seconds: float = DEFAULT_AGENT_RUN_TIMEOUT_SECONDS,
) -> None:
self.sink = sink
self.request = request
@ -200,6 +203,7 @@ class AgentRunRunner:
self.dify_api_http_client = dify_api_http_client
self.layer_providers = layer_providers if layer_providers is not None else create_default_layer_providers()
self.is_cancelled = is_cancelled or (lambda: False)
self.run_timeout_seconds = run_timeout_seconds
async def run(self) -> None:
"""Execute the run and emit the documented event sequence."""
@ -334,13 +338,22 @@ class AgentRunRunner:
tools=tools,
output_type=_resolve_agent_output_type(output_contract.output_type, ask_human_layer is not None),
)
result = await agent.run(
None if deferred_tool_results is not None else normalize_user_input(user_prompts),
message_history=message_history,
deferred_tool_results=deferred_tool_results,
event_stream_handler=handle_events,
usage_limits=UsageLimits(request_limit=_MAX_AGENT_STEPS_PER_RUN),
)
run_timeout = asyncio.timeout(self.run_timeout_seconds)
try:
async with run_timeout:
result = await agent.run(
None if deferred_tool_results is not None else normalize_user_input(user_prompts),
message_history=message_history,
deferred_tool_results=deferred_tool_results,
event_stream_handler=handle_events,
usage_limits=UsageLimits(request_limit=_MAX_AGENT_STEPS_PER_RUN),
)
except TimeoutError as exc:
if not run_timeout.expired():
raise
raise UsageLimitExceeded(
f"Agent run exceeded the configured limit of {self.run_timeout_seconds:g} seconds"
) from exc
complete_usage = model.accumulated_usage if isinstance(model, _HasAccumulatedUsage) else None
usage = _serialize_agent_usage(complete_usage if complete_usage is not None else _result_usage(result))
append_successful_run_history(history_layer, result.new_messages())

View File

@ -111,6 +111,7 @@ def create_app(settings: ServerSettings | None = None) -> FastAPI:
plugin_daemon_http_client=plugin_daemon_http_client,
dify_api_http_client=dify_api_inner_http_client,
shutdown_grace_seconds=resolved_settings.shutdown_grace_seconds,
run_timeout_seconds=resolved_settings.run_timeout_seconds,
layer_providers=layer_providers,
)
state["store"] = store

View File

@ -22,6 +22,7 @@ from dify_agent.agent_stub.server.agent_stub_config import DifyApiAgentStubConfi
from dify_agent.agent_stub.server.agent_stub_drive import DifyApiAgentStubDriveRequestHandler
from dify_agent.agent_stub.server.agent_stub_files import DifyApiAgentStubFileRequestHandler
from dify_agent.agent_stub.server.tokens.agent_stub import AgentStubTokenCodec, decode_server_secret_key
from dify_agent.runtime.runner import DEFAULT_AGENT_RUN_TIMEOUT_SECONDS
from dify_agent.runtime_backend import RuntimeBackendProfile
from dify_agent.runtime_backend.e2b import E2B_MAX_ACTIVE_TIMEOUT_SECONDS
from dify_agent.runtime_backend.profile import (
@ -42,6 +43,7 @@ class ServerSettings(BaseSettings):
redis_prefix: str = "dify-agent"
shutdown_grace_seconds: float = 30
run_retention_seconds: int = Field(default=DEFAULT_RUN_RETENTION_SECONDS, ge=1)
run_timeout_seconds: float = Field(default=DEFAULT_AGENT_RUN_TIMEOUT_SECONDS, gt=0)
plugin_daemon_url: str = "http://localhost:5002"
plugin_daemon_api_key: str = ""
inner_api_url: str = "http://localhost:5001"

View File

@ -814,6 +814,26 @@ def test_async_stream_events_yields_terminal_event() -> None:
asyncio.run(scenario())
def test_async_stream_events_enforces_total_timeout_before_connecting() -> None:
calls = 0
def handler(_request: httpx.Request) -> httpx.Response:
nonlocal calls
calls += 1
return httpx.Response(200, content="")
async def scenario() -> None:
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
client = Client(base_url="http://testserver", async_http_client=http_client)
with pytest.raises(DifyAgentTimeoutError, match="exceeded its timeout"):
_ = [event async for event in client.stream_events("run-1", timeout_seconds=0)]
asyncio.run(scenario())
assert calls == 0
def test_async_stream_events_does_not_reconnect_after_terminal_when_until_terminal_is_false() -> None:
calls = 0

View File

@ -29,6 +29,7 @@ from dify_agent.runtime.event_sink import (
terminal_event_status_fields,
)
from dify_agent.runtime.run_scheduler import RunCancellationConflictError, RunScheduler, SchedulerStoppingError
from dify_agent.runtime.runner import AgentRunRunner
from dify_agent.server.schemas import RunRecord
@ -381,6 +382,26 @@ class FinalizeSuccessOnCancellationRunner:
assert result.applied is True
def test_default_runner_factory_passes_run_timeout_to_runner() -> None:
async def scenario() -> None:
store = FakeStore()
record = await store.create_run()
async with httpx.AsyncClient() as client:
scheduler = RunScheduler(
store=store,
plugin_daemon_http_client=client,
dify_api_http_client=client,
run_timeout_seconds=17,
)
runner = scheduler._default_runner_factory(record, _request(), is_cancelled=lambda: False)
assert isinstance(runner, AgentRunRunner)
assert runner.run_timeout_seconds == 17
asyncio.run(scenario())
def test_create_run_starts_background_task_and_returns_running() -> None:
async def scenario() -> None:
store = FakeStore()

View File

@ -214,11 +214,11 @@ def test_run_failed_error_payload_preserves_knowledge_error_code() -> None:
def test_run_failed_error_payload_classifies_usage_limit() -> None:
exc = UsageLimitExceeded("The next request would exceed the request_limit of 100")
exc = UsageLimitExceeded("The next request would exceed the request_limit of 500")
message, error_type, reason = _run_failed_error_payload(exc)
assert message == "The next request would exceed the request_limit of 100"
assert message == "The next request would exceed the request_limit of 500"
assert error_type is RunFailureType.AGENT_RUN_LIMIT_EXCEEDED
assert reason is None
@ -329,6 +329,24 @@ def _request(
)
def _request_with_shell() -> CreateRunRequest:
request = _request()
request.composition.layers[-1:-1] = [
RunLayerSpec(
name="runtime",
type=DIFY_RUNTIME_LAYER_TYPE_ID,
config=DifyRuntimeLayerConfig(backend_binding_ref="binding-1"),
),
RunLayerSpec(
name="shell",
type=DIFY_SHELL_LAYER_TYPE_ID,
deps={"execution_context": "execution_context", "runtime": "runtime"},
config=DifyShellLayerConfig(),
),
]
return request
def _recursive_output_schema() -> dict[str, object]:
return {
"type": "object",
@ -631,7 +649,7 @@ def test_runner_passes_explicit_step_limit_to_agent(monkeypatch: pytest.MonkeyPa
class FakeAgent:
async def run(self, *_args: object, **kwargs: object) -> FakeAgentRunResult:
usage_limits = cast(UsageLimits, kwargs["usage_limits"])
assert usage_limits.request_limit == 100
assert usage_limits.request_limit == 500
return FakeAgentRunResult("done", [])
monkeypatch.setattr(DifyPluginLLMLayer, "get_model", fake_get_model)
@ -653,6 +671,162 @@ def test_runner_passes_explicit_step_limit_to_agent(monkeypatch: pytest.MonkeyPa
assert sink.statuses["run-explicit-step-limit"] == "succeeded"
def test_runner_timeout_excludes_tool_preparation_and_runtime_cleanup(monkeypatch: pytest.MonkeyPatch) -> None:
shell_client = FakeRunnerShellctlClient()
tools_prepared = False
class SlowLifecycleBackend(FakeRunnerExecutionBindingBackend):
acquired: bool = False
released: bool = False
async def acquire(self, binding_ref: str) -> RuntimeLease:
await asyncio.sleep(0.01)
lease = await super().acquire(binding_ref)
self.acquired = True
return lease
async def release(self, lease: RuntimeLease) -> None:
await asyncio.sleep(0.01)
await super().release(lease)
self.released = True
def fake_get_model(_self: DifyPluginLLMLayer, *, http_client: httpx.AsyncClient, agent_run_id: str):
assert http_client.is_closed is False
assert agent_run_id == "run-lifecycle-outside-timeout"
return TestModel(custom_output_text="unused") # pyright: ignore[reportReturnType]
class ImmediateAgent:
async def run(self, *_args: object, **_kwargs: object) -> FakeAgentRunResult:
return FakeAgentRunResult("done", [])
async def slow_resolve_run_tools(
_run: object,
*,
plugin_daemon_http_client: httpx.AsyncClient,
dify_api_http_client: httpx.AsyncClient,
) -> list[Tool[object]]:
nonlocal tools_prepared
assert plugin_daemon_http_client.is_closed is False
assert dify_api_http_client.is_closed is False
await asyncio.sleep(0.01)
tools_prepared = True
return []
monkeypatch.setattr(DifyPluginLLMLayer, "get_model", fake_get_model)
monkeypatch.setattr("dify_agent.runtime.runner.create_agent", lambda *_args, **_kwargs: ImmediateAgent())
monkeypatch.setattr("dify_agent.runtime.runner._resolve_run_tools", slow_resolve_run_tools)
backend = SlowLifecycleBackend(shell_client)
runtime_backend_profile = RuntimeBackendProfile(
home_snapshots=cast(HomeSnapshotBackend, object()),
execution_bindings=backend,
)
sink = InMemoryRunEventSink()
async def scenario() -> None:
async with httpx.AsyncClient() as client:
await AgentRunRunner(
sink=sink,
request=_request_with_shell(),
run_id="run-lifecycle-outside-timeout",
plugin_daemon_http_client=client,
dify_api_http_client=client,
layer_providers=create_default_layer_providers(runtime_backend_profile=runtime_backend_profile),
run_timeout_seconds=0.001,
).run()
asyncio.run(scenario())
assert sink.statuses["run-lifecycle-outside-timeout"] == "succeeded"
assert tools_prepared is True
assert backend.acquired is True
assert backend.released is True
assert shell_client.closed is True
def test_runner_timeout_cancels_agent_and_releases_runtime_lease(monkeypatch: pytest.MonkeyPatch) -> None:
agent_cancelled = False
shell_client = FakeRunnerShellctlClient()
def fake_get_model(_self: DifyPluginLLMLayer, *, http_client: httpx.AsyncClient, agent_run_id: str):
assert http_client.is_closed is False
assert agent_run_id == "run-timeout"
return TestModel(custom_output_text="unused") # pyright: ignore[reportReturnType]
class FakeAgent:
async def run(self, *_args: object, **_kwargs: object) -> None:
nonlocal agent_cancelled
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
agent_cancelled = True
raise
monkeypatch.setattr(DifyPluginLLMLayer, "get_model", fake_get_model)
monkeypatch.setattr("dify_agent.runtime.runner.create_agent", lambda *_args, **_kwargs: FakeAgent())
runtime_backend_profile = RuntimeBackendProfile(
home_snapshots=cast(HomeSnapshotBackend, object()),
execution_bindings=FakeRunnerExecutionBindingBackend(shell_client),
)
sink = InMemoryRunEventSink()
async def scenario() -> None:
async with httpx.AsyncClient() as client:
with pytest.raises(UsageLimitExceeded, match="0.01 seconds"):
await AgentRunRunner(
sink=sink,
request=_request_with_shell(),
run_id="run-timeout",
plugin_daemon_http_client=client,
dify_api_http_client=client,
layer_providers=create_default_layer_providers(runtime_backend_profile=runtime_backend_profile),
run_timeout_seconds=0.01,
).run()
asyncio.run(scenario())
terminal = sink.events["run-timeout"][-1]
assert isinstance(terminal, RunFailedEvent)
assert terminal.data.error_type is RunFailureType.AGENT_RUN_LIMIT_EXCEEDED
assert sink.statuses["run-timeout"] == "failed"
assert agent_cancelled is True
assert shell_client.closed is True
def test_runner_does_not_classify_nested_timeout_as_agent_limit(monkeypatch: pytest.MonkeyPatch) -> None:
def fake_get_model(_self: DifyPluginLLMLayer, *, http_client: httpx.AsyncClient, agent_run_id: str):
assert http_client.is_closed is False
assert agent_run_id == "run-provider-timeout"
return TestModel(custom_output_text="unused") # pyright: ignore[reportReturnType]
class FakeAgent:
async def run(self, *_args: object, **_kwargs: object) -> FakeAgentRunResult:
raise TimeoutError("provider timed out")
monkeypatch.setattr(DifyPluginLLMLayer, "get_model", fake_get_model)
monkeypatch.setattr("dify_agent.runtime.runner.create_agent", lambda *_args, **_kwargs: FakeAgent())
sink = InMemoryRunEventSink()
async def scenario() -> None:
async with httpx.AsyncClient() as client:
with pytest.raises(TimeoutError, match="provider timed out"):
await AgentRunRunner(
sink=sink,
request=_request(),
run_id="run-provider-timeout",
plugin_daemon_http_client=client,
dify_api_http_client=client,
run_timeout_seconds=1,
).run()
asyncio.run(scenario())
terminal = sink.events["run-provider-timeout"][-1]
assert isinstance(terminal, RunFailedEvent)
assert terminal.data.error == "provider timed out"
assert terminal.data.error_type is None
assert sink.statuses["run-provider-timeout"] == "failed"
def test_runner_emits_deferred_tool_call_and_persists_pending_history(monkeypatch: pytest.MonkeyPatch) -> None:
captured_output_types: list[object] = []
captured_user_prompts: list[object] = []
@ -1981,7 +2155,7 @@ def test_runner_persists_usage_limit_failure_type_in_event_and_status(
)
async def exceed_limit() -> RunSuccessOutcome:
raise UsageLimitExceeded("The next request would exceed the request_limit of 100")
raise UsageLimitExceeded("The next request would exceed the request_limit of 500")
monkeypatch.setattr(runner, "_run_agent", exceed_limit)
with pytest.raises(UsageLimitExceeded):

View File

@ -68,6 +68,7 @@ class FakeRunScheduler:
store: object
shutdown_grace_seconds: float
run_timeout_seconds: float
layer_providers: tuple[DifyAgentLayerProvider, ...]
plugin_daemon_http_client: FakePluginDaemonHttpClient
dify_api_http_client: FakePluginDaemonHttpClient
@ -80,10 +81,12 @@ class FakeRunScheduler:
plugin_daemon_http_client: FakePluginDaemonHttpClient,
dify_api_http_client: FakePluginDaemonHttpClient,
shutdown_grace_seconds: float,
run_timeout_seconds: float,
layer_providers: tuple[DifyAgentLayerProvider, ...],
) -> None:
self.store = store
self.shutdown_grace_seconds = shutdown_grace_seconds
self.run_timeout_seconds = run_timeout_seconds
self.layer_providers = layer_providers
self.plugin_daemon_http_client = plugin_daemon_http_client
self.dify_api_http_client = dify_api_http_client
@ -198,6 +201,7 @@ def test_create_app_creates_scheduler_and_closes_after_shutdown(monkeypatch: pyt
redis_url="redis://example.invalid/0",
redis_prefix="test",
shutdown_grace_seconds=5,
run_timeout_seconds=17,
run_retention_seconds=7,
plugin_daemon_url="http://plugin-daemon",
plugin_daemon_api_key="daemon-secret",
@ -221,6 +225,7 @@ def test_create_app_creates_scheduler_and_closes_after_shutdown(monkeypatch: pyt
assert len(FakeRunScheduler.created) == 1
scheduler = FakeRunScheduler.created[0]
assert scheduler.shutdown_grace_seconds == 5
assert scheduler.run_timeout_seconds == 17
layer_providers = scheduler.layer_providers
assert isinstance(layer_providers, tuple)
execution_context_provider = next(

View File

@ -12,7 +12,8 @@ from dify_agent.agent_stub.server.agent_stub_drive import DifyApiAgentStubDriveR
from dify_agent.agent_stub.server.agent_stub_files import DifyApiAgentStubFileRequestHandler
from dify_agent.agent_stub.server.tokens.agent_stub import AgentStubTokenCodec
from dify_agent.server.settings import ServerSettings
from dify_agent.runtime_backend.e2b import E2BExecutionBindingBackend
from dify_agent.runtime.runner import DEFAULT_AGENT_RUN_TIMEOUT_SECONDS
from dify_agent.runtime_backend.e2b import E2B_MAX_ACTIVE_TIMEOUT_SECONDS, E2BExecutionBindingBackend
from dify_agent.runtime_backend.enterprise import EnterpriseExecutionBindingBackend
from dify_agent.runtime_backend.local import LocalExecutionBindingBackend, LocalHomeSnapshotBackend
@ -49,12 +50,36 @@ def test_server_settings_reads_enterprise_timeouts_from_env(monkeypatch: pytest.
assert settings.enterprise_sandbox_proxy_timeout == 90
def test_server_settings_reads_e2b_active_timeout_from_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS", "900")
def test_server_settings_run_and_e2b_timeouts_default_align_and_override_independently(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
monkeypatch.delenv("DIFY_AGENT_RUN_TIMEOUT_SECONDS", raising=False)
monkeypatch.delenv("DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS", raising=False)
monkeypatch.chdir(tmp_path)
settings = ServerSettings()
assert settings.e2b_active_timeout_seconds == 900
assert settings.run_timeout_seconds == DEFAULT_AGENT_RUN_TIMEOUT_SECONDS == 3600
assert settings.e2b_active_timeout_seconds == E2B_MAX_ACTIVE_TIMEOUT_SECONDS == 3600
monkeypatch.setenv("DIFY_AGENT_RUN_TIMEOUT_SECONDS", "900.5")
run_override_settings = ServerSettings()
assert run_override_settings.run_timeout_seconds == 900.5
assert run_override_settings.e2b_active_timeout_seconds == 3600
monkeypatch.delenv("DIFY_AGENT_RUN_TIMEOUT_SECONDS")
monkeypatch.setenv("DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS", "900")
e2b_override_settings = ServerSettings()
assert e2b_override_settings.run_timeout_seconds == 3600
assert e2b_override_settings.e2b_active_timeout_seconds == 900
def test_server_settings_rejects_non_positive_run_timeout() -> None:
with pytest.raises(ValidationError, match="greater than 0"):
_ = ServerSettings(run_timeout_seconds=0)
def test_server_settings_defaults_shellctl_auth_token_to_none(

View File

@ -260,12 +260,13 @@ AGENT_BACKEND_BASE_URL=http://agent_backend:5050
DIFY_AGENT_API_TOKEN=dify-agent-run-token-for-dev-only
AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS=30
AGENT_BACKEND_STREAM_MAX_RECONNECTS=3
AGENT_BACKEND_RUN_TIMEOUT_SECONDS=1200
# Leave empty to derive from REDIS_PASSWORD.
DIFY_AGENT_REDIS_URL=
DIFY_AGENT_REDIS_PREFIX=dify-agent
DIFY_AGENT_SHUTDOWN_GRACE_SECONDS=30
DIFY_AGENT_RUN_RETENTION_SECONDS=259200
# Pydantic AI run deadline; its default matches the independently configurable E2B active timeout.
DIFY_AGENT_RUN_TIMEOUT_SECONDS=3600
# Leave empty to derive from PLUGIN_DAEMON_URL and PLUGIN_DAEMON_KEY.
DIFY_AGENT_PLUGIN_DAEMON_URL=
DIFY_AGENT_PLUGIN_DAEMON_API_KEY=
@ -280,7 +281,7 @@ DIFY_AGENT_LOCAL_SANDBOX_AUTH_TOKEN=
# E2B_API_KEY and E2B_API_TOKEN remain accepted as deployment-level fallbacks.
DIFY_AGENT_E2B_API_KEY=
DIFY_AGENT_E2B_TEMPLATE=difys-default-team/dify-agent-local-sandbox
# One-hour RuntimeLease limit spanning a complete Agent run.
# RuntimeLease active limit; its default matches the independently configurable Agent run deadline.
DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS=3600
DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN=
DIFY_AGENT_E2B_SHELLCTL_PORT=5004

View File

@ -235,7 +235,6 @@ services:
AGENT_BACKEND_API_TOKEN: ${DIFY_AGENT_API_TOKEN:-dify-agent-run-token-for-dev-only}
AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS: ${AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS:-30}
AGENT_BACKEND_STREAM_MAX_RECONNECTS: ${AGENT_BACKEND_STREAM_MAX_RECONNECTS:-3}
AGENT_BACKEND_RUN_TIMEOUT_SECONDS: ${AGENT_BACKEND_RUN_TIMEOUT_SECONDS:-1200}
depends_on:
init_permissions:
condition: service_completed_successfully
@ -314,7 +313,6 @@ services:
AGENT_BACKEND_API_TOKEN: ${DIFY_AGENT_API_TOKEN:-dify-agent-run-token-for-dev-only}
AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS: ${AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS:-30}
AGENT_BACKEND_STREAM_MAX_RECONNECTS: ${AGENT_BACKEND_STREAM_MAX_RECONNECTS:-3}
AGENT_BACKEND_RUN_TIMEOUT_SECONDS: ${AGENT_BACKEND_RUN_TIMEOUT_SECONDS:-1200}
depends_on:
init_permissions:
condition: service_completed_successfully
@ -677,7 +675,7 @@ services:
DIFY_AGENT_LOCAL_SANDBOX_AUTH_TOKEN: ${DIFY_AGENT_LOCAL_SANDBOX_AUTH_TOKEN:-${DIFY_AGENT_SHELLCTL_AUTH_TOKEN:-}}
DIFY_AGENT_E2B_API_KEY: ${DIFY_AGENT_E2B_API_KEY:-${E2B_API_KEY:-${E2B_API_TOKEN:-}}}
DIFY_AGENT_E2B_TEMPLATE: ${DIFY_AGENT_E2B_TEMPLATE:-difys-default-team/dify-agent-local-sandbox}
# One-hour RuntimeLease limit spanning a complete Agent run.
# RuntimeLease active limit; its default matches the independently configurable Agent run deadline.
DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS: ${DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS:-3600}
DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN: ${DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN:-}
DIFY_AGENT_E2B_SHELLCTL_PORT: ${DIFY_AGENT_E2B_SHELLCTL_PORT:-5004}
@ -690,6 +688,8 @@ services:
DIFY_AGENT_API_TOKEN: ${DIFY_AGENT_API_TOKEN:-dify-agent-run-token-for-dev-only}
DIFY_AGENT_SHUTDOWN_GRACE_SECONDS: ${DIFY_AGENT_SHUTDOWN_GRACE_SECONDS:-30}
DIFY_AGENT_RUN_RETENTION_SECONDS: ${DIFY_AGENT_RUN_RETENTION_SECONDS:-259200}
# Pydantic AI run deadline; its default matches the independently configurable E2B active timeout.
DIFY_AGENT_RUN_TIMEOUT_SECONDS: ${DIFY_AGENT_RUN_TIMEOUT_SECONDS:-3600}
depends_on:
redis:
condition: service_started

View File

@ -241,7 +241,6 @@ services:
AGENT_BACKEND_API_TOKEN: ${DIFY_AGENT_API_TOKEN:-dify-agent-run-token-for-dev-only}
AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS: ${AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS:-30}
AGENT_BACKEND_STREAM_MAX_RECONNECTS: ${AGENT_BACKEND_STREAM_MAX_RECONNECTS:-3}
AGENT_BACKEND_RUN_TIMEOUT_SECONDS: ${AGENT_BACKEND_RUN_TIMEOUT_SECONDS:-1200}
depends_on:
init_permissions:
condition: service_completed_successfully
@ -320,7 +319,6 @@ services:
AGENT_BACKEND_API_TOKEN: ${DIFY_AGENT_API_TOKEN:-dify-agent-run-token-for-dev-only}
AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS: ${AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS:-30}
AGENT_BACKEND_STREAM_MAX_RECONNECTS: ${AGENT_BACKEND_STREAM_MAX_RECONNECTS:-3}
AGENT_BACKEND_RUN_TIMEOUT_SECONDS: ${AGENT_BACKEND_RUN_TIMEOUT_SECONDS:-1200}
depends_on:
init_permissions:
condition: service_completed_successfully
@ -683,7 +681,7 @@ services:
DIFY_AGENT_LOCAL_SANDBOX_AUTH_TOKEN: ${DIFY_AGENT_LOCAL_SANDBOX_AUTH_TOKEN:-${DIFY_AGENT_SHELLCTL_AUTH_TOKEN:-}}
DIFY_AGENT_E2B_API_KEY: ${DIFY_AGENT_E2B_API_KEY:-${E2B_API_KEY:-${E2B_API_TOKEN:-}}}
DIFY_AGENT_E2B_TEMPLATE: ${DIFY_AGENT_E2B_TEMPLATE:-difys-default-team/dify-agent-local-sandbox}
# One-hour RuntimeLease limit spanning a complete Agent run.
# RuntimeLease active limit; its default matches the independently configurable Agent run deadline.
DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS: ${DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS:-3600}
DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN: ${DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN:-}
DIFY_AGENT_E2B_SHELLCTL_PORT: ${DIFY_AGENT_E2B_SHELLCTL_PORT:-5004}
@ -696,6 +694,8 @@ services:
DIFY_AGENT_API_TOKEN: ${DIFY_AGENT_API_TOKEN:-dify-agent-run-token-for-dev-only}
DIFY_AGENT_SHUTDOWN_GRACE_SECONDS: ${DIFY_AGENT_SHUTDOWN_GRACE_SECONDS:-30}
DIFY_AGENT_RUN_RETENTION_SECONDS: ${DIFY_AGENT_RUN_RETENTION_SECONDS:-259200}
# Pydantic AI run deadline; its default matches the independently configurable E2B active timeout.
DIFY_AGENT_RUN_TIMEOUT_SECONDS: ${DIFY_AGENT_RUN_TIMEOUT_SECONDS:-3600}
depends_on:
redis:
condition: service_started

View File

@ -5,13 +5,14 @@
AGENT_BACKEND_BASE_URL=http://agent_backend:5050
AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS=30
AGENT_BACKEND_STREAM_MAX_RECONNECTS=3
AGENT_BACKEND_RUN_TIMEOUT_SECONDS=1200
# Leave empty to derive from REDIS_PASSWORD in Docker Compose.
DIFY_AGENT_REDIS_URL=
DIFY_AGENT_REDIS_PREFIX=dify-agent
DIFY_AGENT_SHUTDOWN_GRACE_SECONDS=30
DIFY_AGENT_RUN_RETENTION_SECONDS=259200
# Pydantic AI run deadline; its default matches the independently configurable E2B active timeout.
DIFY_AGENT_RUN_TIMEOUT_SECONDS=3600
# Leave empty to derive from PLUGIN_DAEMON_URL and PLUGIN_DAEMON_KEY in Docker Compose.
DIFY_AGENT_PLUGIN_DAEMON_URL=
@ -29,7 +30,7 @@ DIFY_AGENT_LOCAL_SANDBOX_AUTH_TOKEN=
# E2B_API_KEY and E2B_API_TOKEN remain accepted as deployment-level fallbacks.
DIFY_AGENT_E2B_API_KEY=
DIFY_AGENT_E2B_TEMPLATE=difys-default-team/dify-agent-local-sandbox
# One-hour RuntimeLease limit spanning a complete Agent run.
# RuntimeLease active limit; its default matches the independently configurable Agent run deadline.
DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS=3600
DIFY_AGENT_E2B_SHELLCTL_AUTH_TOKEN=
DIFY_AGENT_E2B_SHELLCTL_PORT=5004