From 925f97be20eac1954eae30c936baa006eeb9e4f4 Mon Sep 17 00:00:00 2001 From: zyssyz123 <916125788@qq.com> Date: Thu, 9 Jul 2026 18:35:15 +0800 Subject: [PATCH] feat: daily sync (#38593) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 盐粒 Yanli Co-authored-by: yyh Co-authored-by: Joel Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: QuantumGhost Co-authored-by: yyh <92089059+lyzno1@users.noreply.github.com> Co-authored-by: 盐粒 Yanli --- api/.env.example | 3 + api/configs/extra/agent_backend_config.py | 7 +- api/configs/feature/__init__.py | 5 +- api/controllers/files/upload.py | 6 +- api/controllers/inner_api/plugin/plugin.py | 1 + api/core/app/apps/agent_app/app_runner.py | 46 +- .../base_app_generate_response_converter.py | 3 +- .../easy_ui_based_generate_task_pipeline.py | 1 + api/core/plugin/entities/request.py | 1 + api/core/tools/tool_file_manager.py | 15 +- api/core/tools/utils/message_transformer.py | 9 +- .../workflow/nodes/agent_v2/agent_node.py | 16 + .../nodes/agent_v2/output_file_rebacker.py | 4 +- api/factories/file_factory/builders.py | 10 +- api/fields/message_fields.py | 12 +- api/openapi/markdown/console-openapi.md | 6 + api/openapi/markdown/service-openapi.md | 6 + api/openapi/markdown/web-openapi.md | 6 + api/services/file_request_service.py | 3 +- .../unit_tests/configs/test_dify_config.py | 20 + .../console/explore/test_message.py | 2 + .../controllers/files/test_upload.py | 15 +- .../inner_api/plugin/test_plugin.py | 2 + .../app/apps/agent_app/test_app_runner.py | 139 ++ .../test_based_generate_task_pipeline.py | 8 +- .../core/plugin/entities/test_request.py | 20 + .../core/tools/test_tool_file_manager.py | 57 + .../tools/utils/test_message_transformer.py | 28 +- .../nodes/agent_v2/test_agent_node.py | 41 +- .../agent_v2/test_output_file_rebacker.py | 11 + .../factories/test_build_from_mapping.py | 33 + .../unit_tests/fields/test_message_fields.py | 34 +- .../services/test_file_request_service.py | 25 + api/uv.lock | 37 +- dify-agent/.example.env | 16 +- dify-agent/Dockerfile | 4 +- dify-agent/docker/local-sandbox/Dockerfile | 23 +- .../docs/dify-agent/get-started/index.md | 5 +- dify-agent/docs/dify-agent/guide/index.md | 8 +- .../user-manual/shell-layer/index.md | 17 +- .../proto/dify/agent/stub/v1/agent_stub.proto | 1 + dify-agent/pyproject.toml | 14 +- .../src/dify_agent/adapters/llm/model.py | 18 +- .../src/dify_agent/adapters/shell/shellctl.py | 32 +- .../src/dify_agent/agent_stub/cli/_files.py | 5 +- .../src/dify_agent/agent_stub/cli/main.py | 20 + .../agent_stub/client/_agent_stub.py | 3 + .../agent_stub/client/_agent_stub_grpc.py | 3 +- .../agent_stub/client/_agent_stub_http.py | 5 +- .../grpc/_generated/agent_stub_pb2.py | 23 +- .../grpc/_generated/agent_stub_pb2.pyi | 94 +- .../dify_agent/agent_stub/grpc/conversions.py | 12 +- .../agent_stub/protocol/agent_stub.py | 1 + .../agent_stub/server/agent_stub_files.py | 2 + .../src/dify_agent/layers/shell/layer.py | 13 +- dify-agent/src/dify_agent/runtime/runner.py | 29 +- dify-agent/src/shellctl/__init__.py | 124 ++ dify-agent/src/shellctl/cli.py | 587 ++++++ dify-agent/src/shellctl/client/__init__.py | 12 + dify-agent/src/shellctl/client/sdk.py | 319 ++++ dify-agent/src/shellctl/py.typed | 1 + dify-agent/src/shellctl/server/__init__.py | 58 + dify-agent/src/shellctl/server/__main__.py | 6 + dify-agent/src/shellctl/server/api.py | 198 ++ dify-agent/src/shellctl/server/artifacts.py | 62 + dify-agent/src/shellctl/server/cli.py | 17 + dify-agent/src/shellctl/server/config.py | 105 + dify-agent/src/shellctl/server/db.py | 52 + dify-agent/src/shellctl/server/errors.py | 14 + dify-agent/src/shellctl/server/serve.py | 103 + dify-agent/src/shellctl/server/service.py | 1184 ++++++++++++ dify-agent/src/shellctl/server/tmux.py | 343 ++++ dify-agent/src/shellctl/shared/__init__.py | 182 ++ dify-agent/src/shellctl/shared/constants.py | 49 + dify-agent/src/shellctl/shared/output.py | 120 ++ dify-agent/src/shellctl/shared/runtime.py | 95 + dify-agent/src/shellctl/shared/schemas.py | 213 +++ dify-agent/src/shellctl_runtime/__init__.py | 7 + dify-agent/src/shellctl_runtime/paths.py | 16 + dify-agent/src/shellctl_runtime/py.typed | 1 + .../src/shellctl_runtime/runner_exit.py | 158 ++ dify-agent/src/shellctl_runtime/sanitize.py | 186 ++ .../dify_agent/adapters/llm/test_model.py | 68 +- .../adapters/shell/test_shellctl.py | 63 +- .../dify_agent/agent_stub/cli/test_files.py | 42 + .../dify_agent/agent_stub/cli/test_main.py | 18 + .../client/test_agent_stub_client.py | 42 +- .../protocol/test_grpc_conversions.py | 33 +- .../server/test_agent_stub_files.py | 35 + .../agent_stub/server/test_grpc_service.py | 4 +- .../local/dify_agent/runtime/test_runner.py | 28 +- .../dify_agent/test_import_boundaries.py | 30 +- .../shellctl/golden_shellctl_sanitize.json | 32 + .../tests/local/shellctl/test_shellctl_cli.py | 688 +++++++ .../local/shellctl/test_shellctl_client.py | 566 ++++++ .../local/shellctl/test_shellctl_service.py | 1687 +++++++++++++++++ .../local/shellctl/test_shellctl_shared.py | 133 ++ .../golden_shellctl_sanitize.json | 32 + .../shellctl_runtime/test_shellctl_runtime.py | 531 ++++++ dify-agent/tests/local/test_packaging.py | 33 +- dify-agent/uv.lock | 81 +- docker/.env.example | 24 +- docker/README.md | 2 +- docker/docker-compose-template.yaml | 59 + docker/docker-compose.yaml | 59 + .../envs/core-services/dify-agent.env.example | 28 + docker/envs/core-services/shared.env.example | 4 +- docker/envs/core-services/web.env.example | 2 +- e2e/features/agent-v2/AGENTS.md | 2 +- e2e/features/agent-v2/support/agent.ts | 4 +- .../agent-v2/access-point.steps.ts | 8 +- .../agent-v2/agent-edit.steps.ts | 2 +- .../agent-v2/agent-roster.steps.ts | 4 +- .../agent-v2/configure.steps.ts | 18 +- .../agent-v2/workflow-node.steps.ts | 4 +- eslint-suppressions.json | 5 - .../api/console/installed-apps/types.gen.ts | 37 + .../api/console/installed-apps/zod.gen.ts | 47 + .../generated/api/service/types.gen.ts | 36 + .../generated/api/service/zod.gen.ts | 46 + .../contracts/generated/api/web/types.gen.ts | 37 + .../contracts/generated/api/web/zod.gen.ts | 47 + .../assets/public/thought/imagine.svg | 4 + .../custom-public/icons.json | 7 +- .../custom-public/info.json | 2 +- web/.env.example | 2 +- web/__tests__/proxy-frame-options.spec.ts | 2 +- .../[agentId]/access/page.tsx | 0 .../[agentId]/configure/page.tsx | 0 .../agent => agents}/[agentId]/logs/page.tsx | 0 .../[agentId]/monitoring/page.tsx | 0 .../agent => agents}/[agentId]/page.tsx | 0 .../[agentId]/sidebar-page.tsx | 0 .../[agentId]/access/page.tsx | 0 .../[agentId]/configure/page.tsx | 0 .../agent => agents}/[agentId]/layout.tsx | 0 .../agent => agents}/[agentId]/logs/page.tsx | 0 .../[agentId]/monitoring/page.tsx | 0 .../agent => agents}/[agentId]/page.tsx | 2 +- .../__tests__/feature-guard.spec.ts | 0 .../__tests__/layout.spec.tsx | 0 .../{roster => agents}/feature-guard.ts | 0 .../{roster => agents}/layout.tsx | 0 .../{roster => agents}/page.tsx | 0 .../__tests__/configuration-view.spec.tsx | 2 +- .../app/configuration/configuration-view.tsx | 2 +- .../agent-roster-response-content.spec.tsx | 43 +- .../answer/agent-roster-response-content.tsx | 47 +- .../base/icons/src/public/thought/index.ts | 1 - web/app/components/billing/plan/index.tsx | 2 +- .../main-nav/__tests__/index.spec.tsx | 18 +- .../main-nav/__tests__/layout.spec.tsx | 2 +- web/app/components/main-nav/index.tsx | 4 +- web/app/components/main-nav/routes.ts | 16 +- .../block-selector/__tests__/blocks.spec.tsx | 2 +- .../block-selector/agent-selector.tsx | 2 +- .../nodes/agent-v2/__tests__/panel.spec.tsx | 2 +- .../agent-detail/__tests__/layout.spec.tsx | 2 +- .../__tests__/navigation.spec.tsx | 6 +- .../preview/__tests__/chat.spec.tsx | 56 +- .../components/preview/chat-runtime.tsx | 1 + web/features/agent-v2/agent-detail/layout.tsx | 2 +- .../agent-v2/agent-detail/navigation.tsx | 4 +- web/features/agent-v2/agent-detail/routes.ts | 6 +- .../__tests__/create-agent-dialog.spec.tsx | 2 +- .../roster/components/agent-roster-list.tsx | 2 +- web/features/agent-v2/roster/page.tsx | 5 +- web/i18n/ar-TN/common.json | 1 - web/i18n/de-DE/common.json | 1 - web/i18n/en-US/common.json | 1 - web/i18n/es-ES/common.json | 1 - web/i18n/fa-IR/common.json | 1 - web/i18n/fr-FR/common.json | 1 - web/i18n/hi-IN/common.json | 1 - web/i18n/id-ID/common.json | 1 - web/i18n/it-IT/common.json | 1 - web/i18n/ja-JP/common.json | 1 - web/i18n/ko-KR/common.json | 1 - web/i18n/nl-NL/common.json | 1 - web/i18n/pl-PL/common.json | 1 - web/i18n/pt-BR/common.json | 1 - web/i18n/ro-RO/common.json | 1 - web/i18n/ru-RU/common.json | 1 - web/i18n/sl-SI/common.json | 1 - web/i18n/th-TH/common.json | 1 - web/i18n/tr-TR/common.json | 1 - web/i18n/uk-UA/common.json | 1 - web/i18n/vi-VN/common.json | 1 - web/i18n/zh-Hans/common.json | 1 - web/i18n/zh-Hant/common.json | 1 - 190 files changed, 9739 insertions(+), 345 deletions(-) create mode 100644 dify-agent/src/shellctl/__init__.py create mode 100644 dify-agent/src/shellctl/cli.py create mode 100644 dify-agent/src/shellctl/client/__init__.py create mode 100644 dify-agent/src/shellctl/client/sdk.py create mode 100644 dify-agent/src/shellctl/py.typed create mode 100644 dify-agent/src/shellctl/server/__init__.py create mode 100644 dify-agent/src/shellctl/server/__main__.py create mode 100644 dify-agent/src/shellctl/server/api.py create mode 100644 dify-agent/src/shellctl/server/artifacts.py create mode 100644 dify-agent/src/shellctl/server/cli.py create mode 100644 dify-agent/src/shellctl/server/config.py create mode 100644 dify-agent/src/shellctl/server/db.py create mode 100644 dify-agent/src/shellctl/server/errors.py create mode 100644 dify-agent/src/shellctl/server/serve.py create mode 100644 dify-agent/src/shellctl/server/service.py create mode 100644 dify-agent/src/shellctl/server/tmux.py create mode 100644 dify-agent/src/shellctl/shared/__init__.py create mode 100644 dify-agent/src/shellctl/shared/constants.py create mode 100644 dify-agent/src/shellctl/shared/output.py create mode 100644 dify-agent/src/shellctl/shared/runtime.py create mode 100644 dify-agent/src/shellctl/shared/schemas.py create mode 100644 dify-agent/src/shellctl_runtime/__init__.py create mode 100644 dify-agent/src/shellctl_runtime/paths.py create mode 100644 dify-agent/src/shellctl_runtime/py.typed create mode 100644 dify-agent/src/shellctl_runtime/runner_exit.py create mode 100644 dify-agent/src/shellctl_runtime/sanitize.py create mode 100644 dify-agent/tests/local/shellctl/golden_shellctl_sanitize.json create mode 100644 dify-agent/tests/local/shellctl/test_shellctl_cli.py create mode 100644 dify-agent/tests/local/shellctl/test_shellctl_client.py create mode 100644 dify-agent/tests/local/shellctl/test_shellctl_service.py create mode 100644 dify-agent/tests/local/shellctl/test_shellctl_shared.py create mode 100644 dify-agent/tests/local/shellctl_runtime/golden_shellctl_sanitize.json create mode 100644 dify-agent/tests/local/shellctl_runtime/test_shellctl_runtime.py create mode 100644 docker/envs/core-services/dify-agent.env.example create mode 100644 packages/iconify-collections/assets/public/thought/imagine.svg rename web/app/(commonLayout)/@detailSidebar/{roster/agent => agents}/[agentId]/access/page.tsx (100%) rename web/app/(commonLayout)/@detailSidebar/{roster/agent => agents}/[agentId]/configure/page.tsx (100%) rename web/app/(commonLayout)/@detailSidebar/{roster/agent => agents}/[agentId]/logs/page.tsx (100%) rename web/app/(commonLayout)/@detailSidebar/{roster/agent => agents}/[agentId]/monitoring/page.tsx (100%) rename web/app/(commonLayout)/@detailSidebar/{roster/agent => agents}/[agentId]/page.tsx (100%) rename web/app/(commonLayout)/@detailSidebar/{roster/agent => agents}/[agentId]/sidebar-page.tsx (100%) rename web/app/(commonLayout)/{roster/agent => agents}/[agentId]/access/page.tsx (100%) rename web/app/(commonLayout)/{roster/agent => agents}/[agentId]/configure/page.tsx (100%) rename web/app/(commonLayout)/{roster/agent => agents}/[agentId]/layout.tsx (100%) rename web/app/(commonLayout)/{roster/agent => agents}/[agentId]/logs/page.tsx (100%) rename web/app/(commonLayout)/{roster/agent => agents}/[agentId]/monitoring/page.tsx (100%) rename web/app/(commonLayout)/{roster/agent => agents}/[agentId]/page.tsx (80%) rename web/app/(commonLayout)/{roster => agents}/__tests__/feature-guard.spec.ts (100%) rename web/app/(commonLayout)/{roster => agents}/__tests__/layout.spec.tsx (100%) rename web/app/(commonLayout)/{roster => agents}/feature-guard.ts (100%) rename web/app/(commonLayout)/{roster => agents}/layout.tsx (100%) rename web/app/(commonLayout)/{roster => agents}/page.tsx (100%) delete mode 100644 web/app/components/base/icons/src/public/thought/index.ts diff --git a/api/.env.example b/api/.env.example index 48d8707d1ad..8987d388798 100644 --- a/api/.env.example +++ b/api/.env.example @@ -663,6 +663,9 @@ PLUGIN_MODEL_SCHEMA_CACHE_TTL=3600 PLUGIN_MODEL_PROVIDERS_CACHE_TTL=86400 INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1 +# Dify Agent backend +AGENT_BACKEND_BASE_URL=http://localhost:5050 + # Marketplace configuration MARKETPLACE_ENABLED=true MARKETPLACE_API_URL=https://marketplace.dify.ai diff --git a/api/configs/extra/agent_backend_config.py b/api/configs/extra/agent_backend_config.py index b59350d5f0d..2e6b9814954 100644 --- a/api/configs/extra/agent_backend_config.py +++ b/api/configs/extra/agent_backend_config.py @@ -25,11 +25,10 @@ class AgentBackendConfig(BaseSettings): AGENT_SHELL_ENABLED: bool = Field( description=( "Inject the dify.shell layer (sandboxed bash workspace) into Agent runs. " - "Requires the agent backend to be wired with a shellctl entrypoint; keep it " - "off until shellctl is deployed, otherwise every agent run that includes the " - "shell layer will fail." + "Requires the agent backend to be wired with a shellctl entrypoint before " + "shell-using Agent runs are executed." ), - default=False, + default=True, ) AGENT_APP_TEXT_DELTA_DEBOUNCE_SECONDS: NonNegativeFloat = Field( diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index f664274ba75..4c631d0180c 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -363,7 +363,10 @@ class FileAccessConfig(BaseSettings): INTERNAL_FILES_URL: str = Field( description="Internal base URL for file access within Docker network," " used for plugin daemon and internal service communication." - " Falls back to FILES_URL if not specified.", + " Explicit INTERNAL_FILES_URL takes precedence; otherwise SERVER_CONSOLE_API_URL is used," + " then FILES_URL.", + validation_alias=AliasChoices("INTERNAL_FILES_URL", "SERVER_CONSOLE_API_URL"), + alias_priority=1, default="", ) diff --git a/api/controllers/files/upload.py b/api/controllers/files/upload.py index 249784dee92..3de92589f6a 100644 --- a/api/controllers/files/upload.py +++ b/api/controllers/files/upload.py @@ -1,5 +1,3 @@ -from mimetypes import guess_extension - from flask import request from flask_restx import Resource from flask_restx.api import HTTPStatus @@ -8,7 +6,7 @@ from werkzeug.exceptions import Forbidden import services from core.tools.signature import verify_plugin_file_signature -from core.tools.tool_file_manager import ToolFileManager +from core.tools.tool_file_manager import ToolFileManager, resolve_extension from core.workflow.file_reference import build_file_reference from fields.file_fields import FileResponse @@ -110,7 +108,7 @@ class PluginUploadFileApi(Resource): conversation_id=args.conversation_id, ) - extension = guess_extension(tool_file.mimetype) or ".bin" + extension = resolve_extension(filename=tool_file.name, mimetype=tool_file.mimetype) preview_url = ToolFileManager.sign_file(tool_file_id=tool_file.id, extension=extension) # Create a dictionary with all the necessary attributes diff --git a/api/controllers/inner_api/plugin/plugin.py b/api/controllers/inner_api/plugin/plugin.py index 4e6dbf5dcbe..1171761fc53 100644 --- a/api/controllers/inner_api/plugin/plugin.py +++ b/api/controllers/inner_api/plugin/plugin.py @@ -476,6 +476,7 @@ class PluginDownloadFileRequestApi(Resource): user_from=payload.user_from, invoke_from=payload.invoke_from, file_mapping=payload.file.model_dump(mode="python", exclude_none=True), + for_external=payload.for_external, ) return BaseBackwardsInvocationResponse( data={ diff --git a/api/core/app/apps/agent_app/app_runner.py b/api/core/app/apps/agent_app/app_runner.py index c6546cac569..54e34ced9cf 100644 --- a/api/core/app/apps/agent_app/app_runner.py +++ b/api/core/app/apps/agent_app/app_runner.py @@ -27,6 +27,7 @@ from clients.agent_backend import ( AgentBackendInternalEventType, AgentBackendRunClient, AgentBackendRunEventAdapter, + AgentBackendRunFailedInternalEvent, AgentBackendRunSucceededInternalEvent, AgentBackendStreamInternalEvent, extract_runtime_layer_specs, @@ -57,6 +58,14 @@ from core.workflow.nodes.agent_v2.ask_human_resume import build_deferred_tool_re from extensions.ext_database import db from graphon.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage +from graphon.model_runtime.errors.invoke import ( + InvokeAuthorizationError, + InvokeBadRequestError, + InvokeConnectionError, + InvokeError, + InvokeRateLimitError, + InvokeServerUnavailableError, +) from models.agent_config_entities import AgentSoulConfig from models.enums import CreatorUserRole from models.model import MessageAgentThought @@ -71,6 +80,22 @@ class _DefaultSessionScopeSnapshotId: _DEFAULT_SESSION_SCOPE_SNAPSHOT_ID = _DefaultSessionScopeSnapshotId() +_AGENT_BACKEND_INVOKE_ERROR_BY_REASON: Mapping[str, type[InvokeError]] = { + "InvokeAuthorizationError": InvokeAuthorizationError, + "InvokeBadRequestError": InvokeBadRequestError, + "CredentialsValidateFailedError": InvokeBadRequestError, + "InvokeConnectionError": InvokeConnectionError, + "InvokeRateLimitError": InvokeRateLimitError, + "InvokeServerUnavailableError": InvokeServerUnavailableError, +} + + +def _agent_backend_failure_to_exception(event: AgentBackendRunFailedInternalEvent) -> Exception: + err_cls = _AGENT_BACKEND_INVOKE_ERROR_BY_REASON.get(event.reason or "") + if err_cls is not None: + return err_cls(event.error) + return AgentBackendError(event.error or "Agent backend run did not complete successfully.") + def _prompt_messages_from_query(user_query: str | None) -> list[PromptMessage]: if not user_query: @@ -412,12 +437,15 @@ class _AgentProcessRecorder: def _lookup_tool_thought(self, *, index: int, tool_call_id: str | None) -> str | None: if tool_call_id and tool_call_id in self._tool_by_call_id: return self._tool_by_call_id[tool_call_id] + if index < 0: + return None return self._tool_by_index.get(index) def _remember_tool_thought( self, *, index: int, tool_call_id: str | None, tool_name: str | None, thought_id: str ) -> None: - self._tool_by_index[index] = thought_id + if index >= 0: + self._tool_by_index[index] = thought_id if tool_call_id: self._tool_by_call_id[tool_call_id] = thought_id if tool_name: @@ -433,6 +461,10 @@ class _AgentProcessRecorder: return None def _mark_tool_observed(self, thought_id: str) -> None: + self._tool_by_index = {index: value for index, value in self._tool_by_index.items() if value != thought_id} + self._tool_by_call_id = { + tool_call_id: value for tool_call_id, value in self._tool_by_call_id.items() if value != thought_id + } for open_thought_ids in self._open_tool_by_name.values(): open_thought_ids.discard(thought_id) @@ -530,7 +562,12 @@ def _event_index(data: dict[str, Any]) -> int: def _string_or_none(value: Any) -> str | None: - return value if isinstance(value, str) and value else None + if not isinstance(value, str): + return None + normalized = value.strip() + if not normalized or normalized.lower() in {"none", "null"}: + return None + return normalized def _json_or_text(value: Any) -> str | None: @@ -652,8 +689,9 @@ class AgentAppRunner: return if not isinstance(terminal, AgentBackendRunSucceededInternalEvent): - error = getattr(terminal, "error", None) or "Agent backend run did not complete successfully." - raise AgentBackendError(str(error)) + if isinstance(terminal, AgentBackendRunFailedInternalEvent): + raise _agent_backend_failure_to_exception(terminal) + raise AgentBackendError("Agent backend run did not complete successfully.") answer = self._terminal_output_to_answer(terminal.output) try: diff --git a/api/core/app/apps/base_app_generate_response_converter.py b/api/core/app/apps/base_app_generate_response_converter.py index abcbb2f9436..942dfd56d0e 100644 --- a/api/core/app/apps/base_app_generate_response_converter.py +++ b/api/core/app/apps/base_app_generate_response_converter.py @@ -8,7 +8,7 @@ from pydantic import JsonValue from core.app.entities.app_invoke_entities import InvokeFrom from core.app.entities.task_entities import AppBlockingResponse, AppStreamResponse from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError -from graphon.model_runtime.errors.invoke import InvokeError +from graphon.model_runtime.errors.invoke import InvokeError, InvokeRateLimitError logger = logging.getLogger(__name__) @@ -127,6 +127,7 @@ class AppGenerateResponseConverter[TBlockingResponse: AppBlockingResponse](ABC): }, ModelCurrentlyNotSupportError: {"code": "model_currently_not_support", "status": 400}, InvokeError: {"code": "completion_request_error", "status": 400}, + InvokeRateLimitError: {"code": "rate_limit_error", "status": 429}, } # Determine the response based on the type of exception diff --git a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py index c4224d67442..3af48947f78 100644 --- a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py +++ b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py @@ -420,6 +420,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat message.total_price = usage.total_price message.currency = usage.currency self._task_state.llm_result.usage.latency = message.provider_response_latency + self._task_state.metadata.usage = self._task_state.llm_result.usage message.message_metadata = self._task_state.metadata.model_dump_json() if trace_manager: diff --git a/api/core/plugin/entities/request.py b/api/core/plugin/entities/request.py index 65f7d044ea2..468055faea1 100644 --- a/api/core/plugin/entities/request.py +++ b/api/core/plugin/entities/request.py @@ -276,6 +276,7 @@ class RequestRequestDownloadFile(BaseModel): "validation", ] file: RequestDownloadFileMapping + for_external: bool = True model_config = ConfigDict(extra="forbid") diff --git a/api/core/tools/tool_file_manager.py b/api/core/tools/tool_file_manager.py index 25313dda49b..b6c1f080e19 100644 --- a/api/core/tools/tool_file_manager.py +++ b/api/core/tools/tool_file_manager.py @@ -4,6 +4,7 @@ import hmac import logging import os import time +import urllib.parse from collections.abc import Generator from mimetypes import guess_extension, guess_type from uuid import uuid4 @@ -26,7 +27,7 @@ logger = logging.getLogger(__name__) class ToolFileManager: @staticmethod def _build_graph_file_reference(tool_file: ToolFile) -> File: - extension = guess_extension(tool_file.mimetype) or ".bin" + extension = resolve_extension(filename=tool_file.name, mimetype=tool_file.mimetype) return File( file_type=get_file_type_by_mime_type(tool_file.mimetype), transfer_method=FileTransferMethod.TOOL_FILE, @@ -70,7 +71,7 @@ class ToolFileManager: mimetype: str, filename: str | None = None, ) -> ToolFile: - extension = guess_extension(mimetype) or ".bin" + extension = resolve_extension(filename=filename, mimetype=mimetype) unique_name = uuid4().hex unique_filename = f"{unique_name}{extension}" # default just as before @@ -120,7 +121,8 @@ class ToolFileManager: or response.headers.get("Content-Type", "").split(";")[0].strip() or "application/octet-stream" ) - extension = guess_extension(mimetype) or ".bin" + url_filename = os.path.basename(urllib.parse.urlparse(file_url).path) + extension = resolve_extension(filename=url_filename, mimetype=mimetype) unique_name = uuid4().hex filename = f"{unique_name}{extension}" filepath = f"tools/{tenant_id}/{filename}" @@ -220,4 +222,11 @@ def _factory() -> ToolFileManager: return ToolFileManager() +def resolve_extension(*, filename: str | None, mimetype: str) -> str: + filename_extension = os.path.splitext(filename or "")[1].lower() + if filename_extension: + return filename_extension + return guess_extension(mimetype) or ".bin" + + set_tool_file_manager_factory(_factory) diff --git a/api/core/tools/utils/message_transformer.py b/api/core/tools/utils/message_transformer.py index f2e4f0ea5fd..7511071ae31 100644 --- a/api/core/tools/utils/message_transformer.py +++ b/api/core/tools/utils/message_transformer.py @@ -3,7 +3,6 @@ import re from collections.abc import Generator from datetime import date, datetime from decimal import Decimal -from mimetypes import guess_extension from typing import Any from uuid import UUID @@ -11,7 +10,7 @@ import numpy as np import pytz from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.tool_file_manager import ToolFileManager +from core.tools.tool_file_manager import ToolFileManager, resolve_extension from core.workflow.file_reference import parse_file_reference from graphon.file import File, FileTransferMethod, FileType from libs.login import current_user @@ -91,7 +90,8 @@ class ToolFileMessageTransformer: conversation_id=conversation_id, ) - url = f"/files/tools/{tool_file.id}{guess_extension(tool_file.mimetype) or '.png'}" + extension = resolve_extension(filename=tool_file.name, mimetype=tool_file.mimetype) + url = cls.get_tool_file_url(tool_file_id=tool_file.id, extension=extension) meta = cls._with_tool_file_meta( message.meta, tool_file_id=str(tool_file.id), @@ -136,7 +136,8 @@ class ToolFileMessageTransformer: filename=filename, ) - url = cls.get_tool_file_url(tool_file_id=tool_file.id, extension=guess_extension(tool_file.mimetype)) + extension = resolve_extension(filename=tool_file.name, mimetype=tool_file.mimetype) + url = cls.get_tool_file_url(tool_file_id=tool_file.id, extension=extension) meta = cls._with_tool_file_meta(meta, tool_file_id=str(tool_file.id)) # check if file is image diff --git a/api/core/workflow/nodes/agent_v2/agent_node.py b/api/core/workflow/nodes/agent_v2/agent_node.py index 8e27cf63da4..6f27905f9a6 100644 --- a/api/core/workflow/nodes/agent_v2/agent_node.py +++ b/api/core/workflow/nodes/agent_v2/agent_node.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, override from agenton.compositor import CompositorSessionSnapshot from clients.agent_backend import ( + AgentBackendAgentMessageDeltaInternalEvent, AgentBackendDeferredToolCallInternalEvent, AgentBackendError, AgentBackendHTTPError, @@ -481,6 +482,10 @@ class DifyAgentNode(Node[DifyAgentNodeData]): if isinstance(internal_event, AgentBackendStreamInternalEvent): self._record_stream_metadata(metadata, internal_event) continue + if internal_event.type == AgentBackendInternalEventType.AGENT_MESSAGE_DELTA: + if isinstance(internal_event, AgentBackendAgentMessageDeltaInternalEvent): + self._record_agent_message_delta_metadata(metadata, internal_event) + continue metadata["agent_backend"] = { **dict(metadata.get("agent_backend") or {}), "stream_event_count": stream_event_count, @@ -734,6 +739,17 @@ class DifyAgentNode(Node[DifyAgentNodeData]): agent_backend["usage"] = dict(usage) metadata["agent_backend"] = agent_backend + @staticmethod + def _record_agent_message_delta_metadata( + metadata: dict[str, Any], event: AgentBackendAgentMessageDeltaInternalEvent + ) -> None: + agent_backend = dict(metadata.get("agent_backend") or {}) + agent_backend["agent_message_delta_count"] = int(agent_backend.get("agent_message_delta_count") or 0) + 1 + agent_backend["agent_message_delta_length"] = int(agent_backend.get("agent_message_delta_length") or 0) + len( + event.delta + ) + metadata["agent_backend"] = agent_backend + @classmethod @override def _extract_variable_selector_to_variable_mapping( diff --git a/api/core/workflow/nodes/agent_v2/output_file_rebacker.py b/api/core/workflow/nodes/agent_v2/output_file_rebacker.py index ac39583626a..90a73bc6246 100644 --- a/api/core/workflow/nodes/agent_v2/output_file_rebacker.py +++ b/api/core/workflow/nodes/agent_v2/output_file_rebacker.py @@ -10,13 +10,13 @@ trustworthy metadata. from __future__ import annotations -from mimetypes import guess_extension from uuid import UUID from sqlalchemy import select from sqlalchemy.exc import DataError, SQLAlchemyError from core.db.session_factory import session_factory +from core.tools.tool_file_manager import resolve_extension from core.workflow.file_reference import build_file_reference from graphon.file import File, FileTransferMethod, get_file_type_by_mime_type from models.tools import ToolFile @@ -46,7 +46,7 @@ def reback_tool_file_output(*, tenant_id: str, tool_file_id: str) -> File | None return None mime_type = tool_file.mimetype or "" - extension = guess_extension(mime_type) or ".bin" + extension = resolve_extension(filename=tool_file.name, mimetype=mime_type) return File( type=get_file_type_by_mime_type(mime_type), transfer_method=FileTransferMethod.TOOL_FILE, diff --git a/api/factories/file_factory/builders.py b/api/factories/file_factory/builders.py index 67b1c646db3..91ae554cffe 100644 --- a/api/factories/file_factory/builders.py +++ b/api/factories/file_factory/builders.py @@ -3,6 +3,7 @@ from __future__ import annotations import mimetypes +import os import uuid from collections.abc import Mapping, Sequence from typing import Any, Literal, NotRequired, TypedDict, assert_never, cast @@ -285,7 +286,7 @@ def _build_from_remote_url( raise ValueError("Invalid file url") mime_type, filename, file_size = get_remote_file_info(url) - extension = mimetypes.guess_extension(mime_type) or ("." + filename.split(".")[-1] if "." in filename else ".bin") + extension = os.path.splitext(filename)[1].lower() or mimetypes.guess_extension(mime_type) or ".bin" detected_file_type = standardize_file_type(extension=extension, mime_type=mime_type) file_type = _resolve_file_type( detected_file_type=detected_file_type, @@ -326,7 +327,12 @@ def _build_from_tool_file( if tool_file is None: raise ValueError(f"ToolFile {tool_file_id} not found") - extension = "." + tool_file.file_key.split(".")[-1] if "." in tool_file.file_key else ".bin" + extension = ( + os.path.splitext(tool_file.name)[1].lower() + or mimetypes.guess_extension(tool_file.mimetype) + or os.path.splitext(tool_file.file_key)[1].lower() + or ".bin" + ) detected_file_type = standardize_file_type(extension=extension, mime_type=tool_file.mimetype) file_type = _resolve_file_type( detected_file_type=detected_file_type, diff --git a/api/fields/message_fields.py b/api/fields/message_fields.py index ba63b8bd1f4..a90c6a36e74 100644 --- a/api/fields/message_fields.py +++ b/api/fields/message_fields.py @@ -1,9 +1,10 @@ from __future__ import annotations from datetime import datetime +from decimal import Decimal from uuid import uuid4 -from pydantic import Field, field_validator +from pydantic import Field, computed_field, field_validator from core.entities.execution_extra_content import ExecutionExtraContentDomainModel from fields.base import ResponseModel @@ -55,10 +56,19 @@ class MessageListItem(ResponseModel): created_at: int | None = None agent_thoughts: list[AgentThought] message_files: list[MessageFile] + message_tokens: int = 0 + answer_tokens: int = 0 + provider_response_latency: float = 0 + total_price: Decimal | None = None + currency: str | None = None status: str error: str | None = None extra_contents: list[ExecutionExtraContentDomainModel] + @computed_field + def total_tokens(self) -> int: + return self.message_tokens + self.answer_tokens + @field_validator("inputs", mode="before") @classmethod def _normalize_inputs(cls, value: JSONValueType) -> JSONValueType: diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md index 679fc812cab..556e0045b8d 100644 --- a/api/openapi/markdown/console-openapi.md +++ b/api/openapi/markdown/console-openapi.md @@ -17614,19 +17614,25 @@ Built-in tool icons are URL strings; API-based tool icons are provider-defined p | ---- | ---- | ----------- | -------- | | agent_thoughts | [ [AgentThought](#agentthought) ] | | Yes | | answer | string | | Yes | +| answer_tokens | integer | | No | | conversation_id | string | | Yes | | created_at | integer | | No | +| currency | string | | No | | error | string | | No | | extra_contents | [ [HumanInputContent](#humaninputcontent) ] | | Yes | | feedback | [SimpleFeedback](#simplefeedback) | | No | | id | string | | Yes | | inputs | object | | Yes | | message_files | [ [MessageFile](#messagefile) ] | | Yes | +| message_tokens | integer | | No | | metadata | [JSONValueType](#jsonvaluetype) | | No | | parent_message_id | string | | No | +| provider_response_latency | number | | No | | query | string | | Yes | | retriever_resources | [ [RetrieverResource](#retrieverresource) ] | | Yes | | status | string | | Yes | +| total_price | string | | No | +| total_tokens | integer | | Yes | #### ExternalApiTemplateListQuery diff --git a/api/openapi/markdown/service-openapi.md b/api/openapi/markdown/service-openapi.md index b00edc64b4a..32b9c388506 100644 --- a/api/openapi/markdown/service-openapi.md +++ b/api/openapi/markdown/service-openapi.md @@ -3467,18 +3467,24 @@ Model class for i18n object. | ---- | ---- | ----------- | -------- | | agent_thoughts | [ [AgentThought](#agentthought) ] | | Yes | | answer | string | | Yes | +| answer_tokens | integer | | No | | conversation_id | string | | Yes | | created_at | integer | | No | +| currency | string | | No | | error | string | | No | | extra_contents | [ [HumanInputContent](#humaninputcontent) ] | | Yes | | feedback | [SimpleFeedback](#simplefeedback) | | No | | id | string | | Yes | | inputs | object | | Yes | | message_files | [ [MessageFile](#messagefile) ] | | Yes | +| message_tokens | integer | | No | | parent_message_id | string | | No | +| provider_response_latency | number | | No | | query | string | | Yes | | retriever_resources | [ [RetrieverResource](#retrieverresource) ] | | Yes | | status | string | | Yes | +| total_price | string | | No | +| total_tokens | integer | | Yes | #### MessageListQuery diff --git a/api/openapi/markdown/web-openapi.md b/api/openapi/markdown/web-openapi.md index 007a6d15c24..f6812634762 100644 --- a/api/openapi/markdown/web-openapi.md +++ b/api/openapi/markdown/web-openapi.md @@ -1685,19 +1685,25 @@ in form definiton, or a variable while the workflow is running. | ---- | ---- | ----------- | -------- | | agent_thoughts | [ [AgentThought](#agentthought) ] | | Yes | | answer | string | | Yes | +| answer_tokens | integer | | No | | conversation_id | string | | Yes | | created_at | integer | | No | +| currency | string | | No | | error | string | | No | | extra_contents | [ [HumanInputContent](#humaninputcontent) ] | | Yes | | feedback | [SimpleFeedback](#simplefeedback) | | No | | id | string | | Yes | | inputs | object | | Yes | | message_files | [ [MessageFile](#messagefile) ] | | Yes | +| message_tokens | integer | | No | | metadata | [JSONValueType](#jsonvaluetype) | | No | | parent_message_id | string | | No | +| provider_response_latency | number | | No | | query | string | | Yes | | retriever_resources | [ [RetrieverResource](#retrieverresource) ] | | Yes | | status | string | | Yes | +| total_price | string | | No | +| total_tokens | integer | | Yes | #### WebModelConfigResponse diff --git a/api/services/file_request_service.py b/api/services/file_request_service.py index a4be0bbec88..a795595d095 100644 --- a/api/services/file_request_service.py +++ b/api/services/file_request_service.py @@ -45,6 +45,7 @@ class FileRequestService: user_from: UserFrom | str, invoke_from: InvokeFrom | str, file_mapping: Mapping[str, Any], + for_external: bool = True, ) -> DownloadFileRequestResult: """Resolve one file mapping into signed download metadata. @@ -61,7 +62,7 @@ class FileRequestService: ) with bind_file_access_scope(scope): file = self._build_file(mapping=file_mapping, tenant_id=tenant_id) - download_url = file_helpers.resolve_file_url(file, for_external=True) + download_url = file_helpers.resolve_file_url(file, for_external=for_external) if not download_url: raise ValueError("file does not support signed download") diff --git a/api/tests/unit_tests/configs/test_dify_config.py b/api/tests/unit_tests/configs/test_dify_config.py index 81f3f92a7cc..eaeb15a7090 100644 --- a/api/tests/unit_tests/configs/test_dify_config.py +++ b/api/tests/unit_tests/configs/test_dify_config.py @@ -75,6 +75,7 @@ def test_dify_config(monkeypatch: pytest.MonkeyPatch): # default values assert config.EDITION == "SELF_HOSTED" assert config.API_COMPRESSION_ENABLED is False + assert config.AGENT_SHELL_ENABLED is True assert config.SENTRY_TRACES_SAMPLE_RATE == 1.0 assert config.TEMPLATE_TRANSFORM_MAX_LENGTH == 400_000 @@ -110,6 +111,25 @@ def test_http_timeout_defaults(monkeypatch: pytest.MonkeyPatch): assert config.HTTP_REQUEST_MAX_WRITE_TIMEOUT == 600 +def test_internal_files_url_falls_back_to_server_console_api_url(monkeypatch: pytest.MonkeyPatch): + os.environ.clear() + monkeypatch.setenv("SERVER_CONSOLE_API_URL", "http://api:5001") + + config = DifyConfig(_env_file=None) + + assert config.INTERNAL_FILES_URL == "http://api:5001" + + +def test_internal_files_url_prefers_explicit_value(monkeypatch: pytest.MonkeyPatch): + os.environ.clear() + monkeypatch.setenv("INTERNAL_FILES_URL", "http://files-internal:5001") + monkeypatch.setenv("SERVER_CONSOLE_API_URL", "http://api:5001") + + config = DifyConfig(_env_file=None) + + assert config.INTERNAL_FILES_URL == "http://files-internal:5001" + + # NOTE: If there is a `.env` file in your Workspace, this test might not succeed as expected. # This is due to `pymilvus` loading all the variables from the `.env` file into `os.environ`. def test_flask_configs(monkeypatch: pytest.MonkeyPatch): diff --git a/api/tests/unit_tests/controllers/console/explore/test_message.py b/api/tests/unit_tests/controllers/console/explore/test_message.py index f6182abb987..9a93fe5626f 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_message.py +++ b/api/tests/unit_tests/controllers/console/explore/test_message.py @@ -49,6 +49,8 @@ def make_message(): msg.query = "hello" msg.re_sign_file_url_answer = "" msg.user_feedback = MagicMock(rating=None) + msg.total_price = None + msg.currency = None msg.status = "normal" msg.error = None return msg diff --git a/api/tests/unit_tests/controllers/files/test_upload.py b/api/tests/unit_tests/controllers/files/test_upload.py index 4d5522c0c6c..4f6fcd6f3e0 100644 --- a/api/tests/unit_tests/controllers/files/test_upload.py +++ b/api/tests/unit_tests/controllers/files/test_upload.py @@ -34,11 +34,11 @@ class DummyFile: class DummyToolFile: - def __init__(self): + def __init__(self, name="test.txt", mimetype="text/plain"): self.id = "file-id" - self.name = "test.txt" + self.name = name self.size = 10 - self.mimetype = "text/plain" + self.mimetype = mimetype self.original_url = "http://original" self.user_id = "user-1" self.tenant_id = "tenant-1" @@ -56,7 +56,7 @@ class TestPluginUploadFileApi: mock_get_user, mock_verify_signature, ): - dummy_file = DummyFile() + dummy_file = DummyFile(filename="report.docx", mimetype="application/octet-stream") module.request = fake_request( { @@ -71,7 +71,10 @@ class TestPluginUploadFileApi: ) tool_file_manager_instance = mock_tool_file_manager.return_value - tool_file_manager_instance.create_file_by_raw.return_value = DummyToolFile() + tool_file_manager_instance.create_file_by_raw.return_value = DummyToolFile( + name="report.docx", + mimetype="application/octet-stream", + ) mock_tool_file_manager.sign_file.return_value = "signed-url" @@ -84,10 +87,12 @@ class TestPluginUploadFileApi: assert result["id"] == "file-id" assert result["reference"] == build_file_reference(record_id="file-id") assert result["preview_url"] == "signed-url" + assert result["extension"] == ".docx" mock_verify_signature.assert_called_once() assert mock_verify_signature.call_args.kwargs["conversation_id"] == "conversation-1" tool_file_manager_instance.create_file_by_raw.assert_called_once() assert tool_file_manager_instance.create_file_by_raw.call_args.kwargs["conversation_id"] == "conversation-1" + mock_tool_file_manager.sign_file.assert_called_once_with(tool_file_id="file-id", extension=".docx") def test_missing_file(self): module.request = fake_request( diff --git a/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py b/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py index 1b0a82ed7c5..b3690d26c6c 100644 --- a/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py +++ b/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py @@ -318,6 +318,7 @@ class TestPluginDownloadFileRequestApi: mock_payload.user_id = "user-id" mock_payload.user_from = "account" mock_payload.invoke_from = "debugger" + mock_payload.for_external = False reference = build_file_reference(record_id="tool-file-1") mock_payload.file.model_dump.return_value = { "transfer_method": "tool_file", @@ -333,6 +334,7 @@ class TestPluginDownloadFileRequestApi: user_from="account", invoke_from="debugger", file_mapping={"transfer_method": "tool_file", "reference": reference}, + for_external=False, ) assert result["data"] == { "filename": "report.pdf", diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_app_runner.py b/api/tests/unit_tests/core/app/apps/agent_app/test_app_runner.py index ffef1cf3be8..55d5be6d555 100644 --- a/api/tests/unit_tests/core/app/apps/agent_app/test_app_runner.py +++ b/api/tests/unit_tests/core/app/apps/agent_app/test_app_runner.py @@ -37,6 +37,7 @@ from pydantic_ai.messages import ( from clients.agent_backend import ( AgentBackendError, AgentBackendRunEventAdapter, + AgentBackendRunFailedInternalEvent, AgentBackendStreamInternalEvent, FakeAgentBackendRunClient, FakeAgentBackendScenario, @@ -54,6 +55,7 @@ from core.app.entities.queue_entities import ( QueueMessageEndEvent, ) from core.workflow.nodes.agent_v2.ask_human_resume import AskHumanResumeOutcome +from graphon.model_runtime.errors.invoke import InvokeRateLimitError from models.agent_config_entities import AgentSoulConfig from models.model import MessageAgentThought @@ -1039,6 +1041,130 @@ def test_tool_result_without_call_id_matches_unique_open_tool_name(monkeypatch): assert rows[0].observation == "Knowledge base search results: browser skill" +def test_repeated_tool_calls_without_call_id_or_index_create_distinct_rows(monkeypatch): + fake_session = _FakeDbSession() + monkeypatch.setattr(app_runner_module.db, "session", fake_session) + qm = _FakeQueueManager() + recorder = app_runner_module._AgentProcessRecorder( + dify_context=_dify_ctx(), + message_id="msg-1", + queue_manager=qm, # type: ignore[arg-type] + ) + + recorder.handle_stream_event( + AgentBackendStreamInternalEvent( + run_id="run-1", + data={ + "event_kind": "function_tool_call", + "part": { + "part_kind": "tool-call", + "tool_name": "shell_run", + "args": {"script": "lookup find"}, + }, + }, + ) + ) + recorder.handle_stream_event( + AgentBackendStreamInternalEvent( + run_id="run-1", + data={ + "event_kind": "function_tool_result", + "part": { + "part_kind": "tool-return", + "tool_name": "shell_run", + "content": "find output", + }, + }, + ) + ) + recorder.handle_stream_event( + AgentBackendStreamInternalEvent( + run_id="run-1", + data={ + "event_kind": "function_tool_call", + "part": { + "part_kind": "tool-call", + "tool_name": "shell_run", + "args": {"script": "lookup out"}, + }, + }, + ) + ) + recorder.handle_stream_event( + AgentBackendStreamInternalEvent( + run_id="run-1", + data={ + "event_kind": "function_tool_result", + "part": { + "part_kind": "tool-return", + "tool_name": "shell_run", + "content": "out output", + }, + }, + ) + ) + + rows = sorted(fake_session.rows.values(), key=lambda row: row.position) + assert len(rows) == 2 + assert rows[0].tool == "shell_run" + assert rows[0].tool_input == '{"script": "lookup find"}' + assert rows[0].observation == "find output" + assert rows[1].tool == "shell_run" + assert rows[1].tool_input == '{"script": "lookup out"}' + assert rows[1].observation == "out output" + + +def test_repeated_tool_calls_with_placeholder_call_id_and_reused_index_create_distinct_rows(monkeypatch): + fake_session = _FakeDbSession() + monkeypatch.setattr(app_runner_module.db, "session", fake_session) + qm = _FakeQueueManager() + recorder = app_runner_module._AgentProcessRecorder( + dify_context=_dify_ctx(), + message_id="msg-1", + queue_manager=qm, # type: ignore[arg-type] + ) + + for script, output in (("lookup find", "find output"), ("lookup out", "out output")): + recorder.handle_stream_event( + AgentBackendStreamInternalEvent( + run_id="run-1", + data={ + "event_kind": "function_tool_call", + "index": 0, + "part": { + "part_kind": "tool-call", + "tool_name": "shell_run", + "tool_call_id": "None", + "args": {"script": script}, + }, + }, + ) + ) + recorder.handle_stream_event( + AgentBackendStreamInternalEvent( + run_id="run-1", + data={ + "event_kind": "function_tool_result", + "part": { + "part_kind": "tool-return", + "tool_name": "shell_run", + "tool_call_id": "None", + "content": output, + }, + }, + ) + ) + + rows = sorted(fake_session.rows.values(), key=lambda row: row.position) + assert len(rows) == 2 + assert rows[0].tool == "shell_run" + assert rows[0].tool_input == '{"script": "lookup find"}' + assert rows[0].observation == "find output" + assert rows[1].tool == "shell_run" + assert rows[1].tool_input == '{"script": "lookup out"}' + assert rows[1].observation == "out output" + + def test_prior_session_snapshot_is_threaded_into_request(): prior = CompositorSessionSnapshot(layers=[]) client = FakeAgentBackendRunClient() @@ -1088,6 +1214,19 @@ def test_failed_run_raises_agent_backend_error(): assert store.saved == [] +def test_agent_backend_failure_to_exception_maps_rate_limit_reason(): + err = app_runner_module._agent_backend_failure_to_exception( + AgentBackendRunFailedInternalEvent( + run_id="run-1", + error="quota exceeded", + reason="InvokeRateLimitError", + ) + ) + + assert isinstance(err, InvokeRateLimitError) + assert str(err) == "quota exceeded" + + def test_stopped_task_cancels_agent_backend_run_and_skips_session_save(): client = _RecordingFakeAgentBackendRunClient() store = _FakeSessionStore() diff --git a/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py b/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py index c246f7b7836..06823644b27 100644 --- a/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py +++ b/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py @@ -3,10 +3,11 @@ from unittest.mock import Mock import pytest +from core.app.apps.base_app_generate_response_converter import AppGenerateResponseConverter from core.app.entities.queue_entities import QueueErrorEvent from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTaskPipeline from core.errors.error import QuotaExceededError -from graphon.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError +from graphon.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError, InvokeRateLimitError from models.enums import MessageStatus @@ -68,6 +69,11 @@ class TestBasedGenerateTaskPipeline: assert error_response.task_id == "task-1" assert ping_response.task_id == "task-1" + def test_stream_converter_maps_invoke_rate_limit_error(self): + data = AppGenerateResponseConverter._error_to_stream_response(InvokeRateLimitError("quota exceeded")) + + assert data == {"code": "rate_limit_error", "status": 429, "message": "quota exceeded"} + def test_handle_output_moderation_when_flagged(self, pipeline): handler = Mock() handler.moderation_completion.return_value = ("filtered", True) diff --git a/api/tests/unit_tests/core/plugin/entities/test_request.py b/api/tests/unit_tests/core/plugin/entities/test_request.py index 05ff074ea92..57bc7c9aae8 100644 --- a/api/tests/unit_tests/core/plugin/entities/test_request.py +++ b/api/tests/unit_tests/core/plugin/entities/test_request.py @@ -22,6 +22,7 @@ def test_request_download_file_accepts_tool_file_reference() -> None: assert payload.file.transfer_method == "tool_file" assert payload.file.reference == reference + assert payload.for_external is True def test_request_download_file_accepts_remote_url() -> None: @@ -42,6 +43,25 @@ def test_request_download_file_accepts_remote_url() -> None: assert payload.file.url == "https://example.com/report.pdf" +def test_request_download_file_accepts_internal_download_request() -> None: + reference = build_file_reference(record_id="tool-file-1") + payload = RequestRequestDownloadFile.model_validate( + { + "tenant_id": "tenant-1", + "user_id": "user-1", + "user_from": "account", + "invoke_from": "debugger", + "file": { + "transfer_method": "tool_file", + "reference": reference, + }, + "for_external": False, + } + ) + + assert payload.for_external is False + + def test_request_download_file_rejects_remote_url_without_url() -> None: with pytest.raises(ValidationError, match="url is required"): _ = RequestRequestDownloadFile.model_validate( diff --git a/api/tests/unit_tests/core/tools/test_tool_file_manager.py b/api/tests/unit_tests/core/tools/test_tool_file_manager.py index c44ac0d13ef..7f7997b8884 100644 --- a/api/tests/unit_tests/core/tools/test_tool_file_manager.py +++ b/api/tests/unit_tests/core/tools/test_tool_file_manager.py @@ -60,6 +60,37 @@ def test_create_file_by_raw_stores_file_and_persists_record() -> None: session.refresh.assert_called_once_with(file_model) +def test_create_file_by_raw_prefers_filename_extension_over_mimetype() -> None: + manager = ToolFileManager() + session = Mock() + session.refresh.side_effect = lambda model: setattr(model, "id", "tf-docx") + + def tool_file_factory(**kwargs): + return SimpleNamespace(**kwargs) + + with ( + patch("core.tools.tool_file_manager.storage") as storage, + patch("core.tools.tool_file_manager.ToolFile", side_effect=tool_file_factory), + patch("core.tools.tool_file_manager.uuid4", return_value=SimpleNamespace(hex="abc")), + _patch_session_factory(session), + ): + file_model = manager.create_file_by_raw( + user_id="u1", + tenant_id="t1", + conversation_id="c1", + file_binary=b"docx", + mimetype="application/octet-stream", + filename="report.docx", + ) + + assert file_model.name == "report.docx" + assert file_model.file_key == "tools/t1/abc.docx" + storage.save.assert_called_once_with("tools/t1/abc.docx", b"docx") + session.add.assert_called_once_with(file_model) + session.commit.assert_called_once() + session.refresh.assert_called_once_with(file_model) + + def test_create_file_by_url_downloads_and_persists_record() -> None: manager = ToolFileManager() response = Mock() @@ -88,6 +119,32 @@ def test_create_file_by_url_downloads_and_persists_record() -> None: session.refresh.assert_called_once_with(file_model) +def test_create_file_by_url_prefers_url_extension_over_mimetype() -> None: + manager = ToolFileManager() + response = Mock() + response.content = b"docx" + response.headers = {"Content-Type": "application/octet-stream"} + response.raise_for_status.return_value = None + session = Mock() + + def tool_file_factory(**kwargs): + return SimpleNamespace(**kwargs) + + session.refresh.side_effect = lambda model: setattr(model, "id", "tf-docx") + with ( + patch("core.tools.tool_file_manager.storage") as storage, + patch("core.tools.tool_file_manager.ToolFile", side_effect=tool_file_factory), + patch("core.tools.tool_file_manager.uuid4", return_value=SimpleNamespace(hex="urlabc")), + _patch_session_factory(session), + patch("core.tools.tool_file_manager.remote_fetcher.make_request", return_value=response), + ): + file_model = manager.create_file_by_url("u1", "t1", "https://example.com/report.docx?download=1", "c1") + + assert file_model.file_key == "tools/t1/urlabc.docx" + assert file_model.name == "urlabc.docx" + storage.save.assert_called_once_with("tools/t1/urlabc.docx", b"docx") + + def test_create_file_by_url_raises_on_timeout() -> None: manager = ToolFileManager() diff --git a/api/tests/unit_tests/core/tools/utils/test_message_transformer.py b/api/tests/unit_tests/core/tools/utils/test_message_transformer.py index 354b3955042..c5e3f32cca6 100644 --- a/api/tests/unit_tests/core/tools/utils/test_message_transformer.py +++ b/api/tests/unit_tests/core/tools/utils/test_message_transformer.py @@ -7,9 +7,10 @@ from core.tools.entities.tool_entities import ToolInvokeMessage class _FakeToolFile: - def __init__(self, mimetype: str): + def __init__(self, mimetype: str, name: str | None): self.id = "fake-tool-file-id" self.mimetype = mimetype + self.name = name or "fake-tool-file.bin" class _FakeToolFileManager: @@ -38,7 +39,7 @@ class _FakeToolFileManager: "mimetype": mimetype, "filename": filename, } - return _FakeToolFile(mimetype) + return _FakeToolFile(mimetype, filename) @pytest.fixture(autouse=True) @@ -89,6 +90,29 @@ def test_transform_tool_invoke_messages_mimetype_key_present_but_none(): assert o.meta["tool_file_id"] == "fake-tool-file-id" +def test_transform_tool_invoke_messages_prefers_filename_extension_over_mimetype(): + msg = ToolInvokeMessage( + type=ToolInvokeMessage.MessageType.BLOB, + message=ToolInvokeMessage.BlobMessage(blob=b"docx"), + meta={"mime_type": "application/octet-stream", "filename": "report.docx"}, + ) + + out = list( + mt.ToolFileMessageTransformer.transform_tool_invoke_messages( + messages=_gen([msg]), + user_id="u1", + tenant_id="t1", + conversation_id="c1", + ) + ) + + assert _FakeToolFileManager.last_call is not None + assert _FakeToolFileManager.last_call["filename"] == "report.docx" + assert len(out) == 1 + assert isinstance(out[0].message, ToolInvokeMessage.TextMessage) + assert out[0].message.text.endswith(".docx") + + def test_transform_tool_invoke_messages_parses_existing_tool_file_link_meta(): msg = ToolInvokeMessage( type=ToolInvokeMessage.MessageType.IMAGE_LINK, diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py index 407b5e59f7f..b642ea7780d 100644 --- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py @@ -1,10 +1,12 @@ +from datetime import UTC, datetime from types import SimpleNamespace from typing import cast from unittest.mock import MagicMock, patch from agenton.compositor import CompositorSessionSnapshot from dify_agent.layers.ask_human import AskHumanToolResult -from dify_agent.protocol import RunStartedEvent, RunSucceededEvent, RunSucceededEventData +from dify_agent.protocol import PydanticAIStreamRunEvent, RunStartedEvent, RunSucceededEvent, RunSucceededEventData +from pydantic_ai.messages import PartDeltaEvent, TextPartDelta from clients.agent_backend import ( AgentBackendRunEventAdapter, @@ -190,6 +192,30 @@ class FileOutputBackendClient(FakeAgentBackendRunClient): ) +class AgentMessageDeltaBackendClient(FakeAgentBackendRunClient): + def _events(self, run_id: str): + created_at = datetime(2026, 1, 1, tzinfo=UTC) + return ( + RunStartedEvent(id="1-0", run_id=run_id, created_at=created_at), + PydanticAIStreamRunEvent( + id="2-0", + run_id=run_id, + created_at=created_at, + data=PartDeltaEvent(index=0, delta=TextPartDelta(content_delta="hello ")), + agent_message_delta="hello ", + ), + RunSucceededEvent( + id="3-0", + run_id=run_id, + created_at=created_at, + data=RunSucceededEventData( + output={"text": "hello agent"}, + session_snapshot=CompositorSessionSnapshot(layers=[]), + ), + ), + ) + + def _node( *, scenario: FakeAgentBackendScenario = FakeAgentBackendScenario.SUCCESS, @@ -277,6 +303,19 @@ def test_agent_node_run_maps_successful_agent_backend_run_to_node_result(): assert layers["llm"]["config"]["credentials"] == "[REDACTED]" +def test_agent_node_run_ignores_agent_message_delta_until_terminal_result(): + events = list(_node(agent_backend_client=AgentMessageDeltaBackendClient())._run()) + + assert len(events) == 1 + result = cast(StreamCompletedEvent, events[0]).node_run_result + assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert result.outputs == {"text": "hello agent"} + agent_backend = result.metadata[WorkflowNodeExecutionMetadataKey.AGENT_LOG]["agent_backend"] + assert agent_backend["status"] == "succeeded" + assert agent_backend["agent_message_delta_count"] == 1 + assert agent_backend["agent_message_delta_length"] == len("hello ") + + def test_agent_node_run_normalizes_declared_file_output_with_canonical_mapping(): tool_reference = build_file_reference(record_id="tool-file-1") with patch( diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_output_file_rebacker.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_output_file_rebacker.py index f72b7f806aa..5299fe462c8 100644 --- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_output_file_rebacker.py +++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_output_file_rebacker.py @@ -57,6 +57,17 @@ def test_reback_resolves_tenant_tool_file_to_file(): assert file.extension == ".png" +def test_reback_prefers_filename_extension_over_mimetype(): + tf = _seed(mimetype="application/octet-stream", name="report.docx", size=99) + file = reback_tool_file_output(tenant_id=TENANT, tool_file_id=tf) + + assert file is not None + assert file.filename == "report.docx" + assert file.mime_type == "application/octet-stream" + assert file.extension == ".docx" + assert file.type == FileType.CUSTOM + + def test_reback_other_tenant_returns_none(): tf = _seed() assert reback_tool_file_output(tenant_id="33333333-3333-3333-3333-333333333333", tool_file_id=tf) is None diff --git a/api/tests/unit_tests/factories/test_build_from_mapping.py b/api/tests/unit_tests/factories/test_build_from_mapping.py index e2dd0efee32..90d92989c61 100644 --- a/api/tests/unit_tests/factories/test_build_from_mapping.py +++ b/api/tests/unit_tests/factories/test_build_from_mapping.py @@ -165,6 +165,20 @@ def test_build_from_mapping_accepts_opaque_related_id_for_tool_file(mock_tool_fi assert file.storage_key == "tool_file.pdf" +def test_build_from_mapping_prefers_tool_filename_extension_over_mimetype(mock_tool_file): + mock_tool_file.name = "report.docx" + mock_tool_file.file_key = "tools/test_tenant_id/file.bin" + mock_tool_file.mimetype = "application/octet-stream" + mapping = tool_file_mapping(file_type="document") + + file = build_from_mapping(mapping=mapping, tenant_id=TEST_TENANT_ID) + + assert file.extension == ".docx" + assert file.filename == "report.docx" + assert file.mime_type == "application/octet-stream" + assert file.storage_key == "tools/test_tenant_id/file.bin" + + @pytest.mark.parametrize( ("file_type", "should_pass", "expected_error"), [ @@ -213,6 +227,25 @@ def test_build_from_remote_url(mock_http_head): assert file.size == 2048 +def test_build_from_remote_url_prefers_filename_extension_over_mimetype(): + mapping = { + "transfer_method": "remote_url", + "url": TEST_REMOTE_URL, + "type": "document", + } + + with patch( + "factories.file_factory.builders.get_remote_file_info", + return_value=("application/octet-stream", "report.docx", 99), + ): + file = build_from_mapping(mapping=mapping, tenant_id=TEST_TENANT_ID) + + assert file.filename == "report.docx" + assert file.extension == ".docx" + assert file.mime_type == "application/octet-stream" + assert file.size == 99 + + @pytest.mark.parametrize( ("file_type", "should_pass", "expected_error"), [ diff --git a/api/tests/unit_tests/fields/test_message_fields.py b/api/tests/unit_tests/fields/test_message_fields.py index 8a4eadaf744..2180e0d896c 100644 --- a/api/tests/unit_tests/fields/test_message_fields.py +++ b/api/tests/unit_tests/fields/test_message_fields.py @@ -1,4 +1,6 @@ -from fields.message_fields import ExploreMessageListItem, MessageListItem +from decimal import Decimal + +from fields.message_fields import ExploreMessageListItem, MessageListItem, WebMessageListItem def _base_kwargs(): @@ -34,3 +36,33 @@ class TestExploreMessageListItem: # Guard the public service-API contract: the base item must not leak metadata. payload = MessageListItem(**_base_kwargs()).model_dump(mode="json") assert "metadata" not in payload + + def test_message_list_item_exposes_usage_fields(self): + payload = MessageListItem( + **_base_kwargs(), + message_tokens=7, + answer_tokens=11, + provider_response_latency=1.25, + total_price=Decimal("0.0001234"), + currency="USD", + ).model_dump(mode="json") + + assert payload["message_tokens"] == 7 + assert payload["answer_tokens"] == 11 + assert payload["total_tokens"] == 18 + assert payload["provider_response_latency"] == 1.25 + assert payload["total_price"] == "0.0001234" + assert payload["currency"] == "USD" + + def test_web_message_list_item_exposes_usage_and_metadata(self): + payload = WebMessageListItem( + **_base_kwargs(), + metadata={"usage": {"total_tokens": 18}}, + message_tokens=7, + answer_tokens=11, + ).model_dump(mode="json") + + assert payload["metadata"] == {"usage": {"total_tokens": 18}} + assert payload["message_tokens"] == 7 + assert payload["answer_tokens"] == 11 + assert payload["total_tokens"] == 18 diff --git a/api/tests/unit_tests/services/test_file_request_service.py b/api/tests/unit_tests/services/test_file_request_service.py index 57abdeaa322..3e2ea0a258a 100644 --- a/api/tests/unit_tests/services/test_file_request_service.py +++ b/api/tests/unit_tests/services/test_file_request_service.py @@ -59,6 +59,31 @@ def test_request_download_url_builds_file_under_bound_scope( assert result.download_url == "https://files.example.com/x" +def test_request_download_url_supports_internal_download_urls() -> None: + fake_file = MagicMock(filename="report.pdf", mime_type="application/pdf", size=123) + service = FileRequestService(access_controller=MagicMock()) + + with ( + patch("services.file_request_service.bind_file_access_scope", return_value=nullcontext()), + patch.object(service, "_build_file", return_value=fake_file), + patch( + "services.file_request_service.file_helpers.resolve_file_url", + return_value="http://internal-files/report.pdf", + ) as resolve_file_url, + ): + result = service.request_download_url( + tenant_id="tenant-1", + user_id="user-1", + user_from="account", + invoke_from="debugger", + file_mapping={"transfer_method": "tool_file", "reference": "dify-file-ref:tool-file-1"}, + for_external=False, + ) + + resolve_file_url.assert_called_once_with(fake_file, for_external=False) + assert result.download_url == "http://internal-files/report.pdf" + + def test_request_download_url_rejects_unsupported_files() -> None: service = FileRequestService(access_controller=MagicMock()) diff --git a/api/uv.lock b/api/uv.lock index 30df243450a..8fc561aba49 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -322,16 +322,15 @@ wheels = [ [[package]] name = "anyio" -version = "4.11.0" +version = "4.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, - { name = "sniffio" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c6/78/7d432127c41b50bccba979505f272c16cbcadcc33645d5fa3a738110ae75/anyio-4.11.0.tar.gz", hash = "sha256:82a8d0b81e318cc5ce71a5f1f8b5c4e63619620b63141ef8c995fa0db95a57c4", size = 219094, upload-time = "2025-09-23T09:19:12.58Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/72/5562aabb8dd7181e8e860622a38bea08d17842b99ecd4c91f84ac95251b0/anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e", size = 254831, upload-time = "2026-06-24T20:56:06.017Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" }, + { url = "https://files.pythonhosted.org/packages/b0/7b/90df4a0a816d98d6ea26f559d87836d494a2cf1fcf063be67df50a7bcc30/anyio-4.14.1-py3-none-any.whl", hash = "sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72", size = 124875, upload-time = "2026-06-24T20:56:04.413Z" }, ] [[package]] @@ -1284,7 +1283,9 @@ name = "dify-agent" version = "0.1.0" source = { editable = "../dify-agent" } dependencies = [ + { name = "anyio" }, { name = "httpx" }, + { name = "httpx2" }, { name = "pydantic" }, { name = "pydantic-ai-slim" }, { name = "typer" }, @@ -1293,10 +1294,14 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "aiosqlite", marker = "extra == 'shellctl-server'", specifier = ">=0.21.0,<1.0.0" }, + { name = "anyio", specifier = ">=4.12.1,<5.0.0" }, { name = "fastapi", marker = "extra == 'server'", specifier = "==0.136.0" }, + { name = "fastapi", marker = "extra == 'shellctl-server'", specifier = "==0.136.0" }, { name = "graphon", marker = "extra == 'server'", specifier = "==0.5.2" }, { name = "grpclib", extras = ["protobuf"], marker = "extra == 'grpc'", specifier = ">=0.4.9,<0.5.0" }, { name = "httpx", specifier = "==0.28.1" }, + { name = "httpx2", specifier = ">=2.5.0,<3.0.0" }, { name = "jsonschema", marker = "extra == 'server'", specifier = ">=4.23.0,<5.0.0" }, { name = "jwcrypto", marker = "extra == 'server'", specifier = ">=1.5.6,<2" }, { name = "logfire", extras = ["fastapi", "httpx", "redis"], marker = "extra == 'server'", specifier = ">=4.37.0,<5.0.0" }, @@ -1306,12 +1311,13 @@ requires-dist = [ { name = "pydantic-ai-slim", extras = ["anthropic", "google", "openai"], marker = "extra == 'server'", specifier = ">=1.85.1,<2.0.0" }, { name = "pydantic-settings", marker = "extra == 'server'", specifier = ">=2.12.0,<3.0.0" }, { name = "redis", marker = "extra == 'server'", specifier = ">=7.4.0,<8.0.0" }, - { name = "shell-session-manager", marker = "extra == 'server'", specifier = "==2.4.0" }, + { name = "sqlmodel", marker = "extra == 'shellctl-server'", specifier = ">=0.0.24,<0.1.0" }, { name = "typer", specifier = ">=0.16.1,<0.17" }, { name = "typing-extensions", specifier = ">=4.12.2,<5.0.0" }, { name = "uvicorn", extras = ["standard"], marker = "extra == 'server'", specifier = "==0.46.0" }, + { name = "uvicorn", extras = ["standard"], marker = "extra == 'shellctl-server'", specifier = "==0.46.0" }, ] -provides-extras = ["grpc", "server"] +provides-extras = ["grpc", "server", "shellctl-server"] [package.metadata.requires-dev] dev = [ @@ -3283,15 +3289,15 @@ wheels = [ [[package]] name = "httpcore2" -version = "2.3.0" +version = "2.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "h11" }, { name = "truststore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e6/34/18f1c596e677962f040284246f393b10a1f8ce440b3a7e69c637d0f1c7ad/httpcore2-2.3.0.tar.gz", hash = "sha256:07327e251560960eea8e969d92d4c6a325feb13cca39e25340731336c3baf924", size = 64300, upload-time = "2026-06-01T13:15:02.998Z" } +sdist = { url = "https://files.pythonhosted.org/packages/47/06/5c12df521b5322fb1114a83d46911b2fbcb8855ddb3a635f11c01a214af5/httpcore2-2.5.0.tar.gz", hash = "sha256:88aa170137c17328d5ac44234f9fd10706466d5fb347f3edac4d39b91137b09d", size = 64808, upload-time = "2026-06-25T14:16:56.472Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/dd/3357218c69360d1cecc196c230c9a1d5c9afd5dba362056e23e60a5e64e5/httpcore2-2.3.0-py3-none-any.whl", hash = "sha256:477e9e334f74e5240dcac002e890580f36a57d40ff0fb14cc9655731d23b8415", size = 80024, upload-time = "2026-06-01T13:15:00.001Z" }, + { url = "https://files.pythonhosted.org/packages/c9/a1/7564199d1a8728fe737b0a72e5b3f8d92dfe085a74ddf7cdd83bce5f206d/httpcore2-2.5.0-py3-none-any.whl", hash = "sha256:5ce35188de461d31e8d000bfb8ef8bf22c6c16587a211e5571deaa5e9bdf842a", size = 80330, upload-time = "2026-06-25T14:16:53.634Z" }, ] [[package]] @@ -3355,17 +3361,18 @@ wheels = [ [[package]] name = "httpx2" -version = "2.3.0" +version = "2.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "httpcore2" }, { name = "idna" }, { name = "truststore" }, + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/9a/cca0b9145f13d8ae34b885ae28d403a1469a433abc78e0f94f4ce94e650b/httpx2-2.3.0.tar.gz", hash = "sha256:227e7c41d95a76d4077a52640564132777215fc3394e07b66a3116c33d668fa9", size = 81115, upload-time = "2026-06-01T13:15:04.324Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/e2/b5dedc0cf35aa65de5f541ccd30d2bc1fd7f1d43c9ab09f8ed9a7342317b/httpx2-2.5.0.tar.gz", hash = "sha256:e2df9cb4611021527ff8a675b1c320b610a2ec397acc8d6fe6e91df2d9b33c29", size = 83121, upload-time = "2026-06-25T14:16:57.491Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/87/ce/ae2911859847f9ba1d6b23027e53481cbeb50b93234f355a968d300ca2cb/httpx2-2.3.0-py3-none-any.whl", hash = "sha256:6f393663bdf6dbe7fe90118e3eb5b2bd024a675cae0390ac08cec9198812d8b7", size = 74538, upload-time = "2026-06-01T13:15:01.566Z" }, + { url = "https://files.pythonhosted.org/packages/31/22/859d8252dad9bc9adee34b52e62cde621ece07b042ccb2ab4da1be46695f/httpx2-2.5.0-py3-none-any.whl", hash = "sha256:3d2d4d9cf4b61f1a1f46a95947cfdb47e80cb56a2f91c6256ac8f58e4891df41", size = 76652, upload-time = "2026-06-25T14:16:55.23Z" }, ] [[package]] @@ -3423,11 +3430,11 @@ wheels = [ [[package]] name = "idna" -version = "3.11" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] diff --git a/dify-agent/.example.env b/dify-agent/.example.env index c4637f2c985..a800e531005 100644 --- a/dify-agent/.example.env +++ b/dify-agent/.example.env @@ -20,6 +20,12 @@ DIFY_AGENT_PLUGIN_DAEMON_URL=http://localhost:5002 # API key sent to the Dify plugin daemon. DIFY_AGENT_PLUGIN_DAEMON_API_KEY= +# Dify API inner endpoints +# Base URL for Dify API inner endpoints used by Agent Stub config/file/drive requests. +DIFY_AGENT_INNER_API_URL=http://localhost:5001 +# Must match API/worker INNER_API_KEY_FOR_PLUGIN, not the generic INNER_API_KEY. +DIFY_AGENT_INNER_API_KEY= + # Shell layer # Base URL for the shellctl server used by the dify.shell layer. Leave empty to disable shell layer use. DIFY_AGENT_SHELLCTL_ENTRYPOINT= @@ -30,12 +36,14 @@ DIFY_AGENT_SHELLCTL_AUTH_TOKEN= # Public Agent Stub URL reachable from shellctl-managed remote machines. # Use http(s)://.../agent-stub for HTTP or grpc://host:port for gRPC. # Leave empty to avoid injecting DIFY_AGENT_STUB_* into shell.run jobs. -DIFY_AGENT_STUB_URL= -# Optional bind override used only when DIFY_AGENT_STUB_URL uses grpc://. +DIFY_AGENT_STUB_API_BASE_URL=http://localhost:5050/agent-stub +# Optional bind override used only when DIFY_AGENT_STUB_API_BASE_URL uses grpc://. DIFY_AGENT_STUB_GRPC_BIND_ADDRESS= # Server-wide root secret used to derive Agent Stub JWE keys. -# Required when DIFY_AGENT_STUB_URL is set; must be unpadded base64url for 32 bytes. -DIFY_AGENT_SERVER_SECRET_KEY= +# This is security-sensitive: it derives the JWE encryption key for Agent Stub bearer tokens. +# Replace this development default in production. +# Generate one with: python -c 'import secrets; print(secrets.token_urlsafe(32))' +DIFY_AGENT_SERVER_SECRET_KEY=MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY # Shared plugin-daemon HTTP client timeouts and limits. # Plugin-daemon HTTP connect timeout in seconds. diff --git a/dify-agent/Dockerfile b/dify-agent/Dockerfile index 6d56811981d..0e0b47b12a5 100644 --- a/dify-agent/Dockerfile +++ b/dify-agent/Dockerfile @@ -6,8 +6,8 @@ # cd /app/api && .venv/bin/uvicorn dify_agent.server.app:app --host 0.0.0.0 --port 5050 # # Unlike the dify-api image (which only installs the base `dify-agent` -# dependency), this image installs the `[server]` extra, so jwcrypto, -# shell-session-manager, fastapi, uvicorn, etc. are present and the server can +# dependency), this image installs the `[server]` extra, so jwcrypto, fastapi, +# uvicorn, etc. are present and the server can # actually start. dify-api is intentionally left lean. # base image diff --git a/dify-agent/docker/local-sandbox/Dockerfile b/dify-agent/docker/local-sandbox/Dockerfile index 382e43fb0a6..9996f1f53d7 100644 --- a/dify-agent/docker/local-sandbox/Dockerfile +++ b/dify-agent/docker/local-sandbox/Dockerfile @@ -12,19 +12,15 @@ FROM python:3.12-slim-bookworm AS base ARG NODE_VERSION=22.22.1 ARG PNPM_VERSION=11.9.0 ARG UV_VERSION=0.8.9 -ARG DIFY_AGENT_TOOL_SPEC=.[grpc] -ARG SHELL_SESSION_MANAGER_TOOL_SPEC=shell-session-manager==2.4.0 +ARG DIFY_AGENT_TOOL_SPEC=.[grpc,shellctl-server] ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ - PIP_NO_CACHE_DIR=1 \ - DIFY_AGENT_STUB_DRIVE_BASE=/mnt/drive \ - UV_TOOL_DIR=/opt/dify-agent-tools/envs \ - UV_TOOL_BIN_DIR=/opt/dify-agent-tools/bin -ENV PATH="${UV_TOOL_BIN_DIR}:${PATH}" + PIP_NO_CACHE_DIR=1 RUN apt-get update \ && apt-get install -y --no-install-recommends \ + bash \ ca-certificates \ curl \ file \ @@ -62,24 +58,25 @@ WORKDIR /opt/dify-agent FROM base AS tools ARG DIFY_AGENT_TOOL_SPEC -ARG SHELL_SESSION_MANAGER_TOOL_SPEC COPY pyproject.toml uv.lock README.md ./ COPY src ./src RUN uv export --frozen --no-dev --all-extras --no-emit-project --no-hashes \ > /tmp/dify-agent-constraints.txt \ - && uv tool install --force --python /usr/local/bin/python --no-python-downloads \ + && UV_TOOL_DIR=/opt/dify-agent-tools/envs \ + UV_TOOL_BIN_DIR=/opt/dify-agent-tools/bin \ + uv tool install --force --python /usr/local/bin/python --no-python-downloads \ --constraints /tmp/dify-agent-constraints.txt --link-mode=copy "${DIFY_AGENT_TOOL_SPEC}" \ - && uv tool install --force --python /usr/local/bin/python --no-python-downloads \ - --constraints /tmp/dify-agent-constraints.txt --link-mode=copy "${SHELL_SESSION_MANAGER_TOOL_SPEC}" \ && rm -f /tmp/dify-agent-constraints.txt FROM base AS production -COPY --from=tools ${UV_TOOL_DIR} ${UV_TOOL_DIR} -COPY --from=tools ${UV_TOOL_BIN_DIR} ${UV_TOOL_BIN_DIR} +ENV PATH="/opt/dify-agent-tools/bin:${PATH}" + +COPY --from=tools /opt/dify-agent-tools/envs /opt/dify-agent-tools/envs +COPY --from=tools /opt/dify-agent-tools/bin /opt/dify-agent-tools/bin RUN useradd --create-home --shell /bin/sh dify \ && mkdir -p /mnt/drive \ diff --git a/dify-agent/docs/dify-agent/get-started/index.md b/dify-agent/docs/dify-agent/get-started/index.md index 347e85e3d81..9c0056d772c 100644 --- a/dify-agent/docs/dify-agent/get-started/index.md +++ b/dify-agent/docs/dify-agent/get-started/index.md @@ -75,8 +75,9 @@ See `.example.env` for the full server settings template. If you plan to run `dify.shell`, also configure `DIFY_AGENT_SHELLCTL_ENTRYPOINT` and, when shell jobs need to call back with the `dify-agent` command, set -`DIFY_AGENT_STUB_API_BASE_URL` plus a 32-byte base64url -`DIFY_AGENT_SERVER_SECRET_KEY` as documented in `.example.env`. +`DIFY_AGENT_STUB_API_BASE_URL`. The supplied default configs include a +development `DIFY_AGENT_SERVER_SECRET_KEY`, but production deployments should +override it with a unique 32-byte base64url value as documented in `.example.env`. ## Start the Dify Agent server diff --git a/dify-agent/docs/dify-agent/guide/index.md b/dify-agent/docs/dify-agent/guide/index.md index 89d9e0599c7..d3a41ef7d19 100644 --- a/dify-agent/docs/dify-agent/guide/index.md +++ b/dify-agent/docs/dify-agent/guide/index.md @@ -42,7 +42,7 @@ also reads `.env` and `dify-agent/.env` when present. | `DIFY_AGENT_SHELLCTL_AUTH_TOKEN` | empty | Optional bearer token sent to the shellctl server. | | `DIFY_AGENT_STUB_API_BASE_URL` | empty | Public Agent Stub API base URL reachable from shellctl-managed remote machines. HTTP may be the service root or `/agent-stub`; gRPC must be `grpc://host:port`. Enables `DIFY_AGENT_STUB_*` env injection for user `shell.run` jobs. | | `DIFY_AGENT_STUB_GRPC_BIND_ADDRESS` | empty | Optional `host:port` bind override used only when `DIFY_AGENT_STUB_API_BASE_URL` uses `grpc://`. | -| `DIFY_AGENT_SERVER_SECRET_KEY` | empty | Server-wide root secret used to derive Agent Stub JWE keys; required when `DIFY_AGENT_STUB_API_BASE_URL` is set and must be unpadded base64url for 32 bytes. | +| `DIFY_AGENT_SERVER_SECRET_KEY` | empty | Security-sensitive server-wide root secret used to derive the JWE encryption key for Agent Stub bearer tokens; required when `DIFY_AGENT_STUB_API_BASE_URL` is set. The supplied default config uses a development value; set a unique unpadded base64url 32-byte secret in production. | | `DIFY_AGENT_PLUGIN_DAEMON_CONNECT_TIMEOUT` | `10` | Plugin-daemon HTTP connect timeout in seconds. | | `DIFY_AGENT_PLUGIN_DAEMON_READ_TIMEOUT` | `600` | Plugin-daemon HTTP read timeout in seconds. | | `DIFY_AGENT_PLUGIN_DAEMON_WRITE_TIMEOUT` | `30` | Plugin-daemon HTTP write timeout in seconds. | @@ -64,9 +64,11 @@ DIFY_AGENT_INNER_API_URL=http://localhost:5001 DIFY_AGENT_INNER_API_KEY=replace-with-dify-inner-api-key-for-plugin DIFY_AGENT_SHELLCTL_ENTRYPOINT=http://127.0.0.1:5004 DIFY_AGENT_SHELLCTL_AUTH_TOKEN=replace-with-shellctl-token -# Generate with: python -c 'import base64, secrets; print(base64.urlsafe_b64encode(secrets.token_bytes(32)).rstrip(b"=").decode())' DIFY_AGENT_STUB_API_BASE_URL=https://agent.example.com/agent-stub -DIFY_AGENT_SERVER_SECRET_KEY=replace-with-base64url-32-byte-secret +# This is security-sensitive: it derives the JWE encryption key for Agent Stub bearer tokens. +# Replace this development default in production. +# Generate one with: python -c 'import secrets; print(secrets.token_urlsafe(32))' +DIFY_AGENT_SERVER_SECRET_KEY=MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY ``` Run records and event streams use the same retention. Status writes refresh the diff --git a/dify-agent/docs/dify-agent/user-manual/shell-layer/index.md b/dify-agent/docs/dify-agent/user-manual/shell-layer/index.md index 23efec73205..0b77cf64027 100644 --- a/dify-agent/docs/dify-agent/user-manual/shell-layer/index.md +++ b/dify-agent/docs/dify-agent/user-manual/shell-layer/index.md @@ -56,18 +56,22 @@ with `dify-agent ...`, also enable the Agent Stub: ```env DIFY_AGENT_STUB_API_BASE_URL=https://agent.example.com/agent-stub -DIFY_AGENT_SERVER_SECRET_KEY=replace-with-base64url-32-byte-secret +# This is security-sensitive: it derives the JWE encryption key for Agent Stub bearer tokens. +# Replace this development default in production. +# Generate one with: python -c 'import secrets; print(secrets.token_urlsafe(32))' +DIFY_AGENT_SERVER_SECRET_KEY=MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY ``` HTTP `DIFY_AGENT_STUB_API_BASE_URL` may be either the service root or the explicit `/agent-stub` API root; the server normalizes the service root to `/agent-stub`. Other HTTP paths are rejected at startup. -`DIFY_AGENT_SERVER_SECRET_KEY` must be unpadded base64url text for exactly 32 -decoded bytes. One way to generate it is: +The supplied Docker and `.example.env` configs use a development +`DIFY_AGENT_SERVER_SECRET_KEY`. Override it in production with unpadded base64url +text for exactly 32 decoded bytes. One way to generate it is: ```bash -python -c 'import base64, secrets; print(base64.urlsafe_b64encode(secrets.token_bytes(32)).rstrip(b"=").decode())' +python -c 'import secrets; print(secrets.token_urlsafe(32))' ``` ## Client request shape @@ -230,12 +234,11 @@ The provided `docker/local-sandbox/Dockerfile` installs: - `tmux`, required by `shellctl` to manage shell jobs; - common shell workspace tools: `git`, `openssh-client`, `jq`, `ripgrep`, `unzip`, `zip`, `file`, `procps`, and `less`; -- `shell-session-manager==2.3.1` as a standalone uv tool, which provides the - `shellctl` CLI/server; +- `dify-agent[grpc,shellctl-server]` as a standalone uv tool, which provides + both the Agent Stub client CLI and the built-in `shellctl` CLI/server; - `uv`, so uv shebang scripts with PEP 723 metadata can run inside the shell workspace and Python CLI tools can be installed with isolated tool environments; - `node==22.22.1` and `pnpm==11.9.0`, so JavaScript and TypeScript tooling can run inside the shell workspace without per-job installation; -- the `dify-agent[grpc]` Agent Stub client CLI as a standalone uv tool; - a non-root default user named `dify`. diff --git a/dify-agent/proto/dify/agent/stub/v1/agent_stub.proto b/dify-agent/proto/dify/agent/stub/v1/agent_stub.proto index 11a1a4aa54f..9240c195262 100644 --- a/dify-agent/proto/dify/agent/stub/v1/agent_stub.proto +++ b/dify-agent/proto/dify/agent/stub/v1/agent_stub.proto @@ -36,6 +36,7 @@ message FileMapping { message FileDownloadRequest { FileMapping file = 1; + optional bool for_external = 2; } message FileDownloadResponse { diff --git a/dify-agent/pyproject.toml b/dify-agent/pyproject.toml index 5bc84091a24..0d871a4e281 100644 --- a/dify-agent/pyproject.toml +++ b/dify-agent/pyproject.toml @@ -5,7 +5,9 @@ description = "Add your description here" readme = "README.md" requires-python = ">=3.12,<4.0" dependencies = [ + "anyio>=4.12.1,<5.0.0", "httpx==0.28.1", + "httpx2>=2.5.0,<3.0.0", "pydantic>=2.12.5,<2.13", "pydantic-ai-slim>=1.102.0,<2.0.0", "typer>=0.16.1,<0.17", @@ -15,6 +17,9 @@ dependencies = [ [project.scripts] dify-agent = "dify_agent.agent_stub.cli.main:main" dify-agent-stub-server = "dify_agent.agent_stub.server.cli:main" +shellctl = "shellctl.cli:main" +shellctl-sanitize-pty = "shellctl_runtime.sanitize:main" +shellctl-runner-exit = "shellctl_runtime.runner_exit:main" [project.optional-dependencies] grpc = ["grpclib[protobuf]>=0.4.9,<0.5.0", "protobuf>=6.33.5,<7.0.0"] @@ -27,13 +32,18 @@ server = [ "pydantic-ai-slim[anthropic,google,openai]>=1.85.1,<2.0.0", "pydantic-settings>=2.12.0,<3.0.0", "redis>=7.4.0,<8.0.0", - "shell-session-manager==2.4.0", + "uvicorn[standard]==0.46.0", +] +shellctl-server = [ + "aiosqlite>=0.21.0,<1.0.0", + "fastapi==0.136.0", + "sqlmodel>=0.0.24,<0.1.0", "uvicorn[standard]==0.46.0", ] [tool.setuptools.packages.find] where = ["src"] -include = ["agenton*", "agenton_collections*", "dify_agent*"] +include = ["agenton*", "agenton_collections*", "dify_agent*", "shellctl*", "shellctl_runtime*"] [tool.pyright] include = ["src", "examples", "tests"] diff --git a/dify-agent/src/dify_agent/adapters/llm/model.py b/dify-agent/src/dify_agent/adapters/llm/model.py index e48ccc8bb17..60256446e17 100644 --- a/dify-agent/src/dify_agent/adapters/llm/model.py +++ b/dify-agent/src/dify_agent/adapters/llm/model.py @@ -205,6 +205,7 @@ class DifyStreamedResponse(StreamedResponse): @override async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: + chunk_sequence = 0 async for chunk in self.chunks: if chunk.delta.usage is not None: self._usage: RequestUsage = _map_usage(chunk.delta.usage) @@ -216,8 +217,10 @@ class DifyStreamedResponse(StreamedResponse): chunk, self.provider_name_value, self._embedded_thinking_parser, + chunk_sequence, ): yield event + chunk_sequence += 1 for event in self._embedded_thinking_parser.flush(self._parts_manager, self.provider_name_value): yield event @@ -551,11 +554,21 @@ def _normalize_finish_reason(finish_reason: str) -> FinishReason: return "error" +def _normalize_tool_call_id(tool_call_id: str | None) -> str | None: + if tool_call_id is None: + return None + normalized = tool_call_id.strip() + if not normalized or normalized.lower() in {"none", "null"}: + return None + return normalized + + def _chunk_to_stream_events( parts_manager: ModelResponsePartsManager, chunk: LLMResultChunk, provider_name: str, embedded_thinking_parser: "_EmbeddedThinkingParser", + chunk_sequence: int, ) -> list[ModelResponseStreamEvent]: events: list[ModelResponseStreamEvent] = [] message = chunk.delta.message @@ -571,13 +584,14 @@ def _chunk_to_stream_events( events.append(parts_manager.handle_part(vendor_part_id=None, part=part)) for index, tool_call in enumerate(message.tool_calls): - vendor_id = tool_call.id or f"chunk-{chunk.delta.index}-tool-{index}" + tool_call_id = _normalize_tool_call_id(tool_call.id) + vendor_id = tool_call_id or f"chunk-{chunk_sequence}-tool-{index}" events.append( parts_manager.handle_tool_call_part( vendor_part_id=vendor_id, tool_name=tool_call.function.name, args=tool_call.function.arguments, - tool_call_id=tool_call.id, + tool_call_id=tool_call_id or vendor_id, provider_name=provider_name, ) ) diff --git a/dify-agent/src/dify_agent/adapters/shell/shellctl.py b/dify-agent/src/dify_agent/adapters/shell/shellctl.py index 6cf334245fb..ac57773c884 100644 --- a/dify-agent/src/dify_agent/adapters/shell/shellctl.py +++ b/dify-agent/src/dify_agent/adapters/shell/shellctl.py @@ -1,6 +1,6 @@ """Shellctl-backed shell provider adapter for dify-agent. -The shell-session-manager SDK owns the HTTP timeout policy for long-polling +The built-in shellctl SDK owns the HTTP timeout policy for long-polling shellctl requests. This adapter stays narrowly focused on translating SDK and transport failures into ``ShellProviderError`` so the shell layer can return tool observations instead of aborting the agent loop. @@ -16,9 +16,9 @@ import time from collections.abc import Awaitable from collections.abc import Callable from dataclasses import dataclass -from typing import Protocol, TypeVar +from typing import Protocol, TypeVar, cast -import httpx +import httpx2 as httpx from dify_agent.adapters.shell.protocols import ( ShellCommandProtocol, @@ -228,8 +228,16 @@ class ShellctlFileTransfer(ShellFileTransferProtocol): @dataclass(slots=True) class ShellctlResource(ShellResourceProtocol): client: ShellctlClientProtocol - commands: ShellCommandProtocol - files: ShellFileTransferProtocol + _commands: ShellCommandProtocol + _files: ShellFileTransferProtocol + + @property + def commands(self) -> ShellCommandProtocol: + return self._commands + + @property + def files(self) -> ShellFileTransferProtocol: + return self._files async def close(self) -> None: try: @@ -257,8 +265,8 @@ class ShellctlProvider(ShellProviderProtocol): ) return ShellctlResource( client=client, - commands=ShellctlCommands(client=client), - files=ShellctlFileTransfer(client=client), + _commands=ShellctlCommands(client=client), + _files=ShellctlFileTransfer(client=client), ) @@ -269,9 +277,15 @@ def create_default_shellctl_client_factory( output_limit: int = _SHELLCTL_OUTPUT_LIMIT_BYTES, ) -> ShellctlClientFactory: def factory() -> ShellctlClientProtocol: - from shell_session_manager.shellctl.client import ShellctlClient + from shellctl.client import ShellctlClient - return ShellctlClient(entrypoint, token=token, output_limit=output_limit) + return cast( + ShellctlClientProtocol, + cast( + object, + ShellctlClient(entrypoint, token=token, output_limit=output_limit), + ), + ) return factory diff --git a/dify-agent/src/dify_agent/agent_stub/cli/_files.py b/dify-agent/src/dify_agent/agent_stub/cli/_files.py index 5e3635fde14..1ccd57f26da 100644 --- a/dify-agent/src/dify_agent/agent_stub/cli/_files.py +++ b/dify-agent/src/dify_agent/agent_stub/cli/_files.py @@ -143,6 +143,7 @@ def download_file_from_environment( url=environment.url, auth_jwe=environment.auth_jwe, file=file_mapping, + for_external=False, ) if not hasattr(download_request, "filename") or not isinstance(download_request.filename, str): raise AgentStubTransferError("signed file download response is missing filename") @@ -207,8 +208,10 @@ def _request_uploaded_tool_file_download_url(*, url: str, auth_jwe: str, referen file=AgentStubFileMapping(transfer_method="tool_file", reference=reference), ), ) + if not hasattr(download_request, "download_url") or not isinstance(download_request.download_url, str): + raise AgentStubTransferError("signed file download response is missing download_url") download_url = download_request.download_url - if not isinstance(download_url, str) or not download_url: + if not download_url: raise AgentStubTransferError("signed file download response is missing download_url") return download_url diff --git a/dify-agent/src/dify_agent/agent_stub/cli/main.py b/dify-agent/src/dify_agent/agent_stub/cli/main.py index ee00059d1e5..592ff35253e 100644 --- a/dify-agent/src/dify_agent/agent_stub/cli/main.py +++ b/dify-agent/src/dify_agent/agent_stub/cli/main.py @@ -133,6 +133,26 @@ def config_skills_push( Pass a directory such as ./skills/researcher that contains SKILL.md. Other files in that directory are archived with the skill. Pushing a skill with an existing name replaces that config skill. + + Skill directory requirements: + + - Each PATH must be one skill directory; the directory basename is the config skill name. + + - The directory must contain a top-level SKILL.md. + + - SKILL.md must be non-empty UTF-8 Markdown. + + - SKILL.md must start with YAML frontmatter matching this schema: + + \b + --- + name: + description: + --- + + - Symlinked files are rejected. + + - Dependency/cache folders such as .git, __pycache__, .venv and node_modules should be manually cleared before push. """ _run_config_skills_push(paths=paths) diff --git a/dify-agent/src/dify_agent/agent_stub/client/_agent_stub.py b/dify-agent/src/dify_agent/agent_stub/client/_agent_stub.py index 704b099a4de..1f5150b429e 100644 --- a/dify-agent/src/dify_agent/agent_stub/client/_agent_stub.py +++ b/dify-agent/src/dify_agent/agent_stub/client/_agent_stub.py @@ -97,6 +97,7 @@ def request_agent_stub_file_download_sync( url: str, auth_jwe: str, file: AgentStubFileMapping, + for_external: bool = True, timeout: float | httpx.Timeout = 30.0, sync_http_client: httpx.Client | None = None, ): @@ -109,12 +110,14 @@ def request_agent_stub_file_download_sync( url=endpoint.url, auth_jwe=auth_jwe, file=file, + for_external=for_external, timeout=timeout, ) return request_agent_stub_file_download_http_sync( base_url=endpoint.url, auth_jwe=auth_jwe, file=file, + for_external=for_external, timeout=timeout, sync_http_client=sync_http_client, ) diff --git a/dify-agent/src/dify_agent/agent_stub/client/_agent_stub_grpc.py b/dify-agent/src/dify_agent/agent_stub/client/_agent_stub_grpc.py index 84d02aadc2b..7c5f5c4fbbb 100644 --- a/dify-agent/src/dify_agent/agent_stub/client/_agent_stub_grpc.py +++ b/dify-agent/src/dify_agent/agent_stub/client/_agent_stub_grpc.py @@ -124,6 +124,7 @@ def request_agent_stub_file_download_grpc_sync( url: str, auth_jwe: str, file: AgentStubFileMapping, + for_external: bool = True, timeout: float | httpx.Timeout = 30.0, ): """Request one signed download URL through the gRPC Agent Stub endpoint. @@ -144,7 +145,7 @@ def request_agent_stub_file_download_grpc_sync( auth_jwe=auth_jwe, method_name="CreateFileDownloadRequest", request_factory=lambda runtime: _require_conversions().proto_file_download_request( - runtime.agent_stub_pb2, file=file + runtime.agent_stub_pb2, file=file, for_external=for_external ), response_parser=lambda response: _require_conversions().file_download_response_from_proto(response), timeout=timeout, diff --git a/dify-agent/src/dify_agent/agent_stub/client/_agent_stub_http.py b/dify-agent/src/dify_agent/agent_stub/client/_agent_stub_http.py index beb777c58bd..543446ad894 100644 --- a/dify-agent/src/dify_agent/agent_stub/client/_agent_stub_http.py +++ b/dify-agent/src/dify_agent/agent_stub/client/_agent_stub_http.py @@ -117,13 +117,14 @@ def request_agent_stub_file_download_http_sync( base_url: str, auth_jwe: str, file: AgentStubFileMapping, + for_external: bool = True, timeout: float | httpx.Timeout = 30.0, sync_http_client: httpx.Client | None = None, ) -> AgentStubFileDownloadResponse: """Request one signed download URL from the HTTP Agent Stub endpoint.""" try: - request_model = AgentStubFileDownloadRequest(file=file) + request_model = AgentStubFileDownloadRequest(file=file, for_external=for_external) except ValidationError as exc: raise AgentStubValidationError("invalid Agent Stub file download request") from exc response = _post_agent_stub_json( @@ -131,7 +132,7 @@ def request_agent_stub_file_download_http_sync( auth_jwe=auth_jwe, endpoint_name="file download request", endpoint_url_factory=agent_stub_file_download_request_url, - request_body=request_model.model_dump_json(exclude_none=True), + request_body=request_model.model_dump_json(exclude_none=True, exclude_defaults=True), timeout=timeout, sync_http_client=sync_http_client, ) diff --git a/dify-agent/src/dify_agent/agent_stub/grpc/_generated/agent_stub_pb2.py b/dify-agent/src/dify_agent/agent_stub/grpc/_generated/agent_stub_pb2.py index 5b07bec9eb1..db940989607 100644 --- a/dify-agent/src/dify_agent/agent_stub/grpc/_generated/agent_stub_pb2.py +++ b/dify-agent/src/dify_agent/agent_stub/grpc/_generated/agent_stub_pb2.py @@ -1,6 +1,6 @@ -# pyright: reportAttributeAccessIssue=false # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: dify/agent/stub/v1/agent_stub.proto # Protobuf Python Version: 6.33.5 """Generated protocol buffer code.""" @@ -12,12 +12,7 @@ from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 6, - 33, - 5, - "", - "dify/agent/stub/v1/agent_stub.proto", + _runtime_version.Domain.PUBLIC, 6, 33, 5, "", "dify/agent/stub/v1/agent_stub.proto" ) # @@protoc_insertion_point(imports) @@ -25,12 +20,12 @@ _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n#dify/agent/stub/v1/agent_stub.proto\x12\x12\x64ify.agent.stub.v1"O\n\x0e\x43onnectRequest\x12\x18\n\x10protocol_version\x18\x01 \x01(\x05\x12\x0c\n\x04\x61rgv\x18\x02 \x03(\t\x12\x15\n\rmetadata_json\x18\x03 \x01(\t"8\n\x0f\x43onnectResponse\x12\x15\n\rconnection_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\t"7\n\x11\x46ileUploadRequest\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x10\n\x08mimetype\x18\x02 \x01(\t"(\n\x12\x46ileUploadResponse\x12\x12\n\nupload_url\x18\x01 \x01(\t"f\n\x0b\x46ileMapping\x12\x17\n\x0ftransfer_method\x18\x01 \x01(\t\x12\x16\n\treference\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x10\n\x03url\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_referenceB\x06\n\x04_url"D\n\x13\x46ileDownloadRequest\x12-\n\x04\x66ile\x18\x01 \x01(\x0b\x32\x1f.dify.agent.stub.v1.FileMapping"r\n\x14\x46ileDownloadResponse\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x16\n\tmime_type\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x0c\n\x04size\x18\x03 \x01(\x03\x12\x14\n\x0c\x64ownload_url\x18\x04 \x01(\tB\x0c\n\n_mime_type2\xc0\x02\n\x10\x41gentStubService\x12R\n\x07\x43onnect\x12".dify.agent.stub.v1.ConnectRequest\x1a#.dify.agent.stub.v1.ConnectResponse\x12h\n\x17\x43reateFileUploadRequest\x12%.dify.agent.stub.v1.FileUploadRequest\x1a&.dify.agent.stub.v1.FileUploadResponse\x12n\n\x19\x43reateFileDownloadRequest\x12\'.dify.agent.stub.v1.FileDownloadRequest\x1a(.dify.agent.stub.v1.FileDownloadResponseb\x06proto3' + b'\n#dify/agent/stub/v1/agent_stub.proto\x12\x12\x64ify.agent.stub.v1"O\n\x0e\x43onnectRequest\x12\x18\n\x10protocol_version\x18\x01 \x01(\x05\x12\x0c\n\x04\x61rgv\x18\x02 \x03(\t\x12\x15\n\rmetadata_json\x18\x03 \x01(\t"8\n\x0f\x43onnectResponse\x12\x15\n\rconnection_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\t"7\n\x11\x46ileUploadRequest\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x10\n\x08mimetype\x18\x02 \x01(\t"(\n\x12\x46ileUploadResponse\x12\x12\n\nupload_url\x18\x01 \x01(\t"f\n\x0b\x46ileMapping\x12\x17\n\x0ftransfer_method\x18\x01 \x01(\t\x12\x16\n\treference\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x10\n\x03url\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_referenceB\x06\n\x04_url"p\n\x13\x46ileDownloadRequest\x12-\n\x04\x66ile\x18\x01 \x01(\x0b\x32\x1f.dify.agent.stub.v1.FileMapping\x12\x19\n\x0c\x66or_external\x18\x02 \x01(\x08H\x00\x88\x01\x01\x42\x0f\n\r_for_external"r\n\x14\x46ileDownloadResponse\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x16\n\tmime_type\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x0c\n\x04size\x18\x03 \x01(\x03\x12\x14\n\x0c\x64ownload_url\x18\x04 \x01(\tB\x0c\n\n_mime_type2\xc0\x02\n\x10\x41gentStubService\x12R\n\x07\x43onnect\x12".dify.agent.stub.v1.ConnectRequest\x1a#.dify.agent.stub.v1.ConnectResponse\x12h\n\x17\x43reateFileUploadRequest\x12%.dify.agent.stub.v1.FileUploadRequest\x1a&.dify.agent.stub.v1.FileUploadResponse\x12n\n\x19\x43reateFileDownloadRequest\x12\'.dify.agent.stub.v1.FileDownloadRequest\x1a(.dify.agent.stub.v1.FileDownloadResponseb\x06proto3' ) _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "dify_agent.agent_stub.grpc._generated.agent_stub_pb2", _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "dify.agent.stub.v1.agent_stub_pb2", _globals) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None _globals["_CONNECTREQUEST"]._serialized_start = 59 @@ -44,9 +39,9 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals["_FILEMAPPING"]._serialized_start = 297 _globals["_FILEMAPPING"]._serialized_end = 399 _globals["_FILEDOWNLOADREQUEST"]._serialized_start = 401 - _globals["_FILEDOWNLOADREQUEST"]._serialized_end = 469 - _globals["_FILEDOWNLOADRESPONSE"]._serialized_start = 471 - _globals["_FILEDOWNLOADRESPONSE"]._serialized_end = 585 - _globals["_AGENTSTUBSERVICE"]._serialized_start = 588 - _globals["_AGENTSTUBSERVICE"]._serialized_end = 908 + _globals["_FILEDOWNLOADREQUEST"]._serialized_end = 513 + _globals["_FILEDOWNLOADRESPONSE"]._serialized_start = 515 + _globals["_FILEDOWNLOADRESPONSE"]._serialized_end = 629 + _globals["_AGENTSTUBSERVICE"]._serialized_start = 632 + _globals["_AGENTSTUBSERVICE"]._serialized_end = 952 # @@protoc_insertion_point(module_scope) diff --git a/dify-agent/src/dify_agent/agent_stub/grpc/_generated/agent_stub_pb2.pyi b/dify-agent/src/dify_agent/agent_stub/grpc/_generated/agent_stub_pb2.pyi index 32f1fc443d4..617956bd845 100644 --- a/dify-agent/src/dify_agent/agent_stub/grpc/_generated/agent_stub_pb2.pyi +++ b/dify-agent/src/dify_agent/agent_stub/grpc/_generated/agent_stub_pb2.pyi @@ -1,71 +1,69 @@ -from __future__ import annotations +from google.protobuf.internal import containers as _containers +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Iterable as _Iterable, Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union -from collections.abc import Iterable +DESCRIPTOR: _descriptor.FileDescriptor -from google.protobuf.message import Message - - -class ConnectRequest(Message): +class ConnectRequest(_message.Message): + __slots__ = ("protocol_version", "argv", "metadata_json") + PROTOCOL_VERSION_FIELD_NUMBER: _ClassVar[int] + ARGV_FIELD_NUMBER: _ClassVar[int] + METADATA_JSON_FIELD_NUMBER: _ClassVar[int] protocol_version: int - argv: list[str] + argv: _containers.RepeatedScalarFieldContainer[str] metadata_json: str + def __init__(self, protocol_version: _Optional[int] = ..., argv: _Optional[_Iterable[str]] = ..., metadata_json: _Optional[str] = ...) -> None: ... - def __init__( - self, - *, - protocol_version: int = ..., - argv: Iterable[str] = ..., - metadata_json: str = ..., - ) -> None: ... - - -class ConnectResponse(Message): +class ConnectResponse(_message.Message): + __slots__ = ("connection_id", "status") + CONNECTION_ID_FIELD_NUMBER: _ClassVar[int] + STATUS_FIELD_NUMBER: _ClassVar[int] connection_id: str status: str + def __init__(self, connection_id: _Optional[str] = ..., status: _Optional[str] = ...) -> None: ... - def __init__(self, *, connection_id: str = ..., status: str = ...) -> None: ... - - -class FileUploadRequest(Message): +class FileUploadRequest(_message.Message): + __slots__ = ("filename", "mimetype") + FILENAME_FIELD_NUMBER: _ClassVar[int] + MIMETYPE_FIELD_NUMBER: _ClassVar[int] filename: str mimetype: str + def __init__(self, filename: _Optional[str] = ..., mimetype: _Optional[str] = ...) -> None: ... - def __init__(self, *, filename: str = ..., mimetype: str = ...) -> None: ... - - -class FileUploadResponse(Message): +class FileUploadResponse(_message.Message): + __slots__ = ("upload_url",) + UPLOAD_URL_FIELD_NUMBER: _ClassVar[int] upload_url: str + def __init__(self, upload_url: _Optional[str] = ...) -> None: ... - def __init__(self, *, upload_url: str = ...) -> None: ... - - -class FileMapping(Message): +class FileMapping(_message.Message): + __slots__ = ("transfer_method", "reference", "url") + TRANSFER_METHOD_FIELD_NUMBER: _ClassVar[int] + REFERENCE_FIELD_NUMBER: _ClassVar[int] + URL_FIELD_NUMBER: _ClassVar[int] transfer_method: str reference: str url: str + def __init__(self, transfer_method: _Optional[str] = ..., reference: _Optional[str] = ..., url: _Optional[str] = ...) -> None: ... - def __init__(self, *, transfer_method: str = ..., reference: str = ..., url: str = ...) -> None: ... - def HasField(self, field_name: str) -> bool: ... - - -class FileDownloadRequest(Message): +class FileDownloadRequest(_message.Message): + __slots__ = ("file", "for_external") + FILE_FIELD_NUMBER: _ClassVar[int] + FOR_EXTERNAL_FIELD_NUMBER: _ClassVar[int] file: FileMapping + for_external: bool + def __init__(self, file: _Optional[_Union[FileMapping, _Mapping]] = ..., for_external: _Optional[bool] = ...) -> None: ... - def __init__(self, *, file: FileMapping | None = ...) -> None: ... - - -class FileDownloadResponse(Message): +class FileDownloadResponse(_message.Message): + __slots__ = ("filename", "mime_type", "size", "download_url") + FILENAME_FIELD_NUMBER: _ClassVar[int] + MIME_TYPE_FIELD_NUMBER: _ClassVar[int] + SIZE_FIELD_NUMBER: _ClassVar[int] + DOWNLOAD_URL_FIELD_NUMBER: _ClassVar[int] filename: str mime_type: str size: int download_url: str - - def __init__( - self, - *, - filename: str = ..., - mime_type: str = ..., - size: int = ..., - download_url: str = ..., - ) -> None: ... - def HasField(self, field_name: str) -> bool: ... + def __init__(self, filename: _Optional[str] = ..., mime_type: _Optional[str] = ..., size: _Optional[int] = ..., download_url: _Optional[str] = ...) -> None: ... diff --git a/dify-agent/src/dify_agent/agent_stub/grpc/conversions.py b/dify-agent/src/dify_agent/agent_stub/grpc/conversions.py index 3bfa4c50665..a79e446c85e 100644 --- a/dify-agent/src/dify_agent/agent_stub/grpc/conversions.py +++ b/dify-agent/src/dify_agent/agent_stub/grpc/conversions.py @@ -98,13 +98,19 @@ def file_download_request_from_proto(message: agent_stub_pb2.FileDownloadRequest "reference": message.file.reference if message.file.HasField("reference") else None, "url": message.file.url if message.file.HasField("url") else None, } - return AgentStubFileDownloadRequest.model_validate({"file": file_mapping_kwargs}) + return AgentStubFileDownloadRequest.model_validate( + { + "file": file_mapping_kwargs, + "for_external": message.for_external if message.HasField("for_external") else True, + } + ) def proto_file_download_request( pb2_module, *, file: AgentStubFileMapping, + for_external: bool = True, ) -> agent_stub_pb2.FileDownloadRequest: """Build one protobuf file-download request from the public DTO.""" mapping = pb2_module.FileMapping(transfer_method=file.transfer_method) @@ -112,7 +118,9 @@ def proto_file_download_request( mapping.reference = file.reference if file.url is not None: mapping.url = file.url - return pb2_module.FileDownloadRequest(file=mapping) + request = pb2_module.FileDownloadRequest(file=mapping) + request.for_external = for_external + return request def file_download_response_from_proto(message: agent_stub_pb2.FileDownloadResponse) -> AgentStubFileDownloadResponse: diff --git a/dify-agent/src/dify_agent/agent_stub/protocol/agent_stub.py b/dify-agent/src/dify_agent/agent_stub/protocol/agent_stub.py index 1d257287671..734ca50d53e 100644 --- a/dify-agent/src/dify_agent/agent_stub/protocol/agent_stub.py +++ b/dify-agent/src/dify_agent/agent_stub/protocol/agent_stub.py @@ -249,6 +249,7 @@ class AgentStubFileDownloadRequest(BaseModel): """Request body for one signed download URL allocation.""" file: AgentStubFileMapping + for_external: bool = True model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") diff --git a/dify-agent/src/dify_agent/agent_stub/server/agent_stub_files.py b/dify-agent/src/dify_agent/agent_stub/server/agent_stub_files.py index 06cb78d32fa..f3efe34f00a 100644 --- a/dify-agent/src/dify_agent/agent_stub/server/agent_stub_files.py +++ b/dify-agent/src/dify_agent/agent_stub/server/agent_stub_files.py @@ -161,6 +161,8 @@ class DifyApiAgentStubFileRequestHandler: "invoke_from": execution_context.invoke_from, "file": request.file.model_dump(mode="json", exclude_none=True), } + if request.for_external is False: + payload["for_external"] = False data = await self._post_inner_api("/inner/api/download/file/request", payload) try: return AgentStubFileDownloadResponse.model_validate(data) diff --git a/dify-agent/src/dify_agent/layers/shell/layer.py b/dify-agent/src/dify_agent/layers/shell/layer.py index b2939aadadd..53af60641ce 100644 --- a/dify-agent/src/dify_agent/layers/shell/layer.py +++ b/dify-agent/src/dify_agent/layers/shell/layer.py @@ -141,7 +141,18 @@ shell_run script rules: Tips: -- When using Python, prefer a uv script with a PEP 723 dependency header. +- Python 3.12, uv, pip, Node.js, pnpm, and pnx are preinstalled in the local sandbox. +- For one-off Python dependencies, prefer a uv script with a PEP 723 dependency header or: + `uv run --with python `. +- For reusable Python CLI tools, use `uv tool install `; installed commands land in `$HOME/.local/bin`. + Run them by full path or add `$HOME/.local/bin` to PATH in the command that needs them. +- `python3 -m pip install --user ` also installs into `$HOME/.local`; add `$HOME/.local/bin` to PATH + when you need console scripts. +- For reusable Node.js CLIs, use user-level global installs: + `PNPM_HOME=$HOME/.local/share/pnpm PATH=$HOME/.local/share/pnpm/bin:$PATH pnpm add -g `. + Installed commands land in `$PNPM_HOME/bin`; run them by full path or with the same PATH prefix. +- For one-off Node.js CLIs, prefer `pnx [args]`. +- Do not install new packages into system or image tool paths such as `/usr/local`, `/usr`, or `/opt/dify-agent-tools`. - If you need MCP, install the MCP server in the shell environment and start that server when you use it. Example shell_run script: diff --git a/dify-agent/src/dify_agent/runtime/runner.py b/dify-agent/src/dify_agent/runtime/runner.py index 2ad3aa99d8f..13308fa4f9c 100644 --- a/dify-agent/src/dify_agent/runtime/runner.py +++ b/dify-agent/src/dify_agent/runtime/runner.py @@ -31,13 +31,14 @@ both the JSON-safe final output or deferred tool call and the session snapshot; there are no separate output or snapshot events to correlate. """ -from collections.abc import AsyncIterable, Callable +from collections.abc import AsyncIterable, Callable, Mapping from collections import Counter from dataclasses import dataclass from typing import Any, Literal, Protocol, cast, runtime_checkable import httpx from pydantic import JsonValue, TypeAdapter +from pydantic_ai.exceptions import ModelHTTPError from pydantic_ai.messages import AgentStreamEvent, PartDeltaEvent, PartStartEvent, TextPart, TextPartDelta from pydantic_ai.output import OutputSpec from pydantic_ai.tools import DeferredToolRequests, DeferredToolResults @@ -104,6 +105,28 @@ class AgentRunValidationError(ValueError): """Raised when a run request is valid JSON but cannot execute.""" +def _run_failed_error_payload(exc: Exception) -> tuple[str, str | None]: + """Return the public failed-run error text and structured reason.""" + message = str(exc) or type(exc).__name__ + reason: str | None = None + + if isinstance(exc, ModelHTTPError): + body = exc.body + if isinstance(body, Mapping): + body_message = body.get("message") + if isinstance(body_message, str) and body_message: + message = body_message + + error_type = body.get("error_type") + if isinstance(error_type, str) and error_type: + reason = error_type + + if reason is None and exc.status_code == 429: + reason = "InvokeRateLimitError" + + return message, reason + + def _has_model_layer(request: CreateRunRequest) -> bool: """Return whether the public composition includes the reserved model layer.""" return any(layer.name == DIFY_AGENT_MODEL_LAYER_ID for layer in request.composition.layers) @@ -165,8 +188,8 @@ class AgentRunRunner: try: outcome = await self._run_agent() except Exception as exc: - message = str(exc) or type(exc).__name__ - _ = await emit_run_failed(self.sink, run_id=self.run_id, error=message) + message, reason = _run_failed_error_payload(exc) + _ = await emit_run_failed(self.sink, run_id=self.run_id, error=message, reason=reason) await self.sink.update_status(self.run_id, "failed", message) raise diff --git a/dify-agent/src/shellctl/__init__.py b/dify-agent/src/shellctl/__init__.py new file mode 100644 index 00000000000..3f2237dc690 --- /dev/null +++ b/dify-agent/src/shellctl/__init__.py @@ -0,0 +1,124 @@ +"""Public shellctl package exports. + +This package stays lazy on purpose. Hot-path runtime helpers live outside the +`shellctl` package, and importing this package root should +not pull the full client/server/public DTO surface unless a caller explicitly +asks for those exports. +""" + +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from shellctl.client import ( + ShellctlClient, + ShellctlClientError, + ) + from shellctl.shared import ( + DEFAULT_AUTH_TOKEN_ENV, + DEFAULT_BASE_URL, + DEFAULT_BASE_URL_ENV, + DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS, + DEFAULT_GC_INTERVAL_SECONDS, + DEFAULT_IDLE_FLUSH_SECONDS, + DEFAULT_LIST_LIMIT, + DEFAULT_OUTPUT_LIMIT_BYTES, + DEFAULT_TERMINAL_COLS, + DEFAULT_TERMINAL_ROWS, + DEFAULT_TERMINATE_GRACE_SECONDS, + DEFAULT_TIMEOUT_SECONDS, + DeleteJobResponse, + HealthResponse, + InputJobRequest, + JobInfo, + JobResult, + JobStatusName, + JobStatusView, + ListJobsResponse, + RunJobRequest, + TerminalSize, + TerminateJobRequest, + WaitJobRequest, + generate_job_id, + read_output_window, + tail_output_window, + ) + +__all__ = [ + "DEFAULT_AUTH_TOKEN_ENV", + "DEFAULT_BASE_URL", + "DEFAULT_BASE_URL_ENV", + "DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS", + "DEFAULT_GC_INTERVAL_SECONDS", + "DEFAULT_IDLE_FLUSH_SECONDS", + "DEFAULT_LIST_LIMIT", + "DEFAULT_OUTPUT_LIMIT_BYTES", + "DEFAULT_TERMINAL_COLS", + "DEFAULT_TERMINAL_ROWS", + "DEFAULT_TERMINATE_GRACE_SECONDS", + "DEFAULT_TIMEOUT_SECONDS", + "DeleteJobResponse", + "HealthResponse", + "InputJobRequest", + "JobInfo", + "JobResult", + "JobStatusName", + "JobStatusView", + "ListJobsResponse", + "RunJobRequest", + "ShellctlClient", + "ShellctlClientError", + "TerminalSize", + "TerminateJobRequest", + "WaitJobRequest", + "generate_job_id", + "read_output_window", + "tail_output_window", +] + +_EXPORTS = { + "ShellctlClient": "shellctl.client", + "ShellctlClientError": "shellctl.client", + "DEFAULT_AUTH_TOKEN_ENV": "shellctl.shared", + "DEFAULT_BASE_URL": "shellctl.shared", + "DEFAULT_BASE_URL_ENV": "shellctl.shared", + "DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS": "shellctl.shared", + "DEFAULT_GC_INTERVAL_SECONDS": "shellctl.shared", + "DEFAULT_IDLE_FLUSH_SECONDS": "shellctl.shared", + "DEFAULT_LIST_LIMIT": "shellctl.shared", + "DEFAULT_OUTPUT_LIMIT_BYTES": "shellctl.shared", + "DEFAULT_TERMINAL_COLS": "shellctl.shared", + "DEFAULT_TERMINAL_ROWS": "shellctl.shared", + "DEFAULT_TERMINATE_GRACE_SECONDS": "shellctl.shared", + "DEFAULT_TIMEOUT_SECONDS": "shellctl.shared", + "DeleteJobResponse": "shellctl.shared", + "HealthResponse": "shellctl.shared", + "InputJobRequest": "shellctl.shared", + "JobInfo": "shellctl.shared", + "JobResult": "shellctl.shared", + "JobStatusName": "shellctl.shared", + "JobStatusView": "shellctl.shared", + "ListJobsResponse": "shellctl.shared", + "RunJobRequest": "shellctl.shared", + "TerminalSize": "shellctl.shared", + "TerminateJobRequest": "shellctl.shared", + "WaitJobRequest": "shellctl.shared", + "generate_job_id": "shellctl.shared", + "read_output_window": "shellctl.shared", + "tail_output_window": "shellctl.shared", +} + + +def __getattr__(name: str) -> Any: + if name not in _EXPORTS: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module = import_module(_EXPORTS[name]) + value = getattr(module, name) # noqa: no-new-getattr lazy export proxy + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__)) diff --git a/dify-agent/src/shellctl/cli.py b/dify-agent/src/shellctl/cli.py new file mode 100644 index 00000000000..4dac2e8e9c2 --- /dev/null +++ b/dify-agent/src/shellctl/cli.py @@ -0,0 +1,587 @@ +"""Typer CLI for network-backed shellctl commands. + +Job-management commands in this module intentionally stay on the SDK side of +the boundary: they parse CLI options, call `ShellctlClient`, and render compact +JSON. That keeps `shellctl --help` and `shellctl run --help` free of FastAPI, +SQLAlchemy, tmux, and local runtime bootstrap imports. + +Only `serve` lazily imports server-side modules when that subcommand is +actually invoked. +""" + +from __future__ import annotations + +import json +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import NoReturn + +import anyio +import httpx2 as httpx +import typer +from pydantic import BaseModel, ValidationError + +from shellctl.client import ShellctlClient, ShellctlClientError +from shellctl.shared.constants import ( + DEFAULT_AUTH_TOKEN_ENV, + DEFAULT_BASE_URL, + DEFAULT_BASE_URL_ENV, + DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS, + DEFAULT_GC_INTERVAL_SECONDS, + DEFAULT_IDLE_FLUSH_SECONDS, + DEFAULT_LIST_LIMIT, + DEFAULT_OUTPUT_LIMIT_BYTES, + DEFAULT_TERMINAL_COLS, + DEFAULT_TERMINAL_ROWS, + DEFAULT_TERMINATE_GRACE_SECONDS, + DEFAULT_TIMEOUT_SECONDS, + MAX_LIST_LIMIT, + MAX_OUTPUT_LIMIT_BYTES, +) +from shellctl.shared.schemas import ( + DeleteJobResponse, + HealthResponse, + JobInfo, + JobResult, + JobStatusName, + JobStatusView, + RunJobRequest, + TerminalSize, +) + +cli = typer.Typer( + no_args_is_help=True, + pretty_exceptions_enable=False, + rich_markup_mode=None, +) + + +@cli.command("health") +def health_command( + base_url: str = typer.Option( + DEFAULT_BASE_URL, + "--base-url", + envvar=DEFAULT_BASE_URL_ENV, + help="shellctl server base URL. You can also set SHELLCTL_BASE_URL.", + ), + auth_token: str | None = typer.Option( + None, + "--auth-token", + envvar=DEFAULT_AUTH_TOKEN_ENV, + help="Accepted for CLI consistency but ignored because /healthz is public.", + ), +) -> None: + """Call the public health endpoint and report JSON.""" + + del auth_token + + async def action(client: ShellctlClient) -> HealthResponse: + return await client.health() + + _run_client_action( + base_url=base_url, + auth_token=None, + action=action, + emit=_emit_model, + ) + + +@cli.command("run") +def run_command( + script: str = typer.Argument(...), + base_url: str = typer.Option( + DEFAULT_BASE_URL, + "--base-url", + envvar=DEFAULT_BASE_URL_ENV, + help="shellctl server base URL. You can also set SHELLCTL_BASE_URL.", + ), + auth_token: str | None = typer.Option( + None, + "--auth-token", + envvar=DEFAULT_AUTH_TOKEN_ENV, + help=( + "Bearer token value. You can also set SHELLCTL_AUTH_TOKEN. " + "Leave it unset or empty when the server does not require auth." + ), + ), + cwd: Path | None = typer.Option(None, "--cwd"), + env: list[str] | None = typer.Option(None, "--env"), + timeout: float = typer.Option(DEFAULT_TIMEOUT_SECONDS, "--timeout"), + output_limit: int = typer.Option(DEFAULT_OUTPUT_LIMIT_BYTES, "--output-limit"), + idle_flush_seconds: float = typer.Option( + DEFAULT_IDLE_FLUSH_SECONDS, + "--idle-flush-seconds", + ), + cols: int | None = typer.Option(None, "--cols"), + rows: int | None = typer.Option(None, "--rows"), +) -> None: + """Create a job through the running shellctl server.""" + + request = _build_model( + RunJobRequest, + script=script, + cwd=str(cwd) if cwd is not None else None, + env=_parse_env(env), + terminal=_terminal_size(cols=cols, rows=rows), + timeout=timeout, + output_limit=output_limit, + idle_flush_seconds=idle_flush_seconds, + ) + + async def action(client: ShellctlClient) -> JobResult: + return await client.run( + request.script, + cwd=request.cwd, + env=request.env, + timeout=request.timeout, + terminal=request.terminal, + ) + + _run_client_action( + base_url=base_url, + auth_token=auth_token, + output_limit=output_limit, + idle_flush_seconds=idle_flush_seconds, + action=action, + emit=_emit_model, + ) + + +@cli.command("wait") +def wait_command( + job_id: str = typer.Argument(...), + base_url: str = typer.Option( + DEFAULT_BASE_URL, + "--base-url", + envvar=DEFAULT_BASE_URL_ENV, + help="shellctl server base URL. You can also set SHELLCTL_BASE_URL.", + ), + auth_token: str | None = typer.Option( + None, + "--auth-token", + envvar=DEFAULT_AUTH_TOKEN_ENV, + help=( + "Bearer token value. You can also set SHELLCTL_AUTH_TOKEN. " + "Leave it unset or empty when the server does not require auth." + ), + ), + offset: int = typer.Option(..., "--offset"), + timeout: float = typer.Option(DEFAULT_TIMEOUT_SECONDS, "--timeout"), + output_limit: int = typer.Option(DEFAULT_OUTPUT_LIMIT_BYTES, "--output-limit"), + idle_flush_seconds: float = typer.Option( + DEFAULT_IDLE_FLUSH_SECONDS, + "--idle-flush-seconds", + ), +) -> None: + """Wait for incremental output, completion, truncation, or timeout.""" + + async def action(client: ShellctlClient) -> JobResult: + return await client.wait(job_id, offset=offset, timeout=timeout) + + _run_client_action( + base_url=base_url, + auth_token=auth_token, + output_limit=output_limit, + idle_flush_seconds=idle_flush_seconds, + action=action, + emit=_emit_model, + ) + + +@cli.command("status") +def status_command( + job_id: str = typer.Argument(...), + base_url: str = typer.Option( + DEFAULT_BASE_URL, + "--base-url", + envvar=DEFAULT_BASE_URL_ENV, + help="shellctl server base URL. You can also set SHELLCTL_BASE_URL.", + ), + auth_token: str | None = typer.Option( + None, + "--auth-token", + envvar=DEFAULT_AUTH_TOKEN_ENV, + help=( + "Bearer token value. You can also set SHELLCTL_AUTH_TOKEN. " + "Leave it unset or empty when the server does not require auth." + ), + ), +) -> None: + """Materialize the current status view for one job.""" + + async def action(client: ShellctlClient) -> JobStatusView: + return await client.status(job_id) + + _run_client_action( + base_url=base_url, + auth_token=auth_token, + action=action, + emit=_emit_model, + ) + + +@cli.command("list") +def list_command( + base_url: str = typer.Option( + DEFAULT_BASE_URL, + "--base-url", + envvar=DEFAULT_BASE_URL_ENV, + help="shellctl server base URL. You can also set SHELLCTL_BASE_URL.", + ), + auth_token: str | None = typer.Option( + None, + "--auth-token", + envvar=DEFAULT_AUTH_TOKEN_ENV, + help=( + "Bearer token value. You can also set SHELLCTL_AUTH_TOKEN. " + "Leave it unset or empty when the server does not require auth." + ), + ), + status: JobStatusName | None = typer.Option(None, "--status"), + limit: int = typer.Option(DEFAULT_LIST_LIMIT, "--limit", min=1, max=MAX_LIST_LIMIT), +) -> None: + """List recent jobs, optionally filtered by lifecycle status.""" + + async def action(client: ShellctlClient) -> list[JobInfo]: + return await client.list_jobs(status=status, limit=limit) + + _run_client_action( + base_url=base_url, + auth_token=auth_token, + action=action, + emit=_emit_job_list, + ) + + +@cli.command("input") +def input_command( + job_id: str = typer.Argument(...), + text: str = typer.Argument(...), + base_url: str = typer.Option( + DEFAULT_BASE_URL, + "--base-url", + envvar=DEFAULT_BASE_URL_ENV, + help="shellctl server base URL. You can also set SHELLCTL_BASE_URL.", + ), + auth_token: str | None = typer.Option( + None, + "--auth-token", + envvar=DEFAULT_AUTH_TOKEN_ENV, + help=( + "Bearer token value. You can also set SHELLCTL_AUTH_TOKEN. " + "Leave it unset or empty when the server does not require auth." + ), + ), + offset: int = typer.Option(..., "--offset"), + timeout: float = typer.Option(DEFAULT_TIMEOUT_SECONDS, "--timeout"), + output_limit: int = typer.Option(DEFAULT_OUTPUT_LIMIT_BYTES, "--output-limit"), + idle_flush_seconds: float = typer.Option( + DEFAULT_IDLE_FLUSH_SECONDS, + "--idle-flush-seconds", + ), +) -> None: + """Send text input to a running job and wait for the next result window.""" + + async def action(client: ShellctlClient) -> JobResult: + return await client.input(job_id, text, offset=offset, timeout=timeout) + + _run_client_action( + base_url=base_url, + auth_token=auth_token, + output_limit=output_limit, + idle_flush_seconds=idle_flush_seconds, + action=action, + emit=_emit_model, + ) + + +@cli.command("tail") +def tail_command( + job_id: str = typer.Argument(...), + base_url: str = typer.Option( + DEFAULT_BASE_URL, + "--base-url", + envvar=DEFAULT_BASE_URL_ENV, + help="shellctl server base URL. You can also set SHELLCTL_BASE_URL.", + ), + auth_token: str | None = typer.Option( + None, + "--auth-token", + envvar=DEFAULT_AUTH_TOKEN_ENV, + help=( + "Bearer token value. You can also set SHELLCTL_AUTH_TOKEN. " + "Leave it unset or empty when the server does not require auth." + ), + ), + output_limit: int = typer.Option( + DEFAULT_OUTPUT_LIMIT_BYTES, + "--output-limit", + min=1, + max=MAX_OUTPUT_LIMIT_BYTES, + ), +) -> None: + """Read a UTF-8-safe output tail for one job.""" + + async def action(client: ShellctlClient) -> JobResult: + return await client.tail(job_id) + + _run_client_action( + base_url=base_url, + auth_token=auth_token, + output_limit=output_limit, + action=action, + emit=_emit_model, + ) + + +@cli.command("terminate") +def terminate_command( + job_id: str = typer.Argument(...), + base_url: str = typer.Option( + DEFAULT_BASE_URL, + "--base-url", + envvar=DEFAULT_BASE_URL_ENV, + help="shellctl server base URL. You can also set SHELLCTL_BASE_URL.", + ), + auth_token: str | None = typer.Option( + None, + "--auth-token", + envvar=DEFAULT_AUTH_TOKEN_ENV, + help=( + "Bearer token value. You can also set SHELLCTL_AUTH_TOKEN. " + "Leave it unset or empty when the server does not require auth." + ), + ), + grace_seconds: float = typer.Option( + DEFAULT_TERMINATE_GRACE_SECONDS, + "--grace-seconds", + ), +) -> None: + """Terminate a job and return its materialized status.""" + + async def action(client: ShellctlClient) -> JobStatusView: + return await client.terminate(job_id, grace_seconds=grace_seconds) + + _run_client_action( + base_url=base_url, + auth_token=auth_token, + action=action, + emit=_emit_model, + ) + + +@cli.command("delete") +def delete_command( + job_id: str = typer.Argument(...), + base_url: str = typer.Option( + DEFAULT_BASE_URL, + "--base-url", + envvar=DEFAULT_BASE_URL_ENV, + help="shellctl server base URL. You can also set SHELLCTL_BASE_URL.", + ), + auth_token: str | None = typer.Option( + None, + "--auth-token", + envvar=DEFAULT_AUTH_TOKEN_ENV, + help=( + "Bearer token value. You can also set SHELLCTL_AUTH_TOKEN. " + "Leave it unset or empty when the server does not require auth." + ), + ), + force: bool = typer.Option(False, "--force"), + grace_seconds: float = typer.Option( + DEFAULT_TERMINATE_GRACE_SECONDS, + "--grace-seconds", + ), +) -> None: + """Delete a job row and artifacts, optionally terminating first.""" + + async def action(client: ShellctlClient) -> DeleteJobResponse: + return await client.delete( + job_id, + force=force, + grace_seconds=grace_seconds, + ) + + _run_client_action( + base_url=base_url, + auth_token=auth_token, + action=action, + emit=_emit_model, + ) + + +@cli.command("serve") +def serve_command( + listen: str = "127.0.0.1:8765", + auth_token: str | None = typer.Option( + None, + "--auth-token", + envvar=DEFAULT_AUTH_TOKEN_ENV, + help=( + "Bearer token value. You can also set SHELLCTL_AUTH_TOKEN. " + "Leave it unset or empty to disable HTTP bearer auth." + ), + ), + state_dir: Path | None = None, + runtime_dir: Path | None = None, + gc_interval_seconds: float = typer.Option( + DEFAULT_GC_INTERVAL_SECONDS, + "--gc-interval-seconds", + ), + gc_finished_job_retention_seconds: float = typer.Option( + DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS, + "--gc-finished-job-retention-seconds", + ), +) -> None: + """Run the local shellctl FastAPI server via uvicorn.""" + + from shellctl.server.serve import ( + serve_command as server_serve_command, + ) + + server_serve_command( + listen=listen, + auth_token=auth_token, + state_dir=state_dir, + runtime_dir=runtime_dir, + gc_interval_seconds=gc_interval_seconds, + gc_finished_job_retention_seconds=gc_finished_job_retention_seconds, + ) + + +def main() -> None: + """CLI entrypoint used by the console script and `python -m` invocations.""" + + cli() + + +def _parse_env(values: list[str] | None) -> dict[str, str] | None: + if not values: + return None + + parsed: dict[str, str] = {} + for value in values: + if "=" not in value: + raise typer.BadParameter( + "env entries must use NAME=VALUE format", + param_hint="--env", + ) + name, env_value = value.split("=", 1) + if not name: + raise typer.BadParameter( + "env names must be non-empty", + param_hint="--env", + ) + parsed[name] = env_value + return parsed + + +def _terminal_size(*, cols: int | None, rows: int | None) -> TerminalSize | None: + if cols is None and rows is None: + return None + return _build_model( + TerminalSize, + cols=cols if cols is not None else DEFAULT_TERMINAL_COLS, + rows=rows if rows is not None else DEFAULT_TERMINAL_ROWS, + ) + + +def _build_model[ModelT: BaseModel](model_type: type[ModelT], /, **data: object) -> ModelT: + try: + return model_type(**data) + except ValidationError as exc: + raise typer.BadParameter(_validation_error_message(exc)) from exc + + +async def _with_client[ResponseT]( + base_url: str, + auth_token: str | None, + output_limit: int, + idle_flush_seconds: float, + action: Callable[[ShellctlClient], Awaitable[ResponseT]], +) -> ResponseT: + async with ShellctlClient( + base_url, + output_limit=output_limit, + idle_flush_seconds=idle_flush_seconds, + token=auth_token, + ) as client: + return await action(client) + + +def _run_client_action[ResponseT]( + *, + base_url: str, + auth_token: str | None, + action: Callable[[ShellctlClient], Awaitable[ResponseT]], + emit: Callable[[ResponseT], None], + output_limit: int = DEFAULT_OUTPUT_LIMIT_BYTES, + idle_flush_seconds: float = DEFAULT_IDLE_FLUSH_SECONDS, +) -> None: + try: + payload = anyio.run( + _with_client, + base_url, + auth_token, + output_limit, + idle_flush_seconds, + action, + ) + except ShellctlClientError as exc: + _emit_error_and_exit(exc.code, exc.message) + except httpx.TimeoutException: + _emit_error_and_exit("request_timeout", "request timed out") + except httpx.TransportError as exc: + _emit_error_and_exit("connection_error", str(exc)) + + emit(payload) + + +def _emit_model(model: BaseModel) -> None: + typer.echo(model.model_dump_json(exclude_none=True), color=False) + + +def _emit_job_list(jobs: list[JobInfo]) -> None: + typer.echo( + json.dumps( + [item.model_dump(mode="json", exclude_none=True) for item in jobs], + separators=(",", ":"), + ), + color=False, + ) + + +def _emit_error_and_exit(code: str, message: str) -> NoReturn: + typer.echo( + json.dumps( + {"error": {"code": code, "message": message}}, + separators=(",", ":"), + ), + err=True, + color=False, + ) + raise typer.Exit(code=1) + + +def _validation_error_message(exc: ValidationError) -> str: + detail = exc.errors(include_url=False)[0] + location = ".".join(str(part) for part in detail.get("loc", ())) + message = detail["msg"] + return f"{location}: {message}" if location else str(message) + + +__all__ = [ + "cli", + "delete_command", + "health_command", + "input_command", + "list_command", + "main", + "run_command", + "serve_command", + "status_command", + "tail_command", + "terminate_command", + "wait_command", +] diff --git a/dify-agent/src/shellctl/client/__init__.py b/dify-agent/src/shellctl/client/__init__.py new file mode 100644 index 00000000000..a7ce3224908 --- /dev/null +++ b/dify-agent/src/shellctl/client/__init__.py @@ -0,0 +1,12 @@ +"""Async HTTP client package for shellctl. + +`shellctl.client` remains importable as before, but it is + now a package so future client helpers can live beside the main SDK class. +""" + +from shellctl.client.sdk import ( + ShellctlClient, + ShellctlClientError, +) + +__all__ = ["ShellctlClient", "ShellctlClientError"] diff --git a/dify-agent/src/shellctl/client/sdk.py b/dify-agent/src/shellctl/client/sdk.py new file mode 100644 index 00000000000..d007fff551d --- /dev/null +++ b/dify-agent/src/shellctl/client/sdk.py @@ -0,0 +1,319 @@ +"""Async HTTP client for the shellctl server API. + +The SDK keeps transport-level knobs (`output_limit`, `idle_flush_seconds`, base +URL selection, and bearer-token handling) on the client instance so individual +method calls stay close to the network CLI's high-level workflow. Blocking shell +operations reuse the shared client, but they override the HTTP read timeout per +request so the transport does not fail before the server-side shell wait timeout +or terminate-grace budget does. +""" + +from __future__ import annotations + +import os +from typing import Any + +import httpx2 as httpx + +from shellctl.shared.constants import ( + DEFAULT_AUTH_TOKEN_ENV, + DEFAULT_BASE_URL, + DEFAULT_IDLE_FLUSH_SECONDS, + DEFAULT_LIST_LIMIT, + DEFAULT_OUTPUT_LIMIT_BYTES, + DEFAULT_TERMINATE_GRACE_SECONDS, + DEFAULT_TIMEOUT_SECONDS, +) +from shellctl.shared.schemas import ( + DeleteJobResponse, + HealthResponse, + JobInfo, + JobResult, + JobStatusView, + ListJobsResponse, + RunJobRequest, + TerminalSize, +) + + +class ShellctlClientError(RuntimeError): + """Raised for API-declared failures and response decode/shape problems. + + `ShellctlClient` raises this error when the server returns a structured + error payload, and also when an otherwise successful HTTP response contains + invalid JSON or a top-level payload shape that does not match the SDK + contract. Transport and timeout failures remain raw `httpx2` exceptions so + library callers can decide how to handle network-layer failures. + """ + + def __init__(self, status_code: int, code: str, message: str) -> None: + super().__init__(f"{code} ({status_code}): {message}") + self.status_code = status_code + self.code = code + self.message = message + + +class ShellctlClient: + """Thin async SDK for the shellctl HTTP API. + + The client owns a reusable `httpx.AsyncClient` unless one is injected via the + `client` argument. Callers can therefore either keep one instance for a full + workflow or treat it as an async context manager. Injected clients keep their + original lifecycle; `close()` only closes clients that this SDK created. + """ + + def __init__( + self, + base_url: str = DEFAULT_BASE_URL, + *, + output_limit: int = DEFAULT_OUTPUT_LIMIT_BYTES, + idle_flush_seconds: float = DEFAULT_IDLE_FLUSH_SECONDS, + token: str | None = None, + client: httpx.AsyncClient | None = None, + transport: httpx.AsyncBaseTransport | None = None, + request_timeout_grace_seconds: float = 10.0, + ) -> None: + self.base_url = base_url.rstrip("/") + self.output_limit = output_limit + self.idle_flush_seconds = idle_flush_seconds + self.request_timeout_grace_seconds = request_timeout_grace_seconds + self.token = token if token is not None else os.environ.get(DEFAULT_AUTH_TOKEN_ENV) + self._owns_client = client is None + self._client = client or httpx.AsyncClient( + base_url=self.base_url, + follow_redirects=True, + timeout=httpx.Timeout(DEFAULT_TIMEOUT_SECONDS, connect=DEFAULT_TIMEOUT_SECONDS), + transport=transport, + ) + + async def __aenter__(self) -> ShellctlClient: + return self + + async def __aexit__(self, exc_type: object, exc: object, tb: object) -> None: + await self.close() + + async def close(self) -> None: + """Close the underlying HTTP client if this SDK instance owns it.""" + + if self._owns_client: + await self._client.aclose() + + def _wait_request_timeout(self, timeout: float) -> httpx.Timeout: + """Return a request timeout for blocking shell calls. + + The shellctl server enforces the payload timeout, while the SDK keeps a + small HTTP read-timeout grace so the transport can wait slightly longer + for that response without loosening connect/write/pool timeouts. + """ + + return httpx.Timeout( + connect=DEFAULT_TIMEOUT_SECONDS, + read=timeout + self.request_timeout_grace_seconds, + write=DEFAULT_TIMEOUT_SECONDS, + pool=DEFAULT_TIMEOUT_SECONDS, + ) + + def _terminate_request_timeout(self, grace_seconds: float | None = None) -> httpx.Timeout: + """Return a request timeout for terminate-style calls. + + `terminate()` and forced `delete()` block until the server finishes the + terminate grace window, so their HTTP read timeout must cover that + business wait budget even when the request relies on the API default. + """ + + effective_grace_seconds = DEFAULT_TERMINATE_GRACE_SECONDS if grace_seconds is None else grace_seconds + return self._wait_request_timeout(effective_grace_seconds) + + async def health(self) -> HealthResponse: + """Call the public health endpoint and decode it as `HealthResponse`.""" + + return HealthResponse.model_validate(await self.healthz()) + + async def healthz(self) -> dict[str, Any]: + """Call the public health endpoint without requiring auth.""" + + response = await self._client.get("/healthz") + return self._decode_response(response) + + async def run( + self, + script: str, + *, + cwd: str | None = None, + env: dict[str, str] | None = None, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + terminal: TerminalSize | None = None, + ) -> JobResult: + """Create a new job and wait for initial output or completion. + + `cwd` and `env` preset the script's working directory and environment + overlay on the server side. + """ + + payload = RunJobRequest( + script=script, + cwd=cwd, + env=env, + terminal=terminal, + timeout=timeout, + output_limit=self.output_limit, + idle_flush_seconds=self.idle_flush_seconds, + ) + response = await self._client.post( + "/v1/jobs/run", + json=payload.model_dump(mode="json", exclude_none=True), + headers=self._auth_headers(), + timeout=self._wait_request_timeout(timeout), + ) + return JobResult.model_validate(self._decode_response(response)) + + async def wait( + self, + job_id: str, + *, + offset: int, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + ) -> JobResult: + """Wait for incremental output, completion, truncation, or timeout.""" + + response = await self._client.post( + f"/v1/jobs/{job_id}/wait", + json={ + "offset": offset, + "timeout": timeout, + "output_limit": self.output_limit, + "idle_flush_seconds": self.idle_flush_seconds, + }, + headers=self._auth_headers(), + timeout=self._wait_request_timeout(timeout), + ) + return JobResult.model_validate(self._decode_response(response)) + + async def status(self, job_id: str) -> JobStatusView: + """Fetch the materialized status view for one job.""" + + response = await self._client.get( + f"/v1/jobs/{job_id}", + headers=self._auth_headers(), + ) + return JobStatusView.model_validate(self._decode_response(response)) + + async def list_jobs( + self, + *, + status: str | None = None, + limit: int = DEFAULT_LIST_LIMIT, + ) -> list[JobInfo]: + """List recent jobs, optionally filtered by lifecycle status.""" + + params: dict[str, Any] = {"limit": limit} + if status is not None: + params["status"] = status + response = await self._client.get( + "/v1/jobs", + params=params, + headers=self._auth_headers(), + ) + payload = ListJobsResponse.model_validate(self._decode_response(response)) + return payload.jobs + + async def input( + self, + job_id: str, + text: str, + *, + offset: int, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + ) -> JobResult: + """Send text input to a running job and then wait like `wait()`.""" + + response = await self._client.post( + f"/v1/jobs/{job_id}/input", + json={ + "text": text, + "offset": offset, + "timeout": timeout, + "output_limit": self.output_limit, + "idle_flush_seconds": self.idle_flush_seconds, + }, + headers=self._auth_headers(), + timeout=self._wait_request_timeout(timeout), + ) + return JobResult.model_validate(self._decode_response(response)) + + async def tail(self, job_id: str) -> JobResult: + """Fetch an immediate UTF-8-safe tail snapshot for a job.""" + + response = await self._client.get( + f"/v1/jobs/{job_id}/log/tail", + params={"output_limit": self.output_limit}, + headers=self._auth_headers(), + ) + return JobResult.model_validate(self._decode_response(response)) + + async def terminate( + self, + job_id: str, + grace_seconds: float = DEFAULT_TERMINATE_GRACE_SECONDS, + ) -> JobStatusView: + """Terminate a job, waiting long enough for the grace window to finish.""" + + response = await self._client.post( + f"/v1/jobs/{job_id}/terminate", + json={"grace_seconds": grace_seconds}, + headers=self._auth_headers(), + timeout=self._terminate_request_timeout(grace_seconds), + ) + return JobStatusView.model_validate(self._decode_response(response)) + + async def delete( + self, + job_id: str, + *, + force: bool = False, + grace_seconds: float | None = None, + ) -> DeleteJobResponse: + """Delete job artifacts, optionally waiting for forced termination first.""" + + params: dict[str, Any] = {"force": str(force).lower()} + if grace_seconds is not None: + params["grace_seconds"] = grace_seconds + request_kwargs: dict[str, Any] = { + "params": params, + "headers": self._auth_headers(), + } + if force: + request_kwargs["timeout"] = self._terminate_request_timeout(grace_seconds) + response = await self._client.delete( + f"/v1/jobs/{job_id}", + **request_kwargs, + ) + return DeleteJobResponse.model_validate(self._decode_response(response)) + + def _auth_headers(self) -> dict[str, str]: + if not self.token: + return {} + return {"Authorization": f"Bearer {self.token}"} + + def _decode_response(self, response: httpx.Response) -> dict[str, Any]: + try: + payload = response.json() + except ValueError as exc: # pragma: no cover - network/proxy corruption + raise ShellctlClientError(response.status_code, "invalid_json", response.text) from exc + + if response.is_error: + error = payload.get("error") if isinstance(payload, dict) else None + if isinstance(error, dict): + code = str(error.get("code", "request_failed")) + message = str(error.get("message", response.text)) + else: + code = "request_failed" + message = response.text + raise ShellctlClientError(response.status_code, code, message) + + if not isinstance(payload, dict): + raise ShellctlClientError(response.status_code, "invalid_payload", response.text) + return payload + + +__all__ = ["ShellctlClient", "ShellctlClientError"] diff --git a/dify-agent/src/shellctl/py.typed b/dify-agent/src/shellctl/py.typed new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/dify-agent/src/shellctl/py.typed @@ -0,0 +1 @@ + diff --git a/dify-agent/src/shellctl/server/__init__.py b/dify-agent/src/shellctl/server/__init__.py new file mode 100644 index 00000000000..fcb034f3e30 --- /dev/null +++ b/dify-agent/src/shellctl/server/__init__.py @@ -0,0 +1,58 @@ +"""shellctl server package. + +The server stack sits behind lazy exports so importing the network CLI does not +pull in FastAPI, SQLAlchemy, tmux, or the local service runtime unless a +server-side symbol is actually used. +""" + +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from shellctl.server.api import create_app + from shellctl.server.cli import ( + cli, + main, + ) + from shellctl.server.config import ShellctlConfig + from shellctl.server.db import JobRow + from shellctl.server.errors import ShellctlServerError + from shellctl.server.serve import serve_command + from shellctl.server.service import ShellctlService + +__all__ = [ + "JobRow", + "ShellctlConfig", + "ShellctlServerError", + "ShellctlService", + "cli", + "create_app", + "main", + "serve_command", +] + +_EXPORTS = { + "JobRow": "shellctl.server.db", + "ShellctlConfig": "shellctl.server.config", + "ShellctlServerError": "shellctl.server.errors", + "ShellctlService": "shellctl.server.service", + "cli": "shellctl.server.cli", + "create_app": "shellctl.server.api", + "main": "shellctl.server.cli", + "serve_command": "shellctl.server.serve", +} + + +def __getattr__(name: str) -> Any: + if name not in _EXPORTS: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module = import_module(_EXPORTS[name]) + value = getattr(module, name) # noqa: no-new-getattr lazy export proxy + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__)) diff --git a/dify-agent/src/shellctl/server/__main__.py b/dify-agent/src/shellctl/server/__main__.py new file mode 100644 index 00000000000..b5925643528 --- /dev/null +++ b/dify-agent/src/shellctl/server/__main__.py @@ -0,0 +1,6 @@ +"""Support `python -m shellctl.server`.""" + +from shellctl.cli import main + +if __name__ == "__main__": + main() diff --git a/dify-agent/src/shellctl/server/api.py b/dify-agent/src/shellctl/server/api.py new file mode 100644 index 00000000000..07299e2db63 --- /dev/null +++ b/dify-agent/src/shellctl/server/api.py @@ -0,0 +1,198 @@ +"""FastAPI wiring for shellctl server endpoints.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from typing import Annotated, cast + +from fastapi import Depends, FastAPI, Header, Query, Request +from fastapi.responses import JSONResponse + +from shellctl.server.config import ShellctlConfig +from shellctl.server.errors import ShellctlServerError +from shellctl.server.service import ShellctlService +from shellctl.shared.constants import ( + DEFAULT_HEALTH_STATUS, + DEFAULT_LIST_LIMIT, + DEFAULT_OUTPUT_LIMIT_BYTES, + DEFAULT_TERMINATE_GRACE_SECONDS, + MAX_LIST_LIMIT, + MAX_OUTPUT_LIMIT_BYTES, +) +from shellctl.shared.schemas import ( + DeleteJobResponse, + ErrorDetail, + ErrorResponse, + HealthResponse, + InputJobRequest, + JobResult, + JobStatusName, + JobStatusView, + ListJobsResponse, + RunJobRequest, + TerminateJobRequest, + WaitJobRequest, +) + + +def create_app( + config: ShellctlConfig | None = None, + *, + service: ShellctlService | None = None, +) -> FastAPI: + """Create the FastAPI application used by `shellctl serve`.""" + + resolved_config = config or ShellctlConfig() + resolved_service = service or ShellctlService(resolved_config) + + @asynccontextmanager + async def lifespan(_app: FastAPI): + await resolved_service.initialize() + resolved_service.start_background_gc() + resolved_service.start_background_pipe_monitor() + try: + yield + finally: + await resolved_service.shutdown() + + app = FastAPI(title="shellctl", version="0.1.0", lifespan=lifespan) + app.state.shellctl_service = resolved_service + + @app.exception_handler(ShellctlServerError) + async def handle_shellctl_error( + _request: Request, + exc: ShellctlServerError, + ) -> JSONResponse: + return JSONResponse( + status_code=exc.status_code, + content=ErrorResponse(error=ErrorDetail(code=exc.code, message=exc.message)).model_dump(mode="json"), + ) + + @app.exception_handler(RuntimeError) + async def handle_runtime_error(_request: Request, exc: RuntimeError) -> JSONResponse: + return JSONResponse( + status_code=500, + content=ErrorResponse( + error=ErrorDetail( + code="internal_error", + message=str(exc) or "internal server error", + ) + ).model_dump(mode="json"), + ) + + def get_service() -> ShellctlService: + return cast(ShellctlService, app.state.shellctl_service) + + def verify_auth( + authorization: Annotated[str | None, Header()] = None, + ) -> None: + token = resolved_config.auth_token + if token is None: + return + expected = f"Bearer {token}" + if authorization != expected: + raise ShellctlServerError(401, "unauthorized", "Missing or invalid bearer token") + + @app.get("/healthz", response_model=HealthResponse) + async def healthz() -> HealthResponse: + return HealthResponse(status=DEFAULT_HEALTH_STATUS) + + @app.post( + "/v1/jobs/run", + response_model=JobResult, + dependencies=[Depends(verify_auth)], + ) + async def run_job( + payload: RunJobRequest, + svc: ShellctlService = Depends(get_service), + ) -> JobResult: + return await svc.run_job(payload) + + @app.post( + "/v1/jobs/{job_id}/wait", + response_model=JobResult, + dependencies=[Depends(verify_auth)], + ) + async def wait_job( + job_id: str, + payload: WaitJobRequest, + svc: ShellctlService = Depends(get_service), + ) -> JobResult: + return await svc.wait_job(job_id, payload) + + @app.get( + "/v1/jobs/{job_id}/log/tail", + response_model=JobResult, + dependencies=[Depends(verify_auth)], + ) + async def tail_job( + job_id: str, + output_limit: Annotated[int, Query(ge=1, le=MAX_OUTPUT_LIMIT_BYTES)] = DEFAULT_OUTPUT_LIMIT_BYTES, + svc: ShellctlService = Depends(get_service), + ) -> JobResult: + return await svc.tail_job(job_id, output_limit=output_limit) + + @app.get( + "/v1/jobs/{job_id}", + response_model=JobStatusView, + dependencies=[Depends(verify_auth)], + ) + async def job_status( + job_id: str, + svc: ShellctlService = Depends(get_service), + ) -> JobStatusView: + return await svc.get_job_status(job_id) + + @app.get( + "/v1/jobs", + response_model=ListJobsResponse, + dependencies=[Depends(verify_auth)], + ) + async def list_jobs( + status: Annotated[JobStatusName | None, Query()] = None, + limit: Annotated[int, Query(ge=1, le=MAX_LIST_LIMIT)] = DEFAULT_LIST_LIMIT, + svc: ShellctlService = Depends(get_service), + ) -> ListJobsResponse: + return await svc.list_jobs(status=status, limit=limit) + + @app.post( + "/v1/jobs/{job_id}/input", + response_model=JobResult, + dependencies=[Depends(verify_auth)], + ) + async def input_job( + job_id: str, + payload: InputJobRequest, + svc: ShellctlService = Depends(get_service), + ) -> JobResult: + return await svc.send_input(job_id, payload) + + @app.post( + "/v1/jobs/{job_id}/terminate", + response_model=JobStatusView, + dependencies=[Depends(verify_auth)], + ) + async def terminate_job( + job_id: str, + payload: TerminateJobRequest, + svc: ShellctlService = Depends(get_service), + ) -> JobStatusView: + return await svc.terminate_job(job_id, payload) + + @app.delete( + "/v1/jobs/{job_id}", + response_model=DeleteJobResponse, + dependencies=[Depends(verify_auth)], + ) + async def delete_job( + job_id: str, + force: bool = False, + grace_seconds: float = DEFAULT_TERMINATE_GRACE_SECONDS, + svc: ShellctlService = Depends(get_service), + ) -> DeleteJobResponse: + return await svc.delete_job(job_id, force=force, grace_seconds=grace_seconds) + + return app + + +__all__ = ["create_app"] diff --git a/dify-agent/src/shellctl/server/artifacts.py b/dify-agent/src/shellctl/server/artifacts.py new file mode 100644 index 00000000000..ca013f4a36a --- /dev/null +++ b/dify-agent/src/shellctl/server/artifacts.py @@ -0,0 +1,62 @@ +"""Per-job artifact names used by shellctl server/runtime code. + +Normal job completion is coordinated through small marker files inside each +`jobs//` directory so the tmux output-pipe finalizer can publish the +SQLite `exited(exit_code, ended_at)` state only after PTY output is fully +drained into `output.log`. The same artifact directory also stores the request's +environment overlay so the runner can merge arbitrary key/value pairs without +shell-escaping them into the generated script. Separate failure markers and a +dedicated `pipe-error.log` stderr capture keep startup diagnostics available +when the sanitizer never reaches its ready-file handshake. +""" + +from __future__ import annotations + +from pathlib import Path + +RUNNER_EXIT_CODE_FILENAME = ".runner-exit-code" +RUNNER_ENDED_AT_FILENAME = ".runner-ended-at" +JOB_ENV_FILENAME = ".job-env.json" +PIPE_DRAINED_FILENAME = ".pipe-drained" +PIPE_FAILED_FILENAME = ".pipe-failed" +PIPE_ERROR_LOG_FILENAME = "pipe-error.log" + + +def runner_exit_code_path(job_dir: Path) -> Path: + return job_dir / RUNNER_EXIT_CODE_FILENAME + + +def runner_ended_at_path(job_dir: Path) -> Path: + return job_dir / RUNNER_ENDED_AT_FILENAME + + +def job_env_path(job_dir: Path) -> Path: + return job_dir / JOB_ENV_FILENAME + + +def pipe_drained_path(job_dir: Path) -> Path: + return job_dir / PIPE_DRAINED_FILENAME + + +def pipe_failed_path(job_dir: Path) -> Path: + return job_dir / PIPE_FAILED_FILENAME + + +def pipe_error_log_path(job_dir: Path) -> Path: + return job_dir / PIPE_ERROR_LOG_FILENAME + + +__all__ = [ + "JOB_ENV_FILENAME", + "PIPE_DRAINED_FILENAME", + "PIPE_ERROR_LOG_FILENAME", + "PIPE_FAILED_FILENAME", + "RUNNER_ENDED_AT_FILENAME", + "RUNNER_EXIT_CODE_FILENAME", + "job_env_path", + "pipe_drained_path", + "pipe_error_log_path", + "pipe_failed_path", + "runner_ended_at_path", + "runner_exit_code_path", +] diff --git a/dify-agent/src/shellctl/server/cli.py b/dify-agent/src/shellctl/server/cli.py new file mode 100644 index 00000000000..306981fb696 --- /dev/null +++ b/dify-agent/src/shellctl/server/cli.py @@ -0,0 +1,17 @@ +"""Compatibility shim for historical `shellctl.server.cli` imports. + +Job-management commands now live in `shellctl.cli`. This +module is intentionally a thin legacy re-export for callers that still import +CLI symbols from `shellctl.server.cli`. +""" + +from __future__ import annotations + +from shellctl.cli import cli, main +from shellctl.server.serve import serve_command + +__all__ = [ + "cli", + "main", + "serve_command", +] diff --git a/dify-agent/src/shellctl/server/config.py b/dify-agent/src/shellctl/server/config.py new file mode 100644 index 00000000000..979d61c59b8 --- /dev/null +++ b/dify-agent/src/shellctl/server/config.py @@ -0,0 +1,105 @@ +"""Configuration objects for shellctl server/runtime modules.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import cast + +from shellctl.shared.constants import ( + DEFAULT_AUTH_TOKEN_ENV, + DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS, + DEFAULT_GC_INTERVAL_SECONDS, + DEFAULT_IDLE_FLUSH_SECONDS, + DEFAULT_LIST_LIMIT, + DEFAULT_OUTPUT_LIMIT_BYTES, + DEFAULT_TERMINAL_COLS, + DEFAULT_TERMINAL_ROWS, + DEFAULT_TERMINATE_GRACE_SECONDS, + DEFAULT_TIMEOUT_SECONDS, + MAX_LIST_LIMIT, + MAX_OUTPUT_LIMIT_BYTES, + MAX_WAIT_TIMEOUT_SECONDS, +) +from shellctl.shared.runtime import ( + default_runtime_dir, + default_state_dir, +) +from shellctl_runtime.paths import DEFAULT_SQLITE_BUSY_TIMEOUT_MS + + +@dataclass(slots=True, frozen=True) +class ShellctlConfig: + """Runtime configuration for the shellctl service and CLI. + + The tmux subprocess hooks use dedicated console scripts instead of + `python -m shellctl...` entrypoints so every job does not pay + the shellctl client/server import cost just to sanitize PTY bytes or record + an exit row. `sqlite_busy_timeout_ms` still applies to the out-of-process + `runner-exit` callback, so non-default deployments keep one SQLite timeout + policy across the service and tmux finalizer. + + Bearer auth is opt-in: if the explicit `auth_token` and the fallback + `SHELLCTL_AUTH_TOKEN` environment variable are both missing or empty, + `shellctl serve` accepts requests without checking an Authorization header. + """ + + listen: str = "127.0.0.1:8765" + auth_token: str | None = None + state_dir: Path = field(default_factory=default_state_dir) + runtime_dir: Path | None = None + gc_interval_seconds: float = DEFAULT_GC_INTERVAL_SECONDS + gc_finished_job_retention_seconds: float = DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS + default_timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS + max_wait_timeout_seconds: float = MAX_WAIT_TIMEOUT_SECONDS + idle_flush_seconds: float = DEFAULT_IDLE_FLUSH_SECONDS + default_cwd: Path = field(default_factory=Path.home) + default_terminal_cols: int = DEFAULT_TERMINAL_COLS + default_terminal_rows: int = DEFAULT_TERMINAL_ROWS + default_list_limit: int = DEFAULT_LIST_LIMIT + max_list_limit: int = MAX_LIST_LIMIT + default_output_limit_bytes: int = DEFAULT_OUTPUT_LIMIT_BYTES + max_output_limit_bytes: int = MAX_OUTPUT_LIMIT_BYTES + default_terminate_grace_seconds: float = DEFAULT_TERMINATE_GRACE_SECONDS + poll_interval_seconds: float = 0.05 + pipe_monitor_interval_seconds: float = 1.0 + pipe_ready_timeout_seconds: float = 10.0 + sqlite_busy_timeout_ms: int = DEFAULT_SQLITE_BUSY_TIMEOUT_MS + sanitize_pty_command: tuple[str, ...] = ("shellctl-sanitize-pty",) + runner_exit_command: tuple[str, ...] = ("shellctl-runner-exit",) + + def __post_init__(self) -> None: + if self.runtime_dir is None: + object.__setattr__(self, "runtime_dir", default_runtime_dir(self.state_dir)) + token = self.auth_token + if token is None: + token = os.environ.get(DEFAULT_AUTH_TOKEN_ENV) + if not token: + token = None + object.__setattr__(self, "auth_token", token) + + @property + def jobs_dir(self) -> Path: + return self.state_dir / "jobs" + + @property + def db_path(self) -> Path: + return self.state_dir / "shellctl.db" + + @property + def database_url(self) -> str: + return f"sqlite+aiosqlite:///{self.db_path}" + + @property + def tmux_socket(self) -> Path: + runtime_dir = cast(Path, self.runtime_dir) + return runtime_dir / "tmux.sock" + + @property + def runner_path(self) -> Path: + runtime_dir = cast(Path, self.runtime_dir) + return runtime_dir / "bin" / "shellctl-runner" + + +__all__ = ["ShellctlConfig"] diff --git a/dify-agent/src/shellctl/server/db.py b/dify-agent/src/shellctl/server/db.py new file mode 100644 index 00000000000..88ac516b9fc --- /dev/null +++ b/dify-agent/src/shellctl/server/db.py @@ -0,0 +1,52 @@ +"""SQLite models and engine helpers for shellctl.""" + +from __future__ import annotations + +from typing import Any, cast + +from sqlalchemy import event +from sqlalchemy.ext.asyncio import AsyncEngine +from sqlmodel import Field, SQLModel + + +class JobRow(SQLModel, table=True): + """SQLite source-of-truth row for shellctl jobs. + + The table intentionally combines immutable metadata, mutable lifecycle state, + and exit facts so state transitions can be expressed as single conditional + `UPDATE` statements without synchronizing separate records. + """ + + __tablename__ = cast(Any, "jobs") + + job_id: str = Field(primary_key=True) + script_path: str + output_path: str + cwd: str + terminal_cols: int + terminal_rows: int + status: str = Field(index=True) + session_name: str + pane_target: str + exit_code: int | None = Field(default=None, nullable=True) + reason: str | None = Field(default=None, nullable=True) + message: str | None = Field(default=None, nullable=True) + created_at: str = Field(index=True) + started_at: str | None = Field(default=None, nullable=True) + ended_at: str | None = Field(default=None, nullable=True, index=True) + updated_at: str + + +def configure_sqlite_engine(engine: AsyncEngine, *, busy_timeout_ms: int) -> None: + """Install SQLite pragmas required by the proposal's concurrency model.""" + + @event.listens_for(engine.sync_engine, "connect") + def _set_pragmas(dbapi_connection: Any, _connection_record: Any) -> None: + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA foreign_keys=ON") + cursor.execute(f"PRAGMA busy_timeout={busy_timeout_ms}") + cursor.close() + + +__all__ = ["JobRow", "configure_sqlite_engine"] diff --git a/dify-agent/src/shellctl/server/errors.py b/dify-agent/src/shellctl/server/errors.py new file mode 100644 index 00000000000..3e673860bc1 --- /dev/null +++ b/dify-agent/src/shellctl/server/errors.py @@ -0,0 +1,14 @@ +"""Shared exception types for shellctl server-side modules.""" + + +class ShellctlServerError(RuntimeError): + """Structured server-side error that maps directly to API responses.""" + + def __init__(self, status_code: int, code: str, message: str) -> None: + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + + +__all__ = ["ShellctlServerError"] diff --git a/dify-agent/src/shellctl/server/serve.py b/dify-agent/src/shellctl/server/serve.py new file mode 100644 index 00000000000..de5957254d0 --- /dev/null +++ b/dify-agent/src/shellctl/server/serve.py @@ -0,0 +1,103 @@ +"""Server-side `shellctl serve` implementation. + +This module stays separate from the top-level CLI so ordinary client commands +do not import the FastAPI or uvicorn stack. `serve_command()` performs those +imports lazily because only the long-running server path needs them. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import typer + +from shellctl.shared.constants import ( + DEFAULT_AUTH_TOKEN_ENV, + DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS, + DEFAULT_GC_INTERVAL_SECONDS, +) +from shellctl.shared.runtime import ( + default_state_dir, +) + +if TYPE_CHECKING: + from shellctl.server.config import ShellctlConfig + + +def serve_command( + listen: str = "127.0.0.1:8765", + auth_token: str | None = typer.Option( + None, + "--auth-token", + envvar=DEFAULT_AUTH_TOKEN_ENV, + help=( + "Bearer token value. You can also set SHELLCTL_AUTH_TOKEN. " + "Leave it unset or empty to disable HTTP bearer auth." + ), + ), + state_dir: Path | None = None, + runtime_dir: Path | None = None, + gc_interval_seconds: float = DEFAULT_GC_INTERVAL_SECONDS, + gc_finished_job_retention_seconds: float = DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS, +) -> None: + """Build `ShellctlConfig` from CLI inputs and run the local HTTP server. + + Args: + listen: Host/port pair for the uvicorn listener in `host:port` form. + auth_token: Optional bearer token value. An explicit empty string + disables HTTP auth, an explicit non-empty token enables it, and an + omitted/`None` value may still resolve from `SHELLCTL_AUTH_TOKEN` + through the Typer env var binding or `ShellctlConfig` fallback. + state_dir: Persistent shellctl state directory; defaults to the shared + XDG-style state path when omitted. + runtime_dir: Optional runtime directory override for tmux/runtime + artifacts. + gc_interval_seconds: Background GC wake-up cadence for finished jobs. + gc_finished_job_retention_seconds: Retention window before finished jobs + are eligible for GC. + + This entrypoint is the only CLI path that should pull in the FastAPI and + uvicorn stack. It parses the listener, constructs `ShellctlConfig`, and + then hands the configured app to uvicorn. + """ + + from shellctl.server.config import ShellctlConfig + + host, port = _parse_listen(listen) + config = ShellctlConfig( + listen=listen, + auth_token=auth_token, + state_dir=state_dir or default_state_dir(), + runtime_dir=runtime_dir, + gc_interval_seconds=gc_interval_seconds, + gc_finished_job_retention_seconds=gc_finished_job_retention_seconds, + ) + _uvicorn_run(_create_app(config), host=host, port=port, log_level="info") + + +def _parse_listen(value: str) -> tuple[str, int]: + if ":" not in value: + raise typer.BadParameter("listen must use host:port format") + host, raw_port = value.rsplit(":", 1) + host = host.strip("[]") + try: + port = int(raw_port) + except ValueError as exc: + raise typer.BadParameter(f"invalid port: {raw_port}") from exc + return host, port + + +def _create_app(config: ShellctlConfig) -> Any: + from shellctl.server.api import create_app + + return create_app(config) + + +def _uvicorn_run(app: Any, *, host: str, port: int, log_level: str) -> None: + import uvicorn + + uvicorn.run(app, host=host, port=port, log_level=log_level) + + +__all__ = ["serve_command"] diff --git a/dify-agent/src/shellctl/server/service.py b/dify-agent/src/shellctl/server/service.py new file mode 100644 index 00000000000..741cbb6406d --- /dev/null +++ b/dify-agent/src/shellctl/server/service.py @@ -0,0 +1,1184 @@ +"""SQLite-backed shellctl job service. + +This module owns the long-lived job lifecycle rules: artifact creation, +SQLite compare-and-swap transitions, tmux reconciliation, runner exit +materialization, and GC. API and local server entrypoints should call into +`ShellctlService` rather than duplicating lifecycle logic, choosing either the +lightweight `prepare_runtime()` bootstrap or the full `initialize()` startup +depending on whether they need bounded setup work or long-running maintenance. +""" + +from __future__ import annotations + +import asyncio +import json +import shlex +import shutil +import stat +import sys +from contextlib import suppress +from pathlib import Path +from typing import Any, cast + +import anyio +from sqlalchemy import case, delete, select, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlmodel import SQLModel + +from shellctl.server.artifacts import ( + JOB_ENV_FILENAME, + RUNNER_ENDED_AT_FILENAME, + RUNNER_EXIT_CODE_FILENAME, + job_env_path, + pipe_drained_path, + pipe_error_log_path, + pipe_failed_path, + runner_ended_at_path, + runner_exit_code_path, +) +from shellctl.server.config import ShellctlConfig +from shellctl.server.db import JobRow, configure_sqlite_engine +from shellctl.server.errors import ShellctlServerError +from shellctl.server.tmux import ( + TmuxController, + TmuxControllerProtocol, +) +from shellctl.shared.constants import DEFAULT_AUTH_TOKEN_ENV +from shellctl.shared.output import ( + OutputWindow, + read_output_window, + tail_output_window, +) +from shellctl.shared.runtime import ( + format_timestamp, + generate_job_id, + is_terminal_status, + job_pane_target, + job_session_name, + parse_timestamp, + utc_now, +) +from shellctl.shared.schemas import ( + DeleteJobResponse, + InputJobRequest, + JobInfo, + JobResult, + JobStatusName, + JobStatusView, + ListJobsResponse, + RunJobRequest, + TerminalSize, + TerminateJobRequest, + WaitJobRequest, +) + + +class ShellctlService: + """SQLite-backed shellctl job service used by HTTP and local server entrypoints. + + The service keeps only minimal in-memory coordination state: + + - `_starting_jobs` prevents `created`/`starting` rows from being marked + `lost` while the current startup path is still installing tmux state. + + Everything else uses SQLite conditional updates so concurrent readers and + lifecycle paths converge through the DB rather than through mutable JSON + artifacts. + """ + + def __init__( + self, + config: ShellctlConfig, + *, + tmux: TmuxControllerProtocol | None = None, + ) -> None: + self.config = config + self._tmux = tmux or TmuxController(config) + self._gc_task: asyncio.Task[None] | None = None + self._monitor_task: asyncio.Task[None] | None = None + self._starting_jobs: set[str] = set() + self._engine = create_async_engine(self.config.database_url, future=True) + configure_sqlite_engine( + self._engine, + busy_timeout_ms=self.config.sqlite_busy_timeout_ms, + ) + self._session_factory = async_sessionmaker( + self._engine, + expire_on_commit=False, + class_=AsyncSession, + ) + + async def initialize_database(self) -> None: + """Ensure the SQLite database exists and its schema is initialized.""" + + self._ensure_dir(self.config.state_dir) + async with self._engine.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + + async def prepare_runtime(self) -> None: + """Prepare the minimal local runtime needed for bounded server-side work. + + This startup path intentionally skips reconciliation, GC, and background + tasks. Entry points that only need local runtime artifacts should stay + bounded by the requested operation instead of scanning or maintaining + the full historical job set on every invocation. + """ + + await self.initialize_database() + self._ensure_dir(cast(Path, self.config.runtime_dir)) + self._ensure_dir(self.config.jobs_dir) + self._ensure_dir(self.config.runner_path.parent) + self._install_runner() + await self._tmux.start_server() + + async def initialize(self) -> None: + """Run the full server startup path used by `shellctl serve`. + + Auth is configured at the API layer, so service startup must also work + when `shellctl serve` is intentionally running without a bearer token. + Unlike `prepare_runtime()`, this path also reconciles persisted jobs and + performs one foreground GC pass before the long-running server starts. + """ + + await self.prepare_runtime() + await self.reconcile() + await self.gc_once() + + async def shutdown(self) -> None: + """Stop background work and close the SQLite engine.""" + + for task_name in ("_gc_task", "_monitor_task"): + task = getattr(self, task_name) # noqa: no-new-getattr dynamic task cleanup + if task is not None: + task.cancel() + with anyio.CancelScope(shield=True): + with suppress(asyncio.CancelledError): + await task + setattr(self, task_name, None) + await self._engine.dispose() + + def start_background_gc(self) -> None: + if self._gc_task is None: + self._gc_task = asyncio.create_task(self._gc_loop(), name="shellctl-gc") + + def start_background_pipe_monitor(self) -> None: + if self._monitor_task is None: + self._monitor_task = asyncio.create_task(self._pipe_monitor_loop(), name="shellctl-pipe-monitor") + + async def run_job(self, request: RunJobRequest) -> JobResult: + """Create a tmux-backed job and wait for its initial result window. + + Side effects: + - allocates `jobs//` and writes `script`, `.job-env.json`, and + `output.log` + - inserts a `jobs` row into SQLite, then conditionally transitions it + through `created -> starting -> running` + - creates a dedicated tmux session, installs `pipe-pane`, waits for the + sanitize/output pipeline ready-file handshake, and only then opens the + `start-gate` + - on startup failure, marks the job `failed` in SQLite and cleans up the + tmux session without deleting the job artifacts + + Returns: + The same `JobResult` shape as `wait()`, starting from byte offset 0. + + Notes: + The startup reservation in `_starting_jobs` exists so concurrent + status/list/reconcile calls can observe a fresh row as a legitimate + `created`/`starting` job instead of materializing it to `lost` before + tmux becomes visible. + """ + + cwd = self._resolve_cwd(request.cwd) + env = self._resolve_env(request.env) + terminal = request.terminal or TerminalSize( + cols=self.config.default_terminal_cols, + rows=self.config.default_terminal_rows, + ) + created_at = format_timestamp() + + job_id: str | None = None + job_dir: Path | None = None + while True: + candidate_job_id, candidate_job_dir = self._allocate_job_dir() + try: + self._starting_jobs.add(candidate_job_id) + script_path = candidate_job_dir / "script" + output_path = candidate_job_dir / "output.log" + script_path.write_text(request.script, encoding="utf-8") + job_env_path(candidate_job_dir).write_text( + json.dumps(env, ensure_ascii=False), + encoding="utf-8", + ) + output_path.touch() + row = JobRow( + job_id=candidate_job_id, + script_path=f"jobs/{candidate_job_id}/script", + output_path=f"jobs/{candidate_job_id}/output.log", + cwd=str(cwd), + terminal_cols=terminal.cols, + terminal_rows=terminal.rows, + status=JobStatusName.CREATED.value, + session_name=job_session_name(candidate_job_id), + pane_target=job_pane_target(candidate_job_id), + exit_code=None, + reason=None, + message=None, + created_at=created_at, + started_at=None, + ended_at=None, + updated_at=created_at, + ) + if await self._insert_job_row(row): + job_id = candidate_job_id + job_dir = candidate_job_dir + break + self._starting_jobs.discard(candidate_job_id) + except BaseException: + self._starting_jobs.discard(candidate_job_id) + shutil.rmtree(candidate_job_dir, ignore_errors=True) + raise + shutil.rmtree(candidate_job_dir, ignore_errors=True) + + assert job_id is not None and job_dir is not None + await self._transition_status( + job_id, + allowed_from={JobStatusName.CREATED}, + target=JobStatusName.STARTING, + ) + + try: + pipe_ready_path = job_dir / ".pipe-ready" + await self._tmux.create_job_session( + job_id=job_id, + job_dir=job_dir, + cwd=cwd, + terminal=terminal, + ) + await self._tmux.enable_output_pipe( + job_id=job_id, + job_dir=job_dir, + ready_file=pipe_ready_path, + ) + await self._wait_for_output_pipe_ready( + job_id=job_id, + ready_file=pipe_ready_path, + ) + (job_dir / "start-gate").touch() + await self._transition_status( + job_id, + allowed_from={JobStatusName.STARTING}, + target=JobStatusName.RUNNING, + require_exit_code_null=True, + ) + except Exception as exc: + await self._transition_status( + job_id, + allowed_from={ + JobStatusName.CREATED, + JobStatusName.STARTING, + JobStatusName.RUNNING, + }, + target=JobStatusName.FAILED, + reason=getattr(exc, "code", "start_failed"), # noqa: no-new-getattr exception code fallback + message=str(exc), + ) + await self._tmux.cleanup_session(job_id=job_id) + finally: + (job_dir / ".pipe-ready").unlink(missing_ok=True) + self._starting_jobs.discard(job_id) + + return await self.wait_job( + job_id, + WaitJobRequest( + offset=0, + timeout=request.timeout, + output_limit=request.output_limit, + idle_flush_seconds=request.idle_flush_seconds, + ), + ) + + async def wait_job(self, job_id: str, request: WaitJobRequest) -> JobResult: + """Block until output, completion, truncation, or timeout.""" + + row = await self._get_job_row(job_id) + output_path = self._output_log_path(row) + self._validate_offset(output_path, request.offset) + deadline = anyio.current_time() + request.timeout + last_size = output_path.stat().st_size if output_path.exists() else 0 + saw_output = last_size > request.offset + last_growth_at: float | None = anyio.current_time() if saw_output else None + + while True: + view = await self.get_job_status(job_id) + current_size = output_path.stat().st_size if output_path.exists() else 0 + if request.offset > current_size: + raise ShellctlServerError( + 400, + "invalid_offset", + f"offset {request.offset} exceeds current file size {current_size}", + ) + + if current_size > last_size: + last_size = current_size + if current_size > request.offset: + saw_output = True + last_growth_at = anyio.current_time() + + if view.done: + window = read_output_window( + output_path, + offset=request.offset, + limit=request.output_limit, + ) + return self._job_result_from_view(view, row, window) + + if current_size > request.offset: + window = read_output_window( + output_path, + offset=request.offset, + limit=request.output_limit, + ) + if window.truncated: + return self._job_result_from_view(view, row, window) + if saw_output and last_growth_at is not None: + idle_elapsed = anyio.current_time() - last_growth_at + if idle_elapsed >= request.idle_flush_seconds: + return self._job_result_from_view(view, row, window) + + if anyio.current_time() >= deadline: + window = ( + read_output_window( + output_path, + offset=request.offset, + limit=request.output_limit, + ) + if current_size > request.offset + else OutputWindow(output="", offset=request.offset, truncated=False) + ) + return self._job_result_from_view(view, row, window) + + await anyio.sleep(self.config.poll_interval_seconds) + + async def tail_job(self, job_id: str, *, output_limit: int) -> JobResult: + row = await self._get_job_row(job_id) + view = await self.get_job_status(job_id) + tail = tail_output_window(self._output_log_path(row), limit=output_limit) + return self._job_result_from_view(view, row, tail, truncated=False) + + async def get_job_status(self, job_id: str) -> JobStatusView: + session_exists, pipe_active = await self._live_runtime_state(job_id) + view = await self._materialize_status_view( + job_id, + session_exists=session_exists, + pipe_active=pipe_active, + ) + if view.done: + await self._tmux.cleanup_session(job_id=job_id) + return view + + async def list_jobs( + self, + *, + status: JobStatusName | None = None, + limit: int, + ) -> ListJobsResponse: + """List recent jobs ordered by `created_at` descending. + + This intentionally uses live per-job runtime queries instead of a single + tmux snapshot so concurrent startup cannot turn a healthy job into `lost`. + Rows may also disappear concurrently because delete removes the DB record + before artifact cleanup; such races are treated as benign skips rather + than failing the whole list request. + """ + + rows = await self._list_job_rows() + items: list[JobInfo] = [] + for row in rows: + try: + view = await self.get_job_status(row.job_id) + except ShellctlServerError as exc: + if exc.code == "job_not_found": + continue + raise + if status is not None and view.status != status: + continue + items.append( + JobInfo( + job_id=view.job_id, + status=view.status, + created_at=view.created_at, + started_at=view.started_at, + ended_at=view.ended_at, + ) + ) + if len(items) >= limit: + break + return ListJobsResponse(jobs=items) + + async def send_input(self, job_id: str, request: InputJobRequest) -> JobResult: + """Send input to a running job and then wait using the same semantics as `wait()`. + + If the pane disappears after the initial running-state check, this method + re-materializes the job state before deciding whether to return terminal + `409` semantics or a true internal tmux failure. + """ + + view = await self.get_job_status(job_id) + if view.done: + raise ShellctlServerError(409, "job_not_running", f"Job {job_id} is already terminal") + try: + await self._tmux.send_input(job_id=job_id, text=request.text) + except ShellctlServerError as exc: + if exc.code != "tmux_target_missing": + raise + view = await self.get_job_status(job_id) + if view.done: + raise ShellctlServerError(409, "job_not_running", f"Job {job_id} is already terminal") from exc + raise ShellctlServerError(500, "tmux_input_failed", exc.message) from exc + return await self.wait_job( + job_id, + WaitJobRequest( + offset=request.offset, + timeout=request.timeout, + output_limit=request.output_limit, + idle_flush_seconds=request.idle_flush_seconds, + ), + ) + + async def terminate_job(self, job_id: str, request: TerminateJobRequest) -> JobStatusView: + """Terminate a job, reserving `terminated` before tmux cleanup. + + The initial conditional DB update allows an `exited` row to be overridden + only when it raced after an earlier non-terminal read in this method. That + preserves proposal semantics: once a user terminates a non-terminal job, + the final API status stays `terminated` even if the runner exits during the + same window. + """ + + view = await self.get_job_status(job_id) + if view.done: + await self._tmux.cleanup_session(job_id=job_id) + return view + + await self._transition_status( + job_id, + allowed_from={ + JobStatusName.CREATED, + JobStatusName.STARTING, + JobStatusName.RUNNING, + }, + allow_override_from={JobStatusName.EXITED}, + target=JobStatusName.TERMINATED, + ) + await self._tmux.send_interrupt(job_id=job_id) + if request.grace_seconds > 0: + await anyio.sleep(request.grace_seconds) + await self._tmux.cleanup_session(job_id=job_id) + return await self.get_job_status(job_id) + + async def delete_job( + self, + job_id: str, + *, + force: bool, + grace_seconds: float, + ) -> DeleteJobResponse: + """Delete a job row and its artifacts. + + Behavior: + - if the job is still non-terminal and `force` is false, raises `409` + - if `force` is true, first applies normal terminate semantics + - always best-effort cleans the tmux session before removing persisted + state + + Deletion ordering matters: the SQLite row is removed before best-effort + artifact-directory cleanup. That ordering intentionally avoids exposing a + visible half-deleted state where API listing/status can still see a row + whose `script` / `output.log` artifacts are already gone. + """ + + view = await self.get_job_status(job_id) + if not view.done: + if not force: + raise ShellctlServerError(409, "job_running", f"Job {job_id} is still running") + await self.terminate_job(job_id, TerminateJobRequest(grace_seconds=grace_seconds)) + await self._tmux.cleanup_session(job_id=job_id) + await self._delete_job_row(job_id) + shutil.rmtree(self._artifact_dir(job_id), ignore_errors=True) + return DeleteJobResponse(job_id=job_id) + + async def reconcile(self) -> None: + """Reconcile SQLite rows against live tmux state using per-job probes. + + Reconciliation also uses a live per-job runtime query to avoid turning a + job that is still completing startup into `lost` based on an earlier tmux + snapshot. + Jobs may disappear between the initial row query and the later status + materialization because delete removes the SQLite row first. Reconciliation + treats that race as benign and continues with the remaining jobs. + """ + + for row in await self._list_job_rows(): + try: + view = await self.get_job_status(row.job_id) + except ShellctlServerError as exc: + if exc.code == "job_not_found": + continue + raise + if view.done: + await self._tmux.cleanup_session(job_id=row.job_id) + + async def gc_once(self) -> None: + """Delete expired terminal rows and their artifact directories. + + Like reconciliation, GC tolerates rows disappearing mid-pass due to a + concurrent explicit delete. + + Deletion ordering matches explicit delete semantics: remove the SQLite + row first, then best-effort delete the artifact directory. That keeps the + DB/API source of truth from exposing a row whose filesystem artifacts have + already been removed. + """ + + cutoff = utc_now().timestamp() - self.config.gc_finished_job_retention_seconds + for row in await self._list_job_rows(): + try: + view = await self.get_job_status(row.job_id) + except ShellctlServerError as exc: + if exc.code == "job_not_found": + continue + raise + if not view.done or view.ended_at is None: + continue + if parse_timestamp(view.ended_at).timestamp() >= cutoff: + continue + await self._tmux.cleanup_session(job_id=row.job_id) + try: + await self._delete_job_row(row.job_id) + except ShellctlServerError as exc: + if exc.code == "job_not_found": + continue + raise + shutil.rmtree(self._artifact_dir(row.job_id), ignore_errors=True) + + async def record_runner_exit(self, job_id: str, exit_code: int, ended_at: str) -> None: + """Persist a drained normal-exit fact into SQLite. + + The usual caller is the tmux pipe finalizer after the lightweight PTY + sanitizer reaches EOF and flushes `output.log`. `_materialize_status_view()` + may also call this as a recovery path when `.pipe-drained` plus the + normal exit metadata files exist but SQLite is still non-terminal. The + statement always records `exit_code` and `ended_at`, but it only changes + `status` to `exited` when the current row is still non-terminal. + """ + + nonterminal = [ + JobStatusName.CREATED.value, + JobStatusName.STARTING.value, + JobStatusName.RUNNING.value, + ] + job_id_col = cast(Any, JobRow.job_id) + status_col = cast(Any, JobRow.status) + ended_at_col = cast(Any, JobRow.ended_at) + reason_col = cast(Any, JobRow.reason) + message_col = cast(Any, JobRow.message) + async with self._session_factory() as session: + stmt = ( + update(JobRow) + .where(job_id_col == job_id) + .values( + exit_code=exit_code, + ended_at=case( + (ended_at_col.is_(None), ended_at), + else_=ended_at_col, + ), + updated_at=ended_at, + status=case( + (status_col.in_(nonterminal), JobStatusName.EXITED.value), + else_=status_col, + ), + reason=case( + (status_col.in_(nonterminal), None), + else_=reason_col, + ), + message=case( + (status_col.in_(nonterminal), None), + else_=message_col, + ), + ) + ) + result = await session.execute(stmt) + if cast(Any, result).rowcount == 0: + raise ShellctlServerError(404, "job_not_found", f"Unknown job id: {job_id}") + await session.commit() + + async def check_running_jobs_pipe_health(self) -> None: + """Fail running jobs whose tmux output pipe disappeared after startup.""" + + for row in await self._list_job_rows(statuses={JobStatusName.RUNNING}): + session_exists, pipe_active = await self._live_runtime_state(row.job_id) + view = await self._materialize_status_view( + row.job_id, + session_exists=session_exists, + pipe_active=pipe_active, + ) + if view.done: + await self._tmux.cleanup_session(job_id=row.job_id) + + async def _gc_loop(self) -> None: + while True: + await anyio.sleep(self.config.gc_interval_seconds) + await self.gc_once() + + async def _pipe_monitor_loop(self) -> None: + while True: + await anyio.sleep(self.config.pipe_monitor_interval_seconds) + await self.check_running_jobs_pipe_health() + + async def _wait_for_output_pipe_ready(self, *, job_id: str, ready_file: Path) -> None: + """Confirm the sanitize/output pipeline is live before opening start-gate. + + Timeout failures include the ready-file path, current `#{pane_pipe}` + state, and a summary of `pipe-error.log` so operators can tell the + difference between slow startup, tmux pipe loss, and sanitizer crashes. + """ + + started_at = anyio.current_time() + deadline = started_at + self.config.pipe_ready_timeout_seconds + while True: + waited_seconds = anyio.current_time() - started_at + if ready_file.exists(): + pipe_state = await self._tmux.is_output_pipe_active(job_id=job_id) + if pipe_state is True: + return + if pipe_state is None: + raise ShellctlServerError( + 500, + "pipe_failed", + self._pipe_ready_failure_message( + job_id=job_id, + ready_file=ready_file, + waited_seconds=waited_seconds, + pipe_state=pipe_state, + cause="tmux pane disappeared after the ready-file handshake", + ), + ) + else: + if not await self._tmux.session_exists(job_session_name(job_id)): + pipe_state = await self._tmux.is_output_pipe_active(job_id=job_id) + raise ShellctlServerError( + 500, + "pipe_failed", + self._pipe_ready_failure_message( + job_id=job_id, + ready_file=ready_file, + waited_seconds=waited_seconds, + pipe_state=pipe_state, + cause="tmux session exited before the ready-file handshake", + ), + ) + if anyio.current_time() >= deadline: + pipe_state = await self._tmux.is_output_pipe_active(job_id=job_id) + raise ShellctlServerError( + 500, + "pipe_failed", + self._pipe_ready_failure_message( + job_id=job_id, + ready_file=ready_file, + waited_seconds=waited_seconds, + pipe_state=pipe_state, + cause="timed out waiting for the sanitize/output pipeline handshake", + ), + ) + await anyio.sleep(self.config.poll_interval_seconds) + + def _pipe_ready_failure_message( + self, + *, + job_id: str, + ready_file: Path, + waited_seconds: float, + pipe_state: bool | None, + cause: str, + ) -> str: + """Build a startup error message with enough pipe diagnostics to debug.""" + + return ( + f"Output pipe never became ready for {job_id}: {cause}; " + f"waited {waited_seconds:.3f}s; " + f"ready-file={ready_file}; " + f"tmux #{{pane_pipe}}={self._format_pane_pipe_state(pipe_state)}; " + f"pipe-error.log={self._pipe_error_log_summary(ready_file.parent)}" + ) + + def _format_pane_pipe_state(self, pipe_state: bool | None) -> str: + if pipe_state is True: + return "1" + if pipe_state is False: + return "0" + return "pane-missing" + + def _pipe_error_log_summary(self, job_dir: Path) -> str: + """Summarize sanitizer stderr without flooding API error responses.""" + + error_log = pipe_error_log_path(job_dir) + if not error_log.exists(): + return f"{error_log} missing" + stderr_text = error_log.read_text(encoding="utf-8", errors="replace").strip() + if not stderr_text: + return f"{error_log} empty" + summary = " | ".join(line.strip() for line in stderr_text.splitlines() if line.strip()) + if not summary: + return f"{error_log} contains only whitespace" + if len(summary) > 240: + summary = f"{summary[:237]}..." + return f"{error_log}: {summary}" + + async def _materialize_status_view( + self, + job_id: str, + *, + session_exists: bool, + pipe_active: bool | None, + ) -> JobStatusView: + """Materialize the current API status view from SQLite + live tmux state. + + Priority/invariants: + - terminal SQLite status wins and is preserved + - if SQLite already has an `exit_code` on a non-terminal row, materialize + `exited` + - if normal-exit metadata and `.pipe-drained` already exist, recover the + drained normal exit immediately even when the finalizer has not yet + committed SQLite state + - if normal-exit metadata exists and neither `.pipe-drained` nor + `.pipe-failed` exists, keep the non-terminal row instead of guessing + `lost` while the pipe finalizer is still responsible for the eventual + `runner-exit` commit + - if a live tmux session exists but the output pipe is known-dead, + conditionally materialize `failed(reason=pipe_failed)` + - if no live session exists and the row is not protected by the local + startup reservation, conditionally materialize `lost` + + This helper is the main reconciliation point between live tmux probes and + persisted SQLite state. It must never revive terminal rows and must never + convert an ambiguous startup race into `lost` while the current `run_job` + path still owns that startup window. + """ + + row = await self._get_job_row(job_id) + status = JobStatusName(row.status) + + if is_terminal_status(status): + if row.ended_at is None: + row = await self._transition_status( + job_id, + allowed_from={status}, + target=status, + ended_at=format_timestamp(), + ) + elif row.exit_code is not None: + row = await self._transition_status( + job_id, + allowed_from={ + JobStatusName.CREATED, + JobStatusName.STARTING, + JobStatusName.RUNNING, + }, + target=JobStatusName.EXITED, + ended_at=row.ended_at or format_timestamp(), + ) + elif (drained_exit := self._drained_normal_exit_metadata(job_id)) is not None: + exit_code, ended_at = drained_exit + await self.record_runner_exit(job_id, exit_code, ended_at) + row = await self._get_job_row(job_id) + elif session_exists: + if pipe_active is False: + if ( + status + in { + JobStatusName.CREATED, + JobStatusName.STARTING, + } + and job_id in self._starting_jobs + ): + pass + else: + row = await self._transition_status( + job_id, + allowed_from={ + JobStatusName.CREATED, + JobStatusName.STARTING, + JobStatusName.RUNNING, + }, + target=JobStatusName.FAILED, + reason="pipe_failed", + message="The tmux output pipe stopped while the job was still running.", + ended_at=format_timestamp(), + ) + elif status in {JobStatusName.CREATED, JobStatusName.STARTING} and job_id not in self._starting_jobs: + row = await self._transition_status( + job_id, + allowed_from={JobStatusName.CREATED, JobStatusName.STARTING}, + target=JobStatusName.RUNNING, + require_exit_code_null=True, + ) + else: + if self._normal_exit_commit_pending(job_id): + pass + elif not (status in {JobStatusName.CREATED, JobStatusName.STARTING} and job_id in self._starting_jobs): + row = await self._transition_status( + job_id, + allowed_from={ + JobStatusName.CREATED, + JobStatusName.STARTING, + JobStatusName.RUNNING, + }, + target=JobStatusName.LOST, + reason="tmux_session_missing", + message="The dedicated tmux session is no longer present.", + ended_at=format_timestamp(), + ) + + return self._status_view_from_row(row) + + async def _transition_status( + self, + job_id: str, + *, + allowed_from: set[JobStatusName], + target: JobStatusName, + allow_override_from: set[JobStatusName] | None = None, + require_exit_code_null: bool = False, + reason: str | None = None, + message: str | None = None, + ended_at: str | None = None, + ) -> JobRow: + """Conditionally transition a job row through a SQLite CAS update. + + This is the SQLite replacement for the old JSON+lock compare-and-swap + flow. The method performs a single conditional `UPDATE ... WHERE ...` + against the current row and returns the post-update materialized row. + + Invariants: + - non-terminal states may only advance from explicitly allowed source + states + - callers can require `exit_code IS NULL` so late runner exits do not let + startup paths overwrite a concurrently materialized terminal outcome + - terminal transitions preserve first-writer semantics via the WHERE + clause and fill `ended_at` only when needed + - when the conditional update affects zero rows, the method returns the + current persisted row instead of forcing a stale overwrite + """ + + now = format_timestamp() + allowed_statuses = {status.value for status in allowed_from} + if allow_override_from is not None: + allowed_statuses.update(status.value for status in allow_override_from) + + values: dict[str, Any] = { + "status": target.value, + "updated_at": now, + "reason": reason, + "message": message, + } + status_col = cast(Any, JobRow.status) + job_id_col = cast(Any, JobRow.job_id) + started_at_col = cast(Any, JobRow.started_at) + ended_at_col = cast(Any, JobRow.ended_at) + exit_code_col = cast(Any, JobRow.exit_code) + if target in {JobStatusName.STARTING, JobStatusName.RUNNING}: + values["started_at"] = case( + (started_at_col.is_(None), now), + else_=started_at_col, + ) + if is_terminal_status(target): + values["ended_at"] = case( + (ended_at_col.is_(None), ended_at or now), + else_=ended_at_col, + ) + + async with self._session_factory() as session: + stmt = update(JobRow).where(job_id_col == job_id, status_col.in_(allowed_statuses)) + if require_exit_code_null: + stmt = stmt.where(exit_code_col.is_(None)) + result = await session.execute(stmt.values(**values)) + await session.commit() + if cast(Any, result).rowcount == 0: + return await self._get_job_row(job_id) + return await self._get_job_row(job_id) + + async def _live_runtime_state(self, job_id: str) -> tuple[bool, bool | None]: + """Read tmux session/pipe liveness for one job. + + If the pane disappears between the session probe and the pipe probe, this + helper degrades to `(False, None)` so reconciliation treats the job as a + missing-session case rather than a `pipe_failed` case. + """ + + session_exists = await self._tmux.session_exists(job_session_name(job_id)) + if not session_exists: + return False, None + pipe_active = await self._tmux.is_output_pipe_active(job_id=job_id) + if pipe_active is None: + return False, None + return True, pipe_active + + async def _insert_job_row(self, row: JobRow) -> bool: + async with self._session_factory() as session: + session.add(row) + try: + await session.commit() + except IntegrityError: + await session.rollback() + return False + return True + + async def _delete_job_row(self, job_id: str) -> None: + async with self._session_factory() as session: + result = await session.execute(delete(JobRow).where(cast(Any, JobRow.job_id) == job_id)) + if cast(Any, result).rowcount == 0: + raise ShellctlServerError(404, "job_not_found", f"Unknown job id: {job_id}") + await session.commit() + + async def _get_job_row(self, job_id: str) -> JobRow: + async with self._session_factory() as session: + row = await session.get(JobRow, job_id) + if row is None: + raise ShellctlServerError(404, "job_not_found", f"Unknown job id: {job_id}") + return row + + async def _list_job_rows(self, *, statuses: set[JobStatusName] | None = None) -> list[JobRow]: + created_at_col = cast(Any, JobRow.created_at) + status_col = cast(Any, JobRow.status) + async with self._session_factory() as session: + stmt = select(JobRow).order_by(created_at_col.desc()) + if statuses is not None: + stmt = stmt.where(status_col.in_([status.value for status in statuses])) + rows = (await session.execute(stmt)).scalars().all() + return list(rows) + + def _status_view_from_row(self, row: JobRow) -> JobStatusView: + status = JobStatusName(row.status) + output_path = self._output_log_path(row) + offset = output_path.stat().st_size if output_path.exists() else 0 + return JobStatusView( + job_id=row.job_id, + status=status, + done=is_terminal_status(status), + exit_code=row.exit_code, + created_at=row.created_at, + started_at=row.started_at, + ended_at=row.ended_at, + offset=offset, + ) + + def _job_result_from_view( + self, + view: JobStatusView, + row: JobRow, + window: OutputWindow, + *, + truncated: bool | None = None, + ) -> JobResult: + return JobResult( + job_id=view.job_id, + done=view.done, + status=view.status, + exit_code=view.exit_code, + output_path=str(self._output_log_path(row).resolve()), + output=window.output, + offset=window.offset, + truncated=window.truncated if truncated is None else truncated, + ) + + def _output_log_path(self, row: JobRow) -> Path: + return self.config.state_dir / row.output_path + + def _artifact_dir(self, job_id: str) -> Path: + return self.config.jobs_dir / job_id + + def _normal_exit_commit_pending(self, job_id: str) -> bool: + """Check whether a normal exit is still waiting for pipe drain + commit.""" + + job_dir = self._artifact_dir(job_id) + return ( + runner_exit_code_path(job_dir).exists() + and runner_ended_at_path(job_dir).exists() + and not pipe_drained_path(job_dir).exists() + and not pipe_failed_path(job_dir).exists() + ) + + def _drained_normal_exit_metadata(self, job_id: str) -> tuple[int, str] | None: + """Read drained normal-exit metadata after successful sanitize drain.""" + + job_dir = self._artifact_dir(job_id) + drained_path = pipe_drained_path(job_dir) + exit_code_path = runner_exit_code_path(job_dir) + ended_at_path = runner_ended_at_path(job_dir) + if not drained_path.exists() or not exit_code_path.exists() or not ended_at_path.exists(): + return None + raw_exit_code = exit_code_path.read_text(encoding="utf-8").strip() + raw_ended_at = ended_at_path.read_text(encoding="utf-8").strip() + if not raw_exit_code or not raw_ended_at: + return None + return int(raw_exit_code), raw_ended_at + + def _allocate_job_dir(self) -> tuple[str, Path]: + """Atomically allocate a unique artifact directory for a new job. + + The proposal requires `mkdir jobs/` itself to be the collision + detector. We therefore create the directory inside this helper and retry + on `FileExistsError` instead of using a separate `exists()` pre-check. + """ + + for _ in range(20): + job_id = generate_job_id() + job_dir = self.config.jobs_dir / job_id + try: + job_dir.mkdir(mode=0o700) + return job_id, job_dir + except FileExistsError: + continue + raise ShellctlServerError(500, "job_id_collision", "Failed to allocate a unique job id") + + def _resolve_cwd(self, raw_cwd: str | None) -> Path: + cwd = Path(raw_cwd).expanduser() if raw_cwd is not None else self.config.default_cwd.expanduser() + cwd = cwd.resolve() + if not cwd.exists() or not cwd.is_dir(): + raise ShellctlServerError(400, "invalid_cwd", f"cwd is not a directory: {cwd}") + return cwd + + def _resolve_env(self, raw_env: dict[str, str] | None) -> dict[str, str]: + """Return a detached environment overlay for the generated runner. + + `RunJobRequest` validates names/values before the service sees them. + The overlay augments the runner's inherited environment after shellctl + scrubs its own control variables, so explicit request values can be used + without replacing ambient entries like `PATH`. + """ + + return dict(raw_env or {}) + + def _validate_offset(self, output_path: Path, offset: int) -> None: + size = output_path.stat().st_size if output_path.exists() else 0 + if offset > size: + raise ShellctlServerError( + 400, + "invalid_offset", + f"offset {offset} exceeds current file size {size}", + ) + + @staticmethod + def _ensure_dir(path: Path) -> None: + path.mkdir(mode=0o700, parents=True, exist_ok=True) + path.chmod(0o700) + + def _install_runner(self) -> None: + self.config.runner_path.write_text(self._runner_script_source(), encoding="utf-8") + mode = self.config.runner_path.stat().st_mode + self.config.runner_path.chmod(mode | stat.S_IXUSR) + + def _runner_script_source(self) -> str: + """Build the bash runner installed into the shellctl runtime directory. + + The wrapper still handles gate waiting and exit metadata in bash, but it + delegates launch setup to Python so per-job env overlays can be loaded + from JSON without shell-escaping arbitrary values into `export` + statements. The Python helper then `exec`s the target process instead of + waiting on it, which keeps `SIGINT` behavior identical to running the + script directly in the tmux pane. The bootstrap uses `python -c ...` + instead of a stdin-fed heredoc so the exec'd job still inherits the tmux + pane's PTY on fd 0 and can receive later `send_input()` data. + """ + + auth_env = shlex.quote(DEFAULT_AUTH_TOKEN_ENV) + bootstrap_source = shlex.quote( + """ +import json +import os +import stat +import sys +from pathlib import Path + +script_path = Path(sys.argv[1]) +cwd = sys.argv[2] +env_path = Path(sys.argv[3]) + +env = os.environ.copy() +if env_path.exists(): + env.update(json.loads(env_path.read_text(encoding="utf-8"))) + +try: + os.chdir(cwd) +except OSError: + raise SystemExit(111) + +with script_path.open("r", encoding="utf-8") as handle: + first_line = handle.readline() + +if first_line.startswith("#!"): + script_path.chmod(script_path.stat().st_mode | stat.S_IXUSR) + argv = [str(script_path)] +else: + argv = ["sh", str(script_path)] + +try: + os.execvpe(argv[0], argv, env) +except FileNotFoundError as exc: + print(f"{argv[0]}: {exc.strerror}", file=sys.stderr) + raise SystemExit(127) from exc +except OSError as exc: + print(f"{argv[0]}: {exc.strerror}", file=sys.stderr) + raise SystemExit(126) from exc + """.strip() + ) + return f"""#!/usr/bin/env bash +set -uo pipefail + +JOB_DIR="$1" +JOB_ID="$2" +CWD="$3" +SCRIPT_PATH="$JOB_DIR/script" +ENV_PATH="$JOB_DIR/{JOB_ENV_FILENAME}" +START_GATE="$JOB_DIR/start-gate" +RUNNER_EXIT_CODE_PATH="$JOB_DIR/{RUNNER_EXIT_CODE_FILENAME}" +RUNNER_ENDED_AT_PATH="$JOB_DIR/{RUNNER_ENDED_AT_FILENAME}" + +write_atomic() {{ + local dest="$1" + local value="$2" + local tmp="${{dest}}.tmp.$$" + printf '%s\n' "$value" > "$tmp" + mv "$tmp" "$dest" +}} + +while [ ! -e "$START_GATE" ]; do + sleep 0.05 +done + +unset TMUX +unset SHELLCTL_STATE_DIR +unset SHELLCTL_RUNTIME_DIR +unset SHELLCTL_TMUX_SOCKET +unset SHELLCTL_RUNNER +unset {auth_env} + +{shlex.quote(sys.executable)} -c {bootstrap_source} "$SCRIPT_PATH" "$CWD" "$ENV_PATH" +EXIT_CODE=$? + +ENDED_AT="$({shlex.quote(sys.executable)} - <<'PY' +from datetime import UTC, datetime +print(datetime.now(UTC).replace(microsecond=0).isoformat().replace('+00:00', 'Z')) +PY +)" + +write_atomic "$RUNNER_EXIT_CODE_PATH" "$EXIT_CODE" +write_atomic "$RUNNER_ENDED_AT_PATH" "$ENDED_AT" + +exit "$EXIT_CODE" +""" + + +__all__ = ["ShellctlService", "anyio", "shutil"] diff --git a/dify-agent/src/shellctl/server/tmux.py b/dify-agent/src/shellctl/server/tmux.py new file mode 100644 index 00000000000..e412f812207 --- /dev/null +++ b/dify-agent/src/shellctl/server/tmux.py @@ -0,0 +1,343 @@ +"""tmux control layer for shellctl jobs. + +The rest of shellctl treats this module as the only place that knows tmux CLI + command shapes. Tests can replace `TmuxControllerProtocol` with a fake to + exercise SQLite/output semantics without depending on a local tmux daemon. +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import tempfile +from pathlib import Path +from typing import Protocol, cast + +import anyio + +from shellctl.server.artifacts import ( + pipe_drained_path, + pipe_error_log_path, + pipe_failed_path, + runner_ended_at_path, + runner_exit_code_path, +) +from shellctl.server.config import ShellctlConfig +from shellctl.server.errors import ShellctlServerError +from shellctl.shared.runtime import ( + job_pane_target, + job_session_name, +) +from shellctl.shared.schemas import TerminalSize + + +class TmuxControllerProtocol(Protocol): + """Protocol used by `ShellctlService` for tmux interactions.""" + + async def start_server(self) -> None: ... + + async def list_sessions(self) -> set[str]: ... + + async def session_exists(self, session_name: str) -> bool: ... + + async def is_output_pipe_active(self, *, job_id: str) -> bool | None: ... + + async def create_job_session( + self, + *, + job_id: str, + job_dir: Path, + cwd: Path, + terminal: TerminalSize, + ) -> None: ... + + async def enable_output_pipe(self, *, job_id: str, job_dir: Path, ready_file: Path) -> None: ... + + async def send_input(self, *, job_id: str, text: str) -> None: ... + + async def send_interrupt(self, *, job_id: str) -> None: ... + + async def cleanup_session(self, *, job_id: str) -> None: ... + + +class TmuxController: + """Best-effort wrapper around a dedicated tmux socket. + + The controller always clears `TMUX` from the child environment and always + passes `-S ` so shellctl sessions stay isolated from the user's + default tmux server. + """ + + def __init__(self, config: ShellctlConfig) -> None: + self._config = config + + async def start_server(self) -> None: + await self._run_tmux("start-server") + + async def list_sessions(self) -> set[str]: + result = await self._run_tmux("list-sessions", "-F", "#{session_name}", check=False) + if result.returncode != 0: + stderr = result.stderr.decode("utf-8", errors="replace") + if _tmux_target_missing(stderr): + return set() + raise ShellctlServerError(500, "tmux_error", stderr.strip() or "tmux list-sessions failed") + output = result.stdout.decode("utf-8", errors="replace") + return {line.strip() for line in output.splitlines() if line.strip()} + + async def session_exists(self, session_name: str) -> bool: + return session_name in await self.list_sessions() + + async def is_output_pipe_active(self, *, job_id: str) -> bool | None: + result = await self._run_tmux( + "display-message", + "-p", + "-t", + job_pane_target(job_id), + "#{pane_pipe}", + check=False, + ) + if result.returncode != 0: + stderr = result.stderr.decode("utf-8", errors="replace") + if _tmux_target_missing(stderr): + return None + raise ShellctlServerError( + 500, + "tmux_error", + stderr.strip() or f"Failed to inspect output pipe for {job_id}", + ) + return result.stdout.decode("utf-8", errors="replace").strip() == "1" + + async def create_job_session( + self, + *, + job_id: str, + job_dir: Path, + cwd: Path, + terminal: TerminalSize, + ) -> None: + runner_command = self._shell_join( + [ + str(self._config.runner_path), + str(job_dir), + job_id, + str(cwd), + ] + ) + result = await self._run_tmux( + "-f", + "/dev/null", + "new-session", + "-d", + "-s", + job_session_name(job_id), + "-x", + str(terminal.cols), + "-y", + str(terminal.rows), + runner_command, + check=False, + ) + if result.returncode != 0: + raise ShellctlServerError( + 500, + "tmux_new_session_failed", + result.stderr.decode("utf-8", errors="replace").strip() + or f"Failed to create tmux session for {job_id}", + ) + + async def enable_output_pipe(self, *, job_id: str, job_dir: Path, ready_file: Path) -> None: + output_command = self._pipe_command_source( + job_id=job_id, + job_dir=job_dir, + ready_file=ready_file, + ) + result = await self._run_tmux( + "pipe-pane", + "-o", + "-t", + job_pane_target(job_id), + output_command, + check=False, + ) + if result.returncode != 0: + raise ShellctlServerError( + 500, + "pipe_failed", + result.stderr.decode("utf-8", errors="replace").strip() or f"Failed to attach output pipe for {job_id}", + ) + + def _pipe_command_source(self, *, job_id: str, job_dir: Path, ready_file: Path) -> str: + """Build the tmux `pipe-pane` command that drains and finalizes output. + + For normal exits, the runner now records completion metadata into job + artifacts and the pipe finalizer commits `runner-exit` only after + the lightweight sanitizer reaches EOF and flushes `output.log` + successfully. Sanitizer stderr is captured into `pipe-error.log` so + startup timeouts can distinguish slow imports from subprocess crashes. + If the follow-up `runner-exit` write fails, the drain marker remains in + place and stderr is appended to the same log with an explicit status + line. The pipe still exits with the sanitizer status so a drained job is + not misclassified as `pipe_failed` before reconciliation can recover the + SQLite write from the drained artifacts. + """ + + sanitize_command = self._shell_join( + ( + *self._config.sanitize_pty_command, + "--ready-file", + str(ready_file), + ) + ) + runner_exit_command = self._shell_join( + ( + *self._config.runner_exit_command, + "--state-dir", + str(self._config.state_dir), + "--job-id", + job_id, + "--sqlite-busy-timeout-ms", + str(self._config.sqlite_busy_timeout_ms), + ) + ) + output_path = shlex.quote(str(job_dir / "output.log")) + drained_path = shlex.quote(str(pipe_drained_path(job_dir))) + error_log_path = shlex.quote(str(pipe_error_log_path(job_dir))) + failed_path = shlex.quote(str(pipe_failed_path(job_dir))) + exit_code_path = shlex.quote(str(runner_exit_code_path(job_dir))) + ended_at_path = shlex.quote(str(runner_ended_at_path(job_dir))) + return " ; ".join( + [ + f"{sanitize_command} >> {output_path} 2> {error_log_path}", + "sanitize_status=$?", + "runner_exit_status=0", + ( + 'if [ "$sanitize_status" -eq 0 ]; then ' + f": > {drained_path}; " + f"if [ -s {exit_code_path} ] && [ -s {ended_at_path} ]; then " + f'{runner_exit_command} --exit-code "$(cat {exit_code_path})" ' + f'--ended-at "$(cat {ended_at_path})" 2>> {error_log_path}; ' + "runner_exit_status=$?; " + 'if [ "$runner_exit_status" -ne 0 ]; then ' + f"printf 'runner-exit failed with status %s\\n' \"$runner_exit_status\" >> {error_log_path}; " + "fi; fi; " + f"else : > {failed_path}; fi" + ), + 'if [ "$sanitize_status" -ne 0 ]; then exit "$sanitize_status"; fi', + 'exit "$sanitize_status"', + ] + ) + + async def send_input(self, *, job_id: str, text: str) -> None: + buffer_name = f"shellctl-in-{job_id}" + runtime_dir = cast(Path, self._config.runtime_dir) + runtime_dir.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(prefix=f"shellctl-input-{job_id}-", dir=runtime_dir) + tmp_path = Path(tmp_name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(text) + load_result = await self._run_tmux( + "load-buffer", + "-b", + buffer_name, + str(tmp_path), + check=False, + ) + if load_result.returncode != 0: + stderr = load_result.stderr.decode("utf-8", errors="replace").strip() + if _tmux_target_missing(stderr): + raise ShellctlServerError( + 409, + "tmux_target_missing", + stderr or f"The tmux pane for {job_id} is no longer available", + ) + raise ShellctlServerError( + 500, + "tmux_input_failed", + stderr or f"Failed to load input buffer for {job_id}", + ) + paste_result = await self._run_tmux( + "paste-buffer", + "-t", + job_pane_target(job_id), + "-b", + buffer_name, + check=False, + ) + if paste_result.returncode != 0: + stderr = paste_result.stderr.decode("utf-8", errors="replace").strip() + if _tmux_target_missing(stderr): + raise ShellctlServerError( + 409, + "tmux_target_missing", + stderr or f"The tmux pane for {job_id} is no longer available", + ) + raise ShellctlServerError( + 500, + "tmux_input_failed", + stderr or f"Failed to paste input buffer for {job_id}", + ) + finally: + await self._run_tmux("delete-buffer", "-b", buffer_name, check=False) + tmp_path.unlink(missing_ok=True) + + async def send_interrupt(self, *, job_id: str) -> None: + await self._run_tmux( + "send-keys", + "-t", + job_pane_target(job_id), + "C-c", + check=False, + ) + + async def cleanup_session(self, *, job_id: str) -> None: + await self._run_tmux( + "kill-session", + "-t", + job_session_name(job_id), + check=False, + ) + + async def _run_tmux(self, *args: str, check: bool = True) -> subprocess.CompletedProcess[bytes]: + env = dict(os.environ) + env.pop("TMUX", None) + try: + result = await anyio.run_process( + ["tmux", "-S", str(self._config.tmux_socket), *args], + env=env, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except FileNotFoundError as exc: + raise ShellctlServerError(500, "tmux_not_installed", "tmux executable was not found") from exc + if check and result.returncode != 0: + raise ShellctlServerError( + 500, + "tmux_error", + result.stderr.decode("utf-8", errors="replace").strip() or "tmux command failed", + ) + return result + + @staticmethod + def _shell_join(parts: tuple[str, ...] | list[str]) -> str: + return " ".join(shlex.quote(part) for part in parts) + + +def _tmux_target_missing(stderr: str) -> bool: + normalized = stderr.lower() + return ( + "can't find pane" in normalized + or "can't find session" in normalized + or "no server running" in normalized + or "failed to connect" in normalized + or "server exited unexpectedly" in normalized + ) + + +__all__ = [ + "TmuxController", + "TmuxControllerProtocol", + "_tmux_target_missing", +] diff --git a/dify-agent/src/shellctl/shared/__init__.py b/dify-agent/src/shellctl/shared/__init__.py new file mode 100644 index 00000000000..5f235b19d9f --- /dev/null +++ b/dify-agent/src/shellctl/shared/__init__.py @@ -0,0 +1,182 @@ +"""Shared shellctl transport/runtime helpers. + +This package preserves the historical import surface while keeping the package +root lazy. Lightweight callers can import concrete submodules such as +`shared.runtime` without eagerly importing the pydantic schema layer, output +or helpers outside the shared compatibility surface. +""" + +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from shellctl.shared.constants import ( + DEFAULT_AUTH_TOKEN_ENV, + DEFAULT_BASE_URL, + DEFAULT_BASE_URL_ENV, + DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS, + DEFAULT_GC_INTERVAL_SECONDS, + DEFAULT_HEALTH_STATUS, + DEFAULT_IDLE_FLUSH_SECONDS, + DEFAULT_LIST_LIMIT, + DEFAULT_OUTPUT_LIMIT_BYTES, + DEFAULT_TERMINAL_COLS, + DEFAULT_TERMINAL_ROWS, + DEFAULT_TERMINATE_GRACE_SECONDS, + DEFAULT_TIMEOUT_SECONDS, + JOB_ID_ALPHABET, + JOB_ID_RANDOM_SUFFIX_LENGTH, + MAX_LIST_LIMIT, + MAX_OUTPUT_LIMIT_BYTES, + MAX_WAIT_TIMEOUT_SECONDS, + SESSION_NAME_PREFIX, + ) + from shellctl.shared.output import ( + OutputWindow, + read_output_window, + tail_output_window, + ) + from shellctl.shared.runtime import ( + default_runtime_dir, + default_state_dir, + format_timestamp, + generate_job_id, + is_terminal_status, + job_pane_target, + job_session_name, + parse_timestamp, + utc_now, + ) + from shellctl.shared.schemas import ( + TERMINAL_JOB_STATUSES, + DeleteJobResponse, + ErrorDetail, + ErrorResponse, + HealthResponse, + InputJobRequest, + JobInfo, + JobResult, + JobStatusName, + JobStatusView, + ListJobsResponse, + RunJobRequest, + ShellctlModel, + TerminalSize, + TerminateJobRequest, + WaitJobRequest, + ) + +__all__ = [ + "DEFAULT_AUTH_TOKEN_ENV", + "DEFAULT_BASE_URL", + "DEFAULT_BASE_URL_ENV", + "DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS", + "DEFAULT_GC_INTERVAL_SECONDS", + "DEFAULT_HEALTH_STATUS", + "DEFAULT_IDLE_FLUSH_SECONDS", + "DEFAULT_LIST_LIMIT", + "DEFAULT_OUTPUT_LIMIT_BYTES", + "DEFAULT_TERMINAL_COLS", + "DEFAULT_TERMINAL_ROWS", + "DEFAULT_TERMINATE_GRACE_SECONDS", + "DEFAULT_TIMEOUT_SECONDS", + "JOB_ID_ALPHABET", + "JOB_ID_RANDOM_SUFFIX_LENGTH", + "MAX_LIST_LIMIT", + "MAX_OUTPUT_LIMIT_BYTES", + "MAX_WAIT_TIMEOUT_SECONDS", + "SESSION_NAME_PREFIX", + "TERMINAL_JOB_STATUSES", + "DeleteJobResponse", + "ErrorDetail", + "ErrorResponse", + "HealthResponse", + "InputJobRequest", + "JobInfo", + "JobResult", + "JobStatusName", + "JobStatusView", + "ListJobsResponse", + "OutputWindow", + "RunJobRequest", + "ShellctlModel", + "TerminalSize", + "TerminateJobRequest", + "WaitJobRequest", + "default_runtime_dir", + "default_state_dir", + "format_timestamp", + "generate_job_id", + "is_terminal_status", + "job_pane_target", + "job_session_name", + "parse_timestamp", + "read_output_window", + "tail_output_window", + "utc_now", +] + +_EXPORTS = { + "DEFAULT_AUTH_TOKEN_ENV": "shellctl.shared.constants", + "DEFAULT_BASE_URL": "shellctl.shared.constants", + "DEFAULT_BASE_URL_ENV": "shellctl.shared.constants", + "DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS": "shellctl.shared.constants", + "DEFAULT_GC_INTERVAL_SECONDS": "shellctl.shared.constants", + "DEFAULT_HEALTH_STATUS": "shellctl.shared.constants", + "DEFAULT_IDLE_FLUSH_SECONDS": "shellctl.shared.constants", + "DEFAULT_LIST_LIMIT": "shellctl.shared.constants", + "DEFAULT_OUTPUT_LIMIT_BYTES": "shellctl.shared.constants", + "DEFAULT_TERMINAL_COLS": "shellctl.shared.constants", + "DEFAULT_TERMINAL_ROWS": "shellctl.shared.constants", + "DEFAULT_TERMINATE_GRACE_SECONDS": "shellctl.shared.constants", + "DEFAULT_TIMEOUT_SECONDS": "shellctl.shared.constants", + "JOB_ID_ALPHABET": "shellctl.shared.constants", + "JOB_ID_RANDOM_SUFFIX_LENGTH": "shellctl.shared.constants", + "MAX_LIST_LIMIT": "shellctl.shared.constants", + "MAX_OUTPUT_LIMIT_BYTES": "shellctl.shared.constants", + "MAX_WAIT_TIMEOUT_SECONDS": "shellctl.shared.constants", + "SESSION_NAME_PREFIX": "shellctl.shared.constants", + "OutputWindow": "shellctl.shared.output", + "read_output_window": "shellctl.shared.output", + "tail_output_window": "shellctl.shared.output", + "default_runtime_dir": "shellctl.shared.runtime", + "default_state_dir": "shellctl.shared.runtime", + "format_timestamp": "shellctl.shared.runtime", + "generate_job_id": "shellctl.shared.runtime", + "is_terminal_status": "shellctl.shared.runtime", + "job_pane_target": "shellctl.shared.runtime", + "job_session_name": "shellctl.shared.runtime", + "parse_timestamp": "shellctl.shared.runtime", + "utc_now": "shellctl.shared.runtime", + "TERMINAL_JOB_STATUSES": "shellctl.shared.schemas", + "DeleteJobResponse": "shellctl.shared.schemas", + "ErrorDetail": "shellctl.shared.schemas", + "ErrorResponse": "shellctl.shared.schemas", + "HealthResponse": "shellctl.shared.schemas", + "InputJobRequest": "shellctl.shared.schemas", + "JobInfo": "shellctl.shared.schemas", + "JobResult": "shellctl.shared.schemas", + "JobStatusName": "shellctl.shared.schemas", + "JobStatusView": "shellctl.shared.schemas", + "ListJobsResponse": "shellctl.shared.schemas", + "RunJobRequest": "shellctl.shared.schemas", + "ShellctlModel": "shellctl.shared.schemas", + "TerminalSize": "shellctl.shared.schemas", + "TerminateJobRequest": "shellctl.shared.schemas", + "WaitJobRequest": "shellctl.shared.schemas", +} + + +def __getattr__(name: str) -> Any: + if name not in _EXPORTS: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module = import_module(_EXPORTS[name]) + value = getattr(module, name) # noqa: no-new-getattr lazy export proxy + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__)) diff --git a/dify-agent/src/shellctl/shared/constants.py b/dify-agent/src/shellctl/shared/constants.py new file mode 100644 index 00000000000..83eb78160f5 --- /dev/null +++ b/dify-agent/src/shellctl/shared/constants.py @@ -0,0 +1,49 @@ +"""Shared shellctl constants. + +This module intentionally contains only literal defaults and bounds so callers +can import stable configuration values without pulling in the heavier DTO, +sanitize, or server packages. The network CLI depends on this file for its +base URL and auth-token environment contract, so keep it import-light. +""" + +DEFAULT_AUTH_TOKEN_ENV = "SHELLCTL_AUTH_TOKEN" +DEFAULT_BASE_URL_ENV = "SHELLCTL_BASE_URL" +DEFAULT_BASE_URL = "http://127.0.0.1:8765" +DEFAULT_OUTPUT_LIMIT_BYTES = 1024 * 8 +MAX_OUTPUT_LIMIT_BYTES = 1024 * 1024 +DEFAULT_TIMEOUT_SECONDS = 30.0 +MAX_WAIT_TIMEOUT_SECONDS = 5.0 * 60.0 +DEFAULT_IDLE_FLUSH_SECONDS = 0.5 +DEFAULT_TERMINATE_GRACE_SECONDS = 5.0 +DEFAULT_TERMINAL_COLS = 120 +DEFAULT_TERMINAL_ROWS = 80 +DEFAULT_LIST_LIMIT = 100 +MAX_LIST_LIMIT = 1000 +DEFAULT_GC_INTERVAL_SECONDS = 10.0 * 60.0 +DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS = 24.0 * 60.0 * 60.0 +JOB_ID_ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz" +JOB_ID_RANDOM_SUFFIX_LENGTH = 3 +SESSION_NAME_PREFIX = "shellctl-job-" +DEFAULT_HEALTH_STATUS = "ok" + +__all__ = [ + "DEFAULT_AUTH_TOKEN_ENV", + "DEFAULT_BASE_URL", + "DEFAULT_BASE_URL_ENV", + "DEFAULT_GC_FINISHED_JOB_RETENTION_SECONDS", + "DEFAULT_GC_INTERVAL_SECONDS", + "DEFAULT_HEALTH_STATUS", + "DEFAULT_IDLE_FLUSH_SECONDS", + "DEFAULT_LIST_LIMIT", + "DEFAULT_OUTPUT_LIMIT_BYTES", + "DEFAULT_TERMINAL_COLS", + "DEFAULT_TERMINAL_ROWS", + "DEFAULT_TERMINATE_GRACE_SECONDS", + "DEFAULT_TIMEOUT_SECONDS", + "JOB_ID_ALPHABET", + "JOB_ID_RANDOM_SUFFIX_LENGTH", + "MAX_LIST_LIMIT", + "MAX_OUTPUT_LIMIT_BYTES", + "MAX_WAIT_TIMEOUT_SECONDS", + "SESSION_NAME_PREFIX", +] diff --git a/dify-agent/src/shellctl/shared/output.py b/dify-agent/src/shellctl/shared/output.py new file mode 100644 index 00000000000..ca24419b795 --- /dev/null +++ b/dify-agent/src/shellctl/shared/output.py @@ -0,0 +1,120 @@ +"""UTF-8-safe output slicing helpers for shellctl.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(slots=True, frozen=True) +class OutputWindow: + """UTF-8-safe slice of `output.log`.""" + + output: str + offset: int + truncated: bool + + +def read_output_window(path: Path, *, offset: int, limit: int) -> OutputWindow: + """Read a forward UTF-8-safe slice from `output.log`.""" + + if offset < 0: + raise ValueError(f"offset must be >= 0; got {offset}") + if limit <= 0: + raise ValueError(f"limit must be > 0; got {limit}") + + if not path.exists(): + if offset == 0: + return OutputWindow(output="", offset=0, truncated=False) + raise ValueError(f"offset {offset} exceeds current file size 0") + + size = path.stat().st_size + if offset > size: + raise ValueError(f"offset {offset} exceeds current file size {size}") + if offset == size: + return OutputWindow(output="", offset=offset, truncated=False) + + with path.open("rb") as handle: + handle.seek(offset) + raw = handle.read(limit + 4) + + start_shift = _advance_to_utf8_boundary(raw, 0) + data = raw[start_shift:] + budget = max(0, limit - start_shift) + consumed = _valid_utf8_prefix_length(data[:budget]) + if consumed == 0 and data: + consumed = _first_complete_utf8_char_length(data) + output_bytes = data[:consumed] + new_offset = offset + start_shift + consumed + truncated = new_offset < size + return OutputWindow( + output=output_bytes.decode("utf-8", errors="strict"), + offset=new_offset, + truncated=truncated, + ) + + +def tail_output_window(path: Path, *, limit: int) -> OutputWindow: + """Read a UTF-8-safe tail snapshot from `output.log`.""" + + if limit <= 0: + raise ValueError(f"limit must be > 0; got {limit}") + if not path.exists(): + return OutputWindow(output="", offset=0, truncated=False) + size = path.stat().st_size + if size == 0: + return OutputWindow(output="", offset=0, truncated=False) + start = max(0, size - limit) + padded_start = max(0, start - 4) + with path.open("rb") as handle: + handle.seek(padded_start) + raw = handle.read(size - padded_start) + relative_start = _advance_to_utf8_boundary(raw, start - padded_start) + payload = raw[relative_start:] + consumed = _valid_utf8_prefix_length(payload) + output_bytes = payload[:consumed] + return OutputWindow( + output=output_bytes.decode("utf-8", errors="strict"), + offset=padded_start + relative_start + consumed, + truncated=False, + ) + + +def _advance_to_utf8_boundary(data: bytes, start: int) -> int: + while start < len(data) and _is_utf8_continuation_byte(data[start]): + start += 1 + return start + + +def _valid_utf8_prefix_length(data: bytes) -> int: + end = len(data) + while end >= 0: + try: + data[:end].decode("utf-8", errors="strict") + return end + except UnicodeDecodeError as exc: + if exc.start < end - 4: + end = exc.start + else: + end -= 1 + return 0 + + +def _is_utf8_continuation_byte(value: int) -> bool: + return (value & 0b1100_0000) == 0b1000_0000 + + +def _first_complete_utf8_char_length(data: bytes) -> int: + for end in range(1, len(data) + 1): + try: + decoded = data[:end].decode("utf-8", errors="strict") + except UnicodeDecodeError: + continue + if len(decoded) == 1: + return end + if decoded: + return len(decoded[0].encode("utf-8")) + return 0 + + +__all__ = ["OutputWindow", "read_output_window", "tail_output_window"] diff --git a/dify-agent/src/shellctl/shared/runtime.py b/dify-agent/src/shellctl/shared/runtime.py new file mode 100644 index 00000000000..b28423befa9 --- /dev/null +++ b/dify-agent/src/shellctl/shared/runtime.py @@ -0,0 +1,95 @@ +"""Stable runtime/naming helpers shared across shellctl modules.""" + +from __future__ import annotations + +import os +import secrets +from datetime import UTC, datetime +from pathlib import Path + +from shellctl.shared.constants import ( + JOB_ID_ALPHABET, + JOB_ID_RANDOM_SUFFIX_LENGTH, + SESSION_NAME_PREFIX, +) +from shellctl.shared.schemas import ( + TERMINAL_JOB_STATUSES, + JobStatusName, +) + + +def utc_now() -> datetime: + """Return the current UTC time with timezone information.""" + + return datetime.now(UTC) + + +def format_timestamp(value: datetime | None = None) -> str: + """Format a UTC timestamp in the stable artifact/API representation.""" + + moment = (value or utc_now()).astimezone(UTC).replace(microsecond=0) + return moment.isoformat().replace("+00:00", "Z") + + +def parse_timestamp(value: str) -> datetime: + """Parse an artifact/API timestamp back into a timezone-aware datetime.""" + + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC) + + +def is_terminal_status(status: JobStatusName) -> bool: + """Check whether a lifecycle state is terminal.""" + + return status in TERMINAL_JOB_STATUSES + + +def generate_job_id(*, now: datetime | None = None) -> str: + """Generate a short, human-readable job id.""" + + timestamp = (now or utc_now()).astimezone(UTC).strftime("%m%d%H%M") + suffix = "".join(secrets.choice(JOB_ID_ALPHABET) for _ in range(JOB_ID_RANDOM_SUFFIX_LENGTH)) + return f"{timestamp}-{suffix}" + + +def job_session_name(job_id: str) -> str: + """Return the dedicated tmux session name for a job.""" + + return f"{SESSION_NAME_PREFIX}{job_id}" + + +def job_pane_target(job_id: str) -> str: + """Return the canonical tmux pane target for a single-pane job session.""" + + return f"{job_session_name(job_id)}:0.0" + + +def default_state_dir() -> Path: + """Resolve the XDG-style default shellctl state directory.""" + + xdg_state_home = os.environ.get("XDG_STATE_HOME") + if xdg_state_home: + return Path(xdg_state_home) / "shellctl" + return Path.home() / ".local" / "state" / "shellctl" + + +def default_runtime_dir(state_dir: Path | None = None) -> Path: + """Resolve the XDG-style default shellctl runtime directory.""" + + xdg_runtime_dir = os.environ.get("XDG_RUNTIME_DIR") + if xdg_runtime_dir: + return Path(xdg_runtime_dir) / "shellctl" + base_state_dir = state_dir or default_state_dir() + return base_state_dir / "run" / "shellctl" + + +__all__ = [ + "default_runtime_dir", + "default_state_dir", + "format_timestamp", + "generate_job_id", + "is_terminal_status", + "job_pane_target", + "job_session_name", + "parse_timestamp", + "utc_now", +] diff --git a/dify-agent/src/shellctl/shared/schemas.py b/dify-agent/src/shellctl/shared/schemas.py new file mode 100644 index 00000000000..0120b8b8a5f --- /dev/null +++ b/dify-agent/src/shellctl/shared/schemas.py @@ -0,0 +1,213 @@ +"""Shared pydantic transport models for shellctl.""" + +from __future__ import annotations + +from enum import StrEnum + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from shellctl.shared.constants import ( + DEFAULT_HEALTH_STATUS, + DEFAULT_IDLE_FLUSH_SECONDS, + DEFAULT_OUTPUT_LIMIT_BYTES, + DEFAULT_TERMINAL_COLS, + DEFAULT_TERMINAL_ROWS, + DEFAULT_TERMINATE_GRACE_SECONDS, + DEFAULT_TIMEOUT_SECONDS, + MAX_OUTPUT_LIMIT_BYTES, + MAX_WAIT_TIMEOUT_SECONDS, +) + + +class ShellctlModel(BaseModel): + """Base pydantic model with strict extra-field handling. + + shellctl uses these DTOs directly for HTTP request/response bodies, so + silently accepting unknown fields would make it harder to detect schema + drift. + """ + + model_config = ConfigDict(extra="forbid") + + +class JobStatusName(StrEnum): + """Lifecycle states materialized into SQLite rows and API responses.""" + + CREATED = "created" + STARTING = "starting" + RUNNING = "running" + EXITED = "exited" + TERMINATED = "terminated" + FAILED = "failed" + LOST = "lost" + + +TERMINAL_JOB_STATUSES = frozenset( + { + JobStatusName.EXITED, + JobStatusName.TERMINATED, + JobStatusName.FAILED, + JobStatusName.LOST, + } +) + + +class TerminalSize(ShellctlModel): + """Requested initial PTY geometry for a job.""" + + cols: int = Field(default=DEFAULT_TERMINAL_COLS, ge=1, le=4096) + rows: int = Field(default=DEFAULT_TERMINAL_ROWS, ge=1, le=4096) + + +class JobResult(ShellctlModel): + """Unified response shape for output-oriented job APIs.""" + + job_id: str + done: bool + status: JobStatusName + exit_code: int | None = None + output_path: str + output: str + offset: int = Field(ge=0) + truncated: bool + + +class JobStatusView(ShellctlModel): + """Materialized lifecycle view returned by status-like APIs.""" + + job_id: str + status: JobStatusName + done: bool + exit_code: int | None = None + created_at: str + started_at: str | None = None + ended_at: str | None = None + offset: int = Field(ge=0) + + +class JobInfo(ShellctlModel): + """Compact job listing record.""" + + job_id: str + status: JobStatusName + created_at: str + started_at: str | None = None + ended_at: str | None = None + + +class ListJobsResponse(ShellctlModel): + """Response body for `GET /v1/jobs`.""" + + jobs: list[JobInfo] + + +class DeleteJobResponse(ShellctlModel): + """Response body for successful delete operations.""" + + job_id: str + deleted: bool = True + + +class HealthResponse(ShellctlModel): + """Public health check response.""" + + status: str = DEFAULT_HEALTH_STATUS + + +class ErrorDetail(ShellctlModel): + """Machine-readable API error payload.""" + + code: str + message: str + + +class ErrorResponse(ShellctlModel): + """Envelope used by server-side exception handlers.""" + + error: ErrorDetail + + +class RunJobRequest(ShellctlModel): + """HTTP request body for `POST /v1/jobs/run`. + + `env` augments the runner's inherited process environment instead of + replacing it, so callers can preset script-local variables without losing + ambient values such as `PATH`. + """ + + script: str + cwd: str | None = None + env: dict[str, str] | None = None + terminal: TerminalSize | None = None + timeout: float = Field(default=DEFAULT_TIMEOUT_SECONDS, gt=0, le=MAX_WAIT_TIMEOUT_SECONDS) + output_limit: int = Field(default=DEFAULT_OUTPUT_LIMIT_BYTES, ge=1, le=MAX_OUTPUT_LIMIT_BYTES) + idle_flush_seconds: float = Field(default=DEFAULT_IDLE_FLUSH_SECONDS, ge=0, le=30) + + @field_validator("env") + @classmethod + def _validate_env(cls, env: dict[str, str] | None) -> dict[str, str] | None: + """Reject env entries that cannot be represented in `execve`. + + shellctl applies `env` as a process environment overlay, so validation + follows the low-level `NAME=value` constraints instead of shell variable + naming rules: names must be non-empty and cannot contain `=` or NUL, + while values cannot contain NUL. + """ + + if env is None: + return None + for name, value in env.items(): + if not name: + raise ValueError("env names must be non-empty") + if "=" in name: + raise ValueError(f"env name must not contain '=': {name!r}") + if "\x00" in name: + raise ValueError(f"env name must not contain NUL: {name!r}") + if "\x00" in value: + raise ValueError(f"env value must not contain NUL: {name!r}") + return env + + +class WaitJobRequest(ShellctlModel): + """HTTP request body for `POST /v1/jobs/{job_id}/wait`.""" + + timeout: float = Field(default=DEFAULT_TIMEOUT_SECONDS, ge=0, le=MAX_WAIT_TIMEOUT_SECONDS) + offset: int = Field(ge=0) + output_limit: int = Field(default=DEFAULT_OUTPUT_LIMIT_BYTES, ge=1, le=MAX_OUTPUT_LIMIT_BYTES) + idle_flush_seconds: float = Field(default=DEFAULT_IDLE_FLUSH_SECONDS, ge=0, le=30) + + +class InputJobRequest(ShellctlModel): + """HTTP request body for `POST /v1/jobs/{job_id}/input`.""" + + text: str + timeout: float = Field(default=DEFAULT_TIMEOUT_SECONDS, gt=0, le=MAX_WAIT_TIMEOUT_SECONDS) + offset: int = Field(ge=0) + output_limit: int = Field(default=DEFAULT_OUTPUT_LIMIT_BYTES, ge=1, le=MAX_OUTPUT_LIMIT_BYTES) + idle_flush_seconds: float = Field(default=DEFAULT_IDLE_FLUSH_SECONDS, ge=0, le=30) + + +class TerminateJobRequest(ShellctlModel): + """HTTP request body for `POST /v1/jobs/{job_id}/terminate`.""" + + grace_seconds: float = Field(default=DEFAULT_TERMINATE_GRACE_SECONDS, ge=0, le=300) + + +__all__ = [ + "TERMINAL_JOB_STATUSES", + "DeleteJobResponse", + "ErrorDetail", + "ErrorResponse", + "HealthResponse", + "InputJobRequest", + "JobInfo", + "JobResult", + "JobStatusName", + "JobStatusView", + "ListJobsResponse", + "RunJobRequest", + "ShellctlModel", + "TerminalSize", + "TerminateJobRequest", + "WaitJobRequest", +] diff --git a/dify-agent/src/shellctl_runtime/__init__.py b/dify-agent/src/shellctl_runtime/__init__.py new file mode 100644 index 00000000000..daa28c47ea5 --- /dev/null +++ b/dify-agent/src/shellctl_runtime/__init__.py @@ -0,0 +1,7 @@ +"""Stdlib-only shellctl subprocess entrypoints. + +This package is reserved for tmux hot-path helpers that must start quickly for +every job. Keep `__init__` import-free so `shellctl-sanitize-pty` and +`shellctl-runner-exit` do not accidentally pull in the main shellctl client or +server stacks. +""" diff --git a/dify-agent/src/shellctl_runtime/paths.py b/dify-agent/src/shellctl_runtime/paths.py new file mode 100644 index 00000000000..4ba2295a658 --- /dev/null +++ b/dify-agent/src/shellctl_runtime/paths.py @@ -0,0 +1,16 @@ +"""Stdlib-only filesystem helpers shared by shellctl runtime commands.""" + +from __future__ import annotations + +from pathlib import Path + +DEFAULT_SQLITE_BUSY_TIMEOUT_MS = 5000 + + +def db_path_from_state_dir(state_dir: Path) -> Path: + """Resolve the SQLite database path for a shellctl state directory.""" + + return state_dir / "shellctl.db" + + +__all__ = ["DEFAULT_SQLITE_BUSY_TIMEOUT_MS", "db_path_from_state_dir"] diff --git a/dify-agent/src/shellctl_runtime/py.typed b/dify-agent/src/shellctl_runtime/py.typed new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/dify-agent/src/shellctl_runtime/py.typed @@ -0,0 +1 @@ + diff --git a/dify-agent/src/shellctl_runtime/runner_exit.py b/dify-agent/src/shellctl_runtime/runner_exit.py new file mode 100644 index 00000000000..b78f1758a73 --- /dev/null +++ b/dify-agent/src/shellctl_runtime/runner_exit.py @@ -0,0 +1,158 @@ +"""Lightweight SQLite updater for drained shellctl job exits. + +This command runs after the tmux output pipe reaches EOF and flushes +`output.log`. It must stay stdlib-only so exit-state materialization does not +pay the FastAPI/SQLAlchemy import cost on every job completion. The CLI accepts +the busy-timeout override from `ShellctlConfig` so non-default deployments keep +the same SQLite locking behavior in and out of process. +""" + +from __future__ import annotations + +import argparse +import sqlite3 +from pathlib import Path + +from shellctl_runtime.paths import ( + DEFAULT_SQLITE_BUSY_TIMEOUT_MS, + db_path_from_state_dir, +) + +TERMINAL_STATUSES = frozenset({"exited", "terminated", "failed", "lost"}) +NONTERMINAL_STATUSES = frozenset({"created", "starting", "running"}) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the tmux finalizer CLI contract.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--state-dir", required=True, type=Path) + parser.add_argument("--job-id", required=True) + parser.add_argument("--exit-code", required=True, type=int) + parser.add_argument("--ended-at", required=True) + parser.add_argument( + "--sqlite-busy-timeout-ms", + type=int, + default=DEFAULT_SQLITE_BUSY_TIMEOUT_MS, + ) + return parser.parse_args(argv) + + +def record_runner_exit( + *, + state_dir: Path, + job_id: str, + exit_code: int, + ended_at: str, + busy_timeout_ms: int = DEFAULT_SQLITE_BUSY_TIMEOUT_MS, +) -> None: + """Persist a drained runner exit directly into the shellctl SQLite DB. + + The update is idempotent for terminal rows: once a job reaches a terminal + state, this helper returns successfully without rewriting the row. The + terminal/non-terminal decision is repeated inside the `UPDATE` itself so a + concurrent writer cannot be clobbered by a stale pre-read. + """ + + db_path = db_path_from_state_dir(state_dir) + if not db_path.exists(): + raise FileNotFoundError(f"shellctl database not found: {db_path}") + + connection = sqlite3.connect(db_path, timeout=busy_timeout_ms / 1000) + try: + connection.execute(f"PRAGMA busy_timeout={busy_timeout_ms}") + row = connection.execute( + "SELECT status FROM jobs WHERE job_id = ?", + (job_id,), + ).fetchone() + if row is None: + raise LookupError(f"Unknown job id: {job_id}") + status = str(row[0]) + if status in TERMINAL_STATUSES: + return + if status not in NONTERMINAL_STATUSES: + raise ValueError(f"Unsupported shellctl job status: {status}") + nonterminal = tuple(NONTERMINAL_STATUSES) + cursor = connection.execute( + """ + UPDATE jobs + SET status = CASE + WHEN status IN (?, ?, ?) THEN ? + ELSE status + END, + exit_code = CASE + WHEN status IN (?, ?, ?) THEN ? + ELSE exit_code + END, + ended_at = CASE + WHEN status IN (?, ?, ?) THEN ? + ELSE ended_at + END, + updated_at = CASE + WHEN status IN (?, ?, ?) THEN ? + ELSE updated_at + END, + reason = CASE + WHEN status IN (?, ?, ?) THEN NULL + ELSE reason + END, + message = CASE + WHEN status IN (?, ?, ?) THEN NULL + ELSE message + END + WHERE job_id = ? + """, + ( + *nonterminal, + "exited", + *nonterminal, + exit_code, + *nonterminal, + ended_at, + *nonterminal, + ended_at, + *nonterminal, + *nonterminal, + job_id, + ), + ) + if cursor.rowcount == 0: + raise LookupError(f"Unknown job id: {job_id}") + connection.commit() + finally: + connection.close() + + +def main(argv: list[str] | None = None) -> None: + """Run the standalone runner-exit command.""" + + args = parse_args(argv) + try: + record_runner_exit( + state_dir=args.state_dir, + job_id=args.job_id, + exit_code=args.exit_code, + ended_at=args.ended_at, + busy_timeout_ms=args.sqlite_busy_timeout_ms, + ) + except ( + FileNotFoundError, + LookupError, + OSError, + sqlite3.DatabaseError, + ValueError, + ) as exc: + raise SystemExit(str(exc)) from exc + + +if __name__ == "__main__": + main() + + +__all__ = [ + "NONTERMINAL_STATUSES", + "TERMINAL_STATUSES", + "main", + "parse_args", + "record_runner_exit", +] diff --git a/dify-agent/src/shellctl_runtime/sanitize.py b/dify-agent/src/shellctl_runtime/sanitize.py new file mode 100644 index 00000000000..f536e9629b1 --- /dev/null +++ b/dify-agent/src/shellctl_runtime/sanitize.py @@ -0,0 +1,186 @@ +"""Lightweight PTY sanitizer used by tmux `pipe-pane`. + +This module stays stdlib-only because shellctl starts it once per job to drain +PTY output into `output.log`. The ready-file handshake must happen before any +stdin reads so the server can distinguish slow startup from a stuck tmux pipe. +""" + +from __future__ import annotations + +import argparse +import codecs +import sys +from pathlib import Path +from typing import BinaryIO + + +class PtySanitizer: + """Incrementally convert PTY bytes into stable, readable UTF-8 text. + + Responsibilities: + - preserve UTF-8 decoder state across chunk boundaries + - strip common ANSI control sequences without leaking partial escape state + - normalize carriage-return progress updates into the final visible line + + This adapter intentionally keeps only minimal terminal state. It aims for a + practical log representation rather than a full terminal emulator. + """ + + def __init__(self) -> None: + self._decoder = codecs.getincrementaldecoder("utf-8")("replace") + self._line_buffer = "" + self._pending_cr = False + self._escape_state = "normal" + + def feed(self, raw: bytes) -> str: + """Consume one PTY byte chunk and return newly stable text.""" + + return self._consume_text(self._decoder.decode(raw, final=False), final=False) + + def flush(self) -> str: + """Flush buffered decoder/line state at end-of-stream.""" + + tail = self._consume_text(self._decoder.decode(b"", final=True), final=True) + if self._line_buffer: + tail += self._line_buffer + self._line_buffer = "" + self._pending_cr = False + self._escape_state = "normal" + return tail + + def _consume_text(self, text: str, *, final: bool) -> str: + parts: list[str] = [] + for char in text: + self._consume_char(char, parts) + if final and self._escape_state != "normal": + self._escape_state = "normal" + return "".join(parts) + + def _consume_char(self, char: str, parts: list[str]) -> None: + state = self._escape_state + if state == "normal": + if char == "\x1b": + self._escape_state = "esc" + return + self._consume_visible_char(char, parts) + return + if state == "esc": + if char == "[": + self._escape_state = "csi" + return + if char == "]": + self._escape_state = "osc" + return + self._escape_state = "normal" + if char.isprintable() and char != "\x1b": + self._consume_visible_char(char, parts) + return + if state == "csi": + if "@" <= char <= "~": + self._escape_state = "normal" + return + if state == "osc": + if char == "\x07": + self._escape_state = "normal" + return + if char == "\x1b": + self._escape_state = "osc_esc" + return + if state == "osc_esc": + self._escape_state = "normal" if char == "\\" else "osc" + + def _consume_visible_char(self, char: str, parts: list[str]) -> None: + if self._pending_cr: + if char == "\n": + parts.append(self._line_buffer) + parts.append("\n") + self._line_buffer = "" + self._pending_cr = False + return + self._line_buffer = "" + self._pending_cr = False + if char == "\r": + self._pending_cr = True + return + if char == "\n": + parts.append(self._line_buffer) + parts.append("\n") + self._line_buffer = "" + return + self._line_buffer += char + + +def sanitize_pty_output(raw: bytes) -> str: + """Sanitize a complete PTY byte string into readable UTF-8 text.""" + + sanitizer = PtySanitizer() + return sanitizer.feed(raw) + sanitizer.flush() + + +def sanitize_pty_stream( + stdin: BinaryIO, + stdout: BinaryIO, + *, + chunk_size: int = 65536, +) -> None: + """Run the streaming PTY sanitizer as a Unix-style filter.""" + + sanitizer = PtySanitizer() + while True: + chunk = stdin.read(chunk_size) + if not chunk: + break + output = sanitizer.feed(chunk) + if output: + stdout.write(output.encode("utf-8")) + if hasattr(stdout, "flush"): + stdout.flush() + tail = sanitizer.flush() + if tail: + stdout.write(tail.encode("utf-8")) + if hasattr(stdout, "flush"): + stdout.flush() + if hasattr(stdout, "flush"): + stdout.flush() + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the tiny CLI contract used by tmux `pipe-pane`.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ready-file", type=Path) + return parser.parse_args(argv) + + +def run_sanitize_pty( + ready_file: Path | None, + *, + stdin: BinaryIO, + stdout: BinaryIO, +) -> None: + """Touch the ready file, then sanitize stdin into stdout.""" + + if ready_file is not None: + ready_file.touch() + sanitize_pty_stream(stdin, stdout) + + +def main(argv: list[str] | None = None) -> None: + """Run the standalone PTY sanitizer module.""" + + args = parse_args(argv) + run_sanitize_pty(args.ready_file, stdin=sys.stdin.buffer, stdout=sys.stdout.buffer) + + +if __name__ == "__main__": + main() + + +__all__ = [ + "PtySanitizer", + "main", + "parse_args", + "run_sanitize_pty", + "sanitize_pty_output", + "sanitize_pty_stream", +] diff --git a/dify-agent/tests/local/dify_agent/adapters/llm/test_model.py b/dify-agent/tests/local/dify_agent/adapters/llm/test_model.py index 69d71f865cf..fac1d79d994 100644 --- a/dify-agent/tests/local/dify_agent/adapters/llm/test_model.py +++ b/dify-agent/tests/local/dify_agent/adapters/llm/test_model.py @@ -459,7 +459,7 @@ class DifyLLMAdapterModelTests(unittest.IsolatedAsyncioTestCase): LLMResultChunk( model="demo-model", delta=LLMResultChunkDelta( - index=1, + index=0, message=AssistantPromptMessage( content="", tool_calls=[ @@ -513,6 +513,72 @@ class DifyLLMAdapterModelTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(response.parts[2].part_kind, "text") self.assertEqual(cast(TextPart, response.parts[2]).content, "world") + async def test_request_stream_assigns_fallback_ids_to_tool_calls_without_ids(self) -> None: + def handler(_request: httpx.Request) -> httpx.Response: + return build_stream_response( + LLMResultChunk( + model="demo-model", + delta=LLMResultChunkDelta( + index=0, + message=AssistantPromptMessage( + content="", + tool_calls=[ + AssistantPromptMessage.ToolCall( + id=None, + type="function", + function=AssistantPromptMessage.ToolCall.ToolCallFunction( + name="shell_run", + arguments='{"script":"lookup find"}', + ), + ) + ], + ), + ), + ), + LLMResultChunk( + model="demo-model", + delta=LLMResultChunkDelta( + index=1, + message=AssistantPromptMessage( + content="", + tool_calls=[ + AssistantPromptMessage.ToolCall( + id=None, + type="function", + function=AssistantPromptMessage.ToolCall.ToolCallFunction( + name="shell_run", + arguments='{"script":"lookup out"}', + ), + ) + ], + ), + ), + ), + ) + + async with self.mock_daemon_stream(httpx.MockTransport(handler)): + adapter = DifyLLMAdapterModel( + "demo-model", + self.make_provider(), + model_provider="openai", + credentials={"api_key": "secret"}, + ) + + async with adapter.request_stream( + [ModelRequest(parts=[UserPromptPart("hello")])], + model_settings=None, + model_request_parameters=ModelRequestParameters(), + ) as stream: + events = [event async for event in stream] + response = stream.get() + + self.assertTrue(events) + self.assertEqual([part.part_kind for part in response.parts], ["tool-call", "tool-call"]) + self.assertEqual(cast(ToolCallPart, response.parts[0]).tool_call_id, "chunk-0-tool-0") + self.assertEqual(cast(ToolCallPart, response.parts[1]).tool_call_id, "chunk-1-tool-0") + self.assertEqual(cast(ToolCallPart, response.parts[0]).args, '{"script":"lookup find"}') + self.assertEqual(cast(ToolCallPart, response.parts[1]).args, '{"script":"lookup out"}') + async def test_request_splits_embedded_thinking_tags_into_parts(self) -> None: def handler(_request: httpx.Request) -> httpx.Response: return build_stream_response(*single_text_chunk("beforereasoningafter")) diff --git a/dify-agent/tests/local/dify_agent/adapters/shell/test_shellctl.py b/dify-agent/tests/local/dify_agent/adapters/shell/test_shellctl.py index 4fcb407555b..390f2edbef1 100644 --- a/dify-agent/tests/local/dify_agent/adapters/shell/test_shellctl.py +++ b/dify-agent/tests/local/dify_agent/adapters/shell/test_shellctl.py @@ -6,15 +6,20 @@ import asyncio import base64 from collections.abc import Callable from dataclasses import dataclass, field +from typing import cast -import httpx +import httpx2 as httpx import pytest from dify_agent.adapters.shell import shellctl from dify_agent.adapters.shell.config import ShellAdapterSettings from dify_agent.adapters.shell.factory import create_shell_provider from dify_agent.adapters.shell.protocols import ShellCommandResult, ShellProviderError -from dify_agent.adapters.shell.shellctl import ShellFileTransferError, ShellctlProvider +from dify_agent.adapters.shell.shellctl import ( + ShellctlClientProtocol, + ShellFileTransferError, + ShellctlProvider, +) @dataclass(slots=True) @@ -66,45 +71,73 @@ class FakeShellctlClient: delete_calls: list[tuple[str, bool, float | None]] = field(default_factory=list) closed: bool = False - async def run(self, script, *, cwd=None, env=None, timeout=30.0): + async def run( + self, + script: str, + *, + cwd: str | None = None, + env: dict[str, str] | None = None, + timeout: float = 30.0, + ) -> _Job: self.run_calls.append(_RunCall(script=script, cwd=cwd, env=env, timeout=timeout)) if self.run_handler is not None: return self.run_handler(script, cwd, env, timeout) return _Job(job_id="job", status="exited", done=True, exit_code=0) - async def wait(self, job_id, *, offset, timeout=30.0): + async def wait(self, job_id: str, *, offset: int, timeout: float = 30.0) -> _Job: self.wait_calls.append((job_id, offset, timeout)) if self.wait_handler is not None: return self.wait_handler(job_id, offset, timeout) return _Job(job_id=job_id, status="exited", done=True, offset=offset, exit_code=0) - async def input(self, job_id, text, *, offset, timeout=30.0): + async def input( + self, + job_id: str, + text: str, + *, + offset: int, + timeout: float = 30.0, + ) -> _Job: self.input_calls.append((job_id, text, offset, timeout)) if self.input_handler is not None: return self.input_handler(job_id, text, offset, timeout) return _Job(job_id=job_id, status="exited", done=True, offset=offset, exit_code=0) - async def tail(self, job_id): + async def tail(self, job_id: str) -> _Job: if self.tail_handler is not None: return self.tail_handler(job_id) return _Job(job_id=job_id, status="exited", done=True, output="", exit_code=0) - async def terminate(self, job_id, grace_seconds=10.0): + async def terminate(self, job_id: str, grace_seconds: float = 10.0) -> _Status: self.terminate_calls.append((job_id, grace_seconds)) if self.terminate_handler is not None: return self.terminate_handler(job_id, grace_seconds) return _Status(job_id=job_id) - async def delete(self, job_id, *, force=False, grace_seconds=None): + async def delete( + self, + job_id: str, + *, + force: bool = False, + grace_seconds: float | None = None, + ) -> None: self.delete_calls.append((job_id, force, grace_seconds)) return None - async def close(self): + async def close(self) -> None: self.closed = True def _provider(client: FakeShellctlClient) -> ShellctlProvider: - return ShellctlProvider(entrypoint="http://shellctl", token="", client_factory=lambda: client) + return ShellctlProvider( + entrypoint="http://shellctl", + token="", + client_factory=lambda: _client_protocol(client), + ) + + +def _client_protocol(client: FakeShellctlClient) -> ShellctlClientProtocol: + return cast(ShellctlClientProtocol, cast(object, client)) def test_factory_unknown_provider_raises() -> None: @@ -354,7 +387,10 @@ def test_file_transfer_timeout_is_an_end_to_end_budget(monkeypatch: pytest.Monke client = FakeShellctlClient(run_handler=run_handler, wait_handler=wait_handler) async def scenario() -> None: - transfer = shellctl.ShellctlFileTransfer(client=client, timeout=5.0) + transfer = shellctl.ShellctlFileTransfer( + client=_client_protocol(client), + timeout=5.0, + ) await transfer.upload(content=b"payload", remote_path="out.bin") asyncio.run(scenario()) @@ -380,7 +416,10 @@ def test_file_transfer_timeout_exhaustion_raises_timeout_and_still_deletes_job( client = FakeShellctlClient(run_handler=run_handler) async def scenario() -> None: - transfer = shellctl.ShellctlFileTransfer(client=client, timeout=5.0) + transfer = shellctl.ShellctlFileTransfer( + client=_client_protocol(client), + timeout=5.0, + ) with pytest.raises(ShellProviderError, match="timed out") as exc_info: await transfer.upload(content=b"payload", remote_path="out.bin") assert exc_info.value.code == "timeout" diff --git a/dify-agent/tests/local/dify_agent/agent_stub/cli/test_files.py b/dify-agent/tests/local/dify_agent/agent_stub/cli/test_files.py index 6a65daccb9a..3cf22e04d8e 100644 --- a/dify-agent/tests/local/dify_agent/agent_stub/cli/test_files.py +++ b/dify-agent/tests/local/dify_agent/agent_stub/cli/test_files.py @@ -53,6 +53,7 @@ def test_upload_file_from_environment_requests_signed_url_and_normalizes_output( def fake_request_agent_stub_file_download_sync(**kwargs): captured_download_request["file"] = kwargs["file"] + captured_download_request["for_external"] = kwargs.get("for_external", True) return type( "Response", (), @@ -85,6 +86,7 @@ def test_upload_file_from_environment_requests_signed_url_and_normalizes_output( transfer_method="tool_file", reference=_reference("tool-file-1"), ) + assert captured_download_request["for_external"] is True def test_upload_tool_file_resource_from_environment_preserves_tool_file_id( @@ -256,6 +258,7 @@ def test_download_file_from_environment_supports_mapping_json( def fake_request_download(**kwargs): captured["file"] = kwargs["file"] + captured["for_external"] = kwargs["for_external"] return type( "Response", (), @@ -283,10 +286,49 @@ def test_download_file_from_environment_supports_mapping_json( "reference": _reference("tool-file-1"), "url": None, } + assert captured["for_external"] is False assert result.path == target_dir / "report.pdf" assert result.path.read_bytes() == b"downloaded" +def test_download_file_from_environment_requests_internal_download_url( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + target_dir = tmp_path / "downloads" + target_dir.mkdir() + monkeypatch.setenv("DIFY_AGENT_STUB_API_BASE_URL", "https://agent.example.com/agent-stub") + monkeypatch.setenv("DIFY_AGENT_STUB_AUTH_JWE", "test-jwe") + captured: dict[str, object] = {} + + def fake_request_download(**kwargs): + captured.update(kwargs) + return type( + "Response", + (), + { + "filename": "report.pdf", + "mime_type": "application/pdf", + "size": 12, + "download_url": "http://internal-files/report.pdf", + }, + )() + + monkeypatch.setattr("dify_agent.agent_stub.cli._files.request_agent_stub_file_download_sync", fake_request_download) + monkeypatch.setattr( + "dify_agent.agent_stub.cli._files.download_file_bytes_from_signed_url_sync", + lambda **_kwargs: b"downloaded", + ) + + _ = download_file_from_environment( + transfer_method="tool_file", + reference_or_url=_reference("tool-file-1"), + local_dir=str(target_dir), + ) + + assert captured["for_external"] is False + + def test_download_file_from_environment_requires_mapping_or_positional_pair() -> None: with pytest.raises(AgentStubValidationError, match="requires either --mapping or TRANSFER_METHOD REFERENCE_OR_URL"): _ = download_file_from_environment() diff --git a/dify-agent/tests/local/dify_agent/agent_stub/cli/test_main.py b/dify-agent/tests/local/dify_agent/agent_stub/cli/test_main.py index 3018629e105..02d505b5552 100644 --- a/dify-agent/tests/local/dify_agent/agent_stub/cli/test_main.py +++ b/dify-agent/tests/local/dify_agent/agent_stub/cli/test_main.py @@ -161,6 +161,24 @@ def test_cli_plural_config_groups_expose_pull_push_and_delete( assert "delete" in captured.out +def test_cli_config_skills_push_help_describes_skill_directory_requirements( + capsys: pytest.CaptureFixture[str], +) -> None: + with pytest.raises(SystemExit) as exc_info: + main(["config", "skills", "push", "--help"]) + + captured = capsys.readouterr() + normalized_output = " ".join(captured.out.split()) + assert exc_info.value.code == 0 + assert "Skill directory requirements:" in captured.out + assert "top-level SKILL.md" in captured.out + assert "UTF-8 Markdown" in captured.out + assert "YAML frontmatter matching this schema" in captured.out + assert "--- name: description: ---" in normalized_output + assert "Symlinked files are rejected" in captured.out + assert ".venv and node_modules should be manually cleared before push" in normalized_output + + @pytest.mark.parametrize("argv", [["config", "file", "--help"], ["config", "skill", "--help"]]) def test_cli_hidden_singular_alias_help_exposes_pull_only( capsys: pytest.CaptureFixture[str], diff --git a/dify-agent/tests/local/dify_agent/agent_stub/client/test_agent_stub_client.py b/dify-agent/tests/local/dify_agent/agent_stub/client/test_agent_stub_client.py index 7722ff940bd..be4fb1def39 100644 --- a/dify-agent/tests/local/dify_agent/agent_stub/client/test_agent_stub_client.py +++ b/dify-agent/tests/local/dify_agent/agent_stub/client/test_agent_stub_client.py @@ -157,7 +157,7 @@ def test_request_agent_stub_file_download_sync_posts_download_request() -> None: assert request.method == "POST" assert str(request.url) == "https://agent.example.com/agent-stub/files/download-request" assert json.loads(request.content) == { - "file": {"transfer_method": "tool_file", "reference": _reference("tool-file-1")} + "file": {"transfer_method": "tool_file", "reference": _reference("tool-file-1")}, } return httpx.Response( 200, @@ -183,6 +183,39 @@ def test_request_agent_stub_file_download_sync_posts_download_request() -> None: assert response.download_url == "https://files.example.com/download" +def test_request_agent_stub_file_download_sync_posts_internal_download_request() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert str(request.url) == "https://agent.example.com/agent-stub/files/download-request" + assert json.loads(request.content) == { + "file": {"transfer_method": "tool_file", "reference": _reference("tool-file-1")}, + "for_external": False, + } + return httpx.Response( + 200, + json={ + "filename": "report.pdf", + "mime_type": "application/pdf", + "size": 123, + "download_url": "http://internal-files/report.pdf", + }, + ) + + http_client = httpx.Client(transport=httpx.MockTransport(handler)) + try: + response = request_agent_stub_file_download_sync( + url="https://agent.example.com/agent-stub", + auth_jwe="test-jwe", + file=AgentStubFileMapping(transfer_method="tool_file", reference=_reference("tool-file-1")), + for_external=False, + sync_http_client=http_client, + ) + finally: + http_client.close() + + assert response.download_url == "http://internal-files/report.pdf" + + def test_request_agent_stub_drive_manifest_sync_gets_manifest_request() -> None: def handler(request: httpx.Request) -> httpx.Response: assert request.method == "GET" @@ -456,12 +489,14 @@ def test_request_agent_stub_file_download_sync_dispatches_grpc_urls(monkeypatch: url="grpc://agent.example.com:9091", auth_jwe="token", file=file_mapping, + for_external=False, ) assert captured == { "url": "grpc://agent.example.com:9091", "auth_jwe": "token", "file": file_mapping, + "for_external": False, "timeout": 30.0, } assert response.download_url == "https://files.example.com/download" @@ -613,7 +648,10 @@ def test_request_agent_stub_file_download_grpc_sync_maps_grpc_errors(monkeypatch grpc_module, "_require_conversions", lambda: SimpleNamespace( - proto_file_download_request=lambda _pb2, *, file: {"file": file}, + proto_file_download_request=lambda _pb2, *, file, for_external: { + "file": file, + "for_external": for_external, + }, file_download_response_from_proto=lambda response: response, ), ) diff --git a/dify-agent/tests/local/dify_agent/agent_stub/protocol/test_grpc_conversions.py b/dify-agent/tests/local/dify_agent/agent_stub/protocol/test_grpc_conversions.py index eaefac6a2b5..8c061254151 100644 --- a/dify-agent/tests/local/dify_agent/agent_stub/protocol/test_grpc_conversions.py +++ b/dify-agent/tests/local/dify_agent/agent_stub/protocol/test_grpc_conversions.py @@ -13,9 +13,14 @@ from dify_agent.agent_stub.grpc.conversions import ( connect_response_from_proto, file_download_request_from_proto, proto_connect_request, + proto_file_download_request, proto_file_download_response, ) -from dify_agent.agent_stub.protocol.agent_stub import AgentStubConnectResponse, AgentStubFileDownloadResponse +from dify_agent.agent_stub.protocol.agent_stub import ( + AgentStubConnectResponse, + AgentStubFileDownloadResponse, + AgentStubFileMapping, +) def _reference(record_id: str) -> str: @@ -48,6 +53,32 @@ def test_file_download_request_from_proto_respects_optional_reference() -> None: assert request.file.reference == _reference("tool-file-1") assert request.file.url is None + assert request.for_external is True + + +def test_file_download_request_from_proto_preserves_explicit_internal_audience() -> None: + message = agent_stub_pb2.FileDownloadRequest( + file=agent_stub_pb2.FileMapping( + transfer_method="tool_file", + reference=_reference("tool-file-1"), + ), + for_external=False, + ) + + request = file_download_request_from_proto(message) + + assert request.for_external is False + + +def test_proto_file_download_request_preserves_selected_audience() -> None: + message = proto_file_download_request( + agent_stub_pb2, + file=AgentStubFileMapping(transfer_method="tool_file", reference=_reference("tool-file-1")), + for_external=False, + ) + + assert message.HasField("for_external") is True + assert message.for_external is False def test_connect_request_from_proto_rejects_invalid_metadata_json() -> None: diff --git a/dify-agent/tests/local/dify_agent/agent_stub/server/test_agent_stub_files.py b/dify-agent/tests/local/dify_agent/agent_stub/server/test_agent_stub_files.py index 164fd7487d7..54e7863aeb4 100644 --- a/dify-agent/tests/local/dify_agent/agent_stub/server/test_agent_stub_files.py +++ b/dify-agent/tests/local/dify_agent/agent_stub/server/test_agent_stub_files.py @@ -115,6 +115,41 @@ def test_dify_api_agent_stub_file_handler_injects_execution_context_for_download asyncio.run(scenario()) +def test_dify_api_agent_stub_file_handler_forwards_internal_download_audience(monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "https://api.example.com/inner/api/download/file/request" + assert json.loads(request.content)["for_external"] is False + return httpx.Response( + 200, + json={ + "data": { + "filename": "report.pdf", + "mime_type": "application/pdf", + "size": 123, + "download_url": "http://internal-files/report.pdf", + } + }, + ) + + _patch_async_client(monkeypatch, handler) + file_handler = DifyApiAgentStubFileRequestHandler( + inner_api_url="https://api.example.com", + inner_api_key="inner-secret", + ) + + async def scenario() -> None: + response = await file_handler.create_download_request( + principal=_principal(), + request=AgentStubFileDownloadRequest( + file=AgentStubFileMapping(transfer_method="tool_file", reference=_reference("tool-file-1")), + for_external=False, + ), + ) + assert response.download_url == "http://internal-files/report.pdf" + + asyncio.run(scenario()) + + def test_dify_api_agent_stub_file_handler_rejects_missing_user_id() -> None: file_handler = DifyApiAgentStubFileRequestHandler( inner_api_url="https://api.example.com", diff --git a/dify-agent/tests/local/dify_agent/agent_stub/server/test_grpc_service.py b/dify-agent/tests/local/dify_agent/agent_stub/server/test_grpc_service.py index addcec7cf66..149ce1cbffe 100644 --- a/dify-agent/tests/local/dify_agent/agent_stub/server/test_grpc_service.py +++ b/dify-agent/tests/local/dify_agent/agent_stub/server/test_grpc_service.py @@ -130,6 +130,7 @@ def test_agent_stub_grpc_transport_delegates_file_download_requests() -> None: async def create_download_request(self, *, principal, request): assert principal.execution_context.user_id == "user-1" assert request.file.reference == _reference("tool-file-1") + assert request.for_external is False return type( "Response", (), @@ -158,7 +159,8 @@ def test_agent_stub_grpc_transport_delegates_file_download_requests() -> None: file=agent_stub_pb2.FileMapping( transfer_method="tool_file", reference=_reference("tool-file-1"), - ) + ), + for_external=False, ), metadata=(("authorization", f"Bearer {token}"),), ) diff --git a/dify-agent/tests/local/dify_agent/runtime/test_runner.py b/dify-agent/tests/local/dify_agent/runtime/test_runner.py index 87e12854496..7b7762a0ace 100644 --- a/dify-agent/tests/local/dify_agent/runtime/test_runner.py +++ b/dify-agent/tests/local/dify_agent/runtime/test_runner.py @@ -6,7 +6,7 @@ import httpx import pytest from pydantic import JsonValue from pydantic_ai import Tool -from pydantic_ai.exceptions import UnexpectedModelBehavior +from pydantic_ai.exceptions import ModelHTTPError, UnexpectedModelBehavior from pydantic_ai.messages import ( ToolReturnPart, ModelMessage, @@ -63,8 +63,8 @@ from dify_agent.protocol.schemas import ( ) from dify_agent.runtime.event_sink import InMemoryRunEventSink from dify_agent.runtime.compositor_factory import create_default_layer_providers -from dify_agent.runtime.runner import AgentRunRunner, AgentRunValidationError -from shell_session_manager.shellctl.shared import DeleteJobResponse, JobResult, JobStatusName, JobStatusView +from dify_agent.runtime.runner import AgentRunRunner, AgentRunValidationError, _run_failed_error_payload +from shellctl.shared import DeleteJobResponse, JobResult, JobStatusName, JobStatusView class StaticToolsTestLayer(ToolsLayer): @@ -127,6 +127,28 @@ class FakeRunnerShellctlClient: return DeleteJobResponse(job_id=job_id) +def test_run_failed_error_payload_preserves_plugin_rate_limit_error() -> None: + exc = ModelHTTPError( + 429, + "gpt-4o-mini", + {"error_type": "InvokeRateLimitError", "message": "quota exceeded"}, + ) + + message, reason = _run_failed_error_payload(exc) + + assert message == "quota exceeded" + assert reason == "InvokeRateLimitError" + + +def test_run_failed_error_payload_infers_rate_limit_reason_from_status_code() -> None: + exc = ModelHTTPError(429, "gpt-4o-mini", {"message": "too many requests"}) + + message, reason = _run_failed_error_payload(exc) + + assert message == "too many requests" + assert reason == "InvokeRateLimitError" + + def _request( user: str | list[str] = "hello", *, diff --git a/dify-agent/tests/local/dify_agent/test_import_boundaries.py b/dify-agent/tests/local/dify_agent/test_import_boundaries.py index 5f2d8dc5fe8..82fbda61667 100644 --- a/dify-agent/tests/local/dify_agent/test_import_boundaries.py +++ b/dify-agent/tests/local/dify_agent/test_import_boundaries.py @@ -121,8 +121,8 @@ def test_protocol_and_dify_plugin_exports_do_not_import_server_only_modules() -> "openai", "pydantic_settings", "redis", - "shell_session_manager.shellctl.client", - "shell_session_manager.shellctl.server", + "shellctl.client", + "shellctl.server", ], imports=[ "dify_agent.protocol", @@ -158,7 +158,7 @@ def test_agent_stub_cli_main_import_is_client_safe() -> None: "jwcrypto", "pydantic_settings", "redis", - "shell_session_manager", + "shellctl.server", ], imports=[ "dify_agent.agent_stub.client", @@ -238,7 +238,7 @@ def test_agent_stub_cli_help_render_does_not_load_server_modules() -> None: "jwcrypto", "pydantic_settings", "redis", - "shell_session_manager", + "shellctl.server", ] script = "\n".join( [ @@ -269,6 +269,28 @@ def test_agent_stub_cli_help_render_does_not_load_server_modules() -> None: _run_python_script(script) +def test_shellctl_client_imports_do_not_import_server_modules() -> None: + _run_import_check( + blocked_imports=[ + "aiosqlite", + "fastapi", + "sqlalchemy", + "sqlmodel", + "shellctl.server.api", + "shellctl.server.service", + "shellctl.server.tmux", + "uvicorn", + ], + imports=["shellctl", "shellctl.client", "shellctl.shared", "shellctl.cli"], + assertions=[ + "assert hasattr(shellctl, 'ShellctlClient')", + "assert hasattr(shellctl_client, 'ShellctlClient')", + "assert hasattr(shellctl_shared, 'JobResult')", + "assert hasattr(shellctl_cli, 'cli')", + ], + ) + + def test_server_settings_import_does_not_import_agent_stub_app() -> None: try: __import__("pydantic_settings") diff --git a/dify-agent/tests/local/shellctl/golden_shellctl_sanitize.json b/dify-agent/tests/local/shellctl/golden_shellctl_sanitize.json new file mode 100644 index 00000000000..10eba3678ef --- /dev/null +++ b/dify-agent/tests/local/shellctl/golden_shellctl_sanitize.json @@ -0,0 +1,32 @@ +[ + { + "name": "utf8_passthrough", + "chunks_hex": ["68656c6c6f20e4b896e7958cf09f99820a"], + "expected": "hello 世界🙂\n" + }, + { + "name": "split_utf8_chunks", + "chunks_hex": ["e4bda0e5a5bd20f09f", "99820a"], + "expected": "你好 🙂\n" + }, + { + "name": "invalid_utf8_replacement", + "chunks_hex": ["666f80ff6f0a"], + "expected": "fo��o\n" + }, + { + "name": "ansi_sequences_removed", + "chunks_hex": ["1b5b33316d7265641b5b306d1b5b4b0a"], + "expected": "red\n" + }, + { + "name": "carriage_return_progress", + "chunks_hex": ["30250d3530250d313030250a"], + "expected": "100%\n" + }, + { + "name": "crlf_normalized", + "chunks_hex": ["6f6b0d0a6e6578740d0a"], + "expected": "ok\nnext\n" + } +] diff --git a/dify-agent/tests/local/shellctl/test_shellctl_cli.py b/dify-agent/tests/local/shellctl/test_shellctl_cli.py new file mode 100644 index 00000000000..ac8eed51843 --- /dev/null +++ b/dify-agent/tests/local/shellctl/test_shellctl_cli.py @@ -0,0 +1,688 @@ +from __future__ import annotations + +import importlib +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import ClassVar, TypeVar, cast + +import click +import httpx2 as httpx +import pytest +import typer.main +from typer.testing import CliRunner + +from shellctl.client import ShellctlClientError +from shellctl.shared.constants import DEFAULT_BASE_URL +from shellctl.shared.schemas import ( + DeleteJobResponse, + HealthResponse, + JobInfo, + JobResult, + JobStatusName, + JobStatusView, + TerminalSize, +) + +cli_module = importlib.import_module("shellctl.cli") +cli = cli_module.cli +runner = CliRunner() +ResultT = TypeVar("ResultT") + + +def _click_command(command_name: str) -> click.Command: + return cast(click.Group, typer.main.get_command(cli)).commands[command_name] + + +def _command_option_names(command_name: str) -> set[str]: + command = _click_command(command_name) + return { + option + for parameter in command.params + for option in getattr(parameter, "opts", []) # noqa: no-new-getattr click option introspection + } + + +def _command_option(command_name: str, option_name: str) -> click.Option: + command = _click_command(command_name) + for parameter in command.params: + if isinstance(parameter, click.Option) and option_name in parameter.opts: + return parameter + raise AssertionError(f"{option_name} not found on {command_name}") + + +class RecordingShellctlClient: + init_calls: ClassVar[list[dict[str, object]]] = [] + calls: ClassVar[list[tuple[str, tuple[object, ...], dict[str, object]]]] = [] + results: ClassVar[dict[str, object]] = {} + error: ClassVar[BaseException | None] = None + + def __init__( + self, + base_url: str, + *, + output_limit: int, + idle_flush_seconds: float, + token: str | None = None, + client: object | None = None, + transport: object | None = None, + ) -> None: + del client, transport + type(self).init_calls.append( + { + "base_url": base_url, + "output_limit": output_limit, + "idle_flush_seconds": idle_flush_seconds, + "token": token, + } + ) + + @classmethod + def reset(cls) -> None: + cls.init_calls = [] + cls.calls = [] + cls.results = {} + cls.error = None + + async def __aenter__(self) -> RecordingShellctlClient: + return self + + async def __aexit__(self, exc_type: object, exc: object, tb: object) -> None: + return None + + def _result(self, method: str, result_type: type[ResultT]) -> ResultT: + del result_type + error = type(self).error + if error is not None: + raise error + return cast(ResultT, type(self).results[method]) + + async def health(self) -> HealthResponse: + type(self).calls.append(("health", (), {})) + return self._result("health", HealthResponse) + + async def run( + self, + script: str, + *, + cwd: str | None = None, + env: dict[str, str] | None = None, + timeout: float, + terminal: TerminalSize | None = None, + ) -> JobResult: + type(self).calls.append( + ( + "run", + (script,), + { + "cwd": cwd, + "env": env, + "timeout": timeout, + "terminal": terminal, + }, + ) + ) + return self._result("run", JobResult) + + async def wait(self, job_id: str, *, offset: int, timeout: float) -> JobResult: + type(self).calls.append(("wait", (job_id,), {"offset": offset, "timeout": timeout})) + return self._result("wait", JobResult) + + async def status(self, job_id: str) -> JobStatusView: + type(self).calls.append(("status", (job_id,), {})) + return self._result("status", JobStatusView) + + async def list_jobs( + self, + *, + status: JobStatusName | None = None, + limit: int, + ) -> list[JobInfo]: + type(self).calls.append(("list_jobs", (), {"status": status, "limit": limit})) + return cast(list[JobInfo], self._result("list_jobs", list)) + + async def input( + self, + job_id: str, + text: str, + *, + offset: int, + timeout: float, + ) -> JobResult: + type(self).calls.append( + ( + "input", + (job_id, text), + {"offset": offset, "timeout": timeout}, + ) + ) + return self._result("input", JobResult) + + async def tail(self, job_id: str) -> JobResult: + type(self).calls.append(("tail", (job_id,), {})) + return self._result("tail", JobResult) + + async def terminate(self, job_id: str, grace_seconds: float) -> JobStatusView: + type(self).calls.append(("terminate", (job_id,), {"grace_seconds": grace_seconds})) + return self._result("terminate", JobStatusView) + + async def delete( + self, + job_id: str, + *, + force: bool = False, + grace_seconds: float | None = None, + ) -> DeleteJobResponse: + type(self).calls.append( + ( + "delete", + (job_id,), + {"force": force, "grace_seconds": grace_seconds}, + ) + ) + return self._result("delete", DeleteJobResponse) + + +@pytest.fixture +def patched_client( + monkeypatch: pytest.MonkeyPatch, +) -> type[RecordingShellctlClient]: + RecordingShellctlClient.reset() + monkeypatch.setattr(cli_module, "ShellctlClient", RecordingShellctlClient) + return RecordingShellctlClient + + +def _package_env() -> dict[str, str]: + package_root = Path(__file__).resolve().parents[3] + src_path = package_root / "src" + env = dict(os.environ) + current_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = f"{src_path}{os.pathsep}{current_pythonpath}" if current_pythonpath else str(src_path) + return env + + +def test_shellctl_help_lists_network_commands() -> None: + result = runner.invoke(cli, ["--help"]) + + assert result.exit_code == 0, result.stderr + assert "health" in result.stdout + assert "run" in result.stdout + assert "wait" in result.stdout + assert "status" in result.stdout + assert "list" in result.stdout + assert "input" in result.stdout + assert "tail" in result.stdout + assert "terminate" in result.stdout + assert "delete" in result.stdout + assert "serve" in result.stdout + + +def test_shellctl_run_and_serve_help_show_the_new_option_boundaries() -> None: + run_result = runner.invoke(cli, ["run", "--help"]) + serve_result = runner.invoke(cli, ["serve", "--help"]) + + assert run_result.exit_code == 0, run_result.stderr + assert "--base-url" in run_result.stdout + assert "--auth-token" in run_result.stdout + assert "--state-dir" not in run_result.stdout + assert "--runtime-dir" not in run_result.stdout + + assert serve_result.exit_code == 0, serve_result.stderr + assert "--state-dir" in serve_result.stdout + assert "--runtime-dir" in serve_result.stdout + assert "--auth-token" in serve_result.stdout + + +def test_shellctl_health_uses_sdk_health_and_ignores_auth_token_input( + monkeypatch: pytest.MonkeyPatch, + patched_client: type[RecordingShellctlClient], +) -> None: + patched_client.results["health"] = HealthResponse(status="ok") + monkeypatch.setenv("SHELLCTL_AUTH_TOKEN", "from-env") + + result = runner.invoke(cli, ["health", "--auth-token", "flag-token"]) + + assert result.exit_code == 0, result.stderr + assert json.loads(result.stdout) == {"status": "ok"} + assert patched_client.calls == [("health", (), {})] + assert patched_client.init_calls == [ + { + "base_url": DEFAULT_BASE_URL, + "output_limit": 8192, + "idle_flush_seconds": 0.5, + "token": None, + } + ] + + +def test_shellctl_run_builds_sdk_request_and_emits_json( + patched_client: type[RecordingShellctlClient], + tmp_path: Path, +) -> None: + patched_client.results["run"] = JobResult( + job_id="job-run", + done=False, + status=JobStatusName.RUNNING, + output_path="/tmp/output.log", + output="hello\n", + offset=6, + truncated=False, + ) + + result = runner.invoke( + cli, + [ + "run", + "printf hello\\n", + "--cwd", + str(tmp_path / "workspace"), + "--env", + "A=1", + "--env", + "EMPTY=", + "--timeout", + "12", + "--output-limit", + "4096", + "--idle-flush-seconds", + "0.25", + "--cols", + "90", + ], + ) + + assert result.exit_code == 0, result.stderr + assert json.loads(result.stdout) == { + "job_id": "job-run", + "done": False, + "status": "running", + "output_path": "/tmp/output.log", + "output": "hello\n", + "offset": 6, + "truncated": False, + } + assert patched_client.init_calls == [ + { + "base_url": DEFAULT_BASE_URL, + "output_limit": 4096, + "idle_flush_seconds": 0.25, + "token": None, + } + ] + assert patched_client.calls[0][0] == "run" + assert patched_client.calls[0][1] == ("printf hello\\n",) + assert patched_client.calls[0][2]["cwd"] == str(tmp_path / "workspace") + assert patched_client.calls[0][2]["env"] == {"A": "1", "EMPTY": ""} + assert patched_client.calls[0][2]["timeout"] == 12.0 + terminal = patched_client.calls[0][2]["terminal"] + assert isinstance(terminal, TerminalSize) + assert terminal.cols == 90 + assert terminal.rows == 80 + + +def test_shellctl_wait_and_input_require_offset() -> None: + assert _command_option("wait", "--offset").required is True + assert _command_option("input", "--offset").required is True + + +def test_shellctl_wait_and_input_map_requests( + patched_client: type[RecordingShellctlClient], +) -> None: + patched_client.results["wait"] = JobResult( + job_id="job-1", + done=False, + status=JobStatusName.RUNNING, + output_path="/tmp/wait.log", + output="chunk", + offset=5, + truncated=False, + ) + + wait_result = runner.invoke( + cli, + [ + "wait", + "job-1", + "--offset", + "3", + "--timeout", + "9", + "--output-limit", + "2048", + "--idle-flush-seconds", + "0.1", + ], + ) + + assert wait_result.exit_code == 0, wait_result.stderr + assert patched_client.calls[0] == ( + "wait", + ("job-1",), + {"offset": 3, "timeout": 9.0}, + ) + assert patched_client.init_calls[0]["output_limit"] == 2048 + assert patched_client.init_calls[0]["idle_flush_seconds"] == 0.1 + + RecordingShellctlClient.reset() + patched_client.results["input"] = JobResult( + job_id="job-1", + done=False, + status=JobStatusName.RUNNING, + output_path="/tmp/input.log", + output="reply", + offset=8, + truncated=False, + ) + + input_result = runner.invoke( + cli, + [ + "input", + "job-1", + "hello\n", + "--offset", + "5", + "--timeout", + "4", + "--output-limit", + "512", + "--idle-flush-seconds", + "0", + ], + ) + + assert input_result.exit_code == 0, input_result.stderr + assert patched_client.calls[0] == ( + "input", + ("job-1", "hello\n"), + {"offset": 5, "timeout": 4.0}, + ) + assert patched_client.init_calls[0]["output_limit"] == 512 + assert patched_client.init_calls[0]["idle_flush_seconds"] == 0.0 + + +def test_shellctl_list_tail_status_terminate_and_delete_map_arguments( + patched_client: type[RecordingShellctlClient], +) -> None: + patched_client.results["list_jobs"] = [ + JobInfo( + job_id="job-2", + status=JobStatusName.RUNNING, + created_at="2026-05-21T15:30:12Z", + ) + ] + + list_result = runner.invoke(cli, ["list", "--status", "running", "--limit", "5"]) + + assert list_result.exit_code == 0, list_result.stderr + assert json.loads(list_result.stdout) == [ + { + "job_id": "job-2", + "status": "running", + "created_at": "2026-05-21T15:30:12Z", + } + ] + assert patched_client.calls[0] == ( + "list_jobs", + (), + {"status": JobStatusName.RUNNING, "limit": 5}, + ) + + RecordingShellctlClient.reset() + patched_client.results["tail"] = JobResult( + job_id="job-2", + done=False, + status=JobStatusName.RUNNING, + output_path="/tmp/tail.log", + output="tail", + offset=4, + truncated=False, + ) + tail_result = runner.invoke(cli, ["tail", "job-2", "--output-limit", "16"]) + assert tail_result.exit_code == 0, tail_result.stderr + assert patched_client.calls[0] == ("tail", ("job-2",), {}) + assert patched_client.init_calls[0]["output_limit"] == 16 + + RecordingShellctlClient.reset() + patched_client.results["status"] = JobStatusView( + job_id="job-2", + status=JobStatusName.RUNNING, + done=False, + created_at="2026-05-21T15:30:12Z", + started_at="2026-05-21T15:30:13Z", + offset=4, + ) + status_result = runner.invoke(cli, ["status", "job-2"]) + assert status_result.exit_code == 0, status_result.stderr + assert patched_client.calls[0] == ("status", ("job-2",), {}) + + RecordingShellctlClient.reset() + patched_client.results["terminate"] = JobStatusView( + job_id="job-2", + status=JobStatusName.TERMINATED, + done=True, + created_at="2026-05-21T15:30:12Z", + started_at="2026-05-21T15:30:13Z", + ended_at="2026-05-21T15:30:18Z", + offset=4, + ) + terminate_result = runner.invoke( + cli, + ["terminate", "job-2", "--grace-seconds", "0.25"], + ) + assert terminate_result.exit_code == 0, terminate_result.stderr + assert patched_client.calls[0] == ( + "terminate", + ("job-2",), + {"grace_seconds": 0.25}, + ) + + RecordingShellctlClient.reset() + patched_client.results["delete"] = DeleteJobResponse(job_id="job-2") + delete_result = runner.invoke( + cli, + ["delete", "job-2", "--force", "--grace-seconds", "0.5"], + ) + assert delete_result.exit_code == 0, delete_result.stderr + assert patched_client.calls[0] == ( + "delete", + ("job-2",), + {"force": True, "grace_seconds": 0.5}, + ) + + +def test_shellctl_run_rejects_invalid_env_entry() -> None: + result = runner.invoke(cli, ["run", "printf bad", "--env", "MISSING_EQUALS"]) + + assert result.exit_code == 2 + assert "env entries must use NAME=VALUE format" in result.stderr + + +def test_shellctl_commands_render_sdk_errors_as_json_stderr( + patched_client: type[RecordingShellctlClient], +) -> None: + patched_client.error = ShellctlClientError( + 404, + "job_not_found", + "Job missing is not found", + ) + + result = runner.invoke(cli, ["status", "missing"]) + + assert result.exit_code == 1 + assert result.stdout == "" + assert json.loads(result.stderr) == { + "error": { + "code": "job_not_found", + "message": "Job missing is not found", + } + } + + +def test_shellctl_commands_render_transport_errors_as_json_stderr( + patched_client: type[RecordingShellctlClient], +) -> None: + patched_client.error = httpx.TransportError("connection refused") + + result = runner.invoke(cli, ["status", "missing"]) + + assert result.exit_code == 1 + assert result.stdout == "" + assert json.loads(result.stderr) == { + "error": { + "code": "connection_error", + "message": "connection refused", + } + } + + +def test_shellctl_commands_render_timeouts_as_json_stderr( + patched_client: type[RecordingShellctlClient], +) -> None: + patched_client.error = httpx.TimeoutException("slow server") + + result = runner.invoke(cli, ["status", "missing"]) + + assert result.exit_code == 1 + assert result.stdout == "" + assert json.loads(result.stderr) == { + "error": { + "code": "request_timeout", + "message": "request timed out", + } + } + + +def test_shellctl_base_url_and_auth_token_flags_override_environment( + monkeypatch: pytest.MonkeyPatch, + patched_client: type[RecordingShellctlClient], +) -> None: + patched_client.results["status"] = JobStatusView( + job_id="job-2", + status=JobStatusName.RUNNING, + done=False, + created_at="2026-05-21T15:30:12Z", + started_at="2026-05-21T15:30:13Z", + offset=4, + ) + monkeypatch.setenv("SHELLCTL_BASE_URL", "http://from-env:9999") + monkeypatch.setenv("SHELLCTL_AUTH_TOKEN", "from-env-token") + + result = runner.invoke( + cli, + [ + "status", + "job-2", + "--base-url", + "http://override:8765", + "--auth-token", + "flag-token", + ], + ) + + assert result.exit_code == 0, result.stderr + assert patched_client.init_calls == [ + { + "base_url": "http://override:8765", + "output_limit": 8192, + "idle_flush_seconds": 0.5, + "token": "flag-token", + } + ] + + +def test_shellctl_cli_controller_module_is_removed() -> None: + assert importlib.util.find_spec("shellctl.server.cli_controller") is None + + +def test_importing_shellctl_cli_for_run_help_skips_server_stack() -> None: + package_root = Path(__file__).resolve().parents[3] + command = """ +import json +import sys +from typer.testing import CliRunner +from shellctl.cli import cli + +result = CliRunner().invoke(cli, ["run", "--help"]) +print(json.dumps({ + "exit_code": result.exit_code, + "stdout": result.stdout, + "modules": { + "fastapi": "fastapi" in sys.modules, + "uvicorn": "uvicorn" in sys.modules, + "sqlalchemy": "sqlalchemy" in sys.modules, + "sqlmodel": "sqlmodel" in sys.modules, + "service": "shellctl.server.service" in sys.modules, + "api": "shellctl.server.api" in sys.modules, + "tmux": "shellctl.server.tmux" in sys.modules, + "shared_runtime": "shellctl.shared.runtime" in sys.modules, + "shared_output": "shellctl.shared.output" in sys.modules, + "shared_sanitize": "shellctl.shared.sanitize" in sys.modules, + }, +})) +""" + + result = subprocess.run( + [sys.executable, "-c", command], + capture_output=True, + text=True, + check=False, + cwd=package_root, + env=_package_env(), + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert payload["exit_code"] == 0 + assert "--base-url" in payload["stdout"] + assert payload["modules"] == { + "fastapi": False, + "uvicorn": False, + "sqlalchemy": False, + "sqlmodel": False, + "service": False, + "api": False, + "tmux": False, + "shared_runtime": False, + "shared_output": False, + "shared_sanitize": False, + } + + +def test_importing_server_serve_command_skips_cli_and_sdk_modules() -> None: + package_root = Path(__file__).resolve().parents[3] + command = """ +import json +import sys +from shellctl.server import serve_command + +print(json.dumps({ + "callable": callable(serve_command), + "modules": { + "cli": "shellctl.cli" in sys.modules, + "sdk": "shellctl.client.sdk" in sys.modules, + "client": "shellctl.client" in sys.modules, + }, +})) +""" + + result = subprocess.run( + [sys.executable, "-c", command], + capture_output=True, + text=True, + check=False, + cwd=package_root, + env=_package_env(), + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert payload["callable"] is True + assert payload["modules"] == { + "cli": False, + "sdk": False, + "client": False, + } diff --git a/dify-agent/tests/local/shellctl/test_shellctl_client.py b/dify-agent/tests/local/shellctl/test_shellctl_client.py new file mode 100644 index 00000000000..fad5d147a27 --- /dev/null +++ b/dify-agent/tests/local/shellctl/test_shellctl_client.py @@ -0,0 +1,566 @@ +import json +from collections.abc import Awaitable, Callable +from typing import ClassVar, TypedDict, cast + +import httpx2 as httpx +import pytest + +from shellctl.client import ShellctlClient, ShellctlClientError +from shellctl.client import sdk as shellctl_sdk +from shellctl.shared import ( + DEFAULT_TERMINATE_GRACE_SECONDS, + HealthResponse, + JobStatusName, +) + + +class ForcedDeleteKwargs(TypedDict, total=False): + force: bool + grace_seconds: float + + +@pytest.mark.anyio +@pytest.mark.parametrize( + ("call", "expected_path", "expected_json", "wait_timeout"), + [ + ( + lambda client: client.run( + "printf ready\\n", + cwd="/tmp", + env={"HELLO": "world"}, + timeout=12, + ), + "/v1/jobs/run", + { + "script": "printf ready\\n", + "cwd": "/tmp", + "env": {"HELLO": "world"}, + "timeout": 12.0, + "output_limit": 4096, + "idle_flush_seconds": 0.25, + }, + 22.0, + ), + ( + lambda client: client.wait("job-1", offset=3, timeout=7), + "/v1/jobs/job-1/wait", + { + "offset": 3, + "timeout": 7, + "output_limit": 4096, + "idle_flush_seconds": 0.25, + }, + 17.0, + ), + ( + lambda client: client.input("job-1", "ls\\n", offset=5, timeout=9), + "/v1/jobs/job-1/input", + { + "text": "ls\\n", + "offset": 5, + "timeout": 9, + "output_limit": 4096, + "idle_flush_seconds": 0.25, + }, + 19.0, + ), + ], +) +async def test_shellctl_client_blocking_calls_use_grace_read_timeout( + monkeypatch: pytest.MonkeyPatch, + call: Callable[[ShellctlClient], Awaitable[object]], + expected_path: str, + expected_json: dict[str, object], + wait_timeout: float, +) -> None: + captured: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["method"] = request.method + captured["path"] = request.url.path + captured["headers"] = dict(request.headers) + captured["json"] = json.loads(request.content.decode("utf-8")) + captured["timeout"] = request.extensions["timeout"] + return httpx.Response( + 200, + json={ + "job_id": "05211530-k7p", + "done": False, + "status": "running", + "exit_code": None, + "output_path": "/tmp/output.log", + "output": "", + "offset": 0, + "truncated": False, + }, + ) + + monkeypatch.setenv("SHELLCTL_AUTH_TOKEN", "from-env") + transport = httpx.MockTransport(handler) + async with ShellctlClient( + "http://127.0.0.1:8765", + output_limit=4096, + idle_flush_seconds=0.25, + transport=transport, + ) as client: + await call(client) + + headers = cast(dict[str, str], captured["headers"]) + assert captured["method"] == "POST" + assert captured["path"] == expected_path + assert headers["authorization"] == "Bearer from-env" + assert captured["json"] == expected_json + assert captured["timeout"] == { + "connect": 30.0, + "read": wait_timeout, + "write": 30.0, + "pool": 30.0, + } + + +@pytest.mark.anyio +@pytest.mark.parametrize( + ("call", "expected_method", "expected_path", "response_json", "assert_result"), + [ + ( + lambda client: client.tail("job-1"), + "GET", + "/v1/jobs/job-1/log/tail", + { + "job_id": "job-1", + "done": False, + "status": "running", + "exit_code": None, + "output_path": "/tmp/output.log", + "output": "tail", + "offset": 99, + "truncated": False, + }, + lambda result: (result.output, result.offset) == ("tail", 99), + ), + ( + lambda client: client.status("job-1"), + "GET", + "/v1/jobs/job-1", + { + "job_id": "job-1", + "status": "running", + "done": False, + "exit_code": None, + "created_at": "2026-01-01T00:00:00Z", + "started_at": "2026-01-01T00:00:01Z", + "ended_at": None, + "offset": 99, + }, + lambda result: (result.status, result.offset) == ("running", 99), + ), + ( + lambda client: client.delete("job-1", force=False), + "DELETE", + "/v1/jobs/job-1", + {"job_id": "job-1", "deleted": True}, + lambda result: (result.job_id, result.deleted) == ("job-1", True), + ), + ], +) +async def test_shellctl_client_control_calls_use_default_timeout( + call: Callable[[ShellctlClient], Awaitable[object]], + expected_method: str, + expected_path: str, + response_json: dict[str, object], + assert_result: Callable[[object], bool], +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == expected_method + assert request.url.path == expected_path + assert request.extensions["timeout"] == { + "connect": 30.0, + "read": 30.0, + "write": 30.0, + "pool": 30.0, + } + return httpx.Response(200, json=response_json) + + transport = httpx.MockTransport(handler) + async with ShellctlClient( + "http://127.0.0.1:8765", + output_limit=1234, + token="secret", + transport=transport, + ) as client: + result = await call(client) + + assert assert_result(result) + + +@pytest.mark.anyio +async def test_shellctl_client_terminate_uses_grace_read_timeout() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert request.url.path == "/v1/jobs/job-1/terminate" + assert json.loads(request.content.decode("utf-8")) == {"grace_seconds": 7.0} + assert request.extensions["timeout"] == { + "connect": 30.0, + "read": 17.0, + "write": 30.0, + "pool": 30.0, + } + return httpx.Response( + 200, + json={ + "job_id": "job-1", + "status": "terminated", + "done": True, + "exit_code": 130, + "created_at": "2026-01-01T00:00:00Z", + "started_at": "2026-01-01T00:00:01Z", + "ended_at": "2026-01-01T00:00:08Z", + "offset": 99, + }, + ) + + transport = httpx.MockTransport(handler) + async with ShellctlClient( + "http://127.0.0.1:8765", + token="secret", + transport=transport, + ) as client: + result = await client.terminate("job-1", grace_seconds=7.0) + + assert (result.status, result.exit_code) == ("terminated", 130) + + +@pytest.mark.anyio +@pytest.mark.parametrize( + ("kwargs", "expected_params", "expected_read_timeout"), + [ + ( + {"force": True, "grace_seconds": 3.5}, + {"force": "true", "grace_seconds": "3.5"}, + 13.5, + ), + ( + {"force": True}, + {"force": "true"}, + DEFAULT_TERMINATE_GRACE_SECONDS + 10.0, + ), + ( + {"force": True, "grace_seconds": 0.0}, + {"force": "true", "grace_seconds": "0.0"}, + 10.0, + ), + ], +) +async def test_shellctl_client_forced_delete_uses_terminate_grace_timeout( + kwargs: ForcedDeleteKwargs, + expected_params: dict[str, str], + expected_read_timeout: float, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "DELETE" + assert request.url.path == "/v1/jobs/job-1" + assert dict(request.url.params) == expected_params + assert request.extensions["timeout"] == { + "connect": 30.0, + "read": expected_read_timeout, + "write": 30.0, + "pool": 30.0, + } + return httpx.Response(200, json={"job_id": "job-1", "deleted": True}) + + transport = httpx.MockTransport(handler) + async with ShellctlClient( + "http://127.0.0.1:8765", + token="secret", + transport=transport, + ) as client: + result = await client.delete("job-1", **kwargs) + + assert (result.job_id, result.deleted) == ("job-1", True) + + +@pytest.mark.anyio +async def test_shellctl_client_closes_owned_client_on_close_and_context_exit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"status": "ok"}) + + class TrackingAsyncClient(httpx.AsyncClient): + created_clients: ClassVar[list["TrackingAsyncClient"]] = [] + + def __init__(self, *args: object, **kwargs: object) -> None: + super().__init__(*args, **kwargs) + self.close_calls = 0 + self.__class__.created_clients.append(self) + + async def aclose(self) -> None: + self.close_calls += 1 + await super().aclose() + + monkeypatch.setattr(shellctl_sdk.httpx, "AsyncClient", TrackingAsyncClient) + close_client = ShellctlClient( + "http://127.0.0.1:8765", + transport=httpx.MockTransport(handler), + ) + direct_owned_client = TrackingAsyncClient.created_clients[-1] + await close_client.close() + assert direct_owned_client.close_calls == 1 + + async with ShellctlClient( + "http://127.0.0.1:8765", + transport=httpx.MockTransport(handler), + ) as context_client: + context_owned_client = TrackingAsyncClient.created_clients[-1] + await context_client.healthz() + + assert context_owned_client.close_calls == 1 + + +@pytest.mark.anyio +async def test_shellctl_client_list_jobs_uses_query_params_and_auth() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" + assert request.url.path == "/v1/jobs" + assert request.url.params["status"] == "running" + assert request.url.params["limit"] == "5" + assert request.headers["authorization"] == "Bearer secret" + return httpx.Response( + 200, + json={ + "jobs": [ + { + "job_id": "job-1", + "status": "running", + "created_at": "2026-05-21T15:30:12Z", + } + ] + }, + ) + + transport = httpx.MockTransport(handler) + async with ShellctlClient("http://127.0.0.1:8765", token="secret", transport=transport) as client: + result = await client.list_jobs(status="running", limit=5) + + assert len(result) == 1 + assert result[0].job_id == "job-1" + assert result[0].status == JobStatusName.RUNNING + + +@pytest.mark.anyio +async def test_shellctl_client_wait_uses_body_and_omits_auth_without_token() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert request.url.path == "/v1/jobs/job-1/wait" + assert "authorization" not in request.headers + assert json.loads(request.content.decode("utf-8")) == { + "offset": 3, + "timeout": 9.0, + "output_limit": 2048, + "idle_flush_seconds": 0.1, + } + return httpx.Response( + 200, + json={ + "job_id": "job-1", + "done": False, + "status": "running", + "exit_code": None, + "output_path": "/tmp/wait.log", + "output": "chunk", + "offset": 8, + "truncated": False, + }, + ) + + transport = httpx.MockTransport(handler) + async with ShellctlClient( + "http://127.0.0.1:8765", + output_limit=2048, + idle_flush_seconds=0.1, + transport=transport, + ) as client: + result = await client.wait("job-1", offset=3, timeout=9) + + assert result.job_id == "job-1" + assert result.offset == 8 + assert result.output == "chunk" + + +@pytest.mark.anyio +async def test_shellctl_client_input_uses_body_and_auth() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert request.url.path == "/v1/jobs/job-1/input" + assert request.headers["authorization"] == "Bearer secret" + assert json.loads(request.content.decode("utf-8")) == { + "text": "ls\n", + "offset": 5, + "timeout": 4.0, + "output_limit": 512, + "idle_flush_seconds": 0.0, + } + return httpx.Response( + 200, + json={ + "job_id": "job-1", + "done": False, + "status": "running", + "exit_code": None, + "output_path": "/tmp/input.log", + "output": "reply", + "offset": 10, + "truncated": False, + }, + ) + + transport = httpx.MockTransport(handler) + async with ShellctlClient( + "http://127.0.0.1:8765", + output_limit=512, + idle_flush_seconds=0.0, + token="secret", + transport=transport, + ) as client: + result = await client.input("job-1", "ls\n", offset=5, timeout=4) + + assert result.output == "reply" + assert result.offset == 10 + + +@pytest.mark.anyio +async def test_shellctl_client_terminate_uses_body_and_auth() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert request.url.path == "/v1/jobs/job-1/terminate" + assert request.headers["authorization"] == "Bearer secret" + assert json.loads(request.content.decode("utf-8")) == {"grace_seconds": 0.25} + return httpx.Response( + 200, + json={ + "job_id": "job-1", + "status": "terminated", + "done": True, + "created_at": "2026-05-21T15:30:12Z", + "started_at": "2026-05-21T15:30:13Z", + "ended_at": "2026-05-21T15:30:18Z", + "offset": 12, + }, + ) + + transport = httpx.MockTransport(handler) + async with ShellctlClient("http://127.0.0.1:8765", token="secret", transport=transport) as client: + result = await client.terminate("job-1", 0.25) + + assert result.status == JobStatusName.TERMINATED + assert result.done is True + + +@pytest.mark.anyio +async def test_shellctl_client_delete_uses_query_params_and_auth() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "DELETE" + assert request.url.path == "/v1/jobs/job-1" + assert request.url.params["force"] == "true" + assert request.url.params["grace_seconds"] == "0.5" + assert request.headers["authorization"] == "Bearer secret" + return httpx.Response(200, json={"job_id": "job-1", "deleted": True}) + + transport = httpx.MockTransport(handler) + async with ShellctlClient("http://127.0.0.1:8765", token="secret", transport=transport) as client: + result = await client.delete("job-1", force=True, grace_seconds=0.5) + + assert result.job_id == "job-1" + assert result.deleted is True + + +@pytest.mark.anyio +async def test_shellctl_client_raises_structured_errors() -> None: + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 409, + json={"error": {"code": "job_not_running", "message": "already done"}}, + ) + + transport = httpx.MockTransport(handler) + async with ShellctlClient("http://127.0.0.1:8765", token="secret", transport=transport) as client: + with pytest.raises(ShellctlClientError, match="job_not_running"): + await client.input("job-1", "ls\n", offset=0) + + +@pytest.mark.anyio +async def test_shellctl_client_does_not_close_injected_client() -> None: + async def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"status": "ok"}) + + class TrackingAsyncClient(httpx.AsyncClient): + def __init__(self) -> None: + super().__init__( + base_url="http://127.0.0.1:8765", + transport=httpx.MockTransport(handler), + ) + self.close_calls = 0 + + async def aclose(self) -> None: + self.close_calls += 1 + await super().aclose() + + injected_client = TrackingAsyncClient() + + async with ShellctlClient( + "http://127.0.0.1:8765", + client=injected_client, + ) as client: + await client.healthz() + + assert injected_client.close_calls == 0 + await injected_client.aclose() + assert injected_client.close_calls == 1 + + +@pytest.mark.anyio +async def test_shellctl_client_raises_invalid_json_errors() -> None: + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "application/json"}, + content=b"{", + ) + + transport = httpx.MockTransport(handler) + async with ShellctlClient("http://127.0.0.1:8765", transport=transport) as client: + with pytest.raises(ShellctlClientError) as exc_info: + await client.status("job-1") + + assert exc_info.value.code == "invalid_json" + assert exc_info.value.message == "{" + + +@pytest.mark.anyio +async def test_shellctl_client_raises_invalid_payload_errors() -> None: + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=["not", "a", "dict"]) + + transport = httpx.MockTransport(handler) + async with ShellctlClient("http://127.0.0.1:8765", transport=transport) as client: + with pytest.raises(ShellctlClientError) as exc_info: + await client.healthz() + + assert exc_info.value.code == "invalid_payload" + assert exc_info.value.message == '["not","a","dict"]' + + +@pytest.mark.anyio +async def test_shellctl_client_health_returns_model_and_healthz_stays_compatible() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" + assert request.url.path == "/healthz" + return httpx.Response(200, json={"status": "ok"}) + + transport = httpx.MockTransport(handler) + async with ShellctlClient("http://127.0.0.1:8765", transport=transport) as client: + health = await client.health() + healthz = await client.healthz() + + assert health == HealthResponse(status="ok") + assert healthz == {"status": "ok"} diff --git a/dify-agent/tests/local/shellctl/test_shellctl_service.py b/dify-agent/tests/local/shellctl/test_shellctl_service.py new file mode 100644 index 00000000000..6cfc30106ea --- /dev/null +++ b/dify-agent/tests/local/shellctl/test_shellctl_service.py @@ -0,0 +1,1687 @@ +from __future__ import annotations + +import importlib +import json +import os +import re +import shutil +import sqlite3 +import subprocess +import sys +from collections.abc import Awaitable, Callable +from pathlib import Path + +import anyio +import httpx2 as httpx +import pytest +from typer.testing import CliRunner + +import shellctl.server.service as server_service_module +from shellctl.server import ( + JobRow, + ShellctlConfig, + ShellctlServerError, + ShellctlService, + cli, + create_app, +) +from shellctl.server.artifacts import ( + JOB_ENV_FILENAME, + PIPE_DRAINED_FILENAME, + PIPE_ERROR_LOG_FILENAME, + PIPE_FAILED_FILENAME, + RUNNER_ENDED_AT_FILENAME, + RUNNER_EXIT_CODE_FILENAME, + job_env_path, + pipe_error_log_path, +) +from shellctl.server.tmux import TmuxController +from shellctl.shared import ( + DEFAULT_AUTH_TOKEN_ENV, + InputJobRequest, + JobStatusName, + JobStatusView, + RunJobRequest, + TerminalSize, + TerminateJobRequest, + WaitJobRequest, + job_pane_target, + job_session_name, +) + +server_serve_module = importlib.import_module("shellctl.server.serve") + + +class FakeTmuxController: + def __init__(self) -> None: + self.sessions: set[str] = set() + self.pipe_active: dict[str, bool | None] = {} + self.cleaned: list[str] = [] + self.touch_ready_on_enable = True + self.pipe_active_on_enable: bool | None = True + self.pipe_error_log_text: str | None = None + self.on_send_interrupt: Callable[[str], Awaitable[None]] | None = None + self.on_send_input: Callable[[str, str], Awaitable[None]] | None = None + + async def start_server(self) -> None: + return None + + async def list_sessions(self) -> set[str]: + return set(self.sessions) + + async def session_exists(self, session_name: str) -> bool: + return session_name in self.sessions + + async def is_output_pipe_active(self, *, job_id: str) -> bool | None: + return self.pipe_active.get(job_id) + + async def create_job_session(self, *, job_id: str, job_dir: Path, cwd: Path, terminal: TerminalSize) -> None: + del job_dir, cwd, terminal + self.sessions.add(job_session_name(job_id)) + self.pipe_active[job_id] = False + + async def enable_output_pipe(self, *, job_id: str, job_dir: Path, ready_file: Path) -> None: + self.pipe_active[job_id] = self.pipe_active_on_enable + if self.pipe_error_log_text is not None: + pipe_error_log_path(job_dir).write_text( + self.pipe_error_log_text, + encoding="utf-8", + ) + if self.touch_ready_on_enable: + ready_file.touch() + + async def send_input(self, *, job_id: str, text: str) -> None: + if self.on_send_input is not None: + await self.on_send_input(job_id, text) + + async def send_interrupt(self, *, job_id: str) -> None: + if self.on_send_interrupt is not None: + await self.on_send_interrupt(job_id) + + async def cleanup_session(self, *, job_id: str) -> None: + self.cleaned.append(job_id) + self.sessions.discard(job_session_name(job_id)) + self.pipe_active.pop(job_id, None) + + +async def _create_service( + tmp_path: Path, *, auth_token: str | None = None +) -> tuple[ShellctlService, FakeTmuxController]: + fake_tmux = FakeTmuxController() + service = ShellctlService( + ShellctlConfig( + auth_token=auth_token, + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + ), + tmux=fake_tmux, + ) + await service.initialize_database() + service._ensure_dir(service.config.jobs_dir) + return service, fake_tmux + + +async def _create_real_service( + tmp_path: Path, + *, + runner_exit_command: tuple[str, ...] | None = None, + sanitize_pty_command: tuple[str, ...] | None = None, +) -> ShellctlService: + defaults = ShellctlConfig( + state_dir=tmp_path / "default-state", + runtime_dir=tmp_path / "default-run", + ) + config = ShellctlConfig( + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + runner_exit_command=runner_exit_command or defaults.runner_exit_command, + sanitize_pty_command=sanitize_pty_command or defaults.sanitize_pty_command, + ) + service = ShellctlService(config) + await service.initialize() + return service + + +def _write_delayed_sanitize_wrapper( + tmp_path: Path, + *, + delay_seconds: float, + sanitize_exit_code: int | None = None, +) -> tuple[str, ...]: + wrapper_path = tmp_path / "delayed-sanitize-wrapper.py" + wrapper_path.write_text( + "\n".join( + [ + "from __future__ import annotations", + "", + "import subprocess", + "import sys", + "import time", + "from pathlib import Path", + "", + (f"REAL = [{sys.executable!r}, '-m', 'shellctl_runtime.sanitize']"), + "args = sys.argv[1:]", + "if '--ready-file' in args:", + " ready_file = Path(args[args.index('--ready-file') + 1])", + " ready_file.touch()", + f" time.sleep({delay_seconds!r})", + (f" raise SystemExit({sanitize_exit_code!r})" if sanitize_exit_code is not None else ""), + "raise SystemExit(subprocess.run(REAL + args, check=False).returncode)", + "", + ] + ), + encoding="utf-8", + ) + return (sys.executable, str(wrapper_path)) + + +def _capture_serve_config( + monkeypatch: pytest.MonkeyPatch, +) -> tuple[dict[str, object], CliRunner]: + captured: dict[str, object] = {} + + def fake_create_app(config: ShellctlConfig) -> object: + captured["config"] = config + return object() + + def fake_run(app: object, *, host: str, port: int, log_level: str) -> None: + captured["app"] = app + captured["host"] = host + captured["port"] = port + captured["log_level"] = log_level + + monkeypatch.setattr(server_serve_module, "_create_app", fake_create_app) + monkeypatch.setattr(server_serve_module, "_uvicorn_run", fake_run) + return captured, CliRunner() + + +async def _seed_job( + service: ShellctlService, + *, + job_id: str, + status: JobStatusName, + exit_code: int | None = None, + ended_at: str | None = None, + created_at: str = "2026-05-21T15:30:12Z", + started_at: str | None = "2026-05-21T15:30:13Z", +) -> JobRow: + job_dir = service.config.jobs_dir / job_id + job_dir.mkdir(mode=0o700) + (job_dir / "script").write_text("printf ready\n", encoding="utf-8") + (job_dir / "output.log").write_text("hello\n", encoding="utf-8") + row = JobRow( + job_id=job_id, + script_path=f"jobs/{job_id}/script", + output_path=f"jobs/{job_id}/output.log", + cwd="/tmp", + terminal_cols=120, + terminal_rows=80, + status=status.value, + session_name=job_session_name(job_id), + pane_target=job_pane_target(job_id), + exit_code=exit_code, + reason=None, + message=None, + created_at=created_at, + started_at=started_at, + ended_at=ended_at, + updated_at=created_at, + ) + inserted = await service._insert_job_row(row) + assert inserted is True + return row + + +class RecordingService(ShellctlService): + def __init__(self, config: ShellctlConfig, *, tmux: FakeTmuxController) -> None: + super().__init__(config, tmux=tmux) + self.deleted_rows: list[str] = [] + + async def _delete_job_row(self, job_id: str) -> None: + self.deleted_rows.append(job_id) + await super()._delete_job_row(job_id) + + +class StartupObservationService(RecordingService): + def __init__(self, config: ShellctlConfig, *, tmux: FakeTmuxController) -> None: + super().__init__(config, tmux=tmux) + self.observed_startup_statuses: list[JobStatusName] = [] + + async def _insert_job_row(self, row: JobRow) -> bool: + inserted = await super()._insert_job_row(row) + if inserted: + startup_view = await self.get_job_status(row.job_id) + self.observed_startup_statuses.append(startup_view.status) + return inserted + + +class ConcurrentDeleteRaceService(RecordingService): + def __init__(self, config: ShellctlConfig, *, tmux: FakeTmuxController) -> None: + super().__init__(config, tmux=tmux) + self.disappear_on_status: set[str] = set() + self.disappear_on_delete: set[str] = set() + + async def get_job_status(self, job_id: str): # type: ignore[override] + if job_id in self.disappear_on_status: + self.disappear_on_status.remove(job_id) + await super()._delete_job_row(job_id) + return await super().get_job_status(job_id) + + async def _delete_job_row(self, job_id: str) -> None: + if job_id in self.disappear_on_delete: + self.disappear_on_delete.remove(job_id) + raise ShellctlServerError(404, "job_not_found", f"Unknown job id: {job_id}") + await super()._delete_job_row(job_id) + + +class InsertConflictService(RecordingService): + def __init__(self, config: ShellctlConfig, *, tmux: FakeTmuxController) -> None: + super().__init__(config, tmux=tmux) + self.failed_insert_job_ids: list[str] = [] + + async def _insert_job_row(self, row: JobRow) -> bool: + if not self.failed_insert_job_ids: + self.failed_insert_job_ids.append(row.job_id) + return False + return await super()._insert_job_row(row) + + +@pytest.mark.anyio +async def test_sqlite_persistence_survives_service_restart(tmp_path: Path) -> None: + service, _fake_tmux = await _create_service(tmp_path) + await _seed_job( + service, + job_id="05211530-k7p", + status=JobStatusName.EXITED, + exit_code=0, + ended_at="2026-05-21T15:30:20Z", + ) + await service.shutdown() + + restarted, _fake_tmux = await _create_service(tmp_path) + row = await restarted._get_job_row("05211530-k7p") + + assert row.status == JobStatusName.EXITED.value + assert row.exit_code == 0 + assert restarted.config.db_path.exists() + await restarted.shutdown() + + +@pytest.mark.anyio +async def test_run_job_does_not_materialize_fresh_row_to_lost_during_startup_window( + tmp_path: Path, +) -> None: + fake_tmux = FakeTmuxController() + service = StartupObservationService( + ShellctlConfig(state_dir=tmp_path / "state", runtime_dir=tmp_path / "run"), + tmux=fake_tmux, + ) + await service.initialize_database() + service._ensure_dir(service.config.jobs_dir) + fake_tmux.touch_ready_on_enable = False + service.config = ShellctlConfig( + state_dir=service.config.state_dir, + runtime_dir=service.config.runtime_dir, + poll_interval_seconds=0.001, + pipe_ready_timeout_seconds=0.01, + ) + + await service.run_job( + RunJobRequest( + script="printf ready\n", + cwd=str(tmp_path), + terminal=TerminalSize(), + timeout=0.01, + output_limit=8192, + idle_flush_seconds=0.01, + ) + ) + + assert len(service.observed_startup_statuses) == 1 + assert service.observed_startup_statuses[0] in { + JobStatusName.CREATED, + JobStatusName.STARTING, + JobStatusName.RUNNING, + } + await service.shutdown() + + +@pytest.mark.anyio +async def test_run_job_happy_path_persists_running_row_and_artifacts( + tmp_path: Path, +) -> None: + fake_tmux = FakeTmuxController() + service = RecordingService( + ShellctlConfig(state_dir=tmp_path / "state", runtime_dir=tmp_path / "run"), + tmux=fake_tmux, + ) + await service.initialize_database() + service._ensure_dir(service.config.jobs_dir) + + result = await service.run_job( + RunJobRequest( + script="printf ready\n", + cwd=str(tmp_path), + env={"HELLO": "world", "UNICODE": "盐粒"}, + terminal=TerminalSize(cols=100, rows=40), + timeout=0.01, + output_limit=8192, + idle_flush_seconds=0.01, + ) + ) + + row = await service._get_job_row(result.job_id) + job_dir = service.config.jobs_dir / result.job_id + + assert result.status is JobStatusName.RUNNING + assert result.done is False + assert result.output == "" + assert result.offset == 0 + assert row.status == JobStatusName.RUNNING.value + assert row.cwd == str(tmp_path.resolve()) + assert row.script_path == f"jobs/{result.job_id}/script" + assert row.output_path == f"jobs/{result.job_id}/output.log" + assert row.started_at is not None + assert (job_dir / "script").exists() + assert json.loads(job_env_path(job_dir).read_text(encoding="utf-8")) == { + "HELLO": "world", + "UNICODE": "盐粒", + } + assert (job_dir / "output.log").exists() + assert (job_dir / "start-gate").exists() + assert job_session_name(result.job_id) in fake_tmux.sessions + assert fake_tmux.pipe_active[result.job_id] is True + assert await service.get_job_status(result.job_id) == JobStatusView( + job_id=result.job_id, + status=JobStatusName.RUNNING, + done=False, + exit_code=None, + created_at=row.created_at, + started_at=row.started_at, + ended_at=None, + offset=0, + ) + await service.shutdown() + + +@pytest.mark.anyio +@pytest.mark.skipif(shutil.which("tmux") is None, reason="tmux is required") +async def test_run_job_env_overlay_is_visible_without_replacing_inherited_env( + tmp_path: Path, +) -> None: + service = await _create_real_service(tmp_path) + + try: + result = await service.run_job( + RunJobRequest( + script=( + "python3 - <<'PY'\n" + "import os\n" + "print(os.environ['SHELLCTL_PRESET'])\n" + "print('PATH' in os.environ)\n" + "PY\n" + ), + cwd=str(tmp_path), + env={"SHELLCTL_PRESET": "from-client"}, + terminal=TerminalSize(), + timeout=5, + output_limit=8192, + idle_flush_seconds=1, + ) + ) + + assert result.done is True + assert result.status is JobStatusName.EXITED + assert result.exit_code == 0 + assert result.output == "from-client\nTrue\n" + finally: + await service.shutdown() + + +@pytest.mark.anyio +@pytest.mark.skipif(shutil.which("tmux") is None, reason="tmux is required") +async def test_run_job_env_overlay_overrides_inherited_value( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SHELLCTL_PRESET", "parent") + service = await _create_real_service(tmp_path) + + try: + result = await service.run_job( + RunJobRequest( + script=("python3 - <<'PY'\nimport os\nprint(os.environ['SHELLCTL_PRESET'])\nPY\n"), + cwd=str(tmp_path), + env={"SHELLCTL_PRESET": "from-client"}, + terminal=TerminalSize(), + timeout=5, + output_limit=8192, + idle_flush_seconds=1, + ) + ) + + assert result.done is True + assert result.status is JobStatusName.EXITED + assert result.exit_code == 0 + assert result.output == "from-client\n" + finally: + await service.shutdown() + + +@pytest.mark.anyio +@pytest.mark.skipif(shutil.which("tmux") is None, reason="tmux is required") +async def test_send_input_reaches_real_job_stdin_after_env_bootstrap( + tmp_path: Path, +) -> None: + service = await _create_real_service(tmp_path) + + try: + initial = await service.run_job( + RunJobRequest( + script=("printf 'ready\\n'\nIFS= read -r line\nprintf 'got:%s\\n' \"$line\"\n"), + cwd=str(tmp_path), + terminal=TerminalSize(), + timeout=5, + output_limit=8192, + idle_flush_seconds=0.01, + ) + ) + + ready_window = initial + if ready_window.output == "": + ready_window = await service.wait_job( + initial.job_id, + WaitJobRequest( + offset=initial.offset, + timeout=1, + output_limit=8192, + idle_flush_seconds=0.01, + ), + ) + + result = await service.send_input( + initial.job_id, + InputJobRequest( + text="hello from stdin\n", + offset=ready_window.offset, + timeout=1, + output_limit=8192, + idle_flush_seconds=0.01, + ), + ) + + output_after_input = result.output + final = result + if not final.done: + # `send_input()` shares wait semantics with `wait_job()`: it may + # return after the post-input output flushes but before tmux exit + # artifacts have materialized into a terminal DB status on slower CI. + final = await service.wait_job( + initial.job_id, + WaitJobRequest( + offset=result.offset, + timeout=5, + output_limit=8192, + idle_flush_seconds=0.01, + ), + ) + output_after_input += final.output + + assert ready_window.done is False + assert final.done is True + assert final.status is JobStatusName.EXITED + assert final.exit_code == 0 + assert output_after_input.endswith("got:hello from stdin\n") + finally: + await service.shutdown() + + +@pytest.mark.anyio +@pytest.mark.skipif(shutil.which("tmux") is None, reason="tmux is required") +async def test_terminated_job_does_not_emit_python_wrapper_traceback( + tmp_path: Path, +) -> None: + service = await _create_real_service(tmp_path) + + try: + initial = await service.run_job( + RunJobRequest( + script=("trap 'exit 130' INT\nprintf 'ready\\n'\nwhile :; do sleep 1; done\n"), + cwd=str(tmp_path), + terminal=TerminalSize(), + timeout=5, + output_limit=8192, + idle_flush_seconds=0.01, + ) + ) + + ready_window = initial + if ready_window.output == "": + ready_window = await service.wait_job( + initial.job_id, + WaitJobRequest( + offset=initial.offset, + timeout=1, + output_limit=8192, + idle_flush_seconds=0.01, + ), + ) + assert ready_window.done is False + + terminated = await service.terminate_job( + initial.job_id, + TerminateJobRequest(grace_seconds=0.2), + ) + final = await service.wait_job( + initial.job_id, + WaitJobRequest( + offset=ready_window.offset, + timeout=1, + output_limit=8192, + idle_flush_seconds=0.01, + ), + ) + + assert terminated.status is JobStatusName.TERMINATED + assert final.done is True + assert "KeyboardInterrupt" not in final.output + assert "Traceback (most recent call last)" not in final.output + finally: + await service.shutdown() + + +@pytest.mark.anyio +@pytest.mark.skipif(shutil.which("tmux") is None, reason="tmux is required") +async def test_run_job_returns_flushed_output_on_first_terminal_result( + tmp_path: Path, +) -> None: + sanitize_pty_command = _write_delayed_sanitize_wrapper(tmp_path, delay_seconds=0.2) + service = await _create_real_service( + tmp_path, + sanitize_pty_command=sanitize_pty_command, + ) + + try: + result = await service.run_job( + RunJobRequest( + script="printf 'delayed-flush\\n'\n", + cwd=str(tmp_path), + terminal=TerminalSize(), + timeout=3, + output_limit=8192, + idle_flush_seconds=1, + ) + ) + + reread = await service.wait_job( + result.job_id, + WaitJobRequest( + offset=0, + timeout=0.001, + output_limit=8192, + idle_flush_seconds=0.01, + ), + ) + + assert result.done is True + assert result.status is JobStatusName.EXITED + assert result.exit_code == 0 + assert result.output == "delayed-flush\n" + assert reread.output == "delayed-flush\n" + assert reread.offset == result.offset + finally: + await service.shutdown() + + +@pytest.mark.anyio +@pytest.mark.skipif(shutil.which("tmux") is None, reason="tmux is required") +async def test_sanitize_failure_does_not_commit_normal_exit( + tmp_path: Path, +) -> None: + sanitize_pty_command = _write_delayed_sanitize_wrapper( + tmp_path, + delay_seconds=0.2, + sanitize_exit_code=7, + ) + service = await _create_real_service( + tmp_path, + sanitize_pty_command=sanitize_pty_command, + ) + + try: + result = await service.run_job( + RunJobRequest( + script="printf 'missing-output\\n'\n", + cwd=str(tmp_path), + terminal=TerminalSize(), + timeout=3, + output_limit=8192, + idle_flush_seconds=1, + ) + ) + + reread = await service.wait_job( + result.job_id, + WaitJobRequest( + offset=0, + timeout=0.001, + output_limit=8192, + idle_flush_seconds=0.01, + ), + ) + job_dir = service.config.jobs_dir / result.job_id + + assert result.done is True + assert result.status is not JobStatusName.EXITED + assert result.exit_code is None + assert result.output == "" + assert reread.output == "" + assert not (job_dir / PIPE_DRAINED_FILENAME).exists() + assert (job_dir / PIPE_FAILED_FILENAME).exists() + finally: + await service.shutdown() + + +@pytest.mark.anyio +@pytest.mark.skipif(shutil.which("tmux") is None, reason="tmux is required") +@pytest.mark.skipif(shutil.which("uv") is None, reason="uv is required") +async def test_uv_quiet_shebang_returns_output_in_first_terminal_result( + tmp_path: Path, +) -> None: + service = await _create_real_service(tmp_path) + + try: + result = await service.run_job( + RunJobRequest( + script=( + "#!/usr/bin/env -S uv run --script --quiet\n" + "# /// script\n" + '# requires-python = ">=3.12"\n' + "# dependencies = []\n" + "# ///\n" + 'print("hello")\n' + ), + cwd=str(tmp_path), + terminal=TerminalSize(), + timeout=10, + output_limit=8192, + idle_flush_seconds=1, + ) + ) + + assert result.done is True + assert result.status is JobStatusName.EXITED + assert result.exit_code == 0 + assert result.output == "hello\n" + finally: + await service.shutdown() + + +@pytest.mark.anyio +@pytest.mark.skipif(shutil.which("tmux") is None, reason="tmux is required") +@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is required") +@pytest.mark.parametrize( + ("script", "expected_output"), + [ + ("printf 'no-shebang\\n'\n", "no-shebang\n"), + ("#!/bin/sh\nprintf 'sh-shebang\\n'\n", "sh-shebang\n"), + ( + "#!/usr/bin/env bash\nprintf 'bash-shebang\\n'\n", + "bash-shebang\n", + ), + ], +) +async def test_existing_script_modes_still_return_output( + tmp_path: Path, + script: str, + expected_output: str, +) -> None: + service = await _create_real_service(tmp_path) + + try: + result = await service.run_job( + RunJobRequest( + script=script, + cwd=str(tmp_path), + terminal=TerminalSize(), + timeout=5, + output_limit=8192, + idle_flush_seconds=1, + ) + ) + + assert result.done is True + assert result.status is JobStatusName.EXITED + assert result.exit_code == 0 + assert result.output == expected_output + finally: + await service.shutdown() + + +@pytest.mark.anyio +async def test_run_job_retries_after_sqlite_insert_conflict_and_cleans_artifacts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fake_tmux = FakeTmuxController() + service = InsertConflictService( + ShellctlConfig(state_dir=tmp_path / "state", runtime_dir=tmp_path / "run"), + tmux=fake_tmux, + ) + await service.initialize_database() + service._ensure_dir(service.config.jobs_dir) + generated_ids = iter(["05211530-k7p", "05211530-abc"]) + + monkeypatch.setattr(server_service_module, "generate_job_id", lambda now=None: next(generated_ids)) + + result = await service.run_job( + RunJobRequest( + script="printf ready\n", + cwd=str(tmp_path), + terminal=TerminalSize(), + timeout=0.01, + output_limit=8192, + idle_flush_seconds=0.01, + ) + ) + + failed_job_id = service.failed_insert_job_ids[0] + assert failed_job_id == "05211530-k7p" + assert result.job_id == "05211530-abc" + assert not (service.config.jobs_dir / failed_job_id).exists() + assert (service.config.jobs_dir / result.job_id).exists() + with pytest.raises(ShellctlServerError, match="Unknown job id"): + await service._get_job_row(failed_job_id) + await service.shutdown() + + +@pytest.mark.anyio +async def test_transition_status_does_not_overwrite_terminal_state( + tmp_path: Path, +) -> None: + service, _fake_tmux = await _create_service(tmp_path) + await _seed_job( + service, + job_id="05211530-k7p", + status=JobStatusName.TERMINATED, + ended_at="2026-05-21T15:30:20Z", + ) + + row = await service._transition_status( + "05211530-k7p", + allowed_from={JobStatusName.RUNNING}, + target=JobStatusName.RUNNING, + ) + + assert row.status == JobStatusName.TERMINATED.value + await service.shutdown() + + +@pytest.mark.anyio +async def test_wait_job_flushes_preexisting_unread_output_after_idle( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + service, fake_tmux = await _create_service(tmp_path) + await _seed_job(service, job_id="05211530-k7p", status=JobStatusName.RUNNING) + fake_tmux.sessions.add(job_session_name("05211530-k7p")) + fake_tmux.pipe_active["05211530-k7p"] = True + + async def fail_sleep(_delay: float) -> None: + raise AssertionError("wait_job should not sleep when idle_flush_seconds=0") + + monkeypatch.setattr(server_service_module.anyio, "sleep", fail_sleep) + + result = await service.wait_job( + "05211530-k7p", + WaitJobRequest(offset=0, timeout=0.2, output_limit=8192, idle_flush_seconds=0), + ) + + assert result.output == "hello\n" + assert result.done is False + await service.shutdown() + + +@pytest.mark.anyio +async def test_wait_job_returns_immediately_for_terminal_row_and_reads_remaining_output( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + service, _fake_tmux = await _create_service(tmp_path) + job_id = "05211530-k7p" + await _seed_job( + service, + job_id=job_id, + status=JobStatusName.EXITED, + exit_code=0, + ended_at="2026-05-21T15:30:20Z", + ) + output_path = service.config.jobs_dir / job_id / "output.log" + output_path.write_text("hello\nworld\n", encoding="utf-8") + + async def fail_sleep(_delay: float) -> None: + raise AssertionError("terminal wait should not sleep") + + monkeypatch.setattr(server_service_module.anyio, "sleep", fail_sleep) + + result = await service.wait_job( + job_id, + WaitJobRequest( + offset=len(b"hello\n"), + timeout=0.5, + output_limit=8192, + idle_flush_seconds=0.01, + ), + ) + + assert result.done is True + assert result.status is JobStatusName.EXITED + assert result.exit_code == 0 + assert result.output == "world\n" + assert result.offset == len(b"hello\nworld\n") + await service.shutdown() + + +@pytest.mark.anyio +async def test_starting_job_is_not_marked_lost_during_local_startup_window( + tmp_path: Path, +) -> None: + service, _fake_tmux = await _create_service(tmp_path) + await _seed_job(service, job_id="05211530-k7p", status=JobStatusName.STARTING) + service._starting_jobs.add("05211530-k7p") + + view = await service.get_job_status("05211530-k7p") + + assert view.status is JobStatusName.STARTING + assert view.done is False + await service.shutdown() + + +@pytest.mark.anyio +async def test_stale_starting_job_with_dead_pipe_materializes_failed( + tmp_path: Path, +) -> None: + service, fake_tmux = await _create_service(tmp_path) + await _seed_job(service, job_id="05211530-k7p", status=JobStatusName.STARTING) + fake_tmux.sessions.add(job_session_name("05211530-k7p")) + fake_tmux.pipe_active["05211530-k7p"] = False + + view = await service.get_job_status("05211530-k7p") + row = await service._get_job_row("05211530-k7p") + + assert view.status is JobStatusName.FAILED + assert row.reason == "pipe_failed" + await service.shutdown() + + +@pytest.mark.anyio +async def test_terminate_beats_concurrent_runner_exit(tmp_path: Path) -> None: + service, fake_tmux = await _create_service(tmp_path) + await _seed_job(service, job_id="05211530-k7p", status=JobStatusName.RUNNING) + fake_tmux.sessions.add(job_session_name("05211530-k7p")) + fake_tmux.pipe_active["05211530-k7p"] = True + + async def record_exit(job_id: str) -> None: + await service.record_runner_exit(job_id, 130, "2026-05-21T15:30:20Z") + + fake_tmux.on_send_interrupt = record_exit + view = await service.terminate_job("05211530-k7p", TerminateJobRequest(grace_seconds=0)) + + assert view.status is JobStatusName.TERMINATED + assert view.exit_code == 130 + await service.shutdown() + + +@pytest.mark.anyio +async def test_record_runner_exit_materializes_nonterminal_row_to_exited( + tmp_path: Path, +) -> None: + service, _fake_tmux = await _create_service(tmp_path) + await _seed_job(service, job_id="05211530-k7p", status=JobStatusName.RUNNING) + + await service.record_runner_exit("05211530-k7p", 7, "2026-05-21T15:30:20Z") + + row = await service._get_job_row("05211530-k7p") + view = await service.get_job_status("05211530-k7p") + + assert row.status == JobStatusName.EXITED.value + assert row.exit_code == 7 + assert row.ended_at == "2026-05-21T15:30:20Z" + assert view.status is JobStatusName.EXITED + assert view.done is True + assert view.exit_code == 7 + assert view.ended_at == "2026-05-21T15:30:20Z" + await service.shutdown() + + +@pytest.mark.anyio +async def test_pipe_monitor_marks_running_job_failed_when_pipe_disappears( + tmp_path: Path, +) -> None: + service, fake_tmux = await _create_service(tmp_path) + await _seed_job(service, job_id="05211530-k7p", status=JobStatusName.RUNNING) + fake_tmux.sessions.add(job_session_name("05211530-k7p")) + fake_tmux.pipe_active["05211530-k7p"] = False + + await service.check_running_jobs_pipe_health() + row = await service._get_job_row("05211530-k7p") + + assert row.status == JobStatusName.FAILED.value + assert row.reason == "pipe_failed" + await service.shutdown() + + +@pytest.mark.anyio +async def test_session_disappearing_between_probes_materializes_lost_not_pipe_failed( + tmp_path: Path, +) -> None: + service, fake_tmux = await _create_service(tmp_path) + await _seed_job(service, job_id="05211530-k7p", status=JobStatusName.RUNNING) + fake_tmux.sessions.add(job_session_name("05211530-k7p")) + fake_tmux.pipe_active["05211530-k7p"] = None + + view = await service.get_job_status("05211530-k7p") + row = await service._get_job_row("05211530-k7p") + + assert view.status is JobStatusName.LOST + assert row.reason == "tmux_session_missing" + await service.shutdown() + + +@pytest.mark.anyio +async def test_drained_uncommitted_normal_exit_self_commits_to_exited( + tmp_path: Path, +) -> None: + service, _fake_tmux = await _create_service(tmp_path) + job_id = "05211530-k7p" + await _seed_job(service, job_id=job_id, status=JobStatusName.RUNNING) + job_dir = service.config.jobs_dir / job_id + (job_dir / RUNNER_EXIT_CODE_FILENAME).write_text("0\n", encoding="utf-8") + (job_dir / RUNNER_ENDED_AT_FILENAME).write_text("2026-05-21T15:30:20Z\n", encoding="utf-8") + (job_dir / PIPE_DRAINED_FILENAME).touch() + + view = await service.get_job_status(job_id) + row = await service._get_job_row(job_id) + + assert view.status is JobStatusName.EXITED + assert view.done is True + assert view.exit_code == 0 + assert row.status == JobStatusName.EXITED.value + assert row.exit_code == 0 + assert row.ended_at == "2026-05-21T15:30:20Z" + await service.shutdown() + + +@pytest.mark.anyio +async def test_list_jobs_ignores_half_created_directory_and_uses_db_rows( + tmp_path: Path, +) -> None: + service, fake_tmux = await _create_service(tmp_path) + (service.config.jobs_dir / "half-created").mkdir(mode=0o700) + await _seed_job(service, job_id="05211530-k7p", status=JobStatusName.RUNNING) + fake_tmux.sessions.add(job_session_name("05211530-k7p")) + fake_tmux.pipe_active["05211530-k7p"] = True + + jobs = await service.list_jobs(limit=10) + + assert [job.job_id for job in jobs.jobs] == ["05211530-k7p"] + assert fake_tmux.cleaned == [] + await service.shutdown() + + +@pytest.mark.anyio +async def test_reconcile_and_gc_query_sqlite_rows(tmp_path: Path) -> None: + service, fake_tmux = await _create_service(tmp_path) + await _seed_job( + service, + job_id="05211530-old", + status=JobStatusName.EXITED, + exit_code=0, + ended_at="2020-05-21T15:30:20Z", + created_at="2020-05-21T15:30:12Z", + ) + await _seed_job(service, job_id="05211530-run", status=JobStatusName.RUNNING) + fake_tmux.sessions.add(job_session_name("05211530-run")) + fake_tmux.pipe_active["05211530-run"] = True + + await service.reconcile() + await service.gc_once() + + with pytest.raises(ShellctlServerError, match="Unknown job id"): + await service._get_job_row("05211530-old") + row = await service._get_job_row("05211530-run") + assert row.status == JobStatusName.RUNNING.value + assert not (service.config.jobs_dir / "05211530-old").exists() + await service.shutdown() + + +@pytest.mark.anyio +async def test_delete_job_removes_db_row_before_artifact_cleanup( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fake_tmux = FakeTmuxController() + service = RecordingService( + ShellctlConfig(state_dir=tmp_path / "state", runtime_dir=tmp_path / "run"), + tmux=fake_tmux, + ) + await service.initialize_database() + service._ensure_dir(service.config.jobs_dir) + await _seed_job( + service, + job_id="05211530-k7p", + status=JobStatusName.EXITED, + exit_code=0, + ended_at="2026-05-21T15:30:20Z", + ) + + observed_db_missing_before_cleanup = False + original_rmtree = server_service_module.shutil.rmtree + + def assert_db_deleted_then_remove(path: Path, *, ignore_errors: bool = False) -> None: + nonlocal observed_db_missing_before_cleanup + with sqlite3.connect(service.config.db_path) as connection: + count = connection.execute( + "SELECT COUNT(*) FROM jobs WHERE job_id = ?", + ("05211530-k7p",), + ).fetchone()[0] + assert count == 0 + observed_db_missing_before_cleanup = True + original_rmtree(path, ignore_errors=ignore_errors) + + monkeypatch.setattr(server_service_module.shutil, "rmtree", assert_db_deleted_then_remove) + + await service.delete_job("05211530-k7p", force=False, grace_seconds=0) + + assert service.deleted_rows == ["05211530-k7p"] + assert observed_db_missing_before_cleanup is True + with pytest.raises(ShellctlServerError, match="Unknown job id"): + await service._get_job_row("05211530-k7p") + await service.shutdown() + + +@pytest.mark.anyio +async def test_list_jobs_skips_row_deleted_mid_iteration(tmp_path: Path) -> None: + fake_tmux = FakeTmuxController() + service = ConcurrentDeleteRaceService( + ShellctlConfig(state_dir=tmp_path / "state", runtime_dir=tmp_path / "run"), + tmux=fake_tmux, + ) + await service.initialize_database() + service._ensure_dir(service.config.jobs_dir) + await _seed_job(service, job_id="05211530-gone", status=JobStatusName.RUNNING) + await _seed_job(service, job_id="05211530-keep", status=JobStatusName.RUNNING) + fake_tmux.sessions.add(job_session_name("05211530-gone")) + fake_tmux.sessions.add(job_session_name("05211530-keep")) + fake_tmux.pipe_active["05211530-gone"] = True + fake_tmux.pipe_active["05211530-keep"] = True + service.disappear_on_status.add("05211530-gone") + + jobs = await service.list_jobs(limit=10) + + assert [job.job_id for job in jobs.jobs] == ["05211530-keep"] + await service.shutdown() + + +@pytest.mark.anyio +async def test_reconcile_skips_row_deleted_mid_iteration(tmp_path: Path) -> None: + fake_tmux = FakeTmuxController() + service = ConcurrentDeleteRaceService( + ShellctlConfig(state_dir=tmp_path / "state", runtime_dir=tmp_path / "run"), + tmux=fake_tmux, + ) + await service.initialize_database() + service._ensure_dir(service.config.jobs_dir) + await _seed_job(service, job_id="05211530-gone", status=JobStatusName.RUNNING) + await _seed_job(service, job_id="05211530-keep", status=JobStatusName.RUNNING) + fake_tmux.sessions.add(job_session_name("05211530-gone")) + fake_tmux.sessions.add(job_session_name("05211530-keep")) + fake_tmux.pipe_active["05211530-gone"] = True + fake_tmux.pipe_active["05211530-keep"] = True + service.disappear_on_status.add("05211530-gone") + + await service.reconcile() + + row = await service._get_job_row("05211530-keep") + assert row.status == JobStatusName.RUNNING.value + await service.shutdown() + + +@pytest.mark.anyio +async def test_gc_skips_row_deleted_mid_pass(tmp_path: Path) -> None: + fake_tmux = FakeTmuxController() + service = ConcurrentDeleteRaceService( + ShellctlConfig(state_dir=tmp_path / "state", runtime_dir=tmp_path / "run"), + tmux=fake_tmux, + ) + await service.initialize_database() + service._ensure_dir(service.config.jobs_dir) + await _seed_job( + service, + job_id="05211530-gone", + status=JobStatusName.EXITED, + exit_code=0, + ended_at="2020-05-21T15:30:20Z", + created_at="2020-05-21T15:30:12Z", + ) + await _seed_job( + service, + job_id="05211530-keep", + status=JobStatusName.EXITED, + exit_code=0, + ended_at="2020-05-21T15:30:20Z", + created_at="2020-05-21T15:30:13Z", + ) + service.disappear_on_delete.add("05211530-gone") + + await service.gc_once() + + with pytest.raises(ShellctlServerError, match="Unknown job id"): + await service._get_job_row("05211530-keep") + await service.shutdown() + + +@pytest.mark.anyio +async def test_run_job_keeps_start_gate_closed_when_pipe_never_becomes_ready( + tmp_path: Path, +) -> None: + service, fake_tmux = await _create_service(tmp_path) + fake_tmux.touch_ready_on_enable = False + service.config = ShellctlConfig( + state_dir=service.config.state_dir, + runtime_dir=service.config.runtime_dir, + poll_interval_seconds=0.001, + pipe_monitor_interval_seconds=0.01, + pipe_ready_timeout_seconds=0.01, + ) + + result = await service.run_job( + RunJobRequest( + script="printf ready\n", + cwd=str(tmp_path), + terminal=TerminalSize(), + timeout=0.05, + output_limit=8192, + idle_flush_seconds=0.01, + ) + ) + + assert result.status is JobStatusName.FAILED + assert result.done is True + assert not (service.config.jobs_dir / result.job_id / "start-gate").exists() + await service.shutdown() + + +@pytest.mark.anyio +async def test_run_job_pipe_ready_timeout_records_diagnostics(tmp_path: Path) -> None: + service, fake_tmux = await _create_service(tmp_path) + fake_tmux.touch_ready_on_enable = False + fake_tmux.pipe_active_on_enable = False + fake_tmux.pipe_error_log_text = "Traceback: sanitizer crashed\nsecond line\n" + service.config = ShellctlConfig( + state_dir=service.config.state_dir, + runtime_dir=service.config.runtime_dir, + poll_interval_seconds=0.001, + pipe_ready_timeout_seconds=0.01, + ) + + result = await service.run_job( + RunJobRequest( + script="printf ready\n", + cwd=str(tmp_path), + terminal=TerminalSize(), + timeout=0.05, + output_limit=8192, + idle_flush_seconds=0.01, + ) + ) + + row = await service._get_job_row(result.job_id) + ready_file = service.config.jobs_dir / result.job_id / ".pipe-ready" + + assert result.status is JobStatusName.FAILED + assert row.message is not None + assert "waited " in row.message + assert str(ready_file) in row.message + assert "tmux #{pane_pipe}=0" in row.message + assert PIPE_ERROR_LOG_FILENAME in row.message + assert "Traceback: sanitizer crashed | second line" in row.message + await service.shutdown() + + +@pytest.mark.anyio +async def test_send_input_terminal_race_returns_conflict_not_500( + tmp_path: Path, +) -> None: + service, fake_tmux = await _create_service(tmp_path) + await _seed_job(service, job_id="05211530-k7p", status=JobStatusName.RUNNING) + fake_tmux.sessions.add(job_session_name("05211530-k7p")) + fake_tmux.pipe_active["05211530-k7p"] = True + + async def finish_before_send(job_id: str, _text: str) -> None: + fake_tmux.sessions.clear() + fake_tmux.pipe_active.pop(job_id, None) + await service.record_runner_exit(job_id, 0, "2026-05-21T15:30:20Z") + raise ShellctlServerError( + 409, + "tmux_target_missing", + "no server running on /tmp/tmux.sock", + ) + + fake_tmux.on_send_input = finish_before_send + + with pytest.raises(ShellctlServerError) as exc_info: + await service.send_input( + "05211530-k7p", + InputJobRequest( + text="pwd\n", + offset=0, + timeout=0.05, + output_limit=8192, + idle_flush_seconds=0.01, + ), + ) + + assert exc_info.value.status_code == 409 + assert exc_info.value.code == "job_not_running" + await service.shutdown() + + +@pytest.mark.anyio +async def test_allocate_job_dir_retries_on_atomic_mkdir_collision( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + service, _fake_tmux = await _create_service(tmp_path) + (service.config.jobs_dir / "05211530-k7p").mkdir(mode=0o700) + generated_ids = iter(["05211530-k7p", "05211530-abc"]) + + monkeypatch.setattr(server_service_module, "generate_job_id", lambda now=None: next(generated_ids)) + + job_id, job_dir = service._allocate_job_dir() + + assert job_id == "05211530-abc" + assert job_dir.exists() + await service.shutdown() + + +@pytest.mark.anyio +async def test_service_initialize_allows_missing_auth_token(tmp_path: Path) -> None: + service = ShellctlService( + ShellctlConfig(state_dir=tmp_path / "state", runtime_dir=tmp_path / "run"), + tmux=FakeTmuxController(), + ) + + await service.initialize() + + assert service.config.runner_path.exists() + await service.shutdown() + + +@pytest.mark.anyio +async def test_http_routes_inject_shellctl_service_dependency( + tmp_path: Path, +) -> None: + service, _fake_tmux = await _create_service(tmp_path, auth_token="route-token") + app = create_app(service.config, service=service) + transport = httpx.ASGITransport(app=app) + + async with httpx.AsyncClient(transport=transport, base_url="http://shellctl.test") as client: + health = await client.get("/healthz") + assert health.status_code == 200 + + unauthenticated = await client.get("/v1/jobs") + assert unauthenticated.status_code == 401 + + authenticated = await client.get("/v1/jobs", headers={"Authorization": "Bearer route-token"}) + + assert authenticated.status_code == 200 + assert authenticated.json() == {"jobs": []} + await service.shutdown() + + +@pytest.mark.anyio +async def test_http_routes_enforce_auth_from_environment_fallback( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(DEFAULT_AUTH_TOKEN_ENV, "route-token") + service, _fake_tmux = await _create_service(tmp_path) + app = create_app(service.config, service=service) + transport = httpx.ASGITransport(app=app) + + async with httpx.AsyncClient(transport=transport, base_url="http://shellctl.test") as client: + unauthenticated = await client.get("/v1/jobs") + authenticated = await client.get("/v1/jobs", headers={"Authorization": "Bearer route-token"}) + + assert unauthenticated.status_code == 401 + assert authenticated.status_code == 200 + await service.shutdown() + + +@pytest.mark.anyio +async def test_create_app_without_explicit_config_reads_auth_token_from_environment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def noop_initialize(self: ShellctlService) -> None: + return None + + monkeypatch.setenv(DEFAULT_AUTH_TOKEN_ENV, "route-token") + monkeypatch.setattr(ShellctlService, "initialize", noop_initialize) + monkeypatch.setattr(ShellctlService, "start_background_gc", lambda self: None) + monkeypatch.setattr(ShellctlService, "start_background_pipe_monitor", lambda self: None) + + app = create_app() + transport = httpx.ASGITransport(app=app) + + async with httpx.AsyncClient(transport=transport, base_url="http://shellctl.test") as client: + response = await client.get("/v1/jobs") + + assert app.state.shellctl_service.config.auth_token == "route-token" + assert response.status_code == 401 + await app.state.shellctl_service.shutdown() + + +@pytest.mark.anyio +async def test_http_routes_skip_auth_when_token_missing(tmp_path: Path) -> None: + service, _fake_tmux = await _create_service(tmp_path) + app = create_app(service.config, service=service) + transport = httpx.ASGITransport(app=app) + + async with httpx.AsyncClient(transport=transport, base_url="http://shellctl.test") as client: + response = await client.get("/v1/jobs") + + assert response.status_code == 200 + assert response.json() == {"jobs": []} + await service.shutdown() + + +@pytest.mark.anyio +async def test_http_routes_skip_auth_when_token_is_empty(tmp_path: Path) -> None: + service, _fake_tmux = await _create_service(tmp_path, auth_token="") + app = create_app(service.config, service=service) + transport = httpx.ASGITransport(app=app) + + async with httpx.AsyncClient(transport=transport, base_url="http://shellctl.test") as client: + response = await client.get("/v1/jobs") + + assert response.status_code == 200 + assert response.json() == {"jobs": []} + await service.shutdown() + + +def test_shellctl_config_reads_auth_token_from_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(DEFAULT_AUTH_TOKEN_ENV, "env-token") + + config = ShellctlConfig(state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + + assert config.auth_token == "env-token" + + +def test_shellctl_config_treats_empty_environment_auth_token_as_disabled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(DEFAULT_AUTH_TOKEN_ENV, "") + + config = ShellctlConfig(state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + + assert config.auth_token is None + + +def test_serve_cli_passes_direct_auth_token_to_config(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + captured, runner = _capture_serve_config(monkeypatch) + + result = runner.invoke( + cli, + [ + "serve", + "--listen", + "0.0.0.0:9999", + "--auth-token", + "direct-token", + "--state-dir", + str(tmp_path / "state"), + ], + ) + + assert result.exit_code == 0, result.output + assert captured["host"] == "0.0.0.0" + assert captured["port"] == 9999 + assert captured["log_level"] == "info" + config = captured["config"] + assert isinstance(config, ShellctlConfig) + assert config.auth_token == "direct-token" + + +def test_serve_cli_prefers_explicit_auth_token_over_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(DEFAULT_AUTH_TOKEN_ENV, "env-token") + captured, runner = _capture_serve_config(monkeypatch) + + result = runner.invoke( + cli, + [ + "serve", + "--auth-token", + "direct-token", + "--state-dir", + str(tmp_path / "state"), + ], + ) + + assert result.exit_code == 0, result.output + config = captured["config"] + assert isinstance(config, ShellctlConfig) + assert config.auth_token == "direct-token" + + +def test_serve_cli_treats_empty_auth_token_as_disabled(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + captured, runner = _capture_serve_config(monkeypatch) + + result = runner.invoke( + cli, + [ + "serve", + "--auth-token", + "", + "--state-dir", + str(tmp_path / "state"), + ], + ) + + assert result.exit_code == 0, result.output + config = captured["config"] + assert isinstance(config, ShellctlConfig) + assert config.auth_token is None + + +def test_serve_cli_explicit_empty_auth_token_beats_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(DEFAULT_AUTH_TOKEN_ENV, "env-token") + captured, runner = _capture_serve_config(monkeypatch) + + result = runner.invoke( + cli, + [ + "serve", + "--auth-token", + "", + "--state-dir", + str(tmp_path / "state"), + ], + ) + + assert result.exit_code == 0, result.output + config = captured["config"] + assert isinstance(config, ShellctlConfig) + assert config.auth_token is None + + +def test_serve_cli_reads_auth_token_from_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(DEFAULT_AUTH_TOKEN_ENV, "env-token") + captured, runner = _capture_serve_config(monkeypatch) + + result = runner.invoke( + cli, + [ + "serve", + "--state-dir", + str(tmp_path / "state"), + ], + ) + + assert result.exit_code == 0, result.output + config = captured["config"] + assert isinstance(config, ShellctlConfig) + assert config.auth_token == "env-token" + + +def test_serve_cli_forwards_state_runtime_and_gc_flags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + captured, runner = _capture_serve_config(monkeypatch) + state_dir = tmp_path / "state" + runtime_dir = tmp_path / "runtime" + + result = runner.invoke( + cli, + [ + "serve", + "--state-dir", + str(state_dir), + "--runtime-dir", + str(runtime_dir), + "--gc-interval-seconds", + "12.5", + "--gc-finished-job-retention-seconds", + "345.0", + ], + ) + + assert result.exit_code == 0, result.output + config = captured["config"] + assert isinstance(config, ShellctlConfig) + assert config.state_dir == state_dir + assert config.runtime_dir == runtime_dir + assert config.gc_interval_seconds == 12.5 + assert config.gc_finished_job_retention_seconds == 345.0 + + +def test_runner_script_records_completion_metadata_without_direct_runner_exit( + tmp_path: Path, +) -> None: + service = ShellctlService( + ShellctlConfig(state_dir=tmp_path / "state", runtime_dir=tmp_path / "run"), + tmux=FakeTmuxController(), + ) + source = service._runner_script_source() + + assert JOB_ENV_FILENAME in source + assert RUNNER_EXIT_CODE_FILENAME in source + assert RUNNER_ENDED_AT_FILENAME in source + assert "write_atomic" in source + assert 'mv "$tmp" "$dest"' in source + assert re.search(r"\brunner-exit\s+--state-dir\b", source) is None + anyio.run(service.shutdown) + + +def test_shellctl_config_defaults_to_lightweight_sanitize_entrypoint( + tmp_path: Path, +) -> None: + config = ShellctlConfig(state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + + assert config.pipe_ready_timeout_seconds == 10.0 + assert config.sanitize_pty_command == ("shellctl-sanitize-pty",) + assert config.runner_exit_command == ("shellctl-runner-exit",) + + +def test_pipe_command_finalizer_commits_runner_exit_after_drain(tmp_path: Path) -> None: + controller = TmuxController( + ShellctlConfig( + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + sqlite_busy_timeout_ms=6789, + ) + ) + source = controller._pipe_command_source( + job_id="05211530-k7p", + job_dir=tmp_path / "state" / "jobs" / "05211530-k7p", + ready_file=tmp_path / "state" / "jobs" / "05211530-k7p" / ".pipe-ready", + ) + + assert "shellctl-sanitize-pty" in source + assert PIPE_DRAINED_FILENAME in source + assert PIPE_ERROR_LOG_FILENAME in source + assert PIPE_FAILED_FILENAME in source + assert RUNNER_EXIT_CODE_FILENAME in source + assert RUNNER_ENDED_AT_FILENAME in source + assert "shellctl-runner-exit" in source + assert "--sqlite-busy-timeout-ms 6789" in source + assert "2>>" in source + assert 'exit "$sanitize_status"' in source + assert 'if [ "$sanitize_status" -eq 0 ]' in source + assert "2>" in source + assert source.index("shellctl-sanitize-pty") < source.index(PIPE_DRAINED_FILENAME) + assert source.index(PIPE_DRAINED_FILENAME) < source.index("shellctl-runner-exit") + + +def test_server_cli_no_longer_exposes_sanitize_pty_command() -> None: + result = CliRunner().invoke(cli, ["sanitize-pty"]) + + assert result.exit_code != 0 + + +def test_python_m_shellctl_server_module_entrypoint_shows_cli_help() -> None: + package_root = Path(__file__).resolve().parents[3] + src_path = package_root / "src" + env = dict(os.environ) + current_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = f"{src_path}{os.pathsep}{current_pythonpath}" if current_pythonpath else str(src_path) + + result = subprocess.run( + [sys.executable, "-m", "shellctl.server", "--help"], + capture_output=True, + text=True, + check=False, + cwd=package_root, + env=env, + ) + + assert result.returncode == 0, result.stderr + assert "sanitize-pty" not in result.stdout + assert "runner-exit" not in result.stdout + + +def test_server_package_keeps_config_access_lazy() -> None: + package_root = Path(__file__).resolve().parents[3] + src_path = package_root / "src" + env = dict(os.environ) + current_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = f"{src_path}{os.pathsep}{current_pythonpath}" if current_pythonpath else str(src_path) + + script = """ +import json +import sys + +import shellctl.server as server + +server.ShellctlConfig + +print(json.dumps({ + "deny": sorted( + name for name in sys.modules + if name.split(".", 1)[0] in {"fastapi", "typer", "uvicorn"} + ), + "server_modules": sorted( + name for name in sys.modules + if name.startswith("shellctl.server.") + ), +})) +""" + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + cwd=package_root, + env=env, + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert payload["deny"] == [] + assert "shellctl.server.api" not in payload["server_modules"] + assert "shellctl.server.cli" not in payload["server_modules"] + assert "shellctl.server.service" not in payload["server_modules"] diff --git a/dify-agent/tests/local/shellctl/test_shellctl_shared.py b/dify-agent/tests/local/shellctl/test_shellctl_shared.py new file mode 100644 index 00000000000..abd5167bc7d --- /dev/null +++ b/dify-agent/tests/local/shellctl/test_shellctl_shared.py @@ -0,0 +1,133 @@ +import json +import re +from datetime import UTC, datetime +from io import BytesIO +from pathlib import Path +from typing import BinaryIO, cast + +import pytest +from pydantic import ValidationError + +from shellctl.shared import ( + JOB_ID_ALPHABET, + RunJobRequest, + generate_job_id, + read_output_window, + tail_output_window, +) +from shellctl_runtime.sanitize import ( + PtySanitizer, + sanitize_pty_output, + sanitize_pty_stream, +) + + +def test_generate_job_id_matches_proposal_format() -> None: + job_id = generate_job_id(now=datetime(2026, 5, 21, 15, 30, tzinfo=UTC)) + + assert re.fullmatch(r"05211530-[0-9abcdefghjkmnpqrstvwxyz]{3}", job_id) + assert all(char in f"{JOB_ID_ALPHABET}-" for char in job_id[9:]) + + +def test_sanitize_pty_golden_cases() -> None: + fixture_path = Path(__file__).with_name("golden_shellctl_sanitize.json") + cases = json.loads(fixture_path.read_text(encoding="utf-8")) + + for case in cases: + chunks = [bytes.fromhex(chunk) for chunk in case["chunks_hex"]] + sanitizer = PtySanitizer() + streamed = "".join(sanitizer.feed(chunk) for chunk in chunks) + sanitizer.flush() + batch = sanitize_pty_output(b"".join(chunks)) + + assert streamed == case["expected"], case["name"] + assert batch == case["expected"], case["name"] + + +def test_read_output_window_preserves_utf8_boundaries(tmp_path: Path) -> None: + output_path = tmp_path / "output.log" + output_path.write_text("A🙂B", encoding="utf-8") + + first = read_output_window(output_path, offset=0, limit=3) + second = read_output_window(output_path, offset=first.offset, limit=4) + third = read_output_window(output_path, offset=second.offset, limit=8) + + assert first.output == "A" + assert first.offset == 1 + assert first.truncated is True + assert second.output == "🙂" + assert second.offset == 5 + assert second.truncated is True + assert third.output == "B" + assert third.offset == output_path.stat().st_size + assert third.truncated is False + + +def test_read_output_window_advances_past_wide_char_even_when_limit_is_smaller( + tmp_path: Path, +) -> None: + output_path = tmp_path / "output.log" + output_path.write_text("🙂B", encoding="utf-8") + + first = read_output_window(output_path, offset=0, limit=1) + second = read_output_window(output_path, offset=first.offset, limit=1) + + assert first.output == "🙂" + assert first.offset == len("🙂".encode()) + assert first.truncated is True + assert second.output == "B" + assert second.truncated is False + + +def test_tail_output_window_skips_partial_utf8_prefix(tmp_path: Path) -> None: + output_path = tmp_path / "output.log" + output_path.write_text("a🙂b", encoding="utf-8") + + tail = tail_output_window(output_path, limit=3) + + assert tail.output == "b" + assert tail.offset == output_path.stat().st_size + assert tail.truncated is False + + +def test_sanitize_pty_stream_flushes_incrementally() -> None: + class ChunkedInput: + def __init__(self, chunks: list[bytes]) -> None: + self._chunks = chunks + + def read(self, _size: int) -> bytes: + if not self._chunks: + return b"" + return self._chunks.pop(0) + + class FlushCountingOutput(BytesIO): + def __init__(self) -> None: + super().__init__() + self.flush_count = 0 + + def flush(self) -> None: + self.flush_count += 1 + super().flush() + + stdout = FlushCountingOutput() + sanitize_pty_stream( + cast(BinaryIO, cast(object, ChunkedInput([b"ready\n", b"next\n"]))), + stdout, + chunk_size=5, + ) + + assert stdout.getvalue() == b"ready\nnext\n" + assert stdout.flush_count >= 2 + + +@pytest.mark.parametrize( + ("env", "message"), + [ + ({"": "x"}, "non-empty"), + ({"A=B": "x"}, "must not contain '='"), + ({"A\x00B": "x"}, "must not contain NUL"), + ({"A": "x\x00y"}, "must not contain NUL"), + ], +) +def test_run_job_request_rejects_invalid_env_entries(env: dict[str, str], message: str) -> None: + with pytest.raises(ValidationError, match=message): + RunJobRequest(script="printf ready\n", env=env) diff --git a/dify-agent/tests/local/shellctl_runtime/golden_shellctl_sanitize.json b/dify-agent/tests/local/shellctl_runtime/golden_shellctl_sanitize.json new file mode 100644 index 00000000000..10eba3678ef --- /dev/null +++ b/dify-agent/tests/local/shellctl_runtime/golden_shellctl_sanitize.json @@ -0,0 +1,32 @@ +[ + { + "name": "utf8_passthrough", + "chunks_hex": ["68656c6c6f20e4b896e7958cf09f99820a"], + "expected": "hello 世界🙂\n" + }, + { + "name": "split_utf8_chunks", + "chunks_hex": ["e4bda0e5a5bd20f09f", "99820a"], + "expected": "你好 🙂\n" + }, + { + "name": "invalid_utf8_replacement", + "chunks_hex": ["666f80ff6f0a"], + "expected": "fo��o\n" + }, + { + "name": "ansi_sequences_removed", + "chunks_hex": ["1b5b33316d7265641b5b306d1b5b4b0a"], + "expected": "red\n" + }, + { + "name": "carriage_return_progress", + "chunks_hex": ["30250d3530250d313030250a"], + "expected": "100%\n" + }, + { + "name": "crlf_normalized", + "chunks_hex": ["6f6b0d0a6e6578740d0a"], + "expected": "ok\nnext\n" + } +] diff --git a/dify-agent/tests/local/shellctl_runtime/test_shellctl_runtime.py b/dify-agent/tests/local/shellctl_runtime/test_shellctl_runtime.py new file mode 100644 index 00000000000..1314c629c3d --- /dev/null +++ b/dify-agent/tests/local/shellctl_runtime/test_shellctl_runtime.py @@ -0,0 +1,531 @@ +from __future__ import annotations + +import ast +import importlib.metadata +import importlib.util +import io +import json +import os +import shutil +import sqlite3 +import subprocess +import sys +import textwrap +from pathlib import Path +from types import TracebackType +from typing import BinaryIO, TypedDict, cast + +import anyio +import pytest + +from shellctl.server import ShellctlConfig, ShellctlService +from shellctl_runtime.runner_exit import record_runner_exit +from shellctl_runtime.sanitize import run_sanitize_pty + +PACKAGE_ROOT = Path(__file__).resolve().parents[3] +SRC_PATH = PACKAGE_ROOT / "src" +HEAVY_IMPORT_ROOTS = { + "fastapi", + "httpx", + "pydantic", + "pygments", + "rich", + "sqlalchemy", + "shellctl", + "typer", + "uvicorn", +} + + +class ModuleHelpStats(TypedDict): + code: int + elapsed: float + deny: list[str] + + +def _pythonpath_env() -> dict[str, str]: + env = dict(os.environ) + current_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = f"{SRC_PATH}{os.pathsep}{current_pythonpath}" if current_pythonpath else str(SRC_PATH) + return env + + +async def _initialize_database(state_dir: Path) -> None: + service = ShellctlService(ShellctlConfig(state_dir=state_dir, runtime_dir=state_dir / "run")) + await service.initialize_database() + await service.shutdown() + + +def _insert_job_row( + *, + state_dir: Path, + job_id: str, + status: str, + exit_code: int | None = None, + ended_at: str | None = None, + reason: str | None = None, + message: str | None = None, + updated_at: str = "2026-05-21T15:30:13Z", +) -> None: + db_path = state_dir / "shellctl.db" + with sqlite3.connect(db_path) as connection: + connection.execute( + """ + INSERT INTO jobs ( + job_id, + script_path, + output_path, + cwd, + terminal_cols, + terminal_rows, + status, + session_name, + pane_target, + exit_code, + reason, + message, + created_at, + started_at, + ended_at, + updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + job_id, + f"jobs/{job_id}/script", + f"jobs/{job_id}/output.log", + "/tmp", + 80, + 24, + status, + f"shellctl-{job_id}", + f"shellctl-{job_id}:0.0", + exit_code, + reason, + message, + "2026-05-21T15:30:12Z", + "2026-05-21T15:30:13Z", + ended_at, + updated_at, + ), + ) + connection.commit() + + +def _fetch_job_row(state_dir: Path, job_id: str) -> tuple[object, ...]: + with sqlite3.connect(state_dir / "shellctl.db") as connection: + row = connection.execute( + """ + SELECT status, exit_code, ended_at, updated_at, reason, message + FROM jobs + WHERE job_id = ? + """, + (job_id,), + ).fetchone() + assert row is not None + return row + + +def _measure_module_help(module_name: str) -> ModuleHelpStats: + script = textwrap.dedent( + f""" + import contextlib + import io + import json + import runpy + import sys + import time + + sys.path.insert(0, {str(SRC_PATH)!r}) + sys.argv = [{module_name!r}, "--help"] + start = time.perf_counter() + stdout = io.StringIO() + stderr = io.StringIO() + code = 0 + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + try: + runpy.run_module({module_name!r}, run_name="__main__") + except SystemExit as exc: + if isinstance(exc.code, int): + code = exc.code + elapsed = time.perf_counter() - start + deny = sorted( + name + for name in sys.modules + if name.split(".", 1)[0] in {sorted(HEAVY_IMPORT_ROOTS)!r} + ) + print(json.dumps({{"code": code, "elapsed": elapsed, "deny": deny}})) + """ + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + cwd=PACKAGE_ROOT, + ) + assert result.returncode == 0, result.stderr + return cast(ModuleHelpStats, json.loads(result.stdout)) + + +def test_run_sanitize_pty_touches_ready_file_before_reading(tmp_path: Path) -> None: + ready_file = tmp_path / "ready" + output = io.BytesIO() + + class ReadyCheckingInput: + def __init__(self) -> None: + self._reads = 0 + + def read(self, _size: int) -> bytes: + self._reads += 1 + assert ready_file.exists() + return b"before\rafter\n" if self._reads == 1 else b"" + + run_sanitize_pty( + ready_file, + stdin=cast(BinaryIO, cast(object, ReadyCheckingInput())), + stdout=output, + ) + + assert output.getvalue() == b"after\n" + + +def test_removed_sanitizer_modules_are_not_importable() -> None: + assert importlib.util.find_spec("shellctl.sanitize_pty") is None + assert importlib.util.find_spec("shellctl.shared.sanitize") is None + + +def test_removed_sanitizer_helpers_are_not_shellctl_exports() -> None: + import shellctl as shellctl + import shellctl.shared as shared + + for module in (shellctl, shared): + for name in ("PtySanitizer", "sanitize_pty_output", "sanitize_pty_stream"): + assert not hasattr(module, name) + + +def test_runner_exit_matches_service_state_for_running_job(tmp_path: Path) -> None: + runtime_state_dir = tmp_path / "runtime-state" + service_state_dir = tmp_path / "service-state" + anyio.run(_initialize_database, runtime_state_dir) + anyio.run(_initialize_database, service_state_dir) + _insert_job_row(state_dir=runtime_state_dir, job_id="job-1", status="running") + _insert_job_row(state_dir=service_state_dir, job_id="job-1", status="running") + + record_runner_exit( + state_dir=runtime_state_dir, + job_id="job-1", + exit_code=7, + ended_at="2026-05-21T15:30:20Z", + ) + + async def _record_with_service() -> None: + service = ShellctlService( + ShellctlConfig( + state_dir=service_state_dir, + runtime_dir=service_state_dir / "run", + ) + ) + try: + await service.record_runner_exit("job-1", 7, "2026-05-21T15:30:20Z") + finally: + await service.shutdown() + + anyio.run(_record_with_service) + + assert _fetch_job_row(runtime_state_dir, "job-1") == _fetch_job_row( + service_state_dir, + "job-1", + ) + + +def test_runner_exit_is_idempotent_for_terminal_job(tmp_path: Path) -> None: + state_dir = tmp_path / "state" + anyio.run(_initialize_database, state_dir) + _insert_job_row( + state_dir=state_dir, + job_id="job-2", + status="failed", + exit_code=99, + ended_at="2026-05-21T15:30:18Z", + reason="runner_failed", + message="boom", + updated_at="2026-05-21T15:30:18Z", + ) + + before = _fetch_job_row(state_dir, "job-2") + record_runner_exit( + state_dir=state_dir, + job_id="job-2", + exit_code=7, + ended_at="2026-05-21T15:30:20Z", + ) + + assert _fetch_job_row(state_dir, "job-2") == before + + +def test_runner_exit_uses_explicit_busy_timeout_override(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + state_dir = tmp_path / "state" + anyio.run(_initialize_database, state_dir) + _insert_job_row(state_dir=state_dir, job_id="job-3", status="running") + + captured: dict[str, object] = {} + real_connect = sqlite3.connect + + class RecordingConnection: + def __init__(self, connection: sqlite3.Connection) -> None: + self._connection = connection + + def execute(self, sql: str, parameters: tuple[object, ...] = ()) -> sqlite3.Cursor: + if sql.startswith("PRAGMA busy_timeout="): + captured["pragma"] = sql + return self._connection.execute(sql, parameters) + + def commit(self) -> None: + self._connection.commit() + + def close(self) -> None: + self._connection.close() + + def fake_connect(path: str | Path, timeout: float) -> RecordingConnection: + captured["path"] = str(path) + captured["timeout"] = timeout + return RecordingConnection(real_connect(path, timeout=timeout)) + + monkeypatch.setattr("shellctl_runtime.runner_exit.sqlite3.connect", fake_connect) + + record_runner_exit( + state_dir=state_dir, + job_id="job-3", + exit_code=5, + ended_at="2026-05-21T15:30:20Z", + busy_timeout_ms=6789, + ) + + assert captured["path"] == str(state_dir / "shellctl.db") + assert captured["timeout"] == pytest.approx(6.789) + assert captured["pragma"] == "PRAGMA busy_timeout=6789" + + +def test_runner_exit_does_not_clobber_newer_terminal_state(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + state_dir = tmp_path / "state" + anyio.run(_initialize_database, state_dir) + _insert_job_row(state_dir=state_dir, job_id="job-4", status="running") + + real_connect = sqlite3.connect + + class RacingConnection: + def __init__(self, connection: sqlite3.Connection) -> None: + self._connection = connection + self._raced = False + + def __enter__(self) -> RacingConnection: + self._connection.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> bool | None: + return self._connection.__exit__(exc_type, exc, tb) + + def execute(self, sql: str, parameters: tuple[object, ...] = ()) -> sqlite3.Cursor: + if sql.lstrip().startswith("UPDATE jobs") and not self._raced: + self._raced = True + with real_connect(state_dir / "shellctl.db") as racing_connection: + racing_connection.execute( + """ + UPDATE jobs + SET status = ?, exit_code = ?, ended_at = ?, updated_at = ?, + reason = ?, message = ? + WHERE job_id = ? + """, + ( + "failed", + 91, + "2026-05-21T15:30:19Z", + "2026-05-21T15:30:19Z", + "pipe_failed", + "concurrent winner", + "job-4", + ), + ) + racing_connection.commit() + return self._connection.execute(sql, parameters) + + def commit(self) -> None: + self._connection.commit() + + def close(self) -> None: + self._connection.close() + + def fake_connect(path: str | Path, timeout: float = 5.0, *args: object, **kwargs: object) -> RacingConnection: + return RacingConnection(real_connect(path, timeout=timeout)) + + monkeypatch.setattr("shellctl_runtime.runner_exit.sqlite3.connect", fake_connect) + + record_runner_exit( + state_dir=state_dir, + job_id="job-4", + exit_code=7, + ended_at="2026-05-21T15:30:20Z", + ) + + assert _fetch_job_row(state_dir, "job-4") == ( + "failed", + 91, + "2026-05-21T15:30:19Z", + "2026-05-21T15:30:19Z", + "pipe_failed", + "concurrent winner", + ) + + +def test_python_m_shellctl_runtime_runner_exit_updates_row(tmp_path: Path) -> None: + state_dir = tmp_path / "state" + anyio.run(_initialize_database, state_dir) + _insert_job_row(state_dir=state_dir, job_id="job-cli", status="running") + + result = subprocess.run( + [ + sys.executable, + "-m", + "shellctl_runtime.runner_exit", + "--state-dir", + str(state_dir), + "--job-id", + "job-cli", + "--exit-code", + "7", + "--ended-at", + "2026-05-21T15:30:20Z", + ], + capture_output=True, + text=True, + check=False, + cwd=PACKAGE_ROOT, + env=_pythonpath_env(), + ) + + assert result.returncode == 0, result.stderr + assert _fetch_job_row(state_dir, "job-cli") == ( + "exited", + 7, + "2026-05-21T15:30:20Z", + "2026-05-21T15:30:20Z", + None, + None, + ) + + +def test_python_m_shellctl_runtime_runner_exit_reports_missing_job( + tmp_path: Path, +) -> None: + state_dir = tmp_path / "state" + anyio.run(_initialize_database, state_dir) + + result = subprocess.run( + [ + sys.executable, + "-m", + "shellctl_runtime.runner_exit", + "--state-dir", + str(state_dir), + "--job-id", + "missing-job", + "--exit-code", + "7", + "--ended-at", + "2026-05-21T15:30:20Z", + ], + capture_output=True, + text=True, + check=False, + cwd=PACKAGE_ROOT, + env=_pythonpath_env(), + ) + + assert result.returncode != 0 + assert "Unknown job id: missing-job" in result.stderr or "Unknown job id: missing-job" in result.stdout + + +def test_runtime_console_scripts_are_installed() -> None: + entry_points = { + entry_point.name: entry_point.value for entry_point in importlib.metadata.entry_points(group="console_scripts") + } + + assert entry_points["shellctl-sanitize-pty"] == "shellctl_runtime.sanitize:main" + assert entry_points["shellctl-runner-exit"] == "shellctl_runtime.runner_exit:main" + assert shutil.which("shellctl-sanitize-pty") is not None + assert shutil.which("shellctl-runner-exit") is not None + + +def test_runtime_modules_only_import_stdlib_or_runtime_helpers() -> None: + forbidden_imports = { + "anyio", + "fastapi", + "httpx", + "pydantic", + "pygments", + "rich", + "shellctl", + "sqlalchemy", + "sqlmodel", + "typer", + "uvicorn", + } + + for filename in ("sanitize.py", "runner_exit.py"): + tree = ast.parse((SRC_PATH / "shellctl_runtime" / filename).read_text("utf-8")) + imports: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module is not None: + imports.add(node.module) + assert not any( + imported == forbidden or imported.startswith(f"{forbidden}.") + for imported in imports + for forbidden in forbidden_imports + ), (filename, sorted(imports)) + + +def test_python_m_shellctl_runtime_sanitize_module_stays_lightweight() -> None: + stats = _measure_module_help("shellctl_runtime.sanitize") + + assert stats["code"] == 0 + assert stats["elapsed"] < 0.35 + assert stats["deny"] == [] + + +def test_python_m_shellctl_runtime_runner_exit_module_stays_lightweight() -> None: + stats = _measure_module_help("shellctl_runtime.runner_exit") + + assert stats["code"] == 0 + assert stats["elapsed"] < 0.35 + assert stats["deny"] == [] + + +def test_python_m_shellctl_runtime_sanitize_sanitizes_output(tmp_path: Path) -> None: + ready_file = tmp_path / "ready" + result = subprocess.run( + [ + sys.executable, + "-m", + "shellctl_runtime.sanitize", + "--ready-file", + str(ready_file), + ], + input=b"before\rafter\n", + capture_output=True, + check=False, + cwd=PACKAGE_ROOT, + env=_pythonpath_env(), + ) + + assert result.returncode == 0, result.stderr.decode("utf-8", errors="replace") + assert ready_file.exists() + assert result.stdout.decode("utf-8") == "after\n" diff --git a/dify-agent/tests/local/test_packaging.py b/dify-agent/tests/local/test_packaging.py index 610c4850127..32e688b72bb 100644 --- a/dify-agent/tests/local/test_packaging.py +++ b/dify-agent/tests/local/test_packaging.py @@ -7,7 +7,9 @@ from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parents[2] CLIENT_SHARED_DTO_DEPENDENCIES = { + "anyio>=4.12.1,<5.0.0", "httpx==0.28.1", + "httpx2>=2.5.0,<3.0.0", "pydantic>=2.12.5,<2.13", "pydantic-ai-slim>=1.102.0,<2.0.0", "typer>=0.16.1,<0.17", @@ -23,7 +25,13 @@ SERVER_RUNTIME_DEPENDENCIES = { "pydantic-ai-slim[anthropic,google,openai]>=1.85.1,<2.0.0", "pydantic-settings>=2.12.0,<3.0.0", "redis>=7.4.0,<8.0.0", - "shell-session-manager==2.3.1", + "uvicorn[standard]==0.46.0", +} + +SHELLCTL_SERVER_DEPENDENCIES = { + "aiosqlite>=0.21.0,<1.0.0", + "fastapi==0.136.0", + "sqlmodel>=0.0.24,<0.1.0", "uvicorn[standard]==0.46.0", } @@ -54,6 +62,7 @@ def test_project_dependencies_split_client_and_server_requirements() -> None: assert set(project["dependencies"]) == CLIENT_SHARED_DTO_DEPENDENCIES assert set(project["optional-dependencies"]["grpc"]) == GRPC_RUNTIME_DEPENDENCIES assert set(project["optional-dependencies"]["server"]) == SERVER_RUNTIME_DEPENDENCIES + assert set(project["optional-dependencies"]["shellctl-server"]) == SHELLCTL_SERVER_DEPENDENCIES assert set(pyproject["dependency-groups"]["dev"]) == DEV_DEPENDENCIES @@ -62,6 +71,8 @@ def test_default_package_discovery_excludes_example_packages() -> None: find_config = pyproject["tool"]["setuptools"]["packages"]["find"] assert find_config["where"] == ["src"] + assert "shellctl*" in find_config["include"] + assert "shellctl_runtime*" in find_config["include"] assert "agenton_examples*" not in find_config["include"] assert "dify_agent_examples*" not in find_config["include"] @@ -72,6 +83,9 @@ def test_project_declares_console_scripts() -> None: assert scripts["dify-agent"] == "dify_agent.agent_stub.cli.main:main" assert scripts["dify-agent-stub-server"] == "dify_agent.agent_stub.server.cli:main" + assert scripts["shellctl"] == "shellctl.cli:main" + assert scripts["shellctl-sanitize-pty"] == "shellctl_runtime.sanitize:main" + assert scripts["shellctl-runner-exit"] == "shellctl_runtime.runner_exit:main" def test_local_sandbox_dockerfile_installs_stub_client_and_shellctl() -> None: @@ -80,10 +94,14 @@ def test_local_sandbox_dockerfile_installs_stub_client_and_shellctl() -> None: assert "ARG NODE_VERSION=22.22.1" in dockerfile assert "ARG PNPM_VERSION=11.9.0" in dockerfile assert "ARG UV_VERSION=0.8.9" in dockerfile - assert "ARG DIFY_AGENT_TOOL_SPEC=.[grpc]" in dockerfile - assert "ARG SHELL_SESSION_MANAGER_TOOL_SPEC=shell-session-manager" in dockerfile + assert "ARG DIFY_AGENT_TOOL_SPEC=.[grpc,shellctl-server]" in dockerfile + assert "SHELL_SESSION_MANAGER_TOOL_SPEC" not in dockerfile + assert "DIFY_AGENT_STUB_DRIVE_BASE=/mnt/drive" not in dockerfile + assert 'ENV PATH="${UV_TOOL_BIN_DIR}:${PATH}"' not in dockerfile assert "UV_TOOL_DIR=/opt/dify-agent-tools/envs" in dockerfile assert "UV_TOOL_BIN_DIR=/opt/dify-agent-tools/bin" in dockerfile + assert 'ENV PATH="/opt/dify-agent-tools/bin:${PATH}"' in dockerfile + assert "bash" in dockerfile assert "git" in dockerfile assert "jq" in dockerfile assert "openssh-client" in dockerfile @@ -95,13 +113,8 @@ def test_local_sandbox_dockerfile_installs_stub_client_and_shellctl() -> None: assert "uv export --frozen --no-dev --all-extras --no-emit-project --no-hashes" in dockerfile assert "> /tmp/dify-agent-constraints.txt" in dockerfile assert '--constraints /tmp/dify-agent-constraints.txt --link-mode=copy "${DIFY_AGENT_TOOL_SPEC}"' in dockerfile - assert ( - '--constraints /tmp/dify-agent-constraints.txt --link-mode=copy "${SHELL_SESSION_MANAGER_TOOL_SPEC}"' - in dockerfile - ) - assert "DIFY_AGENT_STUB_DRIVE_BASE=/mnt/drive" in dockerfile - assert "COPY --from=tools ${UV_TOOL_DIR} ${UV_TOOL_DIR}" in dockerfile - assert "COPY --from=tools ${UV_TOOL_BIN_DIR} ${UV_TOOL_BIN_DIR}" in dockerfile + assert "COPY --from=tools /opt/dify-agent-tools/envs /opt/dify-agent-tools/envs" in dockerfile + assert "COPY --from=tools /opt/dify-agent-tools/bin /opt/dify-agent-tools/bin" in dockerfile assert "VIRTUAL_ENV" not in dockerfile assert "uv sync" not in dockerfile assert "mkdir -p /mnt/drive" in dockerfile diff --git a/dify-agent/uv.lock b/dify-agent/uv.lock index c0b7fa8897d..dfdbaaca0da 100644 --- a/dify-agent/uv.lock +++ b/dify-agent/uv.lock @@ -593,7 +593,9 @@ name = "dify-agent" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "anyio" }, { name = "httpx" }, + { name = "httpx2" }, { name = "pydantic" }, { name = "pydantic-ai-slim" }, { name = "typer" }, @@ -614,7 +616,12 @@ server = [ { name = "pydantic-ai-slim", extra = ["anthropic", "google", "openai"] }, { name = "pydantic-settings" }, { name = "redis" }, - { name = "shell-session-manager" }, + { name = "uvicorn", extra = ["standard"] }, +] +shellctl-server = [ + { name = "aiosqlite" }, + { name = "fastapi" }, + { name = "sqlmodel" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -637,10 +644,14 @@ docs = [ [package.metadata] requires-dist = [ + { name = "aiosqlite", marker = "extra == 'shellctl-server'", specifier = ">=0.21.0,<1.0.0" }, + { name = "anyio", specifier = ">=4.12.1,<5.0.0" }, { name = "fastapi", marker = "extra == 'server'", specifier = "==0.136.0" }, + { name = "fastapi", marker = "extra == 'shellctl-server'", specifier = "==0.136.0" }, { name = "graphon", marker = "extra == 'server'", specifier = "==0.5.2" }, { name = "grpclib", extras = ["protobuf"], marker = "extra == 'grpc'", specifier = ">=0.4.9,<0.5.0" }, { name = "httpx", specifier = "==0.28.1" }, + { name = "httpx2", specifier = ">=2.5.0,<3.0.0" }, { name = "jsonschema", marker = "extra == 'server'", specifier = ">=4.23.0,<5.0.0" }, { name = "jwcrypto", marker = "extra == 'server'", specifier = ">=1.5.6,<2" }, { name = "logfire", extras = ["fastapi", "httpx", "redis"], marker = "extra == 'server'", specifier = ">=4.37.0,<5.0.0" }, @@ -650,12 +661,13 @@ requires-dist = [ { name = "pydantic-ai-slim", extras = ["anthropic", "google", "openai"], marker = "extra == 'server'", specifier = ">=1.85.1,<2.0.0" }, { name = "pydantic-settings", marker = "extra == 'server'", specifier = ">=2.12.0,<3.0.0" }, { name = "redis", marker = "extra == 'server'", specifier = ">=7.4.0,<8.0.0" }, - { name = "shell-session-manager", marker = "extra == 'server'", specifier = "==2.4.0" }, + { name = "sqlmodel", marker = "extra == 'shellctl-server'", specifier = ">=0.0.24,<0.1.0" }, { name = "typer", specifier = ">=0.16.1,<0.17" }, { name = "typing-extensions", specifier = ">=4.12.2,<5.0.0" }, { name = "uvicorn", extras = ["standard"], marker = "extra == 'server'", specifier = "==0.46.0" }, + { name = "uvicorn", extras = ["standard"], marker = "extra == 'shellctl-server'", specifier = "==0.46.0" }, ] -provides-extras = ["grpc", "server"] +provides-extras = ["grpc", "server", "shellctl-server"] [package.metadata.requires-dev] dev = [ @@ -1121,6 +1133,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/47/06/5c12df521b5322fb1114a83d46911b2fbcb8855ddb3a635f11c01a214af5/httpcore2-2.5.0.tar.gz", hash = "sha256:88aa170137c17328d5ac44234f9fd10706466d5fb347f3edac4d39b91137b09d", size = 64808, upload-time = "2026-06-25T14:16:56.472Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/a1/7564199d1a8728fe737b0a72e5b3f8d92dfe085a74ddf7cdd83bce5f206d/httpcore2-2.5.0-py3-none-any.whl", hash = "sha256:5ce35188de461d31e8d000bfb8ef8bf22c6c16587a211e5571deaa5e9bdf842a", size = 80330, upload-time = "2026-06-25T14:16:53.634Z" }, +] + [[package]] name = "httptools" version = "0.7.1" @@ -1165,6 +1190,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpcore2" }, + { name = "idna" }, + { name = "truststore" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/e2/b5dedc0cf35aa65de5f541ccd30d2bc1fd7f1d43c9ab09f8ed9a7342317b/httpx2-2.5.0.tar.gz", hash = "sha256:e2df9cb4611021527ff8a675b1c320b610a2ec397acc8d6fe6e91df2d9b33c29", size = 83121, upload-time = "2026-06-25T14:16:57.491Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/22/859d8252dad9bc9adee34b52e62cde621ece07b042ccb2ab4da1be46695f/httpx2-2.5.0-py3-none-any.whl", hash = "sha256:3d2d4d9cf4b61f1a1f46a95947cfdb47e80cb56a2f91c6256ac8f58e4891df41", size = 76652, upload-time = "2026-06-25T14:16:55.23Z" }, +] + [[package]] name = "huggingface-hub" version = "1.11.0" @@ -1196,11 +1237,11 @@ wheels = [ [[package]] name = "idna" -version = "3.12" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/22/12/2948fbe5513d062169bd91f7d7b1cd97bc8894f32946b71fa39f6e63ca0c/idna-3.12.tar.gz", hash = "sha256:724e9952cc9e2bd7550ea784adb098d837ab5267ef67a1ab9cf7846bdbdd8254", size = 194350, upload-time = "2026-04-21T13:32:48.916Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/53/b2/acc33950394b3becb2b664741a0c0889c7ef9f9ffbfa8d47eddb53a50abd/idna-3.12-py3-none-any.whl", hash = "sha256:60ffaa1858fac94c9c124728c24fcde8160f3fb4a7f79aa8cdd33a9d1af60a67", size = 68634, upload-time = "2026-04-21T13:32:47.403Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] @@ -3501,25 +3542,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, ] -[[package]] -name = "shell-session-manager" -version = "2.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiosqlite" }, - { name = "anyio" }, - { name = "fastapi" }, - { name = "httpx" }, - { name = "pydantic" }, - { name = "sqlmodel" }, - { name = "typer" }, - { name = "uvicorn" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/46/c4/5ad9b6a3f8a4ca78acd4fa7a44f55b3f41ce7d539d648d9beee30789dd5b/shell_session_manager-2.4.0.tar.gz", hash = "sha256:cffeff439a149fa0626dd6fe940cd2d010005d97b429769e607a2b09ae64cb84", size = 64571, upload-time = "2026-07-03T19:11:18.931Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/8c/d2f6bb7d461810c22d10b775aeec3b4374eec7fb9fa503d00653d45cd8b3/shell_session_manager-2.4.0-py3-none-any.whl", hash = "sha256:5bb2bb43def96b354817ba6a45ab88fd236897a07c7f6c94b53f3f0cf30fabe6", size = 56513, upload-time = "2026-07-03T19:11:17.506Z" }, -] - [[package]] name = "shellingham" version = "1.5.4" @@ -3900,6 +3922,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/95/0b0218149b0d6f14df35f5b8f676fa83df4f19ed253c3cc447107ef86eca/transformers-5.6.2-py3-none-any.whl", hash = "sha256:f8d3a1bb96778fed9b8aabfd0dd6e19843e4b0f2bb6b59f32b8a92051b0f348f", size = 10364898, upload-time = "2026-04-23T18:33:26.081Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "typer" version = "0.16.1" diff --git a/docker/.env.example b/docker/.env.example index 532c78e75ce..4dd116b6808 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -154,7 +154,7 @@ NEXT_PUBLIC_ENABLE_SINGLE_DOLLAR_LATEX=false # Enable preview features still in development (currently the /create and # /refine slash commands in the "Go to Anything" command palette). NEXT_PUBLIC_ENABLE_FEATURE_PREVIEW=true -ENABLE_AGENT_V2=false +NEXT_PUBLIC_ENABLE_AGENT_V2=true EXPERIMENTAL_ENABLE_VINEXT=false # Storage and default vector store @@ -245,6 +245,28 @@ MARKETPLACE_ENABLED=true MARKETPLACE_API_URL=https://marketplace.dify.ai MARKETPLACE_URL= +# Dify Agent backend +AGENT_BACKEND_BASE_URL=http://agent_backend:5050 +# Leave empty to derive from REDIS_PASSWORD. +DIFY_AGENT_REDIS_URL= +DIFY_AGENT_REDIS_PREFIX=dify-agent +DIFY_AGENT_SHUTDOWN_GRACE_SECONDS=30 +DIFY_AGENT_RUN_RETENTION_SECONDS=259200 +# Leave empty to derive from PLUGIN_DAEMON_URL and PLUGIN_DAEMON_KEY. +DIFY_AGENT_PLUGIN_DAEMON_URL= +DIFY_AGENT_PLUGIN_DAEMON_API_KEY= +# Leave empty to derive from PLUGIN_DIFY_INNER_API_URL and PLUGIN_DIFY_INNER_API_KEY. +# DIFY_AGENT_INNER_API_KEY must match API/worker INNER_API_KEY_FOR_PLUGIN, not INNER_API_KEY. +DIFY_AGENT_INNER_API_URL= +DIFY_AGENT_INNER_API_KEY= +DIFY_AGENT_SHELLCTL_ENTRYPOINT=http://local_sandbox:5004 +DIFY_AGENT_SHELLCTL_AUTH_TOKEN= +DIFY_AGENT_STUB_API_BASE_URL=http://agent_backend:5050/agent-stub +# This is security-sensitive: it derives the JWE encryption key for Agent Stub bearer tokens. +# Replace this development default in production. +# Generate one with: python -c 'import secrets; print(secrets.token_urlsafe(32))' +DIFY_AGENT_SERVER_SECRET_KEY=MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY + # Nginx and Docker Compose NGINX_SERVER_NAME=_ NGINX_HTTPS_ENABLED=false diff --git a/docker/README.md b/docker/README.md index d65b6c624f5..c3b1011bd68 100644 --- a/docker/README.md +++ b/docker/README.md @@ -84,7 +84,7 @@ The root `.env.example` file contains the essential startup settings. Optional a 1. **Common Variables**: - `CONSOLE_API_URL`, `CONSOLE_WEB_URL`, `SERVICE_API_URL`, `APP_API_URL`, `APP_WEB_URL`: public URLs for the API and frontend services. - - `SERVER_CONSOLE_API_URL`: internal URL used by the web service for server-side console API requests. Keep the default `http://api:5001` for standard Docker Compose deployments, and only change it when your web service must reach the API through a different internal address. + - `SERVER_CONSOLE_API_URL`: internal API origin used by web server-side requests and, when `INTERNAL_FILES_URL` is unset, as the default fallback for API-side internal file URL generation. Keep the default `http://api:5001` for standard Docker Compose deployments, and only change it when services must reach the API through a different internal address. - `FILES_URL`, `INTERNAL_FILES_URL`: Public and internal base URLs for file downloads and previews. - `ENDPOINT_URL_TEMPLATE`, `NEXT_PUBLIC_SOCKET_URL`, `TRIGGER_URL`: Additional service URLs. diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 3525e5ff208..7c7e7f571db 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -231,6 +231,7 @@ services: PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} PLUGIN_DAEMON_TIMEOUT: ${PLUGIN_DAEMON_TIMEOUT:-600.0} INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + AGENT_BACKEND_BASE_URL: ${AGENT_BACKEND_BASE_URL:-http://agent_backend:5050} depends_on: init_permissions: condition: service_completed_successfully @@ -248,6 +249,8 @@ services: required: false redis: condition: service_started + agent_backend: + condition: service_started volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage @@ -298,6 +301,7 @@ services: SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + AGENT_BACKEND_BASE_URL: ${AGENT_BACKEND_BASE_URL:-http://agent_backend:5050} depends_on: init_permissions: condition: service_completed_successfully @@ -315,6 +319,8 @@ services: required: false redis: condition: service_started + agent_backend: + condition: service_started volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage @@ -516,6 +522,21 @@ services: networks: - ssrf_proxy_network + # Local sandbox for Dify Agent shell workspaces. + local_sandbox: + image: langgenius/dify-agent-local-sandbox:1.15.0 + restart: always + environment: + SHELLCTL_AUTH_TOKEN: ${DIFY_AGENT_SHELLCTL_AUTH_TOKEN:-} + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5004/healthz"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + networks: + - default + # plugin daemon plugin_daemon: image: langgenius/dify-plugin-daemon:0.6.3-local @@ -603,6 +624,44 @@ services: condition: service_healthy required: false + # Dify Agent backend service. + agent_backend: + image: langgenius/dify-agent-backend:1.15.0 + restart: always + env_file: + - path: ./envs/core-services/dify-agent.env + required: false + - path: ./envs/security.env + required: false + - path: ./envs/databases/redis.env + required: false + - ./.env + environment: + DIFY_AGENT_REDIS_URL: ${DIFY_AGENT_REDIS_URL:-redis://:${REDIS_PASSWORD:-difyai123456}@redis:6379/2} + DIFY_AGENT_REDIS_PREFIX: ${DIFY_AGENT_REDIS_PREFIX:-dify-agent} + DIFY_AGENT_PLUGIN_DAEMON_URL: ${DIFY_AGENT_PLUGIN_DAEMON_URL:-${PLUGIN_DAEMON_URL:-http://plugin_daemon:5002}} + DIFY_AGENT_PLUGIN_DAEMON_API_KEY: ${DIFY_AGENT_PLUGIN_DAEMON_API_KEY:-${PLUGIN_DAEMON_KEY:-lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi}} + DIFY_AGENT_INNER_API_URL: ${DIFY_AGENT_INNER_API_URL:-${PLUGIN_DIFY_INNER_API_URL:-http://api:5001}} + DIFY_AGENT_INNER_API_KEY: ${DIFY_AGENT_INNER_API_KEY:-${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1}} + DIFY_AGENT_SHELLCTL_ENTRYPOINT: ${DIFY_AGENT_SHELLCTL_ENTRYPOINT:-http://local_sandbox:5004} + DIFY_AGENT_SHELLCTL_AUTH_TOKEN: ${DIFY_AGENT_SHELLCTL_AUTH_TOKEN:-} + DIFY_AGENT_STUB_API_BASE_URL: ${DIFY_AGENT_STUB_API_BASE_URL:-http://agent_backend:5050/agent-stub} + # This is security-sensitive: it derives the JWE encryption key for Agent Stub bearer tokens. + # Replace this development default in production. + # Generate one with: python -c 'import secrets; print(secrets.token_urlsafe(32))' + DIFY_AGENT_SERVER_SECRET_KEY: ${DIFY_AGENT_SERVER_SECRET_KEY:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY} + DIFY_AGENT_SHUTDOWN_GRACE_SECONDS: ${DIFY_AGENT_SHUTDOWN_GRACE_SECONDS:-30} + DIFY_AGENT_RUN_RETENTION_SECONDS: ${DIFY_AGENT_RUN_RETENTION_SECONDS:-259200} + depends_on: + redis: + condition: service_started + plugin_daemon: + condition: service_started + local_sandbox: + condition: service_started + networks: + - default + # ssrf_proxy server # for more information, please refer to # https://docs.dify.ai/learn-more/faq/install-faq#18-why-is-ssrf-proxy-needed%3F diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 38d610a4f11..9ce100e4183 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -237,6 +237,7 @@ services: PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} PLUGIN_DAEMON_TIMEOUT: ${PLUGIN_DAEMON_TIMEOUT:-600.0} INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + AGENT_BACKEND_BASE_URL: ${AGENT_BACKEND_BASE_URL:-http://agent_backend:5050} depends_on: init_permissions: condition: service_completed_successfully @@ -254,6 +255,8 @@ services: required: false redis: condition: service_started + agent_backend: + condition: service_started volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage @@ -304,6 +307,7 @@ services: SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + AGENT_BACKEND_BASE_URL: ${AGENT_BACKEND_BASE_URL:-http://agent_backend:5050} depends_on: init_permissions: condition: service_completed_successfully @@ -321,6 +325,8 @@ services: required: false redis: condition: service_started + agent_backend: + condition: service_started volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage @@ -522,6 +528,21 @@ services: networks: - ssrf_proxy_network + # Local sandbox for Dify Agent shell workspaces. + local_sandbox: + image: langgenius/dify-agent-local-sandbox:1.15.0 + restart: always + environment: + SHELLCTL_AUTH_TOKEN: ${DIFY_AGENT_SHELLCTL_AUTH_TOKEN:-} + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5004/healthz"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + networks: + - default + # plugin daemon plugin_daemon: image: langgenius/dify-plugin-daemon:0.6.3-local @@ -609,6 +630,44 @@ services: condition: service_healthy required: false + # Dify Agent backend service. + agent_backend: + image: langgenius/dify-agent-backend:1.15.0 + restart: always + env_file: + - path: ./envs/core-services/dify-agent.env + required: false + - path: ./envs/security.env + required: false + - path: ./envs/databases/redis.env + required: false + - ./.env + environment: + DIFY_AGENT_REDIS_URL: ${DIFY_AGENT_REDIS_URL:-redis://:${REDIS_PASSWORD:-difyai123456}@redis:6379/2} + DIFY_AGENT_REDIS_PREFIX: ${DIFY_AGENT_REDIS_PREFIX:-dify-agent} + DIFY_AGENT_PLUGIN_DAEMON_URL: ${DIFY_AGENT_PLUGIN_DAEMON_URL:-${PLUGIN_DAEMON_URL:-http://plugin_daemon:5002}} + DIFY_AGENT_PLUGIN_DAEMON_API_KEY: ${DIFY_AGENT_PLUGIN_DAEMON_API_KEY:-${PLUGIN_DAEMON_KEY:-lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi}} + DIFY_AGENT_INNER_API_URL: ${DIFY_AGENT_INNER_API_URL:-${PLUGIN_DIFY_INNER_API_URL:-http://api:5001}} + DIFY_AGENT_INNER_API_KEY: ${DIFY_AGENT_INNER_API_KEY:-${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1}} + DIFY_AGENT_SHELLCTL_ENTRYPOINT: ${DIFY_AGENT_SHELLCTL_ENTRYPOINT:-http://local_sandbox:5004} + DIFY_AGENT_SHELLCTL_AUTH_TOKEN: ${DIFY_AGENT_SHELLCTL_AUTH_TOKEN:-} + DIFY_AGENT_STUB_API_BASE_URL: ${DIFY_AGENT_STUB_API_BASE_URL:-http://agent_backend:5050/agent-stub} + # This is security-sensitive: it derives the JWE encryption key for Agent Stub bearer tokens. + # Replace this development default in production. + # Generate one with: python -c 'import secrets; print(secrets.token_urlsafe(32))' + DIFY_AGENT_SERVER_SECRET_KEY: ${DIFY_AGENT_SERVER_SECRET_KEY:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY} + DIFY_AGENT_SHUTDOWN_GRACE_SECONDS: ${DIFY_AGENT_SHUTDOWN_GRACE_SECONDS:-30} + DIFY_AGENT_RUN_RETENTION_SECONDS: ${DIFY_AGENT_RUN_RETENTION_SECONDS:-259200} + depends_on: + redis: + condition: service_started + plugin_daemon: + condition: service_started + local_sandbox: + condition: service_started + networks: + - default + # ssrf_proxy server # for more information, please refer to # https://docs.dify.ai/learn-more/faq/install-faq#18-why-is-ssrf-proxy-needed%3F diff --git a/docker/envs/core-services/dify-agent.env.example b/docker/envs/core-services/dify-agent.env.example new file mode 100644 index 00000000000..be757e456bb --- /dev/null +++ b/docker/envs/core-services/dify-agent.env.example @@ -0,0 +1,28 @@ +# ------------------------------ +# Dify Agent Backend Configuration +# ------------------------------ + +AGENT_BACKEND_BASE_URL=http://agent_backend:5050 + +# Leave empty to derive from REDIS_PASSWORD in Docker Compose. +DIFY_AGENT_REDIS_URL= +DIFY_AGENT_REDIS_PREFIX=dify-agent +DIFY_AGENT_SHUTDOWN_GRACE_SECONDS=30 +DIFY_AGENT_RUN_RETENTION_SECONDS=259200 + +# Leave empty to derive from PLUGIN_DAEMON_URL and PLUGIN_DAEMON_KEY in Docker Compose. +DIFY_AGENT_PLUGIN_DAEMON_URL= +DIFY_AGENT_PLUGIN_DAEMON_API_KEY= + +# Leave empty to derive from PLUGIN_DIFY_INNER_API_URL and PLUGIN_DIFY_INNER_API_KEY in Docker Compose. +# DIFY_AGENT_INNER_API_KEY must match API/worker INNER_API_KEY_FOR_PLUGIN, not INNER_API_KEY. +DIFY_AGENT_INNER_API_URL= +DIFY_AGENT_INNER_API_KEY= + +DIFY_AGENT_SHELLCTL_ENTRYPOINT=http://local_sandbox:5004 +DIFY_AGENT_SHELLCTL_AUTH_TOKEN= +DIFY_AGENT_STUB_API_BASE_URL=http://agent_backend:5050/agent-stub +# This is security-sensitive: it derives the JWE encryption key for Agent Stub bearer tokens. +# Replace this development default in production. +# Generate one with: python -c 'import secrets; print(secrets.token_urlsafe(32))' +DIFY_AGENT_SERVER_SECRET_KEY=MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY diff --git a/docker/envs/core-services/shared.env.example b/docker/envs/core-services/shared.env.example index ca43bd13026..0a6eeecd774 100644 --- a/docker/envs/core-services/shared.env.example +++ b/docker/envs/core-services/shared.env.example @@ -6,8 +6,8 @@ CONSOLE_WEB_URL= SERVICE_API_URL= TRIGGER_URL=http://localhost APP_WEB_URL= -FILES_URL= -INTERNAL_FILES_URL= +# FILES_URL= +# INTERNAL_FILES_URL= LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONIOENCODING=utf-8 diff --git a/docker/envs/core-services/web.env.example b/docker/envs/core-services/web.env.example index 0b75ec7c5b8..c7053350a27 100644 --- a/docker/envs/core-services/web.env.example +++ b/docker/envs/core-services/web.env.example @@ -25,7 +25,7 @@ ENABLE_WEBSITE_FIRECRAWL=true ENABLE_WEBSITE_WATERCRAWL=true NEXT_PUBLIC_ENABLE_SINGLE_DOLLAR_LATEX=false NEXT_PUBLIC_ENABLE_FEATURE_PREVIEW=true -ENABLE_AGENT_V2=false +NEXT_PUBLIC_ENABLE_AGENT_V2=true NEXT_PUBLIC_COOKIE_DOMAIN= NEXT_PUBLIC_BATCH_CONCURRENCY=5 CSP_WHITELIST= diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 33b74103ee7..9f8081970fc 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -8,7 +8,7 @@ Do not add deeper `AGENTS.md` files unless an Agent v2 submodule becomes indepen Agent v2 scenarios live under `features/agent-v2/` and use the `@agent-v2` capability tag. -The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=true` in `scripts/common.ts`, because `/roster` routes are guarded by that feature flag. +The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=true` in `scripts/common.ts`, because `/agents` routes are guarded by that feature flag. Preview/Test Run scenarios are not part of the current build-mode slice unless explicitly requested. Current Agent v2 coverage should prioritize Configure, Build draft, saved configuration display, publish state, Access Point, preflight, files, advanced settings, and other build-mode behavior. Published Web app runtime is not Builder Preview; keep it as a separate `@web-app-runtime` slice because it exercises the public app surface and real model-backed responses after publish. diff --git a/e2e/features/agent-v2/support/agent.ts b/e2e/features/agent-v2/support/agent.ts index db19b11574b..4b3fdcf636e 100644 --- a/e2e/features/agent-v2/support/agent.ts +++ b/e2e/features/agent-v2/support/agent.ts @@ -31,8 +31,8 @@ export type CreateTestAgentOptions = { role?: string } -export const getAgentConfigurePath = (agentId: string) => `/roster/agent/${agentId}/configure` -export const getAgentAccessPath = (agentId: string) => `/roster/agent/${agentId}/access` +export const getAgentConfigurePath = (agentId: string) => `/agents/${agentId}/configure` +export const getAgentAccessPath = (agentId: string) => `/agents/${agentId}/access` export async function createTestAgent({ description = 'Created by Dify E2E.', diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index 0ed6e456a93..ee9e41e7390 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -44,11 +44,11 @@ When( const page = this.getPage() const agent = getPreseededResource(this, agentName, 'agent') - await page.goto('/roster') + await page.goto('/agents') await page.getByRole('link', { name: agentName }).click() - await expect(page).toHaveURL(new RegExp(`/roster/agent/${agent.id}/configure(?:\\?.*)?$`)) + await expect(page).toHaveURL(new RegExp(`/agents/${agent.id}/configure(?:\\?.*)?$`)) await page.getByRole('link', { name: 'Access Point' }).click() - await expect(page).toHaveURL(new RegExp(`/roster/agent/${agent.id}/access(?:\\?.*)?$`)) + await expect(page).toHaveURL(new RegExp(`/agents/${agent.id}/access(?:\\?.*)?$`)) await expect(page.getByRole('region', { name: 'Access Point' })).toBeVisible({ timeout: 30_000, }) @@ -60,7 +60,7 @@ When('I switch to the Agent v2 Access Point section', async function (this: Dify const agentId = getCurrentAgentId(this) await page.getByRole('link', { name: 'Access Point' }).click() - await expect(page).toHaveURL(new RegExp(`/roster/agent/${agentId}/access(?:\\?.*)?$`)) + await expect(page).toHaveURL(new RegExp(`/agents/${agentId}/access(?:\\?.*)?$`)) await expect(page.getByRole('region', { name: 'Access Point' })).toBeVisible() }) diff --git a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts index b2da0b61cb0..5bd818bb1d4 100644 --- a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts +++ b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts @@ -69,7 +69,7 @@ When( const agent = getPreseededAgent(this, agentName) const copyName = createE2EResourceName('Agent', 'copy') - await page.goto('/roster') + await page.goto('/agents') const card = page.locator('article').filter({ has: page.getByRole('link', { name: agentName }), }).first() diff --git a/e2e/features/step-definitions/agent-v2/agent-roster.steps.ts b/e2e/features/step-definitions/agent-v2/agent-roster.steps.ts index 70fd03195e3..c6dc4bd68bb 100644 --- a/e2e/features/step-definitions/agent-v2/agent-roster.steps.ts +++ b/e2e/features/step-definitions/agent-v2/agent-roster.steps.ts @@ -10,7 +10,7 @@ When('I create an Agent v2 test agent from the Agent Roster', async function (th const agentRole = 'E2E roster-created assistant' const agentDescription = 'Created by Dify E2E through the Agent Roster UI.' - await page.goto('/roster') + await page.goto('/agents') await page.getByRole('button', { name: 'Create agent' }).click() const dialog = page.getByRole('dialog', { name: 'Create agent' }) @@ -40,7 +40,7 @@ Then('the created Agent v2 should open in Configure', async function (this: Dify throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') await expect(this.getPage()).toHaveURL( - new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`), + new RegExp(`/agents/${agentId}/configure(?:\\?.*)?$`), { timeout: 30_000 }, ) await expect(this.getPage().getByRole('heading', { name: 'Configure' })) diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 02ed9004d3c..b27b3eee6f4 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -180,15 +180,15 @@ When('I switch to the Agent v2 Configure section', async function (this: DifyWor const agentId = getCurrentAgentId(this) await page.getByRole('link', { name: 'Configure' }).click() - await expect(page).toHaveURL(new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`)) + await expect(page).toHaveURL(new RegExp(`/agents/${agentId}/configure(?:\\?.*)?$`)) await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) }) When('I leave the Agent v2 configure page immediately after editing', async function (this: DifyWorld) { const page = this.getPage() - await page.goto('/roster') - await expect(page).toHaveURL(/\/roster(?:\?.*)?$/) + await page.goto('/agents') + await expect(page).toHaveURL(/\/agents(?:\?.*)?$/) }) When('I open the Agent v2 configure page from the Agent Roster', async function (this: DifyWorld) { @@ -198,9 +198,9 @@ When('I open the Agent v2 configure page from the Agent Roster', async function if (!agentName) throw new Error('No Agent v2 name found. Create an Agent v2 test agent first.') - await page.goto('/roster') + await page.goto('/agents') await page.getByRole('link', { name: agentName }).click() - await expect(page).toHaveURL(new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`)) + await expect(page).toHaveURL(new RegExp(`/agents/${agentId}/configure(?:\\?.*)?$`)) await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) }) @@ -210,9 +210,9 @@ When( const page = this.getPage() const agent = getPreseededAgent(this, agentName) - await page.goto('/roster') + await page.goto('/agents') await page.getByRole('link', { name: agentName }).click() - await expect(page).toHaveURL(new RegExp(`/roster/agent/${agent.id}/configure(?:\\?.*)?$`)) + await expect(page).toHaveURL(new RegExp(`/agents/${agent.id}/configure(?:\\?.*)?$`)) await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) }, ) @@ -235,7 +235,7 @@ When('I open the same Agent v2 configure page in another tab', async function (t this.agentBuilder.configure.concurrentPage = concurrentPage await concurrentPage.goto(getAgentConfigurePath(agentId)) - await expect(concurrentPage).toHaveURL(new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`)) + await expect(concurrentPage).toHaveURL(new RegExp(`/agents/${agentId}/configure(?:\\?.*)?$`)) await expect(concurrentPage.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) }) @@ -276,7 +276,7 @@ Then('I should be on the Agent v2 configure page', async function (this: DifyWor const agentId = getCurrentAgentId(this) await expect(this.getPage()).toHaveURL( - new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`), + new RegExp(`/agents/${agentId}/configure(?:\\?.*)?$`), ) }) diff --git a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts index 8e385485aaa..0df872a0ada 100644 --- a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts +++ b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts @@ -94,7 +94,7 @@ Then( await expect(detailsDialog.getByText(normalAgentPrompt)).toBeVisible() await expect(detailsDialog.getByRole('link', { name: 'Edit in Agent Console' })).toHaveAttribute( 'href', - `/roster/agent/${this.createdAgentIds.at(-1)}/configure`, + `/agents/${this.createdAgentIds.at(-1)}/configure`, ) }, ) @@ -114,7 +114,7 @@ Then( throw new Error('Stable chat model preflight must run before asserting Agent Console.') await expect(agentConsolePage).toHaveURL( - new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`), + new RegExp(`/agents/${agentId}/configure(?:\\?.*)?$`), ) await expect(agentConsolePage.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000, diff --git a/eslint-suppressions.json b/eslint-suppressions.json index 8cc4e96a268..1fb6dc17714 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -1598,11 +1598,6 @@ "count": 6 } }, - "web/app/components/base/icons/src/public/thought/index.ts": { - "no-barrel-files/no-barrel-files": { - "count": 1 - } - }, "web/app/components/base/icons/src/public/tracing/index.ts": { "no-barrel-files/no-barrel-files": { "count": 21 diff --git a/packages/contracts/generated/api/console/installed-apps/types.gen.ts b/packages/contracts/generated/api/console/installed-apps/types.gen.ts index 59b4619c11a..6cba91f3987 100644 --- a/packages/contracts/generated/api/console/installed-apps/types.gen.ts +++ b/packages/contracts/generated/api/console/installed-apps/types.gen.ts @@ -193,8 +193,10 @@ export type JsonValue export type ExploreMessageListItem = { agent_thoughts: Array answer: string + answer_tokens?: number conversation_id: string created_at?: number | null + currency?: string | null error?: string | null extra_contents: Array feedback?: SimpleFeedback | null @@ -203,11 +205,15 @@ export type ExploreMessageListItem = { [key: string]: JsonValueType } message_files: Array + message_tokens?: number metadata?: JsonValueType | null parent_message_id?: string | null + provider_response_latency?: number query: string retriever_resources: Array status: string + total_price?: string | null + readonly total_tokens: number } export type JsonObject = { @@ -410,6 +416,12 @@ export type InstalledAppListResponseWritable = { installed_apps: Array } +export type ExploreMessageInfiniteScrollPaginationWritable = { + data: Array + has_more: boolean + limit: number +} + export type InstalledAppResponseWritable = { app: InstalledAppInfoResponseWritable app_owner_tenant_id: string @@ -420,6 +432,31 @@ export type InstalledAppResponseWritable = { uninstallable: boolean } +export type ExploreMessageListItemWritable = { + agent_thoughts: Array + answer: string + answer_tokens?: number + conversation_id: string + created_at?: number | null + currency?: string | null + error?: string | null + extra_contents: Array + feedback?: SimpleFeedback | null + id: string + inputs: { + [key: string]: JsonValueType + } + message_files: Array + message_tokens?: number + metadata?: JsonValueType | null + parent_message_id?: string | null + provider_response_latency?: number + query: string + retriever_resources: Array + status: string + total_price?: string | null +} + export type InstalledAppInfoResponseWritable = { description?: string | null icon?: string | null diff --git a/packages/contracts/generated/api/console/installed-apps/zod.gen.ts b/packages/contracts/generated/api/console/installed-apps/zod.gen.ts index 60c4f1448c4..2a1d856867b 100644 --- a/packages/contracts/generated/api/console/installed-apps/zod.gen.ts +++ b/packages/contracts/generated/api/console/installed-apps/zod.gen.ts @@ -512,19 +512,28 @@ export const zHumanInputContent = z.object({ export const zExploreMessageListItem = z.object({ agent_thoughts: z.array(zAgentThought), answer: z.string(), + answer_tokens: z.int().optional().default(0), conversation_id: z.string(), created_at: z.int().nullish(), + currency: z.string().nullish(), error: z.string().nullish(), extra_contents: z.array(zHumanInputContent), feedback: zSimpleFeedback.nullish(), id: z.string(), inputs: z.record(z.string(), zJsonValueType), message_files: z.array(zMessageFile), + message_tokens: z.int().optional().default(0), metadata: zJsonValueType.nullish(), parent_message_id: z.string().nullish(), + provider_response_latency: z.number().optional().default(0), query: z.string(), retriever_resources: z.array(zRetrieverResource), status: z.string(), + total_price: z + .string() + .regex(/^(?![-+.]*$)[+-]?\d*(?:\.\d*)?$/) + .nullish(), + total_tokens: z.int().readonly(), }) /** @@ -536,6 +545,44 @@ export const zExploreMessageInfiniteScrollPagination = z.object({ limit: z.int(), }) +/** + * ExploreMessageListItem + */ +export const zExploreMessageListItemWritable = z.object({ + agent_thoughts: z.array(zAgentThought), + answer: z.string(), + answer_tokens: z.int().optional().default(0), + conversation_id: z.string(), + created_at: z.int().nullish(), + currency: z.string().nullish(), + error: z.string().nullish(), + extra_contents: z.array(zHumanInputContent), + feedback: zSimpleFeedback.nullish(), + id: z.string(), + inputs: z.record(z.string(), zJsonValueType), + message_files: z.array(zMessageFile), + message_tokens: z.int().optional().default(0), + metadata: zJsonValueType.nullish(), + parent_message_id: z.string().nullish(), + provider_response_latency: z.number().optional().default(0), + query: z.string(), + retriever_resources: z.array(zRetrieverResource), + status: z.string(), + total_price: z + .string() + .regex(/^(?![-+.]*$)[+-]?\d*(?:\.\d*)?$/) + .nullish(), +}) + +/** + * ExploreMessageInfiniteScrollPagination + */ +export const zExploreMessageInfiniteScrollPaginationWritable = z.object({ + data: z.array(zExploreMessageListItemWritable), + has_more: z.boolean(), + limit: z.int(), +}) + /** * InstalledAppInfoResponse */ diff --git a/packages/contracts/generated/api/service/types.gen.ts b/packages/contracts/generated/api/service/types.gen.ts index 4aface8d7c1..ea06e17478e 100644 --- a/packages/contracts/generated/api/service/types.gen.ts +++ b/packages/contracts/generated/api/service/types.gen.ts @@ -1071,8 +1071,10 @@ export type MessageInfiniteScrollPagination = { export type MessageListItem = { agent_thoughts: Array answer: string + answer_tokens?: number conversation_id: string created_at?: number | null + currency?: string | null error?: string | null extra_contents: Array feedback?: SimpleFeedback | null @@ -1081,10 +1083,14 @@ export type MessageListItem = { [key: string]: JsonValueType } message_files: Array + message_tokens?: number parent_message_id?: string | null + provider_response_latency?: number query: string retriever_resources: Array status: string + total_price?: string | null + readonly total_tokens: number } export type MessageListQuery = { @@ -1691,6 +1697,36 @@ export type HumanInputFormSubmitResponseWritable = { [key: string]: never } +export type MessageInfiniteScrollPaginationWritable = { + data: Array + has_more: boolean + limit: number +} + +export type MessageListItemWritable = { + agent_thoughts: Array + answer: string + answer_tokens?: number + conversation_id: string + created_at?: number | null + currency?: string | null + error?: string | null + extra_contents: Array + feedback?: SimpleFeedback | null + id: string + inputs: { + [key: string]: JsonValueType + } + message_files: Array + message_tokens?: number + parent_message_id?: string | null + provider_response_latency?: number + query: string + retriever_resources: Array + status: string + total_price?: string | null +} + export type SiteWritable = { chat_color_theme?: string | null chat_color_theme_inverted: boolean diff --git a/packages/contracts/generated/api/service/zod.gen.ts b/packages/contracts/generated/api/service/zod.gen.ts index db3f851fc90..72745013ad6 100644 --- a/packages/contracts/generated/api/service/zod.gen.ts +++ b/packages/contracts/generated/api/service/zod.gen.ts @@ -1974,18 +1974,27 @@ export const zHumanInputContent = z.object({ export const zMessageListItem = z.object({ agent_thoughts: z.array(zAgentThought), answer: z.string(), + answer_tokens: z.int().optional().default(0), conversation_id: z.string(), created_at: z.int().nullish(), + currency: z.string().nullish(), error: z.string().nullish(), extra_contents: z.array(zHumanInputContent), feedback: zSimpleFeedback.nullish(), id: z.string(), inputs: z.record(z.string(), zJsonValueType), message_files: z.array(zMessageFile), + message_tokens: z.int().optional().default(0), parent_message_id: z.string().nullish(), + provider_response_latency: z.number().optional().default(0), query: z.string(), retriever_resources: z.array(zRetrieverResource), status: z.string(), + total_price: z + .string() + .regex(/^(?![-+.]*$)[+-]?\d*(?:\.\d*)?$/) + .nullish(), + total_tokens: z.int().readonly(), }) /** @@ -2310,6 +2319,43 @@ export const zGeneratedAppResponseWritable = zJsonValue */ export const zHumanInputFormSubmitResponseWritable = z.record(z.string(), z.never()) +/** + * MessageListItem + */ +export const zMessageListItemWritable = z.object({ + agent_thoughts: z.array(zAgentThought), + answer: z.string(), + answer_tokens: z.int().optional().default(0), + conversation_id: z.string(), + created_at: z.int().nullish(), + currency: z.string().nullish(), + error: z.string().nullish(), + extra_contents: z.array(zHumanInputContent), + feedback: zSimpleFeedback.nullish(), + id: z.string(), + inputs: z.record(z.string(), zJsonValueType), + message_files: z.array(zMessageFile), + message_tokens: z.int().optional().default(0), + parent_message_id: z.string().nullish(), + provider_response_latency: z.number().optional().default(0), + query: z.string(), + retriever_resources: z.array(zRetrieverResource), + status: z.string(), + total_price: z + .string() + .regex(/^(?![-+.]*$)[+-]?\d*(?:\.\d*)?$/) + .nullish(), +}) + +/** + * MessageInfiniteScrollPagination + */ +export const zMessageInfiniteScrollPaginationWritable = z.object({ + data: z.array(zMessageListItemWritable), + has_more: z.boolean(), + limit: z.int(), +}) + /** * Site */ diff --git a/packages/contracts/generated/api/web/types.gen.ts b/packages/contracts/generated/api/web/types.gen.ts index 9a88c36f2b3..17a08c15cd8 100644 --- a/packages/contracts/generated/api/web/types.gen.ts +++ b/packages/contracts/generated/api/web/types.gen.ts @@ -599,8 +599,10 @@ export type WebMessageInfiniteScrollPagination = { export type WebMessageListItem = { agent_thoughts: Array answer: string + answer_tokens?: number conversation_id: string created_at?: number | null + currency?: string | null error?: string | null extra_contents: Array feedback?: SimpleFeedback | null @@ -609,11 +611,15 @@ export type WebMessageListItem = { [key: string]: JsonValueType } message_files: Array + message_tokens?: number metadata?: JsonValueType | null parent_message_id?: string | null + provider_response_latency?: number query: string retriever_resources: Array status: string + total_price?: string | null + readonly total_tokens: number } export type WebModelConfigResponse = { @@ -685,6 +691,37 @@ export type WebAppSiteResponseWritable = { site: WebSiteResponseWritable } +export type WebMessageInfiniteScrollPaginationWritable = { + data: Array + has_more: boolean + limit: number +} + +export type WebMessageListItemWritable = { + agent_thoughts: Array + answer: string + answer_tokens?: number + conversation_id: string + created_at?: number | null + currency?: string | null + error?: string | null + extra_contents: Array + feedback?: SimpleFeedback | null + id: string + inputs: { + [key: string]: JsonValueType + } + message_files: Array + message_tokens?: number + metadata?: JsonValueType | null + parent_message_id?: string | null + provider_response_latency?: number + query: string + retriever_resources: Array + status: string + total_price?: string | null +} + export type WebSiteResponseWritable = { chat_color_theme?: string | null chat_color_theme_inverted: boolean diff --git a/packages/contracts/generated/api/web/zod.gen.ts b/packages/contracts/generated/api/web/zod.gen.ts index 2a3ea96b0ac..23c1d9f4c36 100644 --- a/packages/contracts/generated/api/web/zod.gen.ts +++ b/packages/contracts/generated/api/web/zod.gen.ts @@ -831,19 +831,28 @@ export const zWebAppCustomConfigResponse = z.object({ export const zWebMessageListItem = z.object({ agent_thoughts: z.array(zAgentThought), answer: z.string(), + answer_tokens: z.int().optional().default(0), conversation_id: z.string(), created_at: z.int().nullish(), + currency: z.string().nullish(), error: z.string().nullish(), extra_contents: z.array(zHumanInputContent), feedback: zSimpleFeedback.nullish(), id: z.string(), inputs: z.record(z.string(), zJsonValueType), message_files: z.array(zMessageFile), + message_tokens: z.int().optional().default(0), metadata: zJsonValueType.nullish(), parent_message_id: z.string().nullish(), + provider_response_latency: z.number().optional().default(0), query: z.string(), retriever_resources: z.array(zRetrieverResource), status: z.string(), + total_price: z + .string() + .regex(/^(?![-+.]*$)[+-]?\d*(?:\.\d*)?$/) + .nullish(), + total_tokens: z.int().readonly(), }) /** @@ -943,6 +952,44 @@ export const zGeneratedAppResponseWritable = zJsonValue */ export const zHumanInputFormSubmitResponseWritable = z.record(z.string(), z.unknown()) +/** + * WebMessageListItem + */ +export const zWebMessageListItemWritable = z.object({ + agent_thoughts: z.array(zAgentThought), + answer: z.string(), + answer_tokens: z.int().optional().default(0), + conversation_id: z.string(), + created_at: z.int().nullish(), + currency: z.string().nullish(), + error: z.string().nullish(), + extra_contents: z.array(zHumanInputContent), + feedback: zSimpleFeedback.nullish(), + id: z.string(), + inputs: z.record(z.string(), zJsonValueType), + message_files: z.array(zMessageFile), + message_tokens: z.int().optional().default(0), + metadata: zJsonValueType.nullish(), + parent_message_id: z.string().nullish(), + provider_response_latency: z.number().optional().default(0), + query: z.string(), + retriever_resources: z.array(zRetrieverResource), + status: z.string(), + total_price: z + .string() + .regex(/^(?![-+.]*$)[+-]?\d*(?:\.\d*)?$/) + .nullish(), +}) + +/** + * WebMessageInfiniteScrollPagination + */ +export const zWebMessageInfiniteScrollPaginationWritable = z.object({ + data: z.array(zWebMessageListItemWritable), + has_more: z.boolean(), + limit: z.int(), +}) + /** * WebSiteResponse */ diff --git a/packages/iconify-collections/assets/public/thought/imagine.svg b/packages/iconify-collections/assets/public/thought/imagine.svg new file mode 100644 index 00000000000..ba3fa075205 --- /dev/null +++ b/packages/iconify-collections/assets/public/thought/imagine.svg @@ -0,0 +1,4 @@ + + + + diff --git a/packages/iconify-collections/custom-public/icons.json b/packages/iconify-collections/custom-public/icons.json index bba5a879707..361e5a7a5bd 100644 --- a/packages/iconify-collections/custom-public/icons.json +++ b/packages/iconify-collections/custom-public/icons.json @@ -1,6 +1,6 @@ { "prefix": "custom-public", - "lastModified": 1782215796, + "lastModified": 1783583299, "icons": { "agent-building-blocks": { "body": "" @@ -587,6 +587,11 @@ "width": 12, "height": 12 }, + "thought-imagine": { + "body": "", + "width": 16, + "height": 16 + }, "thought-loading": { "body": "", "width": 12, diff --git a/packages/iconify-collections/custom-public/info.json b/packages/iconify-collections/custom-public/info.json index 1b439b73724..818be4086b8 100644 --- a/packages/iconify-collections/custom-public/info.json +++ b/packages/iconify-collections/custom-public/info.json @@ -1,7 +1,7 @@ { "prefix": "custom-public", "name": "Dify Custom Public", - "total": 145, + "total": 146, "version": "0.0.0-private", "author": { "name": "LangGenius, Inc.", diff --git a/web/.env.example b/web/.env.example index ea80c352352..b745c6953d3 100644 --- a/web/.env.example +++ b/web/.env.example @@ -91,7 +91,7 @@ NEXT_PUBLIC_ENABLE_SINGLE_DOLLAR_LATEX=false NEXT_PUBLIC_ENABLE_FEATURE_PREVIEW=true # Enable Agent v2 frontend entry points. -NEXT_PUBLIC_ENABLE_AGENT_V2=false +NEXT_PUBLIC_ENABLE_AGENT_V2=true # The maximum number of tree node depth for workflow NEXT_PUBLIC_MAX_TREE_DEPTH=50 diff --git a/web/__tests__/proxy-frame-options.spec.ts b/web/__tests__/proxy-frame-options.spec.ts index f6f6abe160c..a2aafa1da20 100644 --- a/web/__tests__/proxy-frame-options.spec.ts +++ b/web/__tests__/proxy-frame-options.spec.ts @@ -14,7 +14,7 @@ describe('proxy frame options', () => { expect(canEmbedPath('/agents')).toBe(false) expect(canEmbedPath('/agent-settings')).toBe(false) expect(canEmbedPath('/agentic')).toBe(false) - expect(canEmbedPath('/roster/agent/agent-1/access')).toBe(false) + expect(canEmbedPath('/agents/agent-1/access')).toBe(false) expect(canEmbedPath('/apps')).toBe(false) }) }) diff --git a/web/app/(commonLayout)/@detailSidebar/roster/agent/[agentId]/access/page.tsx b/web/app/(commonLayout)/@detailSidebar/agents/[agentId]/access/page.tsx similarity index 100% rename from web/app/(commonLayout)/@detailSidebar/roster/agent/[agentId]/access/page.tsx rename to web/app/(commonLayout)/@detailSidebar/agents/[agentId]/access/page.tsx diff --git a/web/app/(commonLayout)/@detailSidebar/roster/agent/[agentId]/configure/page.tsx b/web/app/(commonLayout)/@detailSidebar/agents/[agentId]/configure/page.tsx similarity index 100% rename from web/app/(commonLayout)/@detailSidebar/roster/agent/[agentId]/configure/page.tsx rename to web/app/(commonLayout)/@detailSidebar/agents/[agentId]/configure/page.tsx diff --git a/web/app/(commonLayout)/@detailSidebar/roster/agent/[agentId]/logs/page.tsx b/web/app/(commonLayout)/@detailSidebar/agents/[agentId]/logs/page.tsx similarity index 100% rename from web/app/(commonLayout)/@detailSidebar/roster/agent/[agentId]/logs/page.tsx rename to web/app/(commonLayout)/@detailSidebar/agents/[agentId]/logs/page.tsx diff --git a/web/app/(commonLayout)/@detailSidebar/roster/agent/[agentId]/monitoring/page.tsx b/web/app/(commonLayout)/@detailSidebar/agents/[agentId]/monitoring/page.tsx similarity index 100% rename from web/app/(commonLayout)/@detailSidebar/roster/agent/[agentId]/monitoring/page.tsx rename to web/app/(commonLayout)/@detailSidebar/agents/[agentId]/monitoring/page.tsx diff --git a/web/app/(commonLayout)/@detailSidebar/roster/agent/[agentId]/page.tsx b/web/app/(commonLayout)/@detailSidebar/agents/[agentId]/page.tsx similarity index 100% rename from web/app/(commonLayout)/@detailSidebar/roster/agent/[agentId]/page.tsx rename to web/app/(commonLayout)/@detailSidebar/agents/[agentId]/page.tsx diff --git a/web/app/(commonLayout)/@detailSidebar/roster/agent/[agentId]/sidebar-page.tsx b/web/app/(commonLayout)/@detailSidebar/agents/[agentId]/sidebar-page.tsx similarity index 100% rename from web/app/(commonLayout)/@detailSidebar/roster/agent/[agentId]/sidebar-page.tsx rename to web/app/(commonLayout)/@detailSidebar/agents/[agentId]/sidebar-page.tsx diff --git a/web/app/(commonLayout)/roster/agent/[agentId]/access/page.tsx b/web/app/(commonLayout)/agents/[agentId]/access/page.tsx similarity index 100% rename from web/app/(commonLayout)/roster/agent/[agentId]/access/page.tsx rename to web/app/(commonLayout)/agents/[agentId]/access/page.tsx diff --git a/web/app/(commonLayout)/roster/agent/[agentId]/configure/page.tsx b/web/app/(commonLayout)/agents/[agentId]/configure/page.tsx similarity index 100% rename from web/app/(commonLayout)/roster/agent/[agentId]/configure/page.tsx rename to web/app/(commonLayout)/agents/[agentId]/configure/page.tsx diff --git a/web/app/(commonLayout)/roster/agent/[agentId]/layout.tsx b/web/app/(commonLayout)/agents/[agentId]/layout.tsx similarity index 100% rename from web/app/(commonLayout)/roster/agent/[agentId]/layout.tsx rename to web/app/(commonLayout)/agents/[agentId]/layout.tsx diff --git a/web/app/(commonLayout)/roster/agent/[agentId]/logs/page.tsx b/web/app/(commonLayout)/agents/[agentId]/logs/page.tsx similarity index 100% rename from web/app/(commonLayout)/roster/agent/[agentId]/logs/page.tsx rename to web/app/(commonLayout)/agents/[agentId]/logs/page.tsx diff --git a/web/app/(commonLayout)/roster/agent/[agentId]/monitoring/page.tsx b/web/app/(commonLayout)/agents/[agentId]/monitoring/page.tsx similarity index 100% rename from web/app/(commonLayout)/roster/agent/[agentId]/monitoring/page.tsx rename to web/app/(commonLayout)/agents/[agentId]/monitoring/page.tsx diff --git a/web/app/(commonLayout)/roster/agent/[agentId]/page.tsx b/web/app/(commonLayout)/agents/[agentId]/page.tsx similarity index 80% rename from web/app/(commonLayout)/roster/agent/[agentId]/page.tsx rename to web/app/(commonLayout)/agents/[agentId]/page.tsx index 470fa13f12c..9480283a593 100644 --- a/web/app/(commonLayout)/roster/agent/[agentId]/page.tsx +++ b/web/app/(commonLayout)/agents/[agentId]/page.tsx @@ -9,5 +9,5 @@ export default async function Page({ }: PageProps) { const { agentId } = await params - redirect(`/roster/agent/${agentId}/configure`) + redirect(`/agents/${agentId}/configure`) } diff --git a/web/app/(commonLayout)/roster/__tests__/feature-guard.spec.ts b/web/app/(commonLayout)/agents/__tests__/feature-guard.spec.ts similarity index 100% rename from web/app/(commonLayout)/roster/__tests__/feature-guard.spec.ts rename to web/app/(commonLayout)/agents/__tests__/feature-guard.spec.ts diff --git a/web/app/(commonLayout)/roster/__tests__/layout.spec.tsx b/web/app/(commonLayout)/agents/__tests__/layout.spec.tsx similarity index 100% rename from web/app/(commonLayout)/roster/__tests__/layout.spec.tsx rename to web/app/(commonLayout)/agents/__tests__/layout.spec.tsx diff --git a/web/app/(commonLayout)/roster/feature-guard.ts b/web/app/(commonLayout)/agents/feature-guard.ts similarity index 100% rename from web/app/(commonLayout)/roster/feature-guard.ts rename to web/app/(commonLayout)/agents/feature-guard.ts diff --git a/web/app/(commonLayout)/roster/layout.tsx b/web/app/(commonLayout)/agents/layout.tsx similarity index 100% rename from web/app/(commonLayout)/roster/layout.tsx rename to web/app/(commonLayout)/agents/layout.tsx diff --git a/web/app/(commonLayout)/roster/page.tsx b/web/app/(commonLayout)/agents/page.tsx similarity index 100% rename from web/app/(commonLayout)/roster/page.tsx rename to web/app/(commonLayout)/agents/page.tsx diff --git a/web/app/components/app/configuration/__tests__/configuration-view.spec.tsx b/web/app/components/app/configuration/__tests__/configuration-view.spec.tsx index c185fe2bd40..76ef3bba449 100644 --- a/web/app/components/app/configuration/__tests__/configuration-view.spec.tsx +++ b/web/app/components/app/configuration/__tests__/configuration-view.spec.tsx @@ -301,7 +301,7 @@ describe('ConfigurationView', () => { fireEvent.click(badge) expect(await screen.findByText('appDebug.legacyAgentBadge.description')).toBeInTheDocument() - expect(screen.getByRole('link', { name: /appDebug\.legacyAgentBadge\.action/ })).toHaveAttribute('href', '/roster') + expect(screen.getByRole('link', { name: /appDebug\.legacyAgentBadge\.action/ })).toHaveAttribute('href', '/agents') expect(screen.getByRole('link', { name: /appDebug\.legacyAgentBadge\.action/ })).toHaveAttribute('target', '_blank') expect(screen.getByRole('link', { name: /appDebug\.legacyAgentBadge\.action/ })).toHaveAttribute('rel', 'noopener noreferrer') }) diff --git a/web/app/components/app/configuration/configuration-view.tsx b/web/app/components/app/configuration/configuration-view.tsx index b100b90f190..d350eb09bc5 100644 --- a/web/app/components/app/configuration/configuration-view.tsx +++ b/web/app/components/app/configuration/configuration-view.tsx @@ -70,7 +70,7 @@ function LegacyAgentBadge() { >
{description}
{ + const actual = await vi.importActual('react-i18next') + const { createReactI18nextMock } = await import('../../../../../../../test/i18n-mock') + return { + ...actual, + ...createReactI18nextMock({ + 'common.chat.thought': 'Thought', + }), + } +}) + describe('AgentRosterResponseContent', () => { it('should render historical agent thought answer as markdown instead of thought process', async () => { const item = { @@ -25,6 +37,11 @@ describe('AgentRosterResponseContent', () => { render() + expect(screen.getByRole('button', { name: /workFinished/i })).toBeInTheDocument() + expect(screen.queryByText('history answer')).not.toBeInTheDocument() + + fireEvent.click(screen.getByRole('button', { name: /workFinished/i })) + await waitFor(() => { expect(screen.getByTestId('agent-roster-response-content')).toHaveTextContent('history answer') }) @@ -108,4 +125,28 @@ describe('AgentRosterResponseContent', () => { expect(content.indexOf('first answer')).toBeLessThan(content.indexOf('second thought')) expect(content.indexOf('second thought')).toBeLessThan(content.indexOf('second answer')) }) + + it('should show THOUGHT as the header when a thought process is expanded', () => { + const item = { + id: 'answer-expanded-thought', + content: '', + isAnswer: true, + agent_thoughts: [ + { + id: 'thought-expanded', + thought: 'visible thought summary', + tool: '', + tool_input: '', + observation: '', + message_id: 'answer-expanded-thought', + conversation_id: 'conversation-expanded-thought', + position: 1, + }, + ], + } satisfies ChatItem + + render() + + expect(screen.getByRole('button', { name: 'THOUGHT' })).toBeInTheDocument() + }) }) diff --git a/web/app/components/base/chat/chat/answer/agent-roster-response-content.tsx b/web/app/components/base/chat/chat/answer/agent-roster-response-content.tsx index c6203c2f21a..9e3e8ef30d4 100644 --- a/web/app/components/base/chat/chat/answer/agent-roster-response-content.tsx +++ b/web/app/components/base/chat/chat/answer/agent-roster-response-content.tsx @@ -92,8 +92,19 @@ function getCompletedTitle(latency: NonNullable['latency'] | u return t('agentDetail.configure.answer.workFinished') } -function hasThoughtAnswer(thought: ThoughtItem) { - return !!thought.answer?.trim() +function hashString(value: string) { + let hash = 5381 + for (let i = 0; i < value.length; i++) + hash = ((hash << 5) + hash) ^ value.charCodeAt(i) + + return (hash >>> 0).toString(36) +} + +function getAgentResponsePartBaseKey(part: NonNullable[number]) { + if (part.type === 'message') + return `message-${part.content.length}-${hashString(part.content)}` + + return `thought-${part.thought.id || `${part.thought.message_id}-${part.thought.position}`}` } function useWorkingDuration(enabled?: boolean) { @@ -122,12 +133,14 @@ function useWorkingDuration(enabled?: boolean) { function ProcessShell({ children, collapsed, + expandedTitle, icon, title, defaultOpen = false, }: { children?: ReactNode collapsed?: boolean + expandedTitle?: ReactNode icon: ReactNode title: ReactNode defaultOpen?: boolean @@ -155,7 +168,7 @@ function ProcessShell({ expanded ? 'system-xs-medium-uppercase' : 'system-sm-regular', )} > - {title} + {expanded ? (expandedTitle ?? title) : title} {canExpand && ( expanded @@ -183,12 +196,14 @@ function ThoughtProcess({ defaultOpen?: boolean }) { const { t } = useTranslation() - const summary = thought.thought.trim() || t('chat.thought', { ns: 'common' }) + const thoughtTitle = t('chat.thought', { ns: 'common' }) + const summary = thought.thought.trim() || thoughtTitle return ( } - title={defaultOpen ? t('chat.thought', { ns: 'common' }) : summary} + icon={} + title={summary} + expandedTitle={thoughtTitle.toUpperCase()} defaultOpen={defaultOpen} > @@ -313,15 +328,22 @@ function AgentResponsePartList({ item: ChatItem responding?: boolean }) { + const keyOccurrences = new Map() + return (
{item.agent_response_parts?.map((part, index) => { + const baseKey = getAgentResponsePartBaseKey(part) + const occurrence = keyOccurrences.get(baseKey) ?? 0 + keyOccurrences.set(baseKey, occurrence + 1) + const partKey = occurrence ? `${baseKey}-${occurrence}` : baseKey + if (part.type === 'message') { if (!part.content) return null return ( -
+
) @@ -329,7 +351,7 @@ function AgentResponsePartList({ return ( - {!!agent_thoughts?.length && hasAgentThoughtAnswer && ( - - )} - {!!agent_thoughts?.length && !hasAgentThoughtAnswer && ( + {!!agent_thoughts?.length && ( { expect(screen.getByRole('button', { name: 'common.account.account' })).not.toHaveTextContent(Plan.team) expect(screen.getByRole('link', { name: /common.mainNav.home/ })).toHaveAttribute('href', '/') expect(screen.getByRole('link', { name: /common.menus.apps/ })).toHaveAttribute('href', '/apps') - expect(screen.getByRole('link', { name: /common.menus.roster/ })).toHaveAttribute('href', '/roster') - expect(screen.getByRole('link', { name: /common.menus.roster common.menus.status/ })).toBeInTheDocument() + expect(screen.getByRole('link', { name: /Agents/ })).toHaveAttribute('href', '/agents') + expect(screen.getByRole('link', { name: /Agents common.menus.status/ })).toBeInTheDocument() expect(screen.getByRole('link', { name: /common.menus.datasets/ })).toHaveAttribute('href', '/datasets') expect(screen.getByRole('link', { name: /common.mainNav.integrations/ })).toHaveAttribute('href', '/integrations/model-provider') expect(screen.getByRole('link', { name: /common.mainNav.marketplace/ })).toHaveAttribute('href', '/marketplace') @@ -352,7 +352,7 @@ describe('MainNav', () => { renderMainNav() - expect(screen.queryByRole('link', { name: /common.menus.roster/ })).not.toBeInTheDocument() + expect(screen.queryByRole('link', { name: /Agents/ })).not.toBeInTheDocument() }) it('hides the marketplace entry when marketplace is disabled', () => { @@ -475,7 +475,7 @@ describe('MainNav', () => { expect(screen.getByRole('link', { name: /common.mainNav.home/ })).toHaveAttribute('href', '/') expect(screen.getByRole('link', { name: /common.menus.apps/ })).toHaveAttribute('href', '/apps') - expect(screen.queryByRole('link', { name: /common.menus.roster/ })).not.toBeInTheDocument() + expect(screen.queryByRole('link', { name: /Agents/ })).not.toBeInTheDocument() expect(screen.getByRole('link', { name: /common.menus.datasets/ })).toHaveAttribute('href', '/datasets') expect(screen.getByRole('link', { name: /common.mainNav.integrations/ })).toHaveAttribute('href', '/integrations/model-provider') expect(screen.getByRole('link', { name: /common.mainNav.marketplace/ })).toHaveAttribute('href', '/marketplace') @@ -501,7 +501,7 @@ describe('MainNav', () => { expect(screen.getByRole('link', { name: /common.mainNav.home/ })).toBeInTheDocument() expect(screen.getByRole('link', { name: /common.menus.apps/ })).toBeInTheDocument() - expect(screen.getByRole('link', { name: /common.menus.roster/ })).toBeInTheDocument() + expect(screen.getByRole('link', { name: /Agents/ })).toBeInTheDocument() expect(screen.getByRole('link', { name: /common.menus.datasets/ })).toBeInTheDocument() expect(screen.getByRole('link', { name: /common.mainNav.integrations/ })).toBeInTheDocument() expect(screen.queryByRole('link', { name: /common.menus.deployments/ })).not.toBeInTheDocument() @@ -532,13 +532,13 @@ describe('MainNav', () => { it('keeps roster detail navigation hidden when Agent v2 is disabled', () => { mockIsAgentV2Enabled.mockReturnValue(false) - mockPathname = '/roster/agent/agent-1/configure' + mockPathname = '/agents/agent-1/configure' renderMainNav() expect(screen.queryByTestId('agent-detail-top')).not.toBeInTheDocument() expect(screen.queryByTestId('agent-detail-section')).not.toBeInTheDocument() - expect(screen.queryByRole('link', { name: /common.menus.roster/ })).not.toBeInTheDocument() + expect(screen.queryByRole('link', { name: /Agents/ })).not.toBeInTheDocument() }) it.each([ @@ -582,11 +582,11 @@ describe('MainNav', () => { }) it('marks roster active on roster routes', () => { - mockPathname = '/roster' + mockPathname = '/agents' renderMainNav() - const rosterLink = screen.getByRole('link', { name: /common.menus.roster/ }) + const rosterLink = screen.getByRole('link', { name: /Agents/ }) expect(rosterLink).toHaveClass(activeGradientMaskClassName) expect(rosterLink).toHaveAttribute('aria-current', 'page') }) diff --git a/web/app/components/main-nav/__tests__/layout.spec.tsx b/web/app/components/main-nav/__tests__/layout.spec.tsx index 5a69dc322ef..00c90d89bcb 100644 --- a/web/app/components/main-nav/__tests__/layout.spec.tsx +++ b/web/app/components/main-nav/__tests__/layout.spec.tsx @@ -187,7 +187,7 @@ describe('MainNavLayout', () => { it.each([ { label: 'agent detail route for dataset operators', - pathname: '/roster/agent/agent-1/configure', + pathname: '/agents/agent-1/configure', appContext: { isCurrentWorkspaceDatasetOperator: true, isCurrentWorkspaceEditor: true, diff --git a/web/app/components/main-nav/index.tsx b/web/app/components/main-nav/index.tsx index 7210543d219..08b25b186ee 100644 --- a/web/app/components/main-nav/index.tsx +++ b/web/app/components/main-nav/index.tsx @@ -47,7 +47,7 @@ export function MainNav({ })) .map(route => ({ href: route.href, - label: t(route.labelKey, { ns: 'common' }), + label: 'label' in route ? route.label : t(route.labelKey, { ns: 'common' }), active: route.active, icon: route.icon, activeIcon: route.activeIcon, @@ -93,7 +93,7 @@ export function MainNav({
diff --git a/web/features/agent-v2/agent-detail/routes.ts b/web/features/agent-v2/agent-detail/routes.ts index 313562f5c60..3371b355f67 100644 --- a/web/features/agent-v2/agent-detail/routes.ts +++ b/web/features/agent-v2/agent-detail/routes.ts @@ -3,12 +3,12 @@ import type { AgentDetailSectionKey } from './section' export const getAgentDetailPath = ( agentId: string, section: AgentDetailSectionKey, -) => `/roster/agent/${agentId}/${section}` +) => `/agents/${agentId}/${section}` export const getAgentIdFromPathname = (pathname: string) => { - const [section, type, agentId] = pathname.split('/').filter(Boolean) + const [section, agentId] = pathname.split('/').filter(Boolean) - if (section !== 'roster' || type !== 'agent') + if (section !== 'agents') return undefined return agentId diff --git a/web/features/agent-v2/roster/components/__tests__/create-agent-dialog.spec.tsx b/web/features/agent-v2/roster/components/__tests__/create-agent-dialog.spec.tsx index 54ebe62467c..318f752fc21 100644 --- a/web/features/agent-v2/roster/components/__tests__/create-agent-dialog.spec.tsx +++ b/web/features/agent-v2/roster/components/__tests__/create-agent-dialog.spec.tsx @@ -91,7 +91,7 @@ describe('CreateAgentDialog', () => { }) expect(toastMock.success).toHaveBeenCalledWith('agentV2.roster.createSuccess') - expect(routerPushMock).toHaveBeenCalledWith('/roster/agent/agent-1/configure') + expect(routerPushMock).toHaveBeenCalledWith('/agents/agent-1/configure') }) it('shows a field error when creating with an empty name', async () => { diff --git a/web/features/agent-v2/roster/components/agent-roster-list.tsx b/web/features/agent-v2/roster/components/agent-roster-list.tsx index 683ee8d948c..afc22e21d97 100644 --- a/web/features/agent-v2/roster/components/agent-roster-list.tsx +++ b/web/features/agent-v2/roster/components/agent-roster-list.tsx @@ -130,7 +130,7 @@ function AgentRosterItem({

- {tCommon('menus.roster')} + Agents