diff --git a/api/clients/agent_backend/request_builder.py b/api/clients/agent_backend/request_builder.py index 2f4d09ae9d5..d7a3b8ddc38 100644 --- a/api/clients/agent_backend/request_builder.py +++ b/api/clients/agent_backend/request_builder.py @@ -36,6 +36,11 @@ from dify_agent.layers.knowledge import DIFY_KNOWLEDGE_BASE_LAYER_TYPE_ID, DifyK from dify_agent.layers.output import DIFY_OUTPUT_LAYER_TYPE_ID, DifyOutputLayerConfig from dify_agent.layers.runtime import DIFY_RUNTIME_LAYER_TYPE_ID, DifyRuntimeLayerConfig from dify_agent.layers.shell import DIFY_SHELL_LAYER_TYPE_ID, DifyShellLayerConfig +from dify_agent.layers.user_prompt import ( + DIFY_USER_PROMPT_LAYER_TYPE_ID, + DifyUserPromptFileConfig, + DifyUserPromptLayerConfig, +) from dify_agent.protocol import ( DIFY_AGENT_HISTORY_LAYER_ID, DIFY_AGENT_MODEL_LAYER_ID, @@ -246,6 +251,7 @@ class AgentBackendAgentAppRunInput(BaseModel): execution_context: DifyExecutionContextLayerConfig backend_binding_ref: str = Field(min_length=1) user_prompt: str + user_files: list[DifyUserPromptFileConfig] = Field(default_factory=list) agent_soul_prompt: str | None = None agent_config_version_kind: AgentConfigVersionKind = "snapshot" idempotency_key: str | None = None @@ -310,9 +316,9 @@ class AgentBackendRunRequestBuilder: [ RunLayerSpec( name=AGENT_APP_USER_PROMPT_LAYER_ID, - type=PLAIN_PROMPT_LAYER_TYPE_ID, + type=DIFY_USER_PROMPT_LAYER_TYPE_ID, metadata={**run_input.metadata, "origin": "agent_app_user_prompt"}, - config=PromptLayerConfig(user=run_input.user_prompt), + config=DifyUserPromptLayerConfig(text=run_input.user_prompt, files=run_input.user_files), ), RunLayerSpec( name=DIFY_EXECUTION_CONTEXT_LAYER_ID, @@ -632,7 +638,7 @@ class AgentBackendRunRequestBuilder: ) -_SENSITIVE_KEY_PARTS = ("secret", "credential", "token", "password", "api_key") +_SENSITIVE_KEY_PARTS = ("secret", "credential", "token", "password", "api_key", "base64_data") def redact_for_agent_backend_log(value: object) -> object: @@ -641,9 +647,10 @@ def redact_for_agent_backend_log(value: object) -> object: return redact_for_agent_backend_log(value.model_dump(mode="json", warnings=False)) if isinstance(value, dict): redacted: dict[object, object] = {} + is_multimodal_file = value.get("type") == "image" and "filename" in value and "mime_type" in value for key, item in value.items(): key_text = str(key).lower() - if any(part in key_text for part in _SENSITIVE_KEY_PARTS): + if any(part in key_text for part in _SENSITIVE_KEY_PARTS) or (is_multimodal_file and key_text == "url"): redacted[key] = "[REDACTED]" else: redacted[key] = redact_for_agent_backend_log(item) diff --git a/api/core/app/apps/agent_app/app_generator.py b/api/core/app/apps/agent_app/app_generator.py index 5381cd4b1c3..2d58ca1b4e4 100644 --- a/api/core/app/apps/agent_app/app_generator.py +++ b/api/core/app/apps/agent_app/app_generator.py @@ -11,15 +11,13 @@ changes the runtime exit policy carried to the backend. from __future__ import annotations import contextvars -import json import logging import threading import uuid -from collections.abc import Generator, Mapping, Sequence +from collections.abc import Generator, Mapping from typing import Any, Literal from flask import Flask, current_app -from pydantic import JsonValue from sqlalchemy import and_, or_, select from sqlalchemy.orm import Session @@ -29,6 +27,7 @@ from configs import dify_config from constants import UUID_NIL from core.agent.publish_visibility import agent_has_workflow_callable_active_snapshot from core.app.app_config.easy_ui_based_app.model_config.converter import ModelConfigConverter +from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.app.apps.agent_app.app_config_manager import AgentAppConfigManager from core.app.apps.agent_app.app_runner import AgentAppRunner from core.app.apps.agent_app.errors import ( @@ -52,8 +51,8 @@ from core.app.entities.app_invoke_entities import ( from core.credit_usage import CreditUsageAppType from core.db.session_factory import session_factory from core.ops.ops_trace_manager import TraceQueueManager -from core.workflow.file_reference import build_file_reference, is_canonical_file_reference from extensions.ext_database import db +from factories import file_factory from models import Account, App, AppModelConfig, Conversation, EndUser, Message, MessageAnnotation from models.agent import ( APP_BACKED_AGENT_SOURCES, @@ -75,69 +74,6 @@ from services.conversation_service import ConversationService logger = logging.getLogger(__name__) -_REFERENCE_FILE_TRANSFER_METHODS = {"local_file", "tool_file", "datasource_file"} - - -def _append_prompt_file_mappings(query: str, prompt_file_mappings: Sequence[JsonValue]) -> str: - """Append labeled, prompt-safe file locators to the backend user prompt.""" - prompt_files = _prompt_file_locators(prompt_file_mappings) - if not prompt_files: - return query - payload = json.dumps(prompt_files, ensure_ascii=False, separators=(",", ":")) - return ( - f"{query}\n" - "User provided files: use dify-agent file download with the listed transfer_method and reference/url " - "to get the files and investigate them\n" - f"{payload}" - ) - - -def _prompt_file_locators(prompt_file_mappings: Sequence[JsonValue]) -> list[dict[str, str]]: - locators: list[dict[str, str]] = [] - for file_mapping in prompt_file_mappings: - if not isinstance(file_mapping, Mapping): - continue - locator = _prompt_file_locator(file_mapping) - if locator is not None: - locators.append(locator) - return locators - - -def _prompt_file_locator(file_mapping: Mapping[str, object]) -> dict[str, str] | None: - transfer_method = _string_value(file_mapping, "transfer_method") - if transfer_method == "remote_url": - url = _string_value(file_mapping, "url") or _string_value(file_mapping, "remote_url") - if url is None: - return None - return {"transfer_method": "remote_url", "url": url} - elif transfer_method in _REFERENCE_FILE_TRANSFER_METHODS: - if transfer_method is None: - return None - reference = _canonical_file_reference( - _string_value(file_mapping, "reference") - or _string_value(file_mapping, "upload_file_id") - or _string_value(file_mapping, "file_id") - or _string_value(file_mapping, "id") - ) - if reference is None: - return None - return {"transfer_method": transfer_method, "reference": reference} - else: - return None - - -def _canonical_file_reference(reference: str | None) -> str | None: - if reference is None: - return None - if reference.startswith("dify-file-ref:"): - return reference if is_canonical_file_reference(reference) else None - return build_file_reference(record_id=reference) - - -def _string_value(mapping: Mapping[str, object], key: str) -> str | None: - value = mapping.get(key) - return value if isinstance(value, str) and value else None - class AgentAppGenerator(MessageBasedAppGenerator): def generate( @@ -155,7 +91,7 @@ class AgentAppGenerator(MessageBasedAppGenerator): query = self._require_query(args) inputs = args["inputs"] - prompt_file_mappings = args.get("files") or [] + raw_files = args.get("files") or [] conversation = None conversation_id = args.get("conversation_id") @@ -193,19 +129,33 @@ class AgentAppGenerator(MessageBasedAppGenerator): conversation=conversation, ) model_conf = ModelConfigConverter.convert(app_config) + with self._bind_file_access_scope(tenant_id=app_model.tenant_id, user=user, invoke_from=invoke_from): + file_upload_config = ( + FileUploadConfigManager.convert(app_config.app_model_config_dict, is_vision=True) if raw_files else None + ) + file_objs = ( + file_factory.build_from_mappings( + mappings=raw_files, + tenant_id=app_model.tenant_id, + config=file_upload_config, + access_controller=self._file_access_controller, + ) + if raw_files and file_upload_config is not None + else [] + ) trace_manager = TraceQueueManager(app_model.id, user.id if isinstance(user, Account) else user.session_id) application_generate_entity = AgentAppGenerateEntity( task_id=str(uuid.uuid4()), app_config=app_config, model_conf=model_conf, + file_upload_config=file_upload_config, conversation_id=conversation.id if conversation else None, inputs=self._prepare_user_inputs( user_inputs=inputs, variables=app_config.variables, tenant_id=app_model.tenant_id ), query=query, - files=[], - prompt_file_mappings=prompt_file_mappings, + files=list(file_objs), parent_message_id=( args.get("parent_message_id") if invoke_from not in {InvokeFrom.SERVICE_API, InvokeFrom.OPENAPI} @@ -492,10 +442,6 @@ class AgentAppGenerator(MessageBasedAppGenerator): ) if handled: return - query = _append_prompt_file_mappings( - query=query, - prompt_file_mappings=application_generate_entity.prompt_file_mappings, - ) dify_context = DifyRunContext( tenant_id=app_config.tenant_id, @@ -514,6 +460,14 @@ class AgentAppGenerator(MessageBasedAppGenerator): ) runner = self._build_runner() + image_detail_config = ( + application_generate_entity.file_upload_config.image_config.detail + if ( + application_generate_entity.file_upload_config + and application_generate_entity.file_upload_config.image_config + ) + else None + ) runner.run( dify_context=dify_context, agent_id=application_generate_entity.agent_id, @@ -526,6 +480,8 @@ class AgentAppGenerator(MessageBasedAppGenerator): message_id=message.id, model_name=application_generate_entity.model_conf.model, queue_manager=queue_manager, + files=list(application_generate_entity.files), + image_detail_config=image_detail_config, session_scope_snapshot_id=application_generate_entity.agent_session_scope_config_version_id, build_draft_id=( application_generate_entity.agent_config_snapshot_id diff --git a/api/core/app/apps/agent_app/app_runner.py b/api/core/app/apps/agent_app/app_runner.py index 665af35a035..f155c07a333 100644 --- a/api/core/app/apps/agent_app/app_runner.py +++ b/api/core/app/apps/agent_app/app_runner.py @@ -55,8 +55,14 @@ from core.repositories.human_input_repository import HumanInputFormRepository, H from core.workflow.nodes.agent_v2.ask_human_hitl import AskHumanFormBuildError, create_ask_human_form from core.workflow.nodes.agent_v2.ask_human_resume import build_deferred_tool_results, resolve_ask_human_form from extensions.ext_database import db +from graphon.file import File from graphon.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage -from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage +from graphon.model_runtime.entities.message_entities import ( + AssistantPromptMessage, + ImagePromptMessageContent, + PromptMessage, + UserPromptMessage, +) from graphon.model_runtime.errors.invoke import ( InvokeAuthorizationError, InvokeBadRequestError, @@ -643,6 +649,8 @@ class AgentAppRunner: message_id: str, model_name: str, queue_manager: AppQueueManager, + files: list[File] | None = None, + image_detail_config: ImagePromptMessageContent.DETAIL | None = None, session_scope_snapshot_id: str | None | _DefaultSessionScopeSnapshotId = _DEFAULT_SESSION_SCOPE_SNAPSHOT_ID, build_draft_id: str | None = None, ) -> None: @@ -669,6 +677,8 @@ class AgentAppRunner: backend_binding_ref=stored.backend_binding_ref, conversation_id=conversation_id, query=query, + files=tuple(files or ()), + image_detail_config=image_detail_config, idempotency_key=message_id, stored=stored, message_id=message_id, @@ -796,6 +806,8 @@ class AgentAppRunner: backend_binding_ref: str, conversation_id: str, query: str, + files: tuple[File, ...], + image_detail_config: ImagePromptMessageContent.DETAIL | None, idempotency_key: str, stored: StoredAgentAppSession, message_id: str | None, @@ -815,6 +827,8 @@ class AgentAppRunner: agent_soul=agent_soul, conversation_id=conversation_id, user_query=query, + files=files, + image_detail_config=image_detail_config, idempotency_key=idempotency_key, binding_id=binding_id, backend_binding_ref=backend_binding_ref, diff --git a/api/core/app/apps/agent_app/runtime_request_builder.py b/api/core/app/apps/agent_app/runtime_request_builder.py index 70c813d0fd5..5f05b0c99d6 100644 --- a/api/core/app/apps/agent_app/runtime_request_builder.py +++ b/api/core/app/apps/agent_app/runtime_request_builder.py @@ -19,6 +19,12 @@ from dify_agent.layers.execution_context import ( DifyExecutionContextLayerConfig, DifyExecutionContextUserFrom, ) +from dify_agent.layers.user_prompt import ( + DifyUserPromptDownloadConfig, + DifyUserPromptFileConfig, + DifyUserPromptFileType, + DifyUserPromptImageConfig, +) from dify_agent.protocol import CreateRunRequest, DeferredToolResultsPayload from clients.agent_backend import ( @@ -29,8 +35,9 @@ from clients.agent_backend import ( ) from configs import dify_config from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom -from core.app.llm.model_access import resolve_model_context_window +from core.app.llm.model_access import resolve_model_context_window, resolve_model_supports_vision from core.plugin.provider_identity import normalize_plugin_daemon_provider_identity +from core.workflow.file_reference import build_file_reference, is_canonical_file_reference from core.workflow.nodes.agent_v2.dify_tools_builder import ( WorkflowAgentDifyToolLayersBuilder, WorkflowAgentDifyToolsBuilder, @@ -46,6 +53,8 @@ from core.workflow.nodes.agent_v2.runtime_request_builder import ( build_shell_layer_config, load_runtime_agent_skill_configs, ) +from graphon.file import File, FileTransferMethod, FileType, file_manager +from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent from models.agent_config_entities import AgentSoulConfig, AgentSoulToolsConfig from models.provider_ids import ModelProviderID from services.agent.prompt_mentions import expand_prompt_mentions @@ -61,6 +70,9 @@ class AgentAppRuntimeRequestBuildError(ValueError): super().__init__(message) +type _ReferenceFileTransferMethod = Literal["local_file", "tool_file", "datasource_file"] + + @dataclass(frozen=True, slots=True) class AgentAppRuntimeBuildContext: dify_context: DifyRunContext @@ -72,6 +84,8 @@ class AgentAppRuntimeBuildContext: idempotency_key: str binding_id: str backend_binding_ref: str + files: tuple[File, ...] = () + image_detail_config: ImagePromptMessageContent.DETAIL | None = None agent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot" session_snapshot: CompositorSessionSnapshot | None = None # ENG-638: set when resuming a chat turn after a submitted ask_human form. @@ -150,6 +164,13 @@ class AgentAppRuntimeRequestBuilder: ModelProviderID(agent_soul.model.model_provider), agent_soul.model.plugin_id, ) + user_files = self._build_user_files( + files=context.files, + run_context=context.dify_context, + provider_name=agent_soul.model.model_provider, + model_name=agent_soul.model.model, + image_detail_config=context.image_detail_config, + ) request = self._request_builder.build_for_agent_app( AgentBackendAgentAppRunInput( @@ -180,6 +201,7 @@ class AgentAppRuntimeRequestBuilder: or None, agent_config_version_kind=context.agent_config_version_kind, user_prompt=context.user_query, + user_files=user_files, tools=tool_layers.plugin_tools, core_tools=tool_layers.core_tools, knowledge=knowledge_config, @@ -202,6 +224,27 @@ class AgentAppRuntimeRequestBuilder: binding_id=context.binding_id, ) + @staticmethod + def _build_user_files( + *, + files: tuple[File, ...], + run_context: DifyRunContext, + provider_name: str, + model_name: str, + image_detail_config: ImagePromptMessageContent.DETAIL | None, + ) -> list[DifyUserPromptFileConfig]: + supports_vision = any(file.type == FileType.IMAGE for file in files) and resolve_model_supports_vision( + run_context=run_context, + provider_name=provider_name, + model_name=model_name, + ) + return [ + _build_user_image(file, image_detail_config=image_detail_config) + if supports_vision and file.type == FileType.IMAGE + else _build_user_download(file) + for file in files + ] + @staticmethod def _validate_session_snapshot_layers(request: CreateRunRequest) -> None: """Reject stale snapshots before they reach the Agent backend. @@ -251,6 +294,57 @@ class AgentAppRuntimeRequestBuilder: } +def _build_user_image( + file: File, + *, + image_detail_config: ImagePromptMessageContent.DETAIL | None, +) -> DifyUserPromptImageConfig: + content = file_manager.to_prompt_message_content(file, image_detail_config=image_detail_config) + if not isinstance(content, ImagePromptMessageContent): + raise AgentAppRuntimeRequestBuildError( + "agent_user_file_unsupported", + f"Agent App cannot send file '{file.filename or 'image'}' as vision content.", + ) + detail = content.detail.value + return DifyUserPromptImageConfig( + filename=content.filename or file.filename or f"image.{content.format}", + mime_type=content.mime_type, + format=content.format, + url=content.url or None, + base64_data=content.base64_data or None, + detail=detail if detail in {"low", "high"} else None, + ) + + +def _build_user_download(file: File) -> DifyUserPromptDownloadConfig: + file_type = cast(DifyUserPromptFileType, file.type.value) + if file.transfer_method == FileTransferMethod.REMOTE_URL: + if file.remote_url is None: + raise AgentAppRuntimeRequestBuildError("agent_user_file_invalid", "Remote user file is missing its URL.") + return DifyUserPromptDownloadConfig(type=file_type, transfer_method="remote_url", url=file.remote_url) + if file.reference is None: + raise AgentAppRuntimeRequestBuildError("agent_user_file_invalid", "User file is missing its reference.") + reference = file.reference + if not reference.startswith("dify-file-ref:"): + reference = build_file_reference(record_id=reference) + elif not is_canonical_file_reference(reference): + raise AgentAppRuntimeRequestBuildError("agent_user_file_invalid", "User file reference is invalid.") + transfer_method: _ReferenceFileTransferMethod + match file.transfer_method: + case FileTransferMethod.LOCAL_FILE: + transfer_method = "local_file" + case FileTransferMethod.TOOL_FILE: + transfer_method = "tool_file" + case FileTransferMethod.DATASOURCE_FILE: + transfer_method = "datasource_file" + case _: + raise AgentAppRuntimeRequestBuildError( + "agent_user_file_invalid", + f"User file transfer method '{file.transfer_method.value}' is unsupported.", + ) + return DifyUserPromptDownloadConfig(type=file_type, transfer_method=transfer_method, reference=reference) + + __all__ = [ "AgentAppRuntimeBuildContext", "AgentAppRuntimeRequest", diff --git a/api/core/app/entities/app_invoke_entities.py b/api/core/app/entities/app_invoke_entities.py index 7f983135994..cb3f52d12a0 100644 --- a/api/core/app/entities/app_invoke_entities.py +++ b/api/core/app/entities/app_invoke_entities.py @@ -2,7 +2,7 @@ from collections.abc import Mapping, Sequence from enum import StrEnum from typing import TYPE_CHECKING, Any, Literal -from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationInfo, field_validator +from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator from constants import UUID_NIL from core.app.app_config.entities import EasyUIBasedAppConfig, WorkflowUIBasedAppConfig @@ -286,16 +286,15 @@ class AgentAppGenerateEntity(ChatAppGenerateEntity): ``agent_session_scope_config_version_id`` identifies the draft or immutable config version whose Workspace Binding should be reused for this session. - ``prompt_file_mappings`` preserves the raw request ``files`` array for the - Agent backend prompt. These references are appended to the backend prompt - text while the stored chat message keeps the user's original query. + Uploaded files use the inherited ``files`` field. The Agent App runtime + sends supported images directly to vision models and preserves the sandbox + locator fallback for all other files. """ agent_id: str agent_config_snapshot_id: str agent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot" agent_session_scope_config_version_id: str | None = None - prompt_file_mappings: Sequence[JsonValue] = Field(default_factory=list) agent_llm_gateway_enabled: bool = False diff --git a/api/core/app/llm/model_access.py b/api/core/app/llm/model_access.py index 52fb9add70b..e793200ffb7 100644 --- a/api/core/app/llm/model_access.py +++ b/api/core/app/llm/model_access.py @@ -9,7 +9,7 @@ from core.errors.error import ProviderTokenNotInitError from core.model_manager import ModelInstance, ModelManager from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager from core.provider_manager import ProviderManager -from graphon.model_runtime.entities.model_entities import ModelPropertyKey, ModelType +from graphon.model_runtime.entities.model_entities import ModelFeature, ModelPropertyKey, ModelType from graphon.nodes.llm.entities import ModelConfig from graphon.nodes.llm.exc import LLMModeRequiredError, ModelNotExistError from graphon.nodes.llm.protocols import CredentialsProvider @@ -149,6 +149,18 @@ def resolve_model_context_window( return context_window +def resolve_model_supports_vision( + *, + run_context: DifyRunContext, + provider_name: str, + model_name: str, +) -> bool: + """Return whether the credential-bound model advertises vision support.""" + + model_instance = DifyModelFactory(run_context=run_context).init_model_instance(provider_name, model_name) + return ModelFeature.VISION in (model_instance.get_model_schema().features or []) + + def _normalize_completion_params(completion_params: dict[str, Any]) -> tuple[dict[str, Any], list[str]]: """ Split node-level completion params into provider parameters and stop sequences. diff --git a/api/tests/unit_tests/clients/agent_backend/test_request_builder.py b/api/tests/unit_tests/clients/agent_backend/test_request_builder.py index 978b522d509..29771083de6 100644 --- a/api/tests/unit_tests/clients/agent_backend/test_request_builder.py +++ b/api/tests/unit_tests/clients/agent_backend/test_request_builder.py @@ -20,6 +20,11 @@ from dify_agent.layers.execution_context import DIFY_EXECUTION_CONTEXT_LAYER_TYP from dify_agent.layers.knowledge import DIFY_KNOWLEDGE_BASE_LAYER_TYPE_ID, DifyKnowledgeBaseLayerConfig from dify_agent.layers.output import DIFY_OUTPUT_LAYER_TYPE_ID from dify_agent.layers.shell import DIFY_SHELL_LAYER_TYPE_ID, DifyShellEnvVarConfig, DifyShellLayerConfig +from dify_agent.layers.user_prompt import ( + DIFY_USER_PROMPT_LAYER_TYPE_ID, + DifyUserPromptDownloadConfig, + DifyUserPromptImageConfig, +) from dify_agent.protocol import ( DIFY_AGENT_HISTORY_LAYER_ID, DIFY_AGENT_MODEL_LAYER_ID, @@ -367,6 +372,64 @@ def test_agent_app_request_builder_omits_shell_layer_by_default(): assert DIFY_SHELL_LAYER_ID not in {layer.name for layer in request.composition.layers} +def test_agent_app_request_builder_emits_multimodal_user_prompt_layer(): + run_input = _agent_app_input() + run_input.user_files = [ + DifyUserPromptDownloadConfig( + type="document", transfer_method="remote_url", url="https://example.com/brief.pdf" + ), + DifyUserPromptImageConfig( + filename="earth.png", + mime_type="image/png", + format="png", + url="https://files.example.com/earth.png?sign=secret", + detail="high", + ), + ] + + request = AgentBackendRunRequestBuilder().build_for_agent_app(run_input) + layer = next(layer for layer in request.composition.layers if layer.name == "agent_app_user_prompt") + + assert layer.type == DIFY_USER_PROMPT_LAYER_TYPE_ID + assert layer.config.text == "List files." + assert layer.config.files == run_input.user_files + restored_request = CreateRunRequest.model_validate_json(request.model_dump_json()) + restored_layer = next( + layer for layer in restored_request.composition.layers if layer.name == "agent_app_user_prompt" + ) + assert restored_layer.config == layer.config.model_dump(mode="json") + assert "locators" not in restored_layer.config + + +def test_agent_backend_log_redacts_multimodal_file_transport(): + run_input = _agent_app_input() + run_input.metadata = {"source_url": "https://example.com/docs"} + run_input.user_files = [ + DifyUserPromptImageConfig( + filename="earth.png", + mime_type="image/png", + format="png", + url="https://files.example.com/earth.png?sign=secret", + ), + DifyUserPromptImageConfig( + filename="inline.png", + mime_type="image/png", + format="png", + base64_data="aW1hZ2UtYnl0ZXM=", + ), + ] + + redacted = cast( + dict[str, Any], + redact_for_agent_backend_log(AgentBackendRunRequestBuilder().build_for_agent_app(run_input)), + ) + layer = next(item for item in redacted["composition"]["layers"] if item["name"] == "agent_app_user_prompt") + + assert layer["config"]["files"][0]["url"] == "[REDACTED]" + assert layer["config"]["files"][1]["base64_data"] == "[REDACTED]" + assert redacted["metadata"]["source_url"] == "https://example.com/docs" + + def test_agent_app_request_builder_keeps_build_draft_prompt_when_agent_soul_prompt_is_blank(): run_input = _agent_app_input().model_copy( update={"agent_soul_prompt": " ", "agent_config_version_kind": "build_draft"} diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py index 410efadba33..4a84bcc4689 100644 --- a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py +++ b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py @@ -11,7 +11,6 @@ from __future__ import annotations import contextlib import inspect -import json from decimal import Decimal from types import SimpleNamespace @@ -29,7 +28,6 @@ from core.app.apps.agent_app.errors import AgentSessionSnapshotIncompatibleError from core.app.apps.exc import GenerateTaskStoppedError from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom from core.app.entities.queue_entities import QueueAnnotationReplyEvent -from core.workflow.file_reference import build_file_reference from models import Account, AppModelConfig from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource, AgentStatus from models.agent_config_entities import AgentSoulConfig @@ -209,6 +207,10 @@ class TestGenerateSuccess: return_value=mocker.MagicMock(variables=[], tenant_id="tenant", app_id="app1"), ) mocker.patch(f"{MODULE}.ModelConfigConverter.convert", return_value=mocker.MagicMock(model="gpt-4o-mini")) + file_upload_config = mocker.MagicMock() + mocker.patch(f"{MODULE}.FileUploadConfigManager.convert", return_value=file_upload_config) + parsed_file = mocker.MagicMock() + build_files = mocker.patch(f"{MODULE}.file_factory.build_from_mappings", return_value=[parsed_file]) mocker.patch(f"{MODULE}.TraceQueueManager", return_value=mocker.MagicMock()) generate_entity = mocker.patch( f"{MODULE}.AgentAppGenerateEntity", return_value=mocker.MagicMock(task_id="t", user_id="user") @@ -248,7 +250,10 @@ class TestGenerateSuccess: conversation=None, ) assert session.get(AppModelConfig, "config-1") is config - assert generate_entity.call_args.kwargs["prompt_file_mappings"] == file_mappings + build_files.assert_called_once() + assert build_files.call_args.kwargs["mappings"] == file_mappings + assert generate_entity.call_args.kwargs["files"] == [parsed_file] + assert generate_entity.call_args.kwargs["file_upload_config"] is file_upload_config assert "agent_runtime_exit_intent" not in generate_entity.call_args.kwargs def test_generate_loads_existing_conversation(self, generator: AgentAppGenerator, mocker: MockerFixture): @@ -379,7 +384,8 @@ class TestGenerateWorker: is_resume=False, query="query", session_scope_config_version_id="s", - prompt_file_mappings=(), + files=(), + file_upload_config=None, ): generator._generate_worker( flask_app=mocker.MagicMock(), @@ -391,7 +397,8 @@ class TestGenerateWorker: agent_session_scope_config_version_id=session_scope_config_version_id, model_conf=mocker.MagicMock(model="m"), query=query, - prompt_file_mappings=prompt_file_mappings, + files=files, + file_upload_config=file_upload_config, ), queue_manager=queue_manager, conversation_id="conv", @@ -420,48 +427,22 @@ class TestGenerateWorker: assert runner.run.call_args.kwargs["agent_config_snapshot_id"] == "s" assert runner.run.call_args.kwargs["session_scope_snapshot_id"] is None - def test_worker_appends_prompt_files_to_backend_query(self, generator, mocker: MockerFixture): + def test_worker_passes_files_to_backend_runner_without_rewriting_query(self, generator, mocker: MockerFixture): runner, _ = self._wire(generator, mocker, guard_query="你看得见这张图片吗") queue_manager = mocker.MagicMock() - file_mappings = [ - { - "type": "image", - "transfer_method": "local_file", - "url": "", - "upload_file_id": "upload-file-1", - }, - { - "type": "document", - "transfer_method": "remote_url", - "url": "https://example.com/source.pdf", - "upload_file_id": "ignored", - }, - ] - expected_file_mappings = [ - { - "transfer_method": "local_file", - "reference": build_file_reference(record_id="upload-file-1"), - }, - { - "transfer_method": "remote_url", - "url": "https://example.com/source.pdf", - }, - ] + files = [mocker.MagicMock(), mocker.MagicMock()] self._call( generator, mocker, queue_manager, query="你看得见这张图片吗", - prompt_file_mappings=file_mappings, + files=files, ) - assert runner.run.call_args.kwargs["query"] == ( - "你看得见这张图片吗\nUser provided files: " - "use dify-agent file download with the listed transfer_method and reference/url " - "to get the files and investigate them\n" - f"{json.dumps(expected_file_mappings, ensure_ascii=False, separators=(',', ':'))}" - ) + assert runner.run.call_args.kwargs["query"] == "你看得见这张图片吗" + assert runner.run.call_args.kwargs["files"] == files + assert runner.run.call_args.kwargs["image_detail_config"] is None def test_input_guard_short_circuit_skips_backend(self, generator, mocker: MockerFixture): runner, _ = self._wire(generator, mocker, handled=True) diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py b/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py index db33a1ad03b..b93f0e6ff90 100644 --- a/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py +++ b/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py @@ -12,6 +12,7 @@ from dify_agent.layers.config import DifyConfigSkillConfig from dify_agent.layers.dify_core_tools import DifyCoreToolConfig, DifyCoreToolsLayerConfig from dify_agent.layers.dify_plugin import DifyPluginToolConfig, DifyPluginToolsLayerConfig from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig +from dify_agent.layers.user_prompt import DifyUserPromptLayerConfig from clients.agent_backend import ( DIFY_CONFIG_LAYER_ID, @@ -29,6 +30,9 @@ from core.app.apps.agent_app.runtime_request_builder import ( AgentAppRuntimeRequestBuildError, ) from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom +from core.workflow.file_reference import build_file_reference +from graphon.file import File, FileTransferMethod, FileType +from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent from models.agent_config_entities import AgentSoulConfig from tests.unit_tests.config_override import apply_config_overrides @@ -168,6 +172,8 @@ def _ctx( query: str = "hello", agent_config_version_kind: str = "snapshot", session_snapshot: CompositorSessionSnapshot | None = None, + files: tuple[File, ...] = (), + image_detail_config: ImagePromptMessageContent.DETAIL | None = None, ) -> AgentAppRuntimeBuildContext: dify_context = SimpleNamespace( tenant_id="tenant-1", @@ -188,6 +194,8 @@ def _ctx( backend_binding_ref="binding-ref-1", agent_config_version_kind=agent_config_version_kind, # type: ignore[arg-type] session_snapshot=session_snapshot, + files=files, + image_detail_config=image_detail_config, ) @@ -204,6 +212,32 @@ def _soul_with_model() -> AgentSoulConfig: ) +def _image_file() -> File: + return File( + file_id="file-1", + file_type=FileType.IMAGE, + transfer_method=FileTransferMethod.LOCAL_FILE, + reference="upload-file-1", + filename="earth.png", + extension=".png", + mime_type="image/png", + size=12, + ) + + +def _document_file() -> File: + return File( + file_id="file-2", + file_type=FileType.DOCUMENT, + transfer_method=FileTransferMethod.LOCAL_FILE, + reference="upload-document-1", + filename="brief.pdf", + extension=".pdf", + mime_type="application/pdf", + size=24, + ) + + def _snapshot_for_layer_names(layer_names: list[str]) -> CompositorSessionSnapshot: return CompositorSessionSnapshot( layers=[ @@ -251,6 +285,135 @@ class TestAgentAppRuntimeRequestBuilder: assert "credentials" not in result.redacted_request["composition"]["layers"][-1]["config"] assert result.metadata["conversation_id"] == "conv-1" + def test_build_sends_images_directly_to_vision_model(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + "core.app.apps.agent_app.runtime_request_builder.resolve_model_supports_vision", + lambda **_kwargs: True, + ) + prompt_content_calls: list[tuple[File, ImagePromptMessageContent.DETAIL | None]] = [] + + def to_prompt_message_content( + file: File, + *, + image_detail_config: ImagePromptMessageContent.DETAIL | None, + ) -> ImagePromptMessageContent: + prompt_content_calls.append((file, image_detail_config)) + return ImagePromptMessageContent( + format="png", + url="https://files.example.com/earth.png?sign=secret", + mime_type="image/png", + filename="earth.png", + detail=image_detail_config or ImagePromptMessageContent.DETAIL.LOW, + ) + + monkeypatch.setattr( + "core.app.apps.agent_app.runtime_request_builder.file_manager.to_prompt_message_content", + to_prompt_message_content, + ) + builder = AgentAppRuntimeRequestBuilder(dify_tools_builder=_NoToolsBuilder()) # type: ignore[arg-type] + + result = builder.build( + _ctx( + _soul_with_model(), + query="Describe this image.", + files=(_image_file(),), + image_detail_config=ImagePromptMessageContent.DETAIL.HIGH, + ) + ) + layer = next(item for item in result.request.composition.layers if item.name == "agent_app_user_prompt") + config = DifyUserPromptLayerConfig.model_validate(layer.config) + + assert config.text == "Describe this image." + assert len(config.files) == 1 + image = config.files[0] + assert image.delivery == "multimodal" + assert image.type == "image" + assert image.url == "https://files.example.com/earth.png?sign=secret" + assert image.detail == "high" + assert prompt_content_calls == [(_image_file(), ImagePromptMessageContent.DETAIL.HIGH)] + + def test_build_keeps_image_locator_for_non_vision_model(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + "core.app.apps.agent_app.runtime_request_builder.resolve_model_supports_vision", + lambda **_kwargs: False, + ) + builder = AgentAppRuntimeRequestBuilder(dify_tools_builder=_NoToolsBuilder()) # type: ignore[arg-type] + + result = builder.build(_ctx(_soul_with_model(), query="Inspect the attachment.", files=(_image_file(),))) + layer = next(item for item in result.request.composition.layers if item.name == "agent_app_user_prompt") + config = DifyUserPromptLayerConfig.model_validate(layer.config) + + assert config.text == "Inspect the attachment." + assert [file.model_dump(exclude_none=True) for file in config.files] == [ + { + "delivery": "download", + "type": "image", + "transfer_method": "local_file", + "reference": build_file_reference(record_id="upload-file-1"), + } + ] + + def test_build_preserves_inline_base64_transport_for_vision_model(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + "core.app.apps.agent_app.runtime_request_builder.resolve_model_supports_vision", + lambda **_kwargs: True, + ) + monkeypatch.setattr( + "core.app.apps.agent_app.runtime_request_builder.file_manager.to_prompt_message_content", + lambda *_args, **_kwargs: ImagePromptMessageContent( + format="png", + base64_data="aW1hZ2UtYnl0ZXM=", + mime_type="image/png", + filename="earth.png", + detail="low", + ), + ) + builder = AgentAppRuntimeRequestBuilder(dify_tools_builder=_NoToolsBuilder()) # type: ignore[arg-type] + + result = builder.build(_ctx(_soul_with_model(), query="Describe this image.", files=(_image_file(),))) + layer = next(item for item in result.request.composition.layers if item.name == "agent_app_user_prompt") + config = DifyUserPromptLayerConfig.model_validate(layer.config) + + image = config.files[0] + assert image.delivery == "multimodal" + assert image.url is None + assert image.base64_data == "aW1hZ2UtYnl0ZXM=" + + def test_build_keeps_non_image_locator_when_vision_image_is_direct(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + "core.app.apps.agent_app.runtime_request_builder.resolve_model_supports_vision", + lambda **_kwargs: True, + ) + monkeypatch.setattr( + "core.app.apps.agent_app.runtime_request_builder.file_manager.to_prompt_message_content", + lambda *_args, **_kwargs: ImagePromptMessageContent( + format="png", + url="https://files.example.com/earth.png", + mime_type="image/png", + filename="earth.png", + detail="low", + ), + ) + builder = AgentAppRuntimeRequestBuilder(dify_tools_builder=_NoToolsBuilder()) # type: ignore[arg-type] + + result = builder.build( + _ctx(_soul_with_model(), files=(_image_file(), _document_file()), query="Compare the attachments.") + ) + layer = next(item for item in result.request.composition.layers if item.name == "agent_app_user_prompt") + config = DifyUserPromptLayerConfig.model_validate(layer.config) + + assert config.text == "Compare the attachments." + assert len(config.files) == 2 + image, download = config.files + assert image.delivery == "multimodal" + assert image.filename == "earth.png" + assert download.model_dump(exclude_none=True) == { + "delivery": "download", + "type": "document", + "transfer_method": "local_file", + "reference": build_file_reference(record_id="upload-document-1"), + } + @pytest.mark.parametrize( ("previous_prompt", "current_prompt"), [("", "You are Iris."), ("You are Iris.", "")], diff --git a/api/tests/unit_tests/core/app/llm/test_model_access.py b/api/tests/unit_tests/core/app/llm/test_model_access.py index 8c22d56223e..9bb0ef37712 100644 --- a/api/tests/unit_tests/core/app/llm/test_model_access.py +++ b/api/tests/unit_tests/core/app/llm/test_model_access.py @@ -5,10 +5,15 @@ import pytest from core.app.entities.app_invoke_entities import DifyRunContext from core.app.llm import model_access -from graphon.model_runtime.entities.model_entities import ModelPropertyKey +from graphon.model_runtime.entities.model_entities import ModelFeature, ModelPropertyKey -def _stub_model_factory(monkeypatch: pytest.MonkeyPatch, context_window: object) -> dict[str, object]: +def _stub_model_factory( + monkeypatch: pytest.MonkeyPatch, + context_window: object, + *, + features: list[ModelFeature] | None = None, +) -> dict[str, object]: calls: dict[str, object] = {} class FakeModelFactory: @@ -18,7 +23,10 @@ def _stub_model_factory(monkeypatch: pytest.MonkeyPatch, context_window: object) def init_model_instance(self, provider_name: str, model_name: str) -> object: calls["provider_name"] = provider_name calls["model_name"] = model_name - schema = SimpleNamespace(model_properties={ModelPropertyKey.CONTEXT_SIZE: context_window}) + schema = SimpleNamespace( + model_properties={ModelPropertyKey.CONTEXT_SIZE: context_window}, + features=features, + ) return SimpleNamespace(get_model_schema=lambda: schema) monkeypatch.setattr(model_access, "DifyModelFactory", FakeModelFactory) @@ -58,3 +66,24 @@ def test_resolve_model_context_window_ignores_invalid_schema_values( ) is None ) + + +@pytest.mark.parametrize( + ("features", "expected"), + [([ModelFeature.VISION], True), ([], False), (None, False)], +) +def test_resolve_model_supports_vision_reads_selected_model_schema( + monkeypatch: pytest.MonkeyPatch, + features: list[ModelFeature] | None, + expected: bool, +) -> None: + _stub_model_factory(monkeypatch, 128_000, features=features) + + assert ( + model_access.resolve_model_supports_vision( + run_context=cast(DifyRunContext, object()), + provider_name="openai", + model_name="gpt-4o", + ) + is expected + ) diff --git a/dify-agent/docs/dify-agent/guide/index.md b/dify-agent/docs/dify-agent/guide/index.md index 99ff7b0ac28..bc6eacee02f 100644 --- a/dify-agent/docs/dify-agent/guide/index.md +++ b/dify-agent/docs/dify-agent/guide/index.md @@ -359,6 +359,49 @@ whose Agenton layers provide user input. With the MVP provider set, use effective prompts are rejected during create-run validation before the run is persisted or scheduled. +Agent App callers can use the `dify.user_prompt` layer to send text and images +in the same model turn. Each image must provide exactly one transport: an +HTTP(S) `url` or unprefixed Base64 data in `base64_data`. The image is passed as +structured multimodal content; it is not interpolated into `config.text`. + +```json +{ + "name": "agent_app_user_prompt", + "type": "dify.user_prompt", + "config": { + "text": "Describe this image.", + "files": [ + { + "delivery": "multimodal", + "type": "image", + "filename": "earth.png", + "mime_type": "image/png", + "format": "png", + "url": "https://files.example.com/earth.png", + "base64_data": null, + "detail": "high" + } + ] + } +} +``` + +All attachments use `config.files`. The `delivery` field selects how an attachment +is presented, independently of its file `type` (`image`, `document`, `audio`, +`video`, or `custom`): + +- `delivery: "multimodal"` currently supports `type: "image"` only. Dify API + chooses URL or Base64 transport according to `MULTIMODAL_SEND_FORMAT`. +- `delivery: "download"` preserves the original file `type`, including `image` + when the selected model has no Vision feature. It contains either + `transfer_method: "remote_url"` with `url`, or a `local_file`, `tool_file`, or + `datasource_file` transfer method with a canonical `reference`. + +The layer appends sandbox file-download instructions for download attachments +and adds multimodal attachments as structured model content. Callers keep +`config.text` as the original user text. Omitting `config.files` is equivalent +to an empty list. + The optional Pydantic AI history layer uses the reserved name `history` and persists captured messages in session snapshots for later resume. Resume from a terminal event's `session_snapshot` using the same layer composition, names, and diff --git a/dify-agent/src/dify_agent/layers/user_prompt/__init__.py b/dify-agent/src/dify_agent/layers/user_prompt/__init__.py new file mode 100644 index 00000000000..f675b1c0140 --- /dev/null +++ b/dify-agent/src/dify_agent/layers/user_prompt/__init__.py @@ -0,0 +1,19 @@ +"""Client-safe exports for the Agent App multimodal user prompt layer.""" + +from dify_agent.layers.user_prompt.configs import ( + DIFY_USER_PROMPT_LAYER_TYPE_ID, + DifyUserPromptDownloadConfig, + DifyUserPromptFileConfig, + DifyUserPromptFileType, + DifyUserPromptImageConfig, + DifyUserPromptLayerConfig, +) + +__all__ = [ + "DIFY_USER_PROMPT_LAYER_TYPE_ID", + "DifyUserPromptDownloadConfig", + "DifyUserPromptFileConfig", + "DifyUserPromptFileType", + "DifyUserPromptImageConfig", + "DifyUserPromptLayerConfig", +] diff --git a/dify-agent/src/dify_agent/layers/user_prompt/configs.py b/dify-agent/src/dify_agent/layers/user_prompt/configs.py new file mode 100644 index 00000000000..fc9fa273e7d --- /dev/null +++ b/dify-agent/src/dify_agent/layers/user_prompt/configs.py @@ -0,0 +1,90 @@ +"""Serializable DTOs for one Agent App multimodal user prompt.""" + +from __future__ import annotations + +import base64 +import binascii +from typing import Annotated, ClassVar, Final, Literal, Self + +from pydantic import ConfigDict, Field, model_validator + +from agenton.layers import LayerConfig + + +DIFY_USER_PROMPT_LAYER_TYPE_ID: Final[str] = "dify.user_prompt" +type DifyUserPromptFileType = Literal["image", "document", "audio", "video", "custom"] + + +class DifyUserPromptImageConfig(LayerConfig): + """One image delivered directly to the configured vision model. + + Callers provide either an HTTP(S) URL or base64 bytes. Opaque Dify file + references belong to the sandbox download flow and are not accepted here. + """ + + delivery: Literal["multimodal"] = "multimodal" + type: Literal["image"] = "image" + filename: str = Field(min_length=1) + mime_type: str = Field(pattern=r"^image/[A-Za-z0-9.+-]+$") + format: str = Field(min_length=1) + url: str | None = None + base64_data: str | None = None + detail: Literal["low", "high"] | None = None + + model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") + + @model_validator(mode="after") + def _validate_transport(self) -> Self: + if (self.url is None) == (self.base64_data is None): + raise ValueError("exactly one of url or base64_data must be provided") + if self.url is not None and not self.url.startswith(("http://", "https://")): + raise ValueError("url must use http or https") + if self.base64_data is not None: + try: + base64.b64decode(self.base64_data, validate=True) + except (ValueError, binascii.Error) as exc: + raise ValueError("base64_data must be valid base64") from exc + return self + + +class DifyUserPromptDownloadConfig(LayerConfig): + delivery: Literal["download"] = "download" + type: DifyUserPromptFileType + transfer_method: Literal["remote_url", "local_file", "tool_file", "datasource_file"] + url: str | None = Field(default=None, min_length=1) + reference: str | None = Field(default=None, min_length=1) + + model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") + + @model_validator(mode="after") + def _validate_locator(self) -> Self: + if self.transfer_method == "remote_url": + if self.url is None or self.reference is not None: + raise ValueError("remote_url requires url and must not include reference") + elif self.reference is None or self.url is not None: + raise ValueError("reference is required and url must not be provided") + return self + + +type DifyUserPromptFileConfig = Annotated[ + DifyUserPromptImageConfig | DifyUserPromptDownloadConfig, Field(discriminator="delivery") +] + + +class DifyUserPromptLayerConfig(LayerConfig): + """User text plus images that should be sent in the same model turn.""" + + text: str = Field(min_length=1) + files: list[DifyUserPromptFileConfig] = Field(default_factory=list) + + model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") + + +__all__ = [ + "DIFY_USER_PROMPT_LAYER_TYPE_ID", + "DifyUserPromptDownloadConfig", + "DifyUserPromptFileConfig", + "DifyUserPromptFileType", + "DifyUserPromptImageConfig", + "DifyUserPromptLayerConfig", +] diff --git a/dify-agent/src/dify_agent/layers/user_prompt/layer.py b/dify-agent/src/dify_agent/layers/user_prompt/layer.py new file mode 100644 index 00000000000..edcdf0a45bc --- /dev/null +++ b/dify-agent/src/dify_agent/layers/user_prompt/layer.py @@ -0,0 +1,91 @@ +"""Turn serialized Agent App images into pydantic-ai user content.""" + +from __future__ import annotations + +import base64 +import json +from dataclasses import dataclass +from typing import ClassVar, assert_never + +from pydantic_ai.messages import BinaryContent, ImageUrl, UserContent +from typing_extensions import Self, override + +from agenton.layers import EmptyRuntimeState, NoLayerDeps, PydanticAILayer +from dify_agent.layers.user_prompt.configs import ( + DIFY_USER_PROMPT_LAYER_TYPE_ID, + DifyUserPromptDownloadConfig, + DifyUserPromptImageConfig, + DifyUserPromptLayerConfig, +) + + +@dataclass(slots=True) +class DifyUserPromptLayer(PydanticAILayer[NoLayerDeps, object, DifyUserPromptLayerConfig, EmptyRuntimeState]): + """State-free layer for a text prompt and directly attached images.""" + + type_id: ClassVar[str | None] = DIFY_USER_PROMPT_LAYER_TYPE_ID + config: DifyUserPromptLayerConfig + + @classmethod + @override + def from_config(cls, config: DifyUserPromptLayerConfig) -> Self: + return cls(config=DifyUserPromptLayerConfig.model_validate(config)) + + @property + @override + def user_prompts(self) -> list[UserContent]: + images: list[UserContent] = [] + downloads: list[DifyUserPromptDownloadConfig] = [] + for file in self.config.files: + match file.delivery: + case "multimodal": + images.append(_to_image_content(file)) + case "download": + downloads.append(file) + case _: + assert_never(file) + return [_append_file_downloads(self.config.text, downloads), *images] + + +def _append_file_downloads(text: str, files: list[DifyUserPromptDownloadConfig]) -> str: + if not files: + return text + locators = [file.model_dump(mode="json", exclude={"delivery", "type"}, exclude_none=True) for file in files] + payload = json.dumps(locators, ensure_ascii=False, separators=(",", ":")) + return ( + f"{text}\n" + "User provided files: use dify-agent file download with the listed transfer_method and reference/url " + "to get the files and investigate them\n" + f"{payload}" + ) + + +def _to_image_content(file: DifyUserPromptImageConfig) -> ImageUrl | BinaryContent: + vendor_metadata: dict[str, str] = {"filename": file.filename} + if file.detail is not None: + vendor_metadata["detail"] = file.detail + identifier = _identifier_from_filename(file.filename, file.format) + if file.url is not None: + return ImageUrl( + url=file.url, + media_type=file.mime_type, + identifier=identifier, + vendor_metadata=vendor_metadata, + ) + assert file.base64_data is not None + return BinaryContent( + data=base64.b64decode(file.base64_data, validate=True), + media_type=file.mime_type, + identifier=identifier, + vendor_metadata=vendor_metadata, + ) + + +def _identifier_from_filename(filename: str, file_format: str) -> str: + suffix = f".{file_format}" + if filename.lower().endswith(suffix.lower()): + return filename[: -len(suffix)] or "image" + return filename + + +__all__ = ["DifyUserPromptLayer"] diff --git a/dify-agent/src/dify_agent/runtime/compositor_factory.py b/dify-agent/src/dify_agent/runtime/compositor_factory.py index f56b37e3c17..a22df749ea6 100644 --- a/dify-agent/src/dify_agent/runtime/compositor_factory.py +++ b/dify-agent/src/dify_agent/runtime/compositor_factory.py @@ -10,6 +10,7 @@ layer, and the Dify plugin/knowledge business-layer family: - ``dify.execution_context`` for shared tenant/user/run daemon context, - ``dify.runtime`` for operation-scoped RuntimeLease acquisition, - ``dify.shell`` for command/file capabilities from the active RuntimeLease, +- ``dify.user_prompt`` for structured text-plus-image user turns, - ``dify.plugin.llm`` for plugin-backed model selection, - ``dify.plugin.tools`` for prepared plugin tool exposure, and - ``dify.core.tools`` for API-routed Dify tool exposure, and @@ -55,6 +56,7 @@ from dify_agent.layers.runtime.configs import DifyRuntimeLayerConfig from dify_agent.layers.runtime.layer import DifyRuntimeLayer from dify_agent.layers.shell.configs import DifyShellLayerConfig from dify_agent.layers.shell.layer import DifyShellLayer +from dify_agent.layers.user_prompt.layer import DifyUserPromptLayer from dify_agent.runtime_backend import RuntimeBackendProfile type DifyAgentLayerProvider = LayerProvider[Any] @@ -74,6 +76,7 @@ def create_default_layer_providers( """Return the server provider set of safe config-constructible layers.""" providers: list[DifyAgentLayerProvider] = [ LayerProvider.from_layer_type(PromptLayer), + LayerProvider.from_layer_type(DifyUserPromptLayer), LayerProvider.from_layer_type(PydanticAIHistoryLayer), LayerProvider.from_layer_type(DifyOutputLayer), LayerProvider.from_layer_type(DifyAskHumanLayer), diff --git a/dify-agent/tests/local/dify_agent/layers/user_prompt/test_layer.py b/dify-agent/tests/local/dify_agent/layers/user_prompt/test_layer.py new file mode 100644 index 00000000000..3c8927bd732 --- /dev/null +++ b/dify-agent/tests/local/dify_agent/layers/user_prompt/test_layer.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import base64 + +import pytest +from pydantic import ValidationError +from pydantic_ai.messages import BinaryContent, ImageUrl + +from dify_agent.layers.user_prompt import DifyUserPromptLayerConfig +from dify_agent.layers.user_prompt.layer import DifyUserPromptLayer +from dify_agent.runtime.compositor_factory import create_default_layer_providers + + +@pytest.mark.parametrize("include_image", [False, True]) +@pytest.mark.parametrize("download_type", ["image", "document"]) +def test_user_prompt_layer_injects_file_locators(include_image: bool, download_type: str) -> None: + config = DifyUserPromptLayerConfig.model_validate( + { + "text": "Inspect these files.", + "files": [ + { + "delivery": "download", + "type": download_type, + "transfer_method": "local_file", + "reference": "dify-file-ref:upload-file-1", + }, + { + "delivery": "download", + "type": "document", + "transfer_method": "remote_url", + "url": "https://example.com/说明.pdf", + }, + { + "delivery": "download", + "type": "document", + "transfer_method": "tool_file", + "reference": "dify-file-ref:tool-file-1", + }, + { + "delivery": "download", + "type": "document", + "transfer_method": "datasource_file", + "reference": "dify-file-ref:datasource-file-1", + }, + *( + [ + { + "delivery": "multimodal", + "type": "image", + "filename": "earth.png", + "mime_type": "image/png", + "format": "png", + "url": "https://files.example.com/earth.png", + } + ] + if include_image + else [] + ), + ], + } + ) + layer = DifyUserPromptLayer.from_config(DifyUserPromptLayerConfig.model_validate_json(config.model_dump_json())) + + prompts = layer.user_prompts + + assert prompts[0] == ( + "Inspect these files.\n" + "User provided files: use dify-agent file download with the listed transfer_method and reference/url " + "to get the files and investigate them\n" + '[{"transfer_method":"local_file","reference":"dify-file-ref:upload-file-1"},' + '{"transfer_method":"remote_url","url":"https://example.com/说明.pdf"},' + '{"transfer_method":"tool_file","reference":"dify-file-ref:tool-file-1"},' + '{"transfer_method":"datasource_file","reference":"dify-file-ref:datasource-file-1"}]' + ) + assert layer.config.text == "Inspect these files." + assert "locators" not in layer.config.model_dump() + assert layer.user_prompts == prompts + assert len(prompts) == (2 if include_image else 1) + if include_image: + assert isinstance(prompts[1], ImageUrl) + assert prompts[1].url == "https://files.example.com/earth.png" + + +@pytest.mark.parametrize( + "locator", + [ + {"transfer_method": "remote_url", "reference": "file-1"}, + {"transfer_method": "local_file", "url": "https://example.com/file.pdf"}, + {"transfer_method": "unsupported", "reference": "file-1"}, + {"transfer_method": "local_file"}, + {"transfer_method": "remote_url"}, + {"transfer_method": "local_file", "reference": ""}, + {"transfer_method": "remote_url", "url": ""}, + {"transfer_method": "remote_url", "url": "https://example.com/file.pdf", "reference": "file-1"}, + {"transfer_method": "tool_file", "url": "https://example.com/file.pdf", "reference": "file-1"}, + ], +) +def test_user_prompt_layer_rejects_invalid_locators(locator: dict[str, str]) -> None: + with pytest.raises(ValidationError): + DifyUserPromptLayerConfig.model_validate( + {"text": "Inspect it.", "files": [{"delivery": "download", "type": "document", **locator}]} + ) + + +def test_user_prompt_layer_preserves_text_without_locators() -> None: + layer = DifyUserPromptLayer.from_config(DifyUserPromptLayerConfig(text=" Original prompt.\n")) + + assert layer.user_prompts == [" Original prompt.\n"] + + +def test_user_prompt_layer_restores_image_url_content() -> None: + layer = DifyUserPromptLayer.from_config( + DifyUserPromptLayerConfig.model_validate( + { + "text": "What is in this image?", + "files": [ + { + "delivery": "multimodal", + "type": "image", + "filename": "earth.png", + "mime_type": "image/png", + "format": "png", + "url": "https://files.example.com/earth.png?sign=secret", + "detail": "high", + } + ], + } + ) + ) + + prompts = layer.user_prompts + + assert prompts[0] == "What is in this image?" + assert isinstance(prompts[1], ImageUrl) + assert prompts[1].url == "https://files.example.com/earth.png?sign=secret" + assert prompts[1].media_type == "image/png" + assert prompts[1].vendor_metadata == {"filename": "earth.png", "detail": "high"} + + +def test_user_prompt_layer_restores_inline_binary_content() -> None: + payload = base64.b64encode(b"image-bytes").decode() + layer = DifyUserPromptLayer.from_config( + DifyUserPromptLayerConfig.model_validate( + { + "text": "Describe it.", + "files": [ + { + "delivery": "multimodal", + "type": "image", + "filename": "inline.png", + "mime_type": "image/png", + "format": "png", + "base64_data": payload, + } + ], + } + ) + ) + + content = layer.user_prompts[1] + + assert isinstance(content, BinaryContent) + assert content.data == b"image-bytes" + assert content.media_type == "image/png" + + +@pytest.mark.parametrize( + "file_payload", + [ + { + "delivery": "multimodal", + "type": "image", + "filename": "missing.png", + "mime_type": "image/png", + "format": "png", + }, + { + "delivery": "multimodal", + "type": "image", + "filename": "ambiguous.png", + "mime_type": "image/png", + "format": "png", + "url": "https://files.example.com/ambiguous.png", + "base64_data": "aW1hZ2U=", + }, + ], +) +def test_user_prompt_file_requires_exactly_one_transport(file_payload: dict[str, str]) -> None: + with pytest.raises(ValidationError, match="exactly one"): + DifyUserPromptLayerConfig.model_validate({"text": "Describe it.", "files": [file_payload]}) + + +def test_default_compositor_registers_user_prompt_layer() -> None: + provider = next(provider for provider in create_default_layer_providers() if provider.type_id == "dify.user_prompt") + + layer = provider.create_layer({"text": "Describe it."}) + + assert isinstance(layer, DifyUserPromptLayer) diff --git a/dify-agent/tests/local/dify_agent/test_client_safe_exports.py b/dify-agent/tests/local/dify_agent/test_client_safe_exports.py index ba1493701fb..ec4ca696882 100644 --- a/dify-agent/tests/local/dify_agent/test_client_safe_exports.py +++ b/dify-agent/tests/local/dify_agent/test_client_safe_exports.py @@ -72,6 +72,7 @@ def test_client_public_exports_work_with_default_dependencies_only(tmp_path: Pat plugin_module = importlib.import_module("dify_agent.layers.dify_plugin") ask_human_module = importlib.import_module("dify_agent.layers.ask_human") output_module = importlib.import_module("dify_agent.layers.output") + user_prompt_module = importlib.import_module("dify_agent.layers.user_prompt") assert agenton_layers.ExitIntent is not None assert agenton_layers.LayerConfig is not None @@ -93,6 +94,17 @@ def test_client_public_exports_work_with_default_dependencies_only(tmp_path: Pat assert plugin_module.DifyPluginLLMLayerConfig is not None assert ask_human_module.DifyAskHumanLayerConfig is not None assert output_module.DifyOutputLayerConfig is not None + assert user_prompt_module.DifyUserPromptLayerConfig is not None + assert user_prompt_module.DifyUserPromptFileConfig is not None + download = user_prompt_module.DifyUserPromptDownloadConfig( + type="image", transfer_method="local_file", reference="file-1" + ) + image = user_prompt_module.DifyUserPromptImageConfig( + filename="image.png", mime_type="image/png", format="png", url="https://example.com/image.png" + ) + config = user_prompt_module.DifyUserPromptLayerConfig(text="Inspect it.", files=[download, image]) + assert user_prompt_module.DifyUserPromptLayerConfig.model_validate_json(config.model_dump_json()) == config + assert [file.delivery for file in config.files] == ["download", "multimodal"] unexpectedly_installed = [] for dependency_name in sorted(server_only_dependency_names):