opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-05-09 12:59:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '657eeb65' into sandboxed-agent-rebase

Browse Source 
Made-with: Cursor

# Conflicts:
#	api/core/agent/cot_chat_agent_runner.py
#	api/core/agent/fc_agent_runner.py
#	api/core/memory/token_buffer_memory.py
#	api/core/variables/segments.py
#	api/core/workflow/file/file_manager.py
#	api/core/workflow/nodes/agent/agent_node.py
#	api/core/workflow/nodes/llm/llm_utils.py
#	api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py
#	api/core/workflow/workflow_entry.py
#	api/factories/variable_factory.py
#	api/pyproject.toml
#	api/services/variable_truncator.py
#	api/uv.lock
#	web/app/components/app/app-publisher/index.tsx
#	web/app/components/app/overview/settings/index.tsx
#	web/app/components/apps/app-card.tsx
#	web/app/components/apps/index.tsx
#	web/app/components/apps/list.tsx
#	web/app/components/base/chat/chat-with-history/header-in-mobile.tsx
#	web/app/components/base/features/new-feature-panel/conversation-opener/modal.tsx
#	web/app/components/base/features/new-feature-panel/file-upload/setting-content.tsx
#	web/app/components/base/features/new-feature-panel/moderation/moderation-setting-modal.tsx
#	web/app/components/base/features/new-feature-panel/text-to-speech/param-config-content.tsx
#	web/app/components/base/message-log-modal/index.tsx
#	web/app/components/base/switch/index.tsx
#	web/app/components/base/tab-slider-plain/index.tsx
#	web/app/components/explore/try-app/app-info/index.tsx
#	web/app/components/plugins/plugin-detail-panel/tool-selector/components/reasoning-config-form.tsx
#	web/app/components/workflow/nodes/llm/components/json-schema-config-modal/visual-editor/edit-card/required-switch.tsx
#	web/app/components/workflow/nodes/llm/panel.tsx
#	web/contract/router.ts
#	web/eslint-suppressions.json
#	web/i18n/fa-IR/workflow.json
This commit is contained in:
Novice 2026-03-19 17:38:56 +08:00
commit 89b76d6c33
No known key found for this signature in database
GPG Key ID: A253106A7475AA3E
509 changed files with 39588 additions and 3422 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
.github/dependabot.yml vendored
View File

@ -1,12 +1,25 @@
version: 2
multi-ecosystem-groups:
python:
schedule:
interval: "weekly" # or whatever schedule you want
updates:
- package-ecosystem: "pip"
directory: "/api"
open-pull-requests-limit: 2
patterns: ["*"]
schedule:
interval: "weekly"
- package-ecosystem: "uv"
directory: "/api"
open-pull-requests-limit: 2
patterns: ["*"]
schedule:
interval: "weekly"
- package-ecosystem: "npm"
directory: "/web"
schedule:
interval: "weekly"
open-pull-requests-limit: 2
- package-ecosystem: "uv"
directory: "/api"
schedule:
interval: "weekly"
open-pull-requests-limit: 2

30
api/.importlinter
View File

@ -114,19 +114,15 @@ ignore_imports =
core.workflow.nodes.datasource.datasource_node -> models.model
core.workflow.nodes.datasource.datasource_node -> models.tools
core.workflow.nodes.datasource.datasource_node -> services.datasource_provider_service
core.workflow.nodes.document_extractor.node -> configs
core.workflow.nodes.document_extractor.node -> core.file.file_manager
core.workflow.nodes.document_extractor.node -> core.helper.ssrf_proxy
core.workflow.nodes.http_request.entities -> configs
core.workflow.nodes.http_request.executor -> configs
core.workflow.nodes.http_request.executor -> core.file.file_manager
core.workflow.nodes.http_request.node -> configs
core.workflow.nodes.http_request.node -> core.tools.tool_file_manager
core.workflow.nodes.iteration.iteration_node -> core.app.workflow.node_factory
core.workflow.nodes.knowledge_index.knowledge_index_node -> core.rag.index_processor.index_processor_factory
core.workflow.nodes.llm.llm_utils -> configs
core.workflow.nodes.llm.llm_utils -> core.app.entities.app_invoke_entities
core.workflow.nodes.llm.llm_utils -> core.file.models
core.workflow.nodes.llm.llm_utils -> core.model_manager
core.workflow.nodes.llm.llm_utils -> core.model_runtime.model_providers.__base.large_language_model
core.workflow.nodes.llm.llm_utils -> models.model
@ -162,36 +158,10 @@ ignore_imports =
core.workflow.nodes.llm.llm_utils -> core.entities.provider_entities
core.workflow.nodes.parameter_extractor.parameter_extractor_node -> core.model_manager
core.workflow.nodes.question_classifier.question_classifier_node -> core.model_manager
core.workflow.node_events.node -> core.file
core.workflow.nodes.agent.agent_node -> core.file
core.workflow.nodes.datasource.datasource_node -> core.file
core.workflow.nodes.datasource.datasource_node -> core.file.enums
core.workflow.nodes.document_extractor.node -> core.file
core.workflow.nodes.http_request.executor -> core.file.enums
core.workflow.nodes.http_request.node -> core.file
core.workflow.nodes.http_request.node -> core.file.file_manager
core.workflow.nodes.knowledge_retrieval.knowledge_retrieval_node -> core.file.models
core.workflow.nodes.list_operator.node -> core.file
core.workflow.nodes.llm.file_saver -> core.file
core.workflow.nodes.llm.llm_utils -> core.variables.segments
core.workflow.nodes.llm.node -> core.file
core.workflow.nodes.llm.node -> core.file.file_manager
core.workflow.nodes.llm.node -> core.file.models
core.workflow.nodes.loop.entities -> core.variables.types
core.workflow.nodes.parameter_extractor.parameter_extractor_node -> core.file
core.workflow.nodes.protocols -> core.file
core.workflow.nodes.question_classifier.question_classifier_node -> core.file.models
core.workflow.nodes.tool.tool_node -> core.file
core.workflow.nodes.tool.tool_node -> core.tools.utils.message_transformer
core.workflow.nodes.tool.tool_node -> models
core.workflow.nodes.trigger_webhook.node -> core.file
core.workflow.runtime.variable_pool -> core.file
core.workflow.runtime.variable_pool -> core.file.file_manager
core.workflow.system_variable -> core.file.models
core.workflow.utils.condition.processor -> core.file
core.workflow.utils.condition.processor -> core.file.file_manager
core.workflow.workflow_entry -> core.file.models
core.workflow.workflow_type_encoder -> core.file.models
core.workflow.nodes.agent.agent_node -> models.model
core.workflow.nodes.code.code_node -> core.helper.code_executor.code_node_provider
core.workflow.nodes.code.code_node -> core.helper.code_executor.javascript.javascript_code_provider

16
api/commands.py
View File

@ -31,6 +31,7 @@ from extensions.ext_redis import redis_client
from extensions.ext_storage import storage
from extensions.storage.opendal_storage import OpenDALStorage
from extensions.storage.storage_type import StorageType
from libs.db_migration_lock import DbMigrationAutoRenewLock
from libs.helper import email as email_validate
from libs.password import hash_password, password_pattern, valid_password
from libs.rsa import generate_key_pair
@ -55,6 +56,8 @@ from tasks.remove_app_and_related_data_task import delete_draft_variables_batch
logger = logging.getLogger(__name__)
DB_UPGRADE_LOCK_TTL_SECONDS = 60
@click.command("reset-password", help="Reset the account password.")
@click.option("--email", prompt=True, help="Account email to reset password for")
@ -728,8 +731,15 @@ def create_tenant(email: str, language: str | None = None, name: str | None = No
@click.command("upgrade-db", help="Upgrade the database")
def upgrade_db():
click.echo("Preparing database migration...")
lock = redis_client.lock(name="db_upgrade_lock", timeout=60)
lock = DbMigrationAutoRenewLock(
redis_client=redis_client,
name="db_upgrade_lock",
ttl_seconds=DB_UPGRADE_LOCK_TTL_SECONDS,
logger=logger,
log_context="db_migration",
)
if lock.acquire(blocking=False):
migration_succeeded = False
try:
click.echo(click.style("Starting database migration.", fg="green"))
@ -738,6 +748,7 @@ def upgrade_db():
flask_migrate.upgrade()
migration_succeeded = True
click.echo(click.style("Database migration successful!", fg="green"))
except Exception as e:
@ -745,7 +756,8 @@ def upgrade_db():
click.echo(click.style(f"Database migration failed: {e}", fg="red"))
raise SystemExit(1)
finally:
lock.release()
status = "successful" if migration_succeeded else "failed"
lock.release_safely(status=status)
else:
click.echo("Database migration skipped")

2
api/controllers/common/fields.py
View File

@ -4,7 +4,7 @@ from typing import Any, TypeAlias
from pydantic import BaseModel, ConfigDict, computed_field
from core.file import helpers as file_helpers
from core.workflow.file import helpers as file_helpers
from models.model import IconType
JSONValue: TypeAlias = str | int | float | bool | None | dict[str, Any] | list[Any]

15
api/controllers/console/app/app.py
View File

@ -24,10 +24,10 @@ from controllers.console.wraps import (
is_admin_or_owner_required,
setup_required,
)
from core.file import helpers as file_helpers
from core.ops.ops_trace_manager import OpsTraceManager
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from core.workflow.enums import NodeType, WorkflowExecutionStatus
from core.workflow.file import helpers as file_helpers
from extensions.ext_database import db
from libs.login import current_account_with_tenant, login_required
from models import App, DatasetPermissionEnum, Workflow
@ -679,6 +679,19 @@ class AppCopyApi(Resource):
)
session.commit()
# Inherit web app permission from original app
if result.app_id and FeatureService.get_system_features().webapp_auth.enabled:
try:
# Get the original app's access mode
original_settings = EnterpriseService.WebAppAuth.get_app_access_mode_by_id(app_model.id)
access_mode = original_settings.access_mode
except Exception:
# If original app has no settings (old app), default to public to match fallback behavior
access_mode = "public"
# Apply the same access mode to the copied app
EnterpriseService.WebAppAuth.update_app_access_mode(result.app_id, access_mode)
stmt = select(App).where(App.id == result.app_id)
app = session.scalar(stmt)

2
api/controllers/console/app/workflow.py
View File

@ -20,7 +20,6 @@ from core.app.app_config.features.file_upload.manager import FileUploadConfigMan
from core.app.apps.base_app_queue_manager import AppQueueManager
from core.app.apps.workflow.app_generator import SKIP_PREPARE_USER_INPUTS_KEY
from core.app.entities.app_invoke_entities import InvokeFrom
from core.file.models import File
from core.helper.trace_id_helper import get_external_trace_id
from core.model_runtime.utils.encoders import jsonable_encoder
from core.plugin.impl.exc import PluginInvokeError
@ -31,6 +30,7 @@ from core.trigger.debug.event_selectors import (
select_trigger_debug_events,
)
from core.workflow.enums import NodeType
from core.workflow.file.models import File
from core.workflow.graph_engine.manager import GraphEngineManager
from extensions.ext_database import db
from extensions.ext_redis import redis_client

2
api/controllers/console/app/workflow_draft_variable.py
View File

@ -15,11 +15,11 @@ from controllers.console.app.error import (
from controllers.console.app.wraps import get_app_model
from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required
from controllers.web.error import InvalidArgumentError, NotFoundError
from core.file import helpers as file_helpers
from core.variables.segment_group import SegmentGroup
from core.variables.segments import ArrayFileSegment, ArrayPromptMessageSegment, FileSegment, Segment
from core.variables.types import SegmentType
from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID
from core.workflow.file import helpers as file_helpers
from extensions.ext_database import db
from factories import variable_factory
from factories.file_factory import build_from_mapping, build_from_mappings

40
api/controllers/console/explore/trial.py
View File

@ -10,7 +10,7 @@ import services
from controllers.common.fields import Parameters as ParametersResponse
from controllers.common.fields import Site as SiteResponse
from controllers.common.schema import get_or_create_model
from controllers.console import api, console_ns
from controllers.console import console_ns
from controllers.console.app.error import (
AppUnavailableError,
AudioTooLargeError,
@ -469,7 +469,7 @@ class TrialSitApi(Resource):
"""Resource for trial app sites."""
@trial_feature_enable
@get_app_model_with_trial
@get_app_model_with_trial(None)
def get(self, app_model):
"""Retrieve app site info.
@ -491,7 +491,7 @@ class TrialAppParameterApi(Resource):
"""Resource for app variables."""
@trial_feature_enable
@get_app_model_with_trial
@get_app_model_with_trial(None)
def get(self, app_model):
"""Retrieve app parameters."""
@ -520,7 +520,7 @@ class TrialAppParameterApi(Resource):
class AppApi(Resource):
@trial_feature_enable
@get_app_model_with_trial
@get_app_model_with_trial(None)
@marshal_with(app_detail_with_site_model)
def get(self, app_model):
"""Get app detail"""
@ -533,7 +533,7 @@ class AppApi(Resource):
class AppWorkflowApi(Resource):
@trial_feature_enable
@get_app_model_with_trial
@get_app_model_with_trial(None)
@marshal_with(workflow_model)
def get(self, app_model):
"""Get workflow detail"""
@ -552,7 +552,7 @@ class AppWorkflowApi(Resource):
class DatasetListApi(Resource):
@trial_feature_enable
@get_app_model_with_trial
@get_app_model_with_trial(None)
def get(self, app_model):
page = request.args.get("page", default=1, type=int)
limit = request.args.get("limit", default=20, type=int)
@ -570,27 +570,31 @@ class DatasetListApi(Resource):
return response
api.add_resource(TrialChatApi, "/trial-apps/<uuid:app_id>/chat-messages", endpoint="trial_app_chat_completion")
console_ns.add_resource(TrialChatApi, "/trial-apps/<uuid:app_id>/chat-messages", endpoint="trial_app_chat_completion")
api.add_resource(
console_ns.add_resource(
TrialMessageSuggestedQuestionApi,
"/trial-apps/<uuid:app_id>/messages/<uuid:message_id>/suggested-questions",
endpoint="trial_app_suggested_question",
)
api.add_resource(TrialChatAudioApi, "/trial-apps/<uuid:app_id>/audio-to-text", endpoint="trial_app_audio")
api.add_resource(TrialChatTextApi, "/trial-apps/<uuid:app_id>/text-to-audio", endpoint="trial_app_text")
console_ns.add_resource(TrialChatAudioApi, "/trial-apps/<uuid:app_id>/audio-to-text", endpoint="trial_app_audio")
console_ns.add_resource(TrialChatTextApi, "/trial-apps/<uuid:app_id>/text-to-audio", endpoint="trial_app_text")
api.add_resource(TrialCompletionApi, "/trial-apps/<uuid:app_id>/completion-messages", endpoint="trial_app_completion")
console_ns.add_resource(
TrialCompletionApi, "/trial-apps/<uuid:app_id>/completion-messages", endpoint="trial_app_completion"
)
api.add_resource(TrialSitApi, "/trial-apps/<uuid:app_id>/site")
console_ns.add_resource(TrialSitApi, "/trial-apps/<uuid:app_id>/site")
api.add_resource(TrialAppParameterApi, "/trial-apps/<uuid:app_id>/parameters", endpoint="trial_app_parameters")
console_ns.add_resource(TrialAppParameterApi, "/trial-apps/<uuid:app_id>/parameters", endpoint="trial_app_parameters")
api.add_resource(AppApi, "/trial-apps/<uuid:app_id>", endpoint="trial_app")
console_ns.add_resource(AppApi, "/trial-apps/<uuid:app_id>", endpoint="trial_app")
api.add_resource(TrialAppWorkflowRunApi, "/trial-apps/<uuid:app_id>/workflows/run", endpoint="trial_app_workflow_run")
api.add_resource(TrialAppWorkflowTaskStopApi, "/trial-apps/<uuid:app_id>/workflows/tasks/<string:task_id>/stop")
console_ns.add_resource(
TrialAppWorkflowRunApi, "/trial-apps/<uuid:app_id>/workflows/run", endpoint="trial_app_workflow_run"
)
console_ns.add_resource(TrialAppWorkflowTaskStopApi, "/trial-apps/<uuid:app_id>/workflows/tasks/<string:task_id>/stop")
api.add_resource(AppWorkflowApi, "/trial-apps/<uuid:app_id>/workflows", endpoint="trial_app_workflow")
api.add_resource(DatasetListApi, "/trial-apps/<uuid:app_id>/datasets", endpoint="trial_app_datasets")
console_ns.add_resource(AppWorkflowApi, "/trial-apps/<uuid:app_id>/workflows", endpoint="trial_app_workflow")
console_ns.add_resource(DatasetListApi, "/trial-apps/<uuid:app_id>/datasets", endpoint="trial_app_datasets")

8
api/controllers/console/explore/wraps.py
View File

@ -105,9 +105,9 @@ def trial_app_required(view: Callable[Concatenate[App, P], R] | None = None):
return decorator
def trial_feature_enable(view: Callable[..., R]) -> Callable[..., R]:
def trial_feature_enable(view: Callable[P, R]):
@wraps(view)
def decorated(*args, **kwargs):
def decorated(*args: P.args, **kwargs: P.kwargs):
features = FeatureService.get_system_features()
if not features.enable_trial_app:
abort(403, "Trial app feature is not enabled.")
@ -116,9 +116,9 @@ def trial_feature_enable(view: Callable[..., R]) -> Callable[..., R]:
return decorated
def explore_banner_enabled(view: Callable[..., R]) -> Callable[..., R]:
def explore_banner_enabled(view: Callable[P, R]):
@wraps(view)
def decorated(*args, **kwargs):
def decorated(*args: P.args, **kwargs: P.kwargs):
features = FeatureService.get_system_features()
if not features.enable_explore_banner:
abort(403, "Explore banner feature is not enabled.")

2
api/controllers/console/remote_files.py
View File

@ -12,8 +12,8 @@ from controllers.common.errors import (
UnsupportedFileTypeError,
)
from controllers.console import console_ns
from core.file import helpers as file_helpers
from core.helper import ssrf_proxy
from core.workflow.file import helpers as file_helpers
from extensions.ext_database import db
from fields.file_fields import FileWithSignedUrl, RemoteFileInfo
from libs.login import current_account_with_tenant, login_required

2
api/controllers/files/upload.py
View File

@ -7,8 +7,8 @@ from pydantic import BaseModel, Field
from werkzeug.exceptions import Forbidden
import services
from core.file.helpers import verify_plugin_file_signature
from core.tools.tool_file_manager import ToolFileManager
from core.workflow.file.helpers import verify_plugin_file_signature
from fields.file_fields import FileResponse
from ..common.errors import (

2
api/controllers/inner_api/plugin/plugin.py
View File

@ -4,7 +4,6 @@ from controllers.console.wraps import setup_required
from controllers.inner_api import inner_api_ns
from controllers.inner_api.plugin.wraps import get_user_tenant, plugin_data
from controllers.inner_api.wraps import plugin_inner_api_only
from core.file.helpers import get_signed_file_url_for_plugin
from core.model_runtime.utils.encoders import jsonable_encoder
from core.plugin.backwards_invocation.app import PluginAppBackwardsInvocation
from core.plugin.backwards_invocation.base import BaseBackwardsInvocationResponse
@ -30,6 +29,7 @@ from core.plugin.entities.request import (
RequestRequestUploadFile,
)
from core.tools.entities.tool_entities import ToolProviderType
from core.workflow.file.helpers import get_signed_file_url_for_plugin
from libs.helper import length_prefixed_response
from models import Account, Tenant
from models.model import EndUser

2
api/controllers/web/remote_files.py
View File

@ -10,8 +10,8 @@ from controllers.common.errors import (
RemoteFileUploadError,
UnsupportedFileTypeError,
)
from core.file import helpers as file_helpers
from core.helper import ssrf_proxy
from core.workflow.file import helpers as file_helpers
from extensions.ext_database import db
from fields.file_fields import FileWithSignedUrl, RemoteFileInfo
from services.file_service import FileService

2
api/core/agent/base_agent_runner.py
View File

@ -17,7 +17,6 @@ from core.app.entities.app_invoke_entities import (
)
from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
from core.file import file_manager
from core.memory.token_buffer_memory import TokenBufferMemory
from core.model_manager import ModelInstance
from core.model_runtime.entities import (
@ -40,6 +39,7 @@ from core.tools.entities.tool_entities import (
)
from core.tools.tool_manager import ToolManager
from core.tools.utils.dataset_retriever_tool import DatasetRetrieverTool
from core.workflow.file import file_manager
from extensions.ext_database import db
from factories import file_factory
from models.enums import CreatorUserRole

2
api/core/app/app_config/entities.py
View File

@ -5,9 +5,9 @@ from typing import Any, Literal
from jsonschema import Draft7Validator, SchemaError
from pydantic import BaseModel, Field, field_validator
from core.file import FileTransferMethod, FileType, FileUploadConfig
from core.model_runtime.entities.llm_entities import LLMMode
from core.model_runtime.entities.message_entities import PromptMessageRole
from core.workflow.file import FileTransferMethod, FileType, FileUploadConfig
from models.model import AppMode

2
api/core/app/app_config/features/file_upload/manager.py
View File

@ -2,7 +2,7 @@ from collections.abc import Mapping
from typing import Any
from constants import DEFAULT_FILE_NUMBER_LIMITS
from core.file import FileUploadConfig
from core.workflow.file import FileUploadConfig
class FileUploadConfigManager:

2
api/core/app/apps/base_app_generator.py
View File

@ -5,8 +5,8 @@ from sqlalchemy.orm import Session
from core.app.app_config.entities import VariableEntityType
from core.app.entities.app_invoke_entities import InvokeFrom
from core.file import File, FileUploadConfig
from core.workflow.enums import NodeType
from core.workflow.file import File, FileUploadConfig
from core.workflow.repositories.draft_variable_repository import (
DraftVariableSaver,
DraftVariableSaverFactory,

6
api/core/app/apps/base_app_queue_manager.py
View File

@ -2,7 +2,7 @@ import logging
import queue
import threading
import time
from abc import abstractmethod
from abc import ABC, abstractmethod
from enum import IntEnum, auto
from typing import Any
@ -31,7 +31,7 @@ class PublishFrom(IntEnum):
TASK_PIPELINE = auto()
class AppQueueManager:
class AppQueueManager(ABC):
def __init__(self, task_id: str, user_id: str, invoke_from: InvokeFrom):
if not user_id:
raise ValueError("user is required")
@ -133,7 +133,7 @@ class AppQueueManager:
self._publish(event, pub_from)
@abstractmethod
def _publish(self, event: AppQueueEvent, pub_from: PublishFrom):
def _publish(self, event: AppQueueEvent, pub_from: PublishFrom) -> None:
"""
Publish event to queue
:param event:

4
api/core/app/apps/base_app_runner.py
View File

@ -22,7 +22,6 @@ from core.app.entities.queue_entities import (
from core.app.features.annotation_reply.annotation_reply import AnnotationReplyFeature
from core.app.features.hosting_moderation.hosting_moderation import HostingModerationFeature
from core.external_data_tool.external_data_fetch import ExternalDataFetch
from core.file.enums import FileTransferMethod, FileType
from core.memory.token_buffer_memory import TokenBufferMemory
from core.model_manager import ModelInstance
from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
@ -39,12 +38,13 @@ from core.prompt.advanced_prompt_transform import AdvancedPromptTransform
from core.prompt.entities.advanced_prompt_entities import ChatModelMessage, CompletionModelPromptTemplate, MemoryConfig
from core.prompt.simple_prompt_transform import ModelMode, SimplePromptTransform
from core.tools.tool_file_manager import ToolFileManager
from core.workflow.file.enums import FileTransferMethod, FileType
from extensions.ext_database import db
from models.enums import CreatorUserRole
from models.model import App, AppMode, Message, MessageAnnotation, MessageFile
if TYPE_CHECKING:
from core.file.models import File
from core.workflow.file.models import File
_logger = logging.getLogger(__name__)

2
api/core/app/apps/chat/app_runner.py
View File

@ -11,12 +11,12 @@ from core.app.entities.app_invoke_entities import (
)
from core.app.entities.queue_entities import QueueAnnotationReplyEvent
from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
from core.file import File
from core.memory.token_buffer_memory import TokenBufferMemory
from core.model_manager import ModelInstance
from core.model_runtime.entities.message_entities import ImagePromptMessageContent
from core.moderation.base import ModerationError
from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
from core.workflow.file import File
from extensions.ext_database import db
from models.model import App, Conversation, Message

2
api/core/app/apps/common/workflow_response_converter.py
View File

@ -45,7 +45,6 @@ from core.app.entities.task_entities import (
WorkflowPauseStreamResponse,
WorkflowStartStreamResponse,
)
from core.file import FILE_MODEL_IDENTITY, File
from core.plugin.impl.datasource import PluginDatasourceManager
from core.tools.entities.tool_entities import ToolProviderType
from core.tools.tool_manager import ToolManager
@ -60,6 +59,7 @@ from core.workflow.enums import (
WorkflowNodeExecutionMetadataKey,
WorkflowNodeExecutionStatus,
)
from core.workflow.file import FILE_MODEL_IDENTITY, File
from core.workflow.runtime import GraphRuntimeState
from core.workflow.system_variable import SystemVariable
from core.workflow.workflow_entry import WorkflowEntry

2
api/core/app/apps/completion/app_runner.py
View File

@ -10,11 +10,11 @@ from core.app.entities.app_invoke_entities import (
CompletionAppGenerateEntity,
)
from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
from core.file import File
from core.model_manager import ModelInstance
from core.model_runtime.entities.message_entities import ImagePromptMessageContent
from core.moderation.base import ModerationError
from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
from core.workflow.file import File
from extensions.ext_database import db
from models.model import App, Message

2
api/core/app/entities/app_invoke_entities.py
View File

@ -7,8 +7,8 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validat
from constants import UUID_NIL
from core.app.app_config.entities import EasyUIBasedAppConfig, WorkflowUIBasedAppConfig
from core.entities.provider_configuration import ProviderModelBundle
from core.file import File, FileUploadConfig
from core.model_runtime.entities.model_entities import AIModelEntity
from core.workflow.file import File, FileUploadConfig
if TYPE_CHECKING:
from core.ops.ops_trace_manager import TraceQueueManager

4
api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py
View File

@ -46,8 +46,6 @@ from core.app.entities.task_entities import (
from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTaskPipeline
from core.app.task_pipeline.message_cycle_manager import MessageCycleManager
from core.base.tts import AppGeneratorTTSPublisher, AudioTrunk
from core.file import helpers as file_helpers
from core.file.enums import FileTransferMethod
from core.model_manager import ModelInstance
from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
from core.model_runtime.entities.message_entities import (
@ -60,6 +58,8 @@ from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
from core.prompt.utils.prompt_message_util import PromptMessageUtil
from core.prompt.utils.prompt_template_parser import PromptTemplateParser
from core.tools.signature import sign_tool_file
from core.workflow.file import helpers as file_helpers
from core.workflow.file.enums import FileTransferMethod
from events.message_event import message_was_created
from extensions.ext_database import db
from libs.datetime_utils import naive_utc_now

47
api/core/app/workflow/file_runtime.py Normal file
View File

@ -0,0 +1,47 @@
from __future__ import annotations
from collections.abc import Generator
from configs import dify_config
from core.helper.ssrf_proxy import ssrf_proxy
from core.tools.signature import sign_tool_file
from core.workflow.file.protocols import HttpResponseProtocol, WorkflowFileRuntimeProtocol
from core.workflow.file.runtime import set_workflow_file_runtime
from extensions.ext_storage import storage
class DifyWorkflowFileRuntime(WorkflowFileRuntimeProtocol):
"""Production runtime wiring for ``core.workflow.file``."""
@property
def files_url(self) -> str:
return dify_config.FILES_URL
@property
def internal_files_url(self) -> str | None:
return dify_config.INTERNAL_FILES_URL
@property
def secret_key(self) -> str:
return dify_config.SECRET_KEY
@property
def files_access_timeout(self) -> int:
return dify_config.FILES_ACCESS_TIMEOUT
@property
def multimodal_send_format(self) -> str:
return dify_config.MULTIMODAL_SEND_FORMAT
def http_get(self, url: str, *, follow_redirects: bool = True) -> HttpResponseProtocol:
return ssrf_proxy.get(url, follow_redirects=follow_redirects)
def storage_load(self, path: str, *, stream: bool = False) -> bytes | Generator:
return storage.load(path, stream=stream)
def sign_tool_file(self, *, tool_file_id: str, extension: str, for_external: bool = True) -> str:
return sign_tool_file(tool_file_id=tool_file_id, extension=extension, for_external=for_external)
def bind_dify_workflow_file_runtime() -> None:
set_workflow_file_runtime(DifyWorkflowFileRuntime())

21
api/core/app/workflow/node_factory.py
View File

@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, final
from typing_extensions import override
from configs import dify_config
from core.file.file_manager import file_manager
from core.helper.code_executor.code_executor import CodeExecutor
from core.helper.code_executor.code_node_provider import CodeNodeProvider
from core.helper.ssrf_proxy import ssrf_proxy
@ -12,10 +11,12 @@ from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
from core.tools.tool_file_manager import ToolFileManager
from core.workflow.entities.graph_config import NodeConfigDict
from core.workflow.enums import NodeType
from core.workflow.file.file_manager import file_manager
from core.workflow.graph.graph import NodeFactory
from core.workflow.nodes.base.node import Node
from core.workflow.nodes.code.code_node import CodeNode
from core.workflow.nodes.code.limits import CodeNodeLimits
from core.workflow.nodes.document_extractor import DocumentExtractorNode, UnstructuredApiConfig
from core.workflow.nodes.http_request.node import HttpRequestNode
from core.workflow.nodes.knowledge_retrieval.knowledge_retrieval_node import KnowledgeRetrievalNode
from core.workflow.nodes.node_mapping import LATEST_VERSION, NODE_TYPE_CLASSES_MAPPING
@ -44,7 +45,6 @@ class DifyNodeFactory(NodeFactory):
self,
graph_init_params: "GraphInitParams",
graph_runtime_state: "GraphRuntimeState",
*,
code_executor: type[CodeExecutor] | None = None,
code_providers: Sequence[type[CodeNodeProvider]] | None = None,
code_limits: CodeNodeLimits | None = None,
@ -53,6 +53,7 @@ class DifyNodeFactory(NodeFactory):
http_request_http_client: HttpClientProtocol | None = None,
http_request_tool_file_manager_factory: Callable[[], ToolFileManager] = ToolFileManager,
http_request_file_manager: FileManagerProtocol | None = None,
document_extractor_unstructured_api_config: UnstructuredApiConfig | None = None,
) -> None:
self.graph_init_params = graph_init_params
self.graph_runtime_state = graph_runtime_state
@ -78,6 +79,13 @@ class DifyNodeFactory(NodeFactory):
self._http_request_tool_file_manager_factory = http_request_tool_file_manager_factory
self._http_request_file_manager = http_request_file_manager or file_manager
self._rag_retrieval = DatasetRetrieval()
self._document_extractor_unstructured_api_config = (
document_extractor_unstructured_api_config
or UnstructuredApiConfig(
api_url=dify_config.UNSTRUCTURED_API_URL,
api_key=dify_config.UNSTRUCTURED_API_KEY or "",
)
)
@override
def create_node(self, node_config: NodeConfigDict) -> Node:
@ -152,6 +160,15 @@ class DifyNodeFactory(NodeFactory):
rag_retrieval=self._rag_retrieval,
)
if node_type == NodeType.DOCUMENT_EXTRACTOR:
return DocumentExtractorNode(
id=node_id,
config=node_config,
graph_init_params=self.graph_init_params,
graph_runtime_state=self.graph_runtime_state,
unstructured_api_config=self._document_extractor_unstructured_api_config,
)
return node_class(
id=node_id,
config=node_config,

2
api/core/datasource/datasource_file_manager.py
View File

@ -213,6 +213,6 @@ class DatasourceFileManager:
# init tool_file_parser
# from core.file.datasource_file_parser import datasource_file_manager
# from core.workflow.file.datasource_file_parser import datasource_file_manager
#
# datasource_file_manager["manager"] = DatasourceFileManager

2
api/core/datasource/utils/message_transformer.py
View File

@ -3,8 +3,8 @@ from collections.abc import Generator
from mimetypes import guess_extension, guess_type
from core.datasource.entities.datasource_entities import DatasourceMessage
from core.file import File, FileTransferMethod, FileType
from core.tools.tool_file_manager import ToolFileManager
from core.workflow.file import File, FileTransferMethod, FileType
from models.tools import ToolFile
logger = logging.getLogger(__name__)

2
api/core/entities/mcp_provider.py
View File

@ -10,12 +10,12 @@ from pydantic import BaseModel
from configs import dify_config
from core.entities.provider_entities import BasicProviderConfig
from core.file import helpers as file_helpers
from core.helper import encrypter
from core.helper.provider_cache import NoOpProviderCredentialCache
from core.mcp.types import OAuthClientInformation, OAuthClientMetadata, OAuthTokens
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_entities import ToolProviderType
from core.workflow.file import helpers as file_helpers
if TYPE_CHECKING:
from models.tools import MCPToolProvider

12
api/core/file/tool_file_parser.py
View File

@ -1,12 +0,0 @@
from collections.abc import Callable
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from core.tools.tool_file_manager import ToolFileManager
_tool_file_manager_factory: Callable[[], "ToolFileManager"] | None = None
def set_tool_file_manager_factory(factory: Callable[[], "ToolFileManager"]):
global _tool_file_manager_factory
_tool_file_manager_factory = factory

2
api/core/memory/token_buffer_memory.py
View File

@ -4,7 +4,6 @@ from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
from core.file import file_manager
from core.memory.base import BaseMemory
from core.model_manager import ModelInstance
from core.model_runtime.entities import (
@ -16,6 +15,7 @@ from core.model_runtime.entities import (
)
from core.model_runtime.entities.message_entities import PromptMessageContentUnionTypes
from core.prompt.utils.extract_thread_messages import extract_thread_messages
from core.workflow.file import file_manager
from extensions.ext_database import db
from factories import file_factory
from models.model import AppMode, Conversation, Message, MessageFile

2
api/core/moderation/base.py
View File

@ -39,7 +39,7 @@ class Moderation(Extensible, ABC):
@classmethod
@abstractmethod
def validate_config(cls, tenant_id: str, config: dict):
def validate_config(cls, tenant_id: str, config: dict) -> None:
"""
Validate the incoming form config data.

3
api/core/ops/tencent_trace/client.py
View File

@ -18,8 +18,7 @@ except ImportError:
from importlib_metadata import version # type: ignore[import-not-found]
if TYPE_CHECKING:
from opentelemetry.metrics import Meter
from opentelemetry.metrics._internal.instrument import Histogram
from opentelemetry.metrics import Histogram, Meter
from opentelemetry.sdk.metrics.export import MetricReader
from opentelemetry import trace as trace_api

2
api/core/plugin/utils/converter.py
View File

@ -1,7 +1,7 @@
from typing import Any
from core.file.models import File
from core.tools.entities.tool_entities import ToolSelector
from core.workflow.file.models import File
def convert_parameters_to_plugin_format(parameters: dict[str, Any]) -> dict[str, Any]:

4
api/core/prompt/advanced_prompt_transform.py
View File

@ -2,8 +2,6 @@ from collections.abc import Mapping, Sequence
from typing import cast
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
from core.file import file_manager
from core.file.models import File
from core.helper.code_executor.jinja2.jinja2_formatter import Jinja2Formatter
from core.memory.base import BaseMemory
from core.model_runtime.entities import (
@ -18,6 +16,8 @@ from core.model_runtime.entities.message_entities import ImagePromptMessageConte
from core.prompt.entities.advanced_prompt_entities import ChatModelMessage, CompletionModelPromptTemplate, MemoryConfig
from core.prompt.prompt_transform import PromptTransform
from core.prompt.utils.prompt_template_parser import PromptTemplateParser
from core.workflow.file import file_manager
from core.workflow.file.models import File
from core.workflow.runtime import VariablePool

4
api/core/prompt/simple_prompt_transform.py
View File

@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Any, cast
from core.app.app_config.entities import PromptTemplateEntity
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
from core.file import file_manager
from core.memory.token_buffer_memory import TokenBufferMemory
from core.model_runtime.entities.message_entities import (
ImagePromptMessageContent,
@ -19,10 +18,11 @@ from core.model_runtime.entities.message_entities import (
from core.prompt.entities.advanced_prompt_entities import MemoryConfig
from core.prompt.prompt_transform import PromptTransform
from core.prompt.utils.prompt_template_parser import PromptTemplateParser
from core.workflow.file import file_manager
from models.model import AppMode
if TYPE_CHECKING:
from core.file.models import File
from core.workflow.file.models import File
class ModelMode(StrEnum):

2
api/core/rag/datasource/vdb/vector_base.py
View File

@ -15,7 +15,7 @@ class BaseVector(ABC):
raise NotImplementedError
@abstractmethod
def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs):
def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs) -> list[str] | None:
raise NotImplementedError
@abstractmethod

2
api/core/rag/index_processor/processor/paragraph_index_processor.py
View File

@ -9,7 +9,6 @@ from typing import Any, cast
logger = logging.getLogger(__name__)
from core.entities.knowledge_entities import PreviewDetail
from core.file import File, FileTransferMethod, FileType, file_manager
from core.llm_generator.prompts import DEFAULT_GENERATOR_SUMMARY_PROMPT
from core.model_manager import ModelInstance
from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage
@ -35,6 +34,7 @@ from core.rag.index_processor.index_processor_base import BaseIndexProcessor
from core.rag.models.document import AttachmentDocument, Document, MultimodalGeneralStructureChunk
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from core.tools.utils.text_processing_utils import remove_leading_symbols
from core.workflow.file import File, FileTransferMethod, FileType, file_manager
from core.workflow.nodes.llm import llm_utils
from extensions.ext_database import db
from factories.file_factory import build_from_mapping

2
api/core/rag/models/document.py
View File

@ -4,7 +4,7 @@ from typing import Any
from pydantic import BaseModel, Field
from core.file import File
from core.workflow.file import File
class ChildDocument(BaseModel):

2
api/core/rag/retrieval/dataset_retrieval.py
View File

@ -23,7 +23,6 @@ from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCa
from core.db.session_factory import session_factory
from core.entities.agent_entities import PlanningStrategy
from core.entities.model_entities import ModelStatus
from core.file import File, FileTransferMethod, FileType
from core.memory.token_buffer_memory import TokenBufferMemory
from core.model_manager import ModelInstance, ModelManager
from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage
@ -61,6 +60,7 @@ from core.rag.retrieval.template_prompts import (
)
from core.tools.signature import sign_upload_file
from core.tools.utils.dataset_retriever.dataset_retriever_base_tool import DatasetRetrieverBaseTool
from core.workflow.file import File, FileTransferMethod, FileType
from core.workflow.nodes.knowledge_retrieval import exc
from core.workflow.repositories.rag_retrieval_protocol import (
KnowledgeRetrievalRequest,

4
api/core/tools/builtin_tool/providers/audio/tools/asr.py
View File

@ -2,14 +2,14 @@ import io
from collections.abc import Generator
from typing import Any
from core.file.enums import FileType
from core.file.file_manager import download
from core.model_manager import ModelManager
from core.model_runtime.entities.model_entities import ModelType
from core.plugin.entities.parameters import PluginParameterOption
from core.tools.builtin_tool.tool import BuiltinTool
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter
from core.workflow.file.enums import FileType
from core.workflow.file.file_manager import download
from services.model_provider_service import ModelProviderService

2
api/core/tools/custom_tool/tool.py
View File

@ -7,13 +7,13 @@ from urllib.parse import urlencode
import httpx
from core.file.file_manager import download
from core.helper import ssrf_proxy
from core.tools.__base.tool import Tool
from core.tools.__base.tool_runtime import ToolRuntime
from core.tools.entities.tool_bundle import ApiToolBundle
from core.tools.entities.tool_entities import ToolEntity, ToolInvokeMessage, ToolProviderType
from core.tools.errors import ToolInvokeError, ToolParameterValidationError, ToolProviderCredentialValidationError
from core.workflow.file.file_manager import download
API_TOOL_DEFAULT_TIMEOUT = (
int(getenv("API_TOOL_DEFAULT_CONNECT_TIMEOUT", "10")),

4
api/core/tools/tool_engine.py
View File

@ -12,8 +12,6 @@ from yarl import URL
from core.app.entities.app_invoke_entities import InvokeFrom
from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler
from core.file import FileType
from core.file.models import FileTransferMethod
from core.ops.ops_trace_manager import TraceQueueManager
from core.tools.__base.tool import Tool
from core.tools.entities.tool_entities import (
@ -33,6 +31,8 @@ from core.tools.errors import (
)
from core.tools.utils.message_transformer import ToolFileMessageTransformer, safe_json_value
from core.tools.workflow_as_tool.tool import WorkflowTool
from core.workflow.file import FileType
from core.workflow.file.models import FileTransferMethod
from extensions.ext_database import db
from models.enums import CreatorUserRole
from models.model import Message, MessageFile

2
api/core/tools/tool_file_manager.py
View File

@ -243,7 +243,7 @@ class ToolFileManager:
# init tool_file_parser
from core.file.tool_file_parser import set_tool_file_manager_factory
from core.workflow.file.tool_file_parser import set_tool_file_manager_factory
def _factory() -> ToolFileManager:

2
api/core/tools/utils/message_transformer.py
View File

@ -8,9 +8,9 @@ from uuid import UUID
import numpy as np
import pytz
from core.file import File, FileTransferMethod, FileType
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool_file_manager import ToolFileManager
from core.workflow.file import File, FileTransferMethod, FileType
from libs.login import current_user
from models import Account

2
api/core/tools/workflow_as_tool/tool.py
View File

@ -8,7 +8,6 @@ from typing import Any, cast
from sqlalchemy import select
from core.db.session_factory import session_factory
from core.file import FILE_MODEL_IDENTITY, File, FileTransferMethod
from core.model_runtime.entities.llm_entities import LLMUsage, LLMUsageMetadata
from core.tools.__base.tool import Tool
from core.tools.__base.tool_runtime import ToolRuntime
@ -19,6 +18,7 @@ from core.tools.entities.tool_entities import (
ToolProviderType,
)
from core.tools.errors import ToolInvokeError
from core.workflow.file import FILE_MODEL_IDENTITY, File, FileTransferMethod
from factories.file_factory import build_from_mapping
from models import Account, Tenant
from models.model import App, EndUser

2
api/core/variables/segments.py
View File

@ -5,8 +5,8 @@ from typing import Annotated, Any, TypeAlias
from pydantic import BaseModel, ConfigDict, Discriminator, Tag, field_validator
from core.file import File
from core.model_runtime.entities import PromptMessage
from core.workflow.file import File
from .types import SegmentType

2
api/core/variables/types.py
View File

@ -4,7 +4,7 @@ from collections.abc import Mapping
from enum import StrEnum
from typing import TYPE_CHECKING, Any
from core.file.models import File
from core.workflow.file.models import File
if TYPE_CHECKING:
pass

0
api/core/file/__init__.py → api/core/workflow/file/__init__.py
View File

0
api/core/file/constants.py → api/core/workflow/file/constants.py
View File

0
api/core/file/enums.py → api/core/workflow/file/enums.py
View File

75
api/core/file/file_manager.py → api/core/workflow/file/file_manager.py
View File

@ -1,9 +1,10 @@
from __future__ import annotations
import base64
import logging
from collections.abc import Mapping
from configs import dify_config
from core.helper import ssrf_proxy
from core.model_runtime.entities import (
AudioPromptMessageContent,
DocumentPromptMessageContent,
@ -15,12 +16,11 @@ from core.model_runtime.entities.message_entities import (
MultiModalPromptMessageContent,
PromptMessageContentUnionTypes,
)
from core.tools.signature import sign_tool_file
from extensions.ext_storage import storage
from . import helpers
from .enums import FileAttribute
from .models import File, FileTransferMethod, FileType
from .runtime import get_workflow_file_runtime
logger = logging.getLogger(__name__)
@ -51,26 +51,7 @@ def to_prompt_message_content(
*,
image_detail_config: ImagePromptMessageContent.DETAIL | None = None,
) -> PromptMessageContentUnionTypes:
"""
Convert a file to prompt message content.
This function converts files to their appropriate prompt message content types.
For supported file types (IMAGE, AUDIO, VIDEO, DOCUMENT), it creates the
corresponding message content with proper encoding/URL.
For unsupported file types, instead of raising an error, it returns a
TextPromptMessageContent with a descriptive message about the file.
Args:
f: The file to convert
image_detail_config: Optional detail configuration for image files
Returns:
PromptMessageContentUnionTypes: The appropriate message content type
Raises:
ValueError: If file extension or mime_type is missing
"""
"""Convert a file to prompt message content."""
if f.extension is None:
raise ValueError("Missing file extension")
if f.mime_type is None:
@ -83,15 +64,13 @@ def to_prompt_message_content(
FileType.DOCUMENT: DocumentPromptMessageContent,
}
# Check if file type is supported
if f.type not in prompt_class_map:
# For unsupported file types, return a text description
return TextPromptMessageContent(data=f"[Unsupported file type: {f.filename} ({f.type.value})]")
# Process supported file types
send_format = get_workflow_file_runtime().multimodal_send_format
params = {
"base64_data": _get_encoded_string(f) if dify_config.MULTIMODAL_SEND_FORMAT == "base64" else "",
"url": _to_url(f) if dify_config.MULTIMODAL_SEND_FORMAT == "url" else "",
"base64_data": _get_encoded_string(f) if send_format == "base64" else "",
"url": _to_url(f) if send_format == "url" else "",
"format": f.extension.removeprefix("."),
"mime_type": f.mime_type,
"filename": f.filename or "",
@ -115,7 +94,7 @@ def _encode_file_ref(f: File) -> str | None:
return None
def download(f: File, /):
def download(f: File, /) -> bytes:
if f.transfer_method in (
FileTransferMethod.TOOL_FILE,
FileTransferMethod.LOCAL_FILE,
@ -125,39 +104,26 @@ def download(f: File, /):
elif f.transfer_method == FileTransferMethod.REMOTE_URL:
if f.remote_url is None:
raise ValueError("Missing file remote_url")
response = ssrf_proxy.get(f.remote_url, follow_redirects=True)
response = get_workflow_file_runtime().http_get(f.remote_url, follow_redirects=True)
response.raise_for_status()
return response.content
raise ValueError(f"unsupported transfer method: {f.transfer_method}")
def _download_file_content(path: str, /):
"""
Download and return the contents of a file as bytes.
This function loads the file from storage and ensures it's in bytes format.
Args:
path (str): The path to the file in storage.
Returns:
bytes: The contents of the file as a bytes object.
Raises:
ValueError: If the loaded file is not a bytes object.
"""
data = storage.load(path, stream=False)
def _download_file_content(path: str, /) -> bytes:
"""Download and return a file from storage as bytes."""
data = get_workflow_file_runtime().storage_load(path, stream=False)
if not isinstance(data, bytes):
raise ValueError(f"file {path} is not a bytes object")
return data
def _get_encoded_string(f: File, /):
def _get_encoded_string(f: File, /) -> str:
match f.transfer_method:
case FileTransferMethod.REMOTE_URL:
if f.remote_url is None:
raise ValueError("Missing file remote_url")
response = ssrf_proxy.get(f.remote_url, follow_redirects=True)
response = get_workflow_file_runtime().http_get(f.remote_url, follow_redirects=True)
response.raise_for_status()
data = response.content
case FileTransferMethod.LOCAL_FILE:
@ -167,8 +133,7 @@ def _get_encoded_string(f: File, /):
case FileTransferMethod.DATASOURCE_FILE:
data = _download_file_content(f.storage_key)
encoded_string = base64.b64encode(data).decode("utf-8")
return encoded_string
return base64.b64encode(data).decode("utf-8")
def _to_url(f: File, /):
@ -181,10 +146,9 @@ def _to_url(f: File, /):
raise ValueError("Missing file related_id")
return f.remote_url or helpers.get_signed_file_url(upload_file_id=f.related_id)
elif f.transfer_method == FileTransferMethod.TOOL_FILE:
# add sign url
if f.related_id is None or f.extension is None:
raise ValueError("Missing file related_id or extension")
return sign_tool_file(tool_file_id=f.related_id, extension=f.extension)
return helpers.get_signed_tool_file_url(tool_file_id=f.related_id, extension=f.extension)
else:
raise ValueError(f"Unsupported transfer method: {f.transfer_method}")
@ -315,12 +279,7 @@ def _build_file_from_ref(
class FileManager:
"""
Adapter exposing file manager helpers behind FileManagerProtocol.
This is intentionally a thin wrapper over the existing module-level functions so callers can inject it
where a protocol-typed file manager is expected.
"""
"""Adapter exposing file manager helpers behind FileManagerProtocol."""
def download(self, f: File, /) -> bytes:
return download(f)

43
api/core/file/helpers.py → api/core/workflow/file/helpers.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import base64
import hashlib
import hmac
@ -5,20 +7,21 @@ import os
import time
import urllib.parse
from configs import dify_config
from .runtime import get_workflow_file_runtime
def get_signed_file_url(upload_file_id: str, as_attachment=False, for_external: bool = True) -> str:
base_url = dify_config.FILES_URL if for_external else (dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL)
def get_signed_file_url(upload_file_id: str, as_attachment: bool = False, for_external: bool = True) -> str:
runtime = get_workflow_file_runtime()
base_url = runtime.files_url if for_external else (runtime.internal_files_url or runtime.files_url)
url = f"{base_url}/files/{upload_file_id}/file-preview"
timestamp = str(int(time.time()))
nonce = os.urandom(16).hex()
key = dify_config.SECRET_KEY.encode()
key = runtime.secret_key.encode()
msg = f"file-preview|{upload_file_id}|{timestamp}|{nonce}"
sign = hmac.new(key, msg.encode(), hashlib.sha256).digest()
encoded_sign = base64.urlsafe_b64encode(sign).decode()
query = {"timestamp": timestamp, "nonce": nonce, "sign": encoded_sign}
query: dict[str, str] = {"timestamp": timestamp, "nonce": nonce, "sign": encoded_sign}
if as_attachment:
query["as_attachment"] = "true"
query_string = urllib.parse.urlencode(query)
@ -27,57 +30,63 @@ def get_signed_file_url(upload_file_id: str, as_attachment=False, for_external:
def get_signed_file_url_for_plugin(filename: str, mimetype: str, tenant_id: str, user_id: str) -> str:
# Plugin access should use internal URL for Docker network communication
base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL
runtime = get_workflow_file_runtime()
# Plugin access should use internal URL for Docker network communication.
base_url = runtime.internal_files_url or runtime.files_url
url = f"{base_url}/files/upload/for-plugin"
timestamp = str(int(time.time()))
nonce = os.urandom(16).hex()
key = dify_config.SECRET_KEY.encode()
key = runtime.secret_key.encode()
msg = f"upload|{filename}|{mimetype}|{tenant_id}|{user_id}|{timestamp}|{nonce}"
sign = hmac.new(key, msg.encode(), hashlib.sha256).digest()
encoded_sign = base64.urlsafe_b64encode(sign).decode()
return f"{url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}&user_id={user_id}&tenant_id={tenant_id}"
def get_signed_tool_file_url(tool_file_id: str, extension: str, for_external: bool = True) -> str:
runtime = get_workflow_file_runtime()
return runtime.sign_tool_file(tool_file_id=tool_file_id, extension=extension, for_external=for_external)
def verify_plugin_file_signature(
*, filename: str, mimetype: str, tenant_id: str, user_id: str, timestamp: str, nonce: str, sign: str
) -> bool:
runtime = get_workflow_file_runtime()
data_to_sign = f"upload|{filename}|{mimetype}|{tenant_id}|{user_id}|{timestamp}|{nonce}"
secret_key = dify_config.SECRET_KEY.encode()
secret_key = runtime.secret_key.encode()
recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode()
# verify signature
if sign != recalculated_encoded_sign:
return False
current_time = int(time.time())
return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT
return current_time - int(timestamp) <= runtime.files_access_timeout
def verify_image_signature(*, upload_file_id: str, timestamp: str, nonce: str, sign: str) -> bool:
runtime = get_workflow_file_runtime()
data_to_sign = f"image-preview|{upload_file_id}|{timestamp}|{nonce}"
secret_key = dify_config.SECRET_KEY.encode()
secret_key = runtime.secret_key.encode()
recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode()
# verify signature
if sign != recalculated_encoded_sign:
return False
current_time = int(time.time())
return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT
return current_time - int(timestamp) <= runtime.files_access_timeout
def verify_file_signature(*, upload_file_id: str, timestamp: str, nonce: str, sign: str) -> bool:
runtime = get_workflow_file_runtime()
data_to_sign = f"file-preview|{upload_file_id}|{timestamp}|{nonce}"
secret_key = dify_config.SECRET_KEY.encode()
secret_key = runtime.secret_key.encode()
recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode()
# verify signature
if sign != recalculated_encoded_sign:
return False
current_time = int(time.time())
return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT
return current_time - int(timestamp) <= runtime.files_access_timeout

22
api/core/file/models.py → api/core/workflow/file/models.py
View File

@ -1,16 +1,26 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from typing import Any
from pydantic import BaseModel, Field, model_validator
from core.model_runtime.entities.message_entities import ImagePromptMessageContent
from core.tools.signature import sign_tool_file
from . import helpers
from .constants import FILE_MODEL_IDENTITY
from .enums import FileTransferMethod, FileType
def sign_tool_file(*, tool_file_id: str, extension: str, for_external: bool = True) -> str:
"""Compatibility shim for tests and legacy callers patching ``models.sign_tool_file``."""
return helpers.get_signed_tool_file_url(
tool_file_id=tool_file_id,
extension=extension,
for_external=for_external,
)
class ImageConfig(BaseModel):
"""
NOTE: This part of validation is deprecated, but still used in app features "Image Upload".
@ -122,7 +132,11 @@ class File(BaseModel):
elif self.transfer_method in [FileTransferMethod.TOOL_FILE, FileTransferMethod.DATASOURCE_FILE]:
assert self.related_id is not None
assert self.extension is not None
return sign_tool_file(tool_file_id=self.related_id, extension=self.extension, for_external=for_external)
return sign_tool_file(
tool_file_id=self.related_id,
extension=self.extension,
for_external=for_external,
)
return None
def to_plugin_parameter(self) -> dict[str, Any]:
@ -137,7 +151,7 @@ class File(BaseModel):
}
@model_validator(mode="after")
def validate_after(self):
def validate_after(self) -> File:
match self.transfer_method:
case FileTransferMethod.REMOTE_URL:
if not self.remote_url:
@ -160,5 +174,5 @@ class File(BaseModel):
return self._storage_key
@storage_key.setter
def storage_key(self, value: str):
def storage_key(self, value: str) -> None:
self._storage_key = value

43
api/core/workflow/file/protocols.py Normal file
View File

@ -0,0 +1,43 @@
from __future__ import annotations
from collections.abc import Generator
from typing import Protocol
class HttpResponseProtocol(Protocol):
"""Subset of response behavior needed by workflow file helpers."""
@property
def content(self) -> bytes: ...
def raise_for_status(self) -> object: ...
class WorkflowFileRuntimeProtocol(Protocol):
"""Runtime dependencies required by ``core.workflow.file``.
Implementations are expected to be provided by integration layers (for example,
``core.app.workflow.file_runtime``) so the workflow package avoids importing
application infrastructure modules directly.
"""
@property
def files_url(self) -> str: ...
@property
def internal_files_url(self) -> str | None: ...
@property
def secret_key(self) -> str: ...
@property
def files_access_timeout(self) -> int: ...
@property
def multimodal_send_format(self) -> str: ...
def http_get(self, url: str, *, follow_redirects: bool = True) -> HttpResponseProtocol: ...
def storage_load(self, path: str, *, stream: bool = False) -> bytes | Generator: ...
def sign_tool_file(self, *, tool_file_id: str, extension: str, for_external: bool = True) -> str: ...

58
api/core/workflow/file/runtime.py Normal file
View File

@ -0,0 +1,58 @@
from __future__ import annotations
from collections.abc import Generator
from typing import NoReturn
from .protocols import HttpResponseProtocol, WorkflowFileRuntimeProtocol
class WorkflowFileRuntimeNotConfiguredError(RuntimeError):
"""Raised when workflow file runtime dependencies were not configured."""
class _UnconfiguredWorkflowFileRuntime(WorkflowFileRuntimeProtocol):
def _raise(self) -> NoReturn:
raise WorkflowFileRuntimeNotConfiguredError(
"workflow file runtime is not configured, call set_workflow_file_runtime(...) first"
)
@property
def files_url(self) -> str:
self._raise()
@property
def internal_files_url(self) -> str | None:
self._raise()
@property
def secret_key(self) -> str:
self._raise()
@property
def files_access_timeout(self) -> int:
self._raise()
@property
def multimodal_send_format(self) -> str:
self._raise()
def http_get(self, url: str, *, follow_redirects: bool = True) -> HttpResponseProtocol:
self._raise()
def storage_load(self, path: str, *, stream: bool = False) -> bytes | Generator:
self._raise()
def sign_tool_file(self, *, tool_file_id: str, extension: str, for_external: bool = True) -> str:
self._raise()
_runtime: WorkflowFileRuntimeProtocol = _UnconfiguredWorkflowFileRuntime()
def set_workflow_file_runtime(runtime: WorkflowFileRuntimeProtocol) -> None:
global _runtime
_runtime = runtime
def get_workflow_file_runtime() -> WorkflowFileRuntimeProtocol:
return _runtime

9
api/core/workflow/file/tool_file_parser.py Normal file
View File

@ -0,0 +1,9 @@
from collections.abc import Callable
from typing import Any
_tool_file_manager_factory: Callable[[], Any] | None = None
def set_tool_file_manager_factory(factory: Callable[[], Any]):
global _tool_file_manager_factory
_tool_file_manager_factory = factory

2
api/core/workflow/node_events/node.py
View File

@ -4,11 +4,11 @@ from enum import StrEnum
from pydantic import Field
from core.file import File
from core.model_runtime.entities.llm_entities import LLMUsage
from core.rag.entities.citation_metadata import RetrievalSourceMetadata
from core.workflow.entities import ToolCall, ToolResult
from core.workflow.entities.pause_reason import PauseReason
from core.workflow.file import File
from core.workflow.node_events import NodeRunResult
from .base import NodeEventBase

2
api/core/workflow/nodes/agent/agent_node.py
View File

@ -11,7 +11,6 @@ from sqlalchemy.orm import Session
from core.agent.entities import AgentToolEntity
from core.agent.plugin_entities import AgentStrategyParameter
from core.file import File, FileTransferMethod
from core.memory.base import BaseMemory
from core.memory.node_token_buffer_memory import NodeTokenBufferMemory
from core.memory.token_buffer_memory import TokenBufferMemory
@ -42,6 +41,7 @@ from core.workflow.enums import (
WorkflowNodeExecutionMetadataKey,
WorkflowNodeExecutionStatus,
)
from core.workflow.file import File, FileTransferMethod
from core.workflow.node_events import (
AgentLogEvent,
NodeEventBase,

4
api/core/workflow/nodes/datasource/datasource_node.py
View File

@ -14,13 +14,13 @@ from core.datasource.entities.datasource_entities import (
from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
from core.datasource.utils.message_transformer import DatasourceFileMessageTransformer
from core.file import File
from core.file.enums import FileTransferMethod, FileType
from core.plugin.impl.exc import PluginDaemonClientSideError
from core.variables.segments import ArrayAnySegment
from core.variables.variables import ArrayAnyVariable
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
from core.workflow.enums import NodeExecutionType, NodeType, SystemVariableKey
from core.workflow.file import File
from core.workflow.file.enums import FileTransferMethod, FileType
from core.workflow.node_events import NodeRunResult, StreamChunkEvent, StreamCompletedEvent
from core.workflow.nodes.base.node import Node
from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser

4
api/core/workflow/nodes/document_extractor/__init__.py
View File

@ -1,4 +1,4 @@
from .entities import DocumentExtractorNodeData
from .entities import DocumentExtractorNodeData, UnstructuredApiConfig
from .node import DocumentExtractorNode
__all__ = ["DocumentExtractorNode", "DocumentExtractorNodeData"]
__all__ = ["DocumentExtractorNode", "DocumentExtractorNodeData", "UnstructuredApiConfig"]

7
api/core/workflow/nodes/document_extractor/entities.py
View File

@ -1,7 +1,14 @@
from collections.abc import Sequence
from dataclasses import dataclass
from core.workflow.nodes.base import BaseNodeData
class DocumentExtractorNodeData(BaseNodeData):
variable_selector: Sequence[str]
@dataclass(frozen=True)
class UnstructuredApiConfig:
api_url: str | None = None
api_key: str = ""

120
api/core/workflow/nodes/document_extractor/node.py
View File

@ -5,7 +5,7 @@ import logging
import os
import tempfile
from collections.abc import Mapping, Sequence
from typing import Any
from typing import TYPE_CHECKING, Any
import charset_normalizer
import docx
@ -20,20 +20,23 @@ from docx.oxml.text.paragraph import CT_P
from docx.table import Table
from docx.text.paragraph import Paragraph
from configs import dify_config
from core.file import File, FileTransferMethod, file_manager
from core.helper import ssrf_proxy
from core.variables import ArrayFileSegment
from core.variables.segments import ArrayStringSegment, FileSegment
from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
from core.workflow.file import File, FileTransferMethod, file_manager
from core.workflow.node_events import NodeRunResult
from core.workflow.nodes.base.node import Node
from .entities import DocumentExtractorNodeData
from .entities import DocumentExtractorNodeData, UnstructuredApiConfig
from .exc import DocumentExtractorError, FileDownloadError, TextExtractionError, UnsupportedFileTypeError
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from core.workflow.entities import GraphInitParams
from core.workflow.runtime import GraphRuntimeState
class DocumentExtractorNode(Node[DocumentExtractorNodeData]):
"""
@ -47,6 +50,23 @@ class DocumentExtractorNode(Node[DocumentExtractorNodeData]):
def version(cls) -> str:
return "1"
def __init__(
self,
id: str,
config: Mapping[str, Any],
graph_init_params: "GraphInitParams",
graph_runtime_state: "GraphRuntimeState",
*,
unstructured_api_config: UnstructuredApiConfig | None = None,
) -> None:
super().__init__(
id=id,
config=config,
graph_init_params=graph_init_params,
graph_runtime_state=graph_runtime_state,
)
self._unstructured_api_config = unstructured_api_config or UnstructuredApiConfig()
def _run(self):
variable_selector = self.node_data.variable_selector
variable = self.graph_runtime_state.variable_pool.get(variable_selector)
@ -64,7 +84,10 @@ class DocumentExtractorNode(Node[DocumentExtractorNodeData]):
try:
if isinstance(value, list):
extracted_text_list = list(map(_extract_text_from_file, value))
extracted_text_list = [
_extract_text_from_file(file, unstructured_api_config=self._unstructured_api_config)
for file in value
]
return NodeRunResult(
status=WorkflowNodeExecutionStatus.SUCCEEDED,
inputs=inputs,
@ -72,7 +95,7 @@ class DocumentExtractorNode(Node[DocumentExtractorNodeData]):
outputs={"text": ArrayStringSegment(value=extracted_text_list)},
)
elif isinstance(value, File):
extracted_text = _extract_text_from_file(value)
extracted_text = _extract_text_from_file(value, unstructured_api_config=self._unstructured_api_config)
return NodeRunResult(
status=WorkflowNodeExecutionStatus.SUCCEEDED,
inputs=inputs,
@ -103,7 +126,12 @@ class DocumentExtractorNode(Node[DocumentExtractorNodeData]):
return {node_id + ".files": typed_node_data.variable_selector}
def _extract_text_by_mime_type(*, file_content: bytes, mime_type: str) -> str:
def _extract_text_by_mime_type(
*,
file_content: bytes,
mime_type: str,
unstructured_api_config: UnstructuredApiConfig,
) -> str:
"""Extract text from a file based on its MIME type."""
match mime_type:
case "text/plain" | "text/html" | "text/htm" | "text/markdown" | "text/xml":
@ -111,7 +139,7 @@ def _extract_text_by_mime_type(*, file_content: bytes, mime_type: str) -> str:
case "application/pdf":
return _extract_text_from_pdf(file_content)
case "application/msword":
return _extract_text_from_doc(file_content)
return _extract_text_from_doc(file_content, unstructured_api_config=unstructured_api_config)
case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
return _extract_text_from_docx(file_content)
case "text/csv":
@ -119,11 +147,11 @@ def _extract_text_by_mime_type(*, file_content: bytes, mime_type: str) -> str:
case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" | "application/vnd.ms-excel":
return _extract_text_from_excel(file_content)
case "application/vnd.ms-powerpoint":
return _extract_text_from_ppt(file_content)
return _extract_text_from_ppt(file_content, unstructured_api_config=unstructured_api_config)
case "application/vnd.openxmlformats-officedocument.presentationml.presentation":
return _extract_text_from_pptx(file_content)
return _extract_text_from_pptx(file_content, unstructured_api_config=unstructured_api_config)
case "application/epub+zip":
return _extract_text_from_epub(file_content)
return _extract_text_from_epub(file_content, unstructured_api_config=unstructured_api_config)
case "message/rfc822":
return _extract_text_from_eml(file_content)
case "application/vnd.ms-outlook":
@ -140,7 +168,12 @@ def _extract_text_by_mime_type(*, file_content: bytes, mime_type: str) -> str:
raise UnsupportedFileTypeError(f"Unsupported MIME type: {mime_type}")
def _extract_text_by_file_extension(*, file_content: bytes, file_extension: str) -> str:
def _extract_text_by_file_extension(
*,
file_content: bytes,
file_extension: str,
unstructured_api_config: UnstructuredApiConfig,
) -> str:
"""Extract text from a file based on its file extension."""
match file_extension:
case (
@ -203,7 +236,7 @@ def _extract_text_by_file_extension(*, file_content: bytes, file_extension: str)
case ".pdf":
return _extract_text_from_pdf(file_content)
case ".doc":
return _extract_text_from_doc(file_content)
return _extract_text_from_doc(file_content, unstructured_api_config=unstructured_api_config)
case ".docx":
return _extract_text_from_docx(file_content)
case ".csv":
@ -211,11 +244,11 @@ def _extract_text_by_file_extension(*, file_content: bytes, file_extension: str)
case ".xls" | ".xlsx":
return _extract_text_from_excel(file_content)
case ".ppt":
return _extract_text_from_ppt(file_content)
return _extract_text_from_ppt(file_content, unstructured_api_config=unstructured_api_config)
case ".pptx":
return _extract_text_from_pptx(file_content)
return _extract_text_from_pptx(file_content, unstructured_api_config=unstructured_api_config)
case ".epub":
return _extract_text_from_epub(file_content)
return _extract_text_from_epub(file_content, unstructured_api_config=unstructured_api_config)
case ".eml":
return _extract_text_from_eml(file_content)
case ".msg":
@ -312,14 +345,15 @@ def _extract_text_from_pdf(file_content: bytes) -> str:
raise TextExtractionError(f"Failed to extract text from PDF: {str(e)}") from e
def _extract_text_from_doc(file_content: bytes) -> str:
def _extract_text_from_doc(file_content: bytes, *, unstructured_api_config: UnstructuredApiConfig) -> str:
"""
Extract text from a DOC file.
"""
from unstructured.partition.api import partition_via_api
if not dify_config.UNSTRUCTURED_API_URL:
raise TextExtractionError("UNSTRUCTURED_API_URL must be set")
if not unstructured_api_config.api_url:
raise TextExtractionError("Unstructured API URL is not configured for DOC file processing.")
api_key = unstructured_api_config.api_key or ""
try:
with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as temp_file:
@ -329,8 +363,8 @@ def _extract_text_from_doc(file_content: bytes) -> str:
elements = partition_via_api(
file=file,
metadata_filename=temp_file.name,
api_url=dify_config.UNSTRUCTURED_API_URL,
api_key=dify_config.UNSTRUCTURED_API_KEY, # type: ignore
api_url=unstructured_api_config.api_url,
api_key=api_key,
)
os.unlink(temp_file.name)
return "\n".join([getattr(element, "text", "") for element in elements])
@ -420,12 +454,20 @@ def _download_file_content(file: File) -> bytes:
raise FileDownloadError(f"Error downloading file: {str(e)}") from e
def _extract_text_from_file(file: File):
def _extract_text_from_file(file: File, *, unstructured_api_config: UnstructuredApiConfig) -> str:
file_content = _download_file_content(file)
if file.extension:
extracted_text = _extract_text_by_file_extension(file_content=file_content, file_extension=file.extension)
extracted_text = _extract_text_by_file_extension(
file_content=file_content,
file_extension=file.extension,
unstructured_api_config=unstructured_api_config,
)
elif file.mime_type:
extracted_text = _extract_text_by_mime_type(file_content=file_content, mime_type=file.mime_type)
extracted_text = _extract_text_by_mime_type(
file_content=file_content,
mime_type=file.mime_type,
unstructured_api_config=unstructured_api_config,
)
else:
raise UnsupportedFileTypeError("Unable to determine file type: MIME type or file extension is missing")
return extracted_text
@ -517,12 +559,14 @@ def _extract_text_from_excel(file_content: bytes) -> str:
raise TextExtractionError(f"Failed to extract text from Excel file: {str(e)}") from e
def _extract_text_from_ppt(file_content: bytes) -> str:
def _extract_text_from_ppt(file_content: bytes, *, unstructured_api_config: UnstructuredApiConfig) -> str:
from unstructured.partition.api import partition_via_api
from unstructured.partition.ppt import partition_ppt
api_key = unstructured_api_config.api_key or ""
try:
if dify_config.UNSTRUCTURED_API_URL:
if unstructured_api_config.api_url:
with tempfile.NamedTemporaryFile(suffix=".ppt", delete=False) as temp_file:
temp_file.write(file_content)
temp_file.flush()
@ -530,8 +574,8 @@ def _extract_text_from_ppt(file_content: bytes) -> str:
elements = partition_via_api(
file=file,
metadata_filename=temp_file.name,
api_url=dify_config.UNSTRUCTURED_API_URL,
api_key=dify_config.UNSTRUCTURED_API_KEY, # type: ignore
api_url=unstructured_api_config.api_url,
api_key=api_key,
)
os.unlink(temp_file.name)
else:
@ -543,12 +587,14 @@ def _extract_text_from_ppt(file_content: bytes) -> str:
raise TextExtractionError(f"Failed to extract text from PPTX: {str(e)}") from e
def _extract_text_from_pptx(file_content: bytes) -> str:
def _extract_text_from_pptx(file_content: bytes, *, unstructured_api_config: UnstructuredApiConfig) -> str:
from unstructured.partition.api import partition_via_api
from unstructured.partition.pptx import partition_pptx
api_key = unstructured_api_config.api_key or ""
try:
if dify_config.UNSTRUCTURED_API_URL:
if unstructured_api_config.api_url:
with tempfile.NamedTemporaryFile(suffix=".pptx", delete=False) as temp_file:
temp_file.write(file_content)
temp_file.flush()
@ -556,8 +602,8 @@ def _extract_text_from_pptx(file_content: bytes) -> str:
elements = partition_via_api(
file=file,
metadata_filename=temp_file.name,
api_url=dify_config.UNSTRUCTURED_API_URL,
api_key=dify_config.UNSTRUCTURED_API_KEY, # type: ignore
api_url=unstructured_api_config.api_url,
api_key=api_key,
)
os.unlink(temp_file.name)
else:
@ -568,12 +614,14 @@ def _extract_text_from_pptx(file_content: bytes) -> str:
raise TextExtractionError(f"Failed to extract text from PPTX: {str(e)}") from e
def _extract_text_from_epub(file_content: bytes) -> str:
def _extract_text_from_epub(file_content: bytes, *, unstructured_api_config: UnstructuredApiConfig) -> str:
from unstructured.partition.api import partition_via_api
from unstructured.partition.epub import partition_epub
api_key = unstructured_api_config.api_key or ""
try:
if dify_config.UNSTRUCTURED_API_URL:
if unstructured_api_config.api_url:
with tempfile.NamedTemporaryFile(suffix=".epub", delete=False) as temp_file:
temp_file.write(file_content)
temp_file.flush()
@ -581,8 +629,8 @@ def _extract_text_from_epub(file_content: bytes) -> str:
elements = partition_via_api(
file=file,
metadata_filename=temp_file.name,
api_url=dify_config.UNSTRUCTURED_API_URL,
api_key=dify_config.UNSTRUCTURED_API_KEY, # type: ignore
api_url=unstructured_api_config.api_url,
api_key=api_key,
)
os.unlink(temp_file.name)
else:

8
api/core/workflow/nodes/http_request/executor.py
View File

@ -11,10 +11,10 @@ import httpx
from json_repair import repair_json
from configs import dify_config
from core.file.enums import FileTransferMethod
from core.file.file_manager import file_manager as default_file_manager
from core.helper.ssrf_proxy import ssrf_proxy
from core.variables.segments import ArrayFileSegment, FileSegment
from core.workflow.file.enums import FileTransferMethod
from core.workflow.file.file_manager import file_manager as default_file_manager
from core.workflow.runtime import VariablePool
from ..protocols import FileManagerProtocol, HttpClientProtocol
@ -366,7 +366,9 @@ class Executor:
**request_args,
max_retries=self.max_retries,
)
except (self._http_client.max_retries_exceeded_error, self._http_client.request_error) as e:
except self._http_client.max_retries_exceeded_error as e:
raise HttpRequestNodeError(f"Reached maximum retries for URL {self.url}") from e
except self._http_client.request_error as e:
raise HttpRequestNodeError(str(e)) from e
return response

4
api/core/workflow/nodes/http_request/node.py
View File

@ -4,12 +4,12 @@ from collections.abc import Callable, Mapping, Sequence
from typing import TYPE_CHECKING, Any
from configs import dify_config
from core.file import File, FileTransferMethod
from core.file.file_manager import file_manager as default_file_manager
from core.helper.ssrf_proxy import ssrf_proxy
from core.tools.tool_file_manager import ToolFileManager
from core.variables.segments import ArrayFileSegment
from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
from core.workflow.file import File, FileTransferMethod
from core.workflow.file.file_manager import file_manager as default_file_manager
from core.workflow.node_events import NodeRunResult
from core.workflow.nodes.base import variable_template_parser
from core.workflow.nodes.base.entities import VariableSelector

2
api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py
View File

@ -30,7 +30,7 @@ from .exc import (
)
if TYPE_CHECKING:
from core.file.models import File
from core.workflow.file.models import File
from core.workflow.runtime import GraphRuntimeState
logger = logging.getLogger(__name__)

2
api/core/workflow/nodes/list_operator/node.py
View File

@ -1,10 +1,10 @@
from collections.abc import Callable, Sequence
from typing import Any, TypeAlias, TypeVar
from core.file import File
from core.variables import ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment
from core.variables.segments import ArrayAnySegment, ArrayBooleanSegment, ArraySegment
from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
from core.workflow.file import File
from core.workflow.node_events import NodeRunResult
from core.workflow.nodes.base.node import Node

2
api/core/workflow/nodes/llm/file_saver.py
View File

@ -4,10 +4,10 @@ import typing as tp
from sqlalchemy import Engine
from constants.mimetypes import DEFAULT_EXTENSION, DEFAULT_MIME_TYPE
from core.file import File, FileTransferMethod, FileType
from core.helper import ssrf_proxy
from core.tools.signature import sign_tool_file
from core.tools.tool_file_manager import ToolFileManager
from core.workflow.file import File, FileTransferMethod, FileType
from extensions.ext_database import db as global_db

2
api/core/workflow/nodes/llm/llm_utils.py
View File

@ -7,7 +7,6 @@ from sqlalchemy.orm import Session
from configs import dify_config
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
from core.entities.provider_entities import ProviderQuotaType, QuotaUnit
from core.file.models import File
from core.memory import NodeTokenBufferMemory, TokenBufferMemory
from core.memory.base import BaseMemory
from core.model_manager import ModelInstance, ModelManager
@ -25,6 +24,7 @@ from core.model_runtime.model_providers.__base.large_language_model import Large
from core.prompt.entities.advanced_prompt_entities import MemoryConfig, MemoryMode
from core.variables.segments import ArrayAnySegment, ArrayFileSegment, FileSegment, NoneSegment, StringSegment
from core.workflow.enums import SystemVariableKey
from core.workflow.file.models import File
from core.workflow.nodes.llm.entities import LLMGenerationData, ModelConfig
from core.workflow.runtime import VariablePool
from extensions.ext_database import db

4
api/core/workflow/nodes/llm/node.py
View File

@ -18,7 +18,6 @@ from sqlalchemy import select
from core.agent.entities import AgentEntity, AgentLog, AgentResult, AgentToolEntity, ExecutionContext
from core.agent.patterns import StrategyFactory
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
from core.file import File, FileTransferMethod, FileType, file_manager
from core.helper.code_executor import CodeExecutor, CodeLanguage
from core.llm_generator.output_parser.errors import OutputParserError
from core.llm_generator.output_parser.file_ref import (
@ -91,6 +90,7 @@ from core.workflow.enums import (
WorkflowNodeExecutionMetadataKey,
WorkflowNodeExecutionStatus,
)
from core.workflow.file import File, FileTransferMethod, FileType, file_manager
from core.workflow.node_events import (
AgentLogEvent,
ModelInvokeCompletedEvent,
@ -144,7 +144,7 @@ from .exc import (
from .file_saver import FileSaverImpl, LLMFileSaver
if TYPE_CHECKING:
from core.file.models import File
from core.workflow.file.models import File
from core.workflow.runtime import GraphRuntimeState
logger = logging.getLogger(__name__)

6
api/core/workflow/nodes/loop/loop_node.py
View File

@ -71,9 +71,9 @@ class LoopNode(LLMUsageTrackingMixin, Node[LoopNodeData]):
if self.node_data.loop_variables:
value_processor: dict[Literal["constant", "variable"], Callable[[LoopVariableData], Segment | None]] = {
"constant": lambda var: self._get_segment_for_constant(var.var_type, var.value),
"variable": lambda var: self.graph_runtime_state.variable_pool.get(var.value)
if isinstance(var.value, list)
else None,
"variable": lambda var: (
self.graph_runtime_state.variable_pool.get(var.value) if isinstance(var.value, list) else None
),
}
for loop_variable in self.node_data.loop_variables:
if loop_variable.value_type not in value_processor:

2
api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py
View File

@ -6,7 +6,6 @@ from collections.abc import Mapping, Sequence
from typing import Any, cast
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
from core.file import File
from core.memory.base import BaseMemory
from core.model_manager import ModelInstance
from core.model_runtime.entities import ImagePromptMessageContent
@ -28,6 +27,7 @@ from core.prompt.simple_prompt_transform import ModelMode
from core.prompt.utils.prompt_message_util import PromptMessageUtil
from core.variables.types import ArrayValidation, SegmentType
from core.workflow.enums import NodeType, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus
from core.workflow.file import File
from core.workflow.node_events import NodeRunResult
from core.workflow.nodes.base import variable_template_parser
from core.workflow.nodes.base.node import Node

2
api/core/workflow/nodes/protocols.py
View File

@ -2,7 +2,7 @@ from typing import Any, Protocol
import httpx
from core.file import File
from core.workflow.file import File
class HttpClientProtocol(Protocol):

2
api/core/workflow/nodes/question_classifier/question_classifier_node.py
View File

@ -39,7 +39,7 @@ from .template_prompts import (
)
if TYPE_CHECKING:
from core.file.models import File
from core.workflow.file.models import File
from core.workflow.runtime import GraphRuntimeState

2
api/core/workflow/nodes/tool/tool_node.py
View File

@ -8,7 +8,6 @@ logger = logging.getLogger(__name__)
from sqlalchemy.orm import Session
from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler
from core.file import File, FileTransferMethod
from core.model_runtime.entities.llm_entities import LLMUsage
from core.tools.__base.tool import Tool
from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter
@ -23,6 +22,7 @@ from core.workflow.enums import (
WorkflowNodeExecutionMetadataKey,
WorkflowNodeExecutionStatus,
)
from core.workflow.file import File, FileTransferMethod
from core.workflow.node_events import NodeEventBase, NodeRunResult, StreamChunkEvent, StreamCompletedEvent
from core.workflow.nodes.base.node import Node
from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser

2
api/core/workflow/nodes/trigger_webhook/node.py
View File

@ -2,12 +2,12 @@ import logging
from collections.abc import Mapping
from typing import Any
from core.file import FileTransferMethod
from core.variables.types import SegmentType
from core.variables.variables import FileVariable
from core.workflow.constants import SYSTEM_VARIABLE_NODE_ID
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
from core.workflow.enums import NodeExecutionType, NodeType
from core.workflow.file import FileTransferMethod
from core.workflow.node_events import NodeRunResult
from core.workflow.nodes.base.node import Node
from factories import file_factory

2
api/core/workflow/runtime/variable_pool.py
View File

@ -8,7 +8,6 @@ from typing import Annotated, Any, Union, cast
from pydantic import BaseModel, Field
from core.file import File, FileAttribute, file_manager
from core.variables import Segment, SegmentGroup, VariableBase
from core.variables.consts import SELECTORS_LENGTH
from core.variables.segments import FileSegment, ObjectSegment
@ -19,6 +18,7 @@ from core.workflow.constants import (
RAG_PIPELINE_VARIABLE_NODE_ID,
SYSTEM_VARIABLE_NODE_ID,
)
from core.workflow.file import File, FileAttribute, file_manager
from core.workflow.system_variable import SystemVariable
from factories import variable_factory

2
api/core/workflow/system_variable.py
View File

@ -7,8 +7,8 @@ from uuid import uuid4
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator
from core.file.models import File
from core.workflow.enums import SystemVariableKey
from core.workflow.file.models import File
class SystemVariable(BaseModel):

2
api/core/workflow/utils/condition/processor.py
View File

@ -2,9 +2,9 @@ import json
from collections.abc import Mapping, Sequence
from typing import Literal, NamedTuple
from core.file import FileAttribute, file_manager
from core.variables import ArrayFileSegment
from core.variables.segments import ArrayBooleanSegment, BooleanSegment
from core.workflow.file import FileAttribute, file_manager
from core.workflow.runtime import VariablePool
from .entities import Condition, SubCondition, SupportedComparisonOperator

2
api/core/workflow/workflow_entry.py
View File

@ -9,11 +9,11 @@ from core.app.apps.exc import GenerateTaskStoppedError
from core.app.entities.app_invoke_entities import InvokeFrom
from core.app.workflow.layers.observability import ObservabilityLayer
from core.app.workflow.node_factory import DifyNodeFactory
from core.file.models import File
from core.sandbox import Sandbox
from core.workflow.constants import ENVIRONMENT_VARIABLE_NODE_ID
from core.workflow.entities import GraphInitParams
from core.workflow.errors import WorkflowNodeRunFailedError
from core.workflow.file.models import File
from core.workflow.graph import Graph
from core.workflow.graph_engine import GraphEngine, GraphEngineConfig
from core.workflow.graph_engine.command_channels import InMemoryChannel

2
api/core/workflow/workflow_type_encoder.py
View File

@ -4,8 +4,8 @@ from typing import Any, overload
from pydantic import BaseModel
from core.file.models import File
from core.variables import Segment
from core.workflow.file.models import File
class WorkflowRuntimeTypeConverter:

7
api/extensions/ext_storage.py
View File

@ -94,6 +94,10 @@ class Storage:
@overload
def load(self, filename: str, /, *, stream: Literal[True]) -> Generator: ...
# Keep a bool fallback overload for callers that forward a runtime bool flag.
@overload
def load(self, filename: str, /, *, stream: bool = False) -> Union[bytes, Generator]: ...
def load(self, filename: str, /, *, stream: bool = False) -> Union[bytes, Generator]:
if stream:
return self.load_stream(filename)
@ -133,3 +137,6 @@ storage = Storage()
def init_app(app: DifyApp):
storage.init_app(app)
from core.app.workflow.file_runtime import bind_dify_workflow_file_runtime
bind_dify_workflow_file_runtime()

2
api/extensions/otel/parser/base.py
View File

@ -9,9 +9,9 @@ from opentelemetry.trace import Span
from opentelemetry.trace.status import Status, StatusCode
from pydantic import BaseModel
from core.file.models import File
from core.variables import Segment
from core.workflow.enums import NodeType
from core.workflow.file.models import File
from core.workflow.graph_events import GraphNodeEventBase
from core.workflow.nodes.base.node import Node
from extensions.otel.semconv.gen_ai import ChainAttributes, GenAIAttributes

2
api/factories/file_factory.py
View File

@ -13,8 +13,8 @@ from sqlalchemy.orm import Session
from werkzeug.http import parse_options_header
from constants import AUDIO_EXTENSIONS, DOCUMENT_EXTENSIONS, IMAGE_EXTENSIONS, VIDEO_EXTENSIONS
from core.file import File, FileBelongsTo, FileTransferMethod, FileType, FileUploadConfig, helpers
from core.helper import ssrf_proxy
from core.workflow.file import File, FileBelongsTo, FileTransferMethod, FileType, FileUploadConfig, helpers
from extensions.ext_database import db
from models import MessageFile, ToolFile, UploadFile

2
api/factories/variable_factory.py
View File

@ -3,7 +3,6 @@ from typing import Any, cast
from uuid import uuid4
from configs import dify_config
from core.file import File
from core.model_runtime.entities import PromptMessage
from core.model_runtime.entities.message_entities import (
AssistantPromptMessage,
@ -54,6 +53,7 @@ from core.workflow.constants import (
CONVERSATION_VARIABLE_NODE_ID,
ENVIRONMENT_VARIABLE_NODE_ID,
)
from core.workflow.file import File
class UnsupportedSegmentTypeError(Exception):

2
api/fields/conversation_fields.py
View File

@ -5,7 +5,7 @@ from typing import Any, TypeAlias
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from core.file import File
from core.workflow.file import File
JSONValue: TypeAlias = Any

2
api/fields/member_fields.py
View File

@ -5,7 +5,7 @@ from datetime import datetime
from flask_restx import fields
from pydantic import BaseModel, ConfigDict, computed_field, field_validator
from core.file import helpers as file_helpers
from core.workflow.file import helpers as file_helpers
simple_account_fields = {
"id": fields.String,

2
api/fields/message_fields.py
View File

@ -7,7 +7,7 @@ from uuid import uuid4
from pydantic import BaseModel, ConfigDict, Field, field_validator
from core.entities.execution_extra_content import ExecutionExtraContentDomainModel
from core.file import File
from core.workflow.file import File
from fields.conversation_fields import AgentThought, JSONValue, MessageFile
JSONValueType: TypeAlias = JSONValue

2
api/fields/raws.py
View File

@ -1,6 +1,6 @@
from flask_restx import fields
from core.file import File
from core.workflow.file import File
class FilesContainedField(fields.Raw):

213
api/libs/db_migration_lock.py Normal file
View File

@ -0,0 +1,213 @@
"""
DB migration Redis lock with heartbeat renewal.
This is intentionally migration-specific. Background renewal is a trade-off that makes sense
for unbounded, blocking operations like DB migrations (DDL/DML) where the main thread cannot
periodically refresh the lock TTL.
Do NOT use this as a general-purpose lock primitive for normal application code. Prefer explicit
lock lifecycle management (e.g. redis-py Lock context manager + `extend()` / `reacquire()` from
the same thread) when execution flow is under control.
"""
from __future__ import annotations
import logging
import threading
from typing import Any
from redis.exceptions import LockNotOwnedError, RedisError
logger = logging.getLogger(__name__)
MIN_RENEW_INTERVAL_SECONDS = 0.1
DEFAULT_RENEW_INTERVAL_DIVISOR = 3
MIN_JOIN_TIMEOUT_SECONDS = 0.5
MAX_JOIN_TIMEOUT_SECONDS = 5.0
JOIN_TIMEOUT_MULTIPLIER = 2.0
class DbMigrationAutoRenewLock:
"""
Redis lock wrapper that automatically renews TTL while held (migration-only).
Notes:
- We force `thread_local=False` when creating the underlying redis-py lock, because the
lock token must be accessible from the heartbeat thread for `reacquire()` to work.
- `release_safely()` is best-effort: it never raises, so it won't mask the caller's
primary error/exit code.
"""
_redis_client: Any
_name: str
_ttl_seconds: float
_renew_interval_seconds: float
_log_context: str | None
_logger: logging.Logger
_lock: Any
_stop_event: threading.Event | None
_thread: threading.Thread | None
_acquired: bool
def __init__(
self,
redis_client: Any,
name: str,
ttl_seconds: float = 60,
renew_interval_seconds: float | None = None,
*,
logger: logging.Logger | None = None,
log_context: str | None = None,
) -> None:
self._redis_client = redis_client
self._name = name
self._ttl_seconds = float(ttl_seconds)
self._renew_interval_seconds = (
float(renew_interval_seconds)
if renew_interval_seconds is not None
else max(MIN_RENEW_INTERVAL_SECONDS, self._ttl_seconds / DEFAULT_RENEW_INTERVAL_DIVISOR)
)
self._logger = logger or logging.getLogger(__name__)
self._log_context = log_context
self._lock = None
self._stop_event = None
self._thread = None
self._acquired = False
@property
def name(self) -> str:
return self._name
def acquire(self, *args: Any, **kwargs: Any) -> bool:
"""
Acquire the lock and start heartbeat renewal on success.
Accepts the same args/kwargs as redis-py `Lock.acquire()`.
"""
# Prevent accidental double-acquire which could leave the previous heartbeat thread running.
if self._acquired:
raise RuntimeError("DB migration lock is already acquired; call release_safely() before acquiring again.")
# Reuse the lock object if we already created one.
if self._lock is None:
self._lock = self._redis_client.lock(
name=self._name,
timeout=self._ttl_seconds,
thread_local=False,
)
acquired = bool(self._lock.acquire(*args, **kwargs))
self._acquired = acquired
if acquired:
self._start_heartbeat()
return acquired
def owned(self) -> bool:
if self._lock is None:
return False
try:
return bool(self._lock.owned())
except Exception:
# Ownership checks are best-effort and must not break callers.
return False
def _start_heartbeat(self) -> None:
if self._lock is None:
return
if self._stop_event is not None:
return
self._stop_event = threading.Event()
self._thread = threading.Thread(
target=self._heartbeat_loop,
args=(self._lock, self._stop_event),
daemon=True,
name=f"DbMigrationAutoRenewLock({self._name})",
)
self._thread.start()
def _heartbeat_loop(self, lock: Any, stop_event: threading.Event) -> None:
while not stop_event.wait(self._renew_interval_seconds):
try:
lock.reacquire()
except LockNotOwnedError:
self._logger.warning(
"DB migration lock is no longer owned during heartbeat; stop renewing. log_context=%s",
self._log_context,
exc_info=True,
)
return
except RedisError:
self._logger.warning(
"Failed to renew DB migration lock due to Redis error; will retry. log_context=%s",
self._log_context,
exc_info=True,
)
except Exception:
self._logger.warning(
"Unexpected error while renewing DB migration lock; will retry. log_context=%s",
self._log_context,
exc_info=True,
)
def release_safely(self, *, status: str | None = None) -> None:
"""
Stop heartbeat and release lock. Never raises.
Args:
status: Optional caller-provided status (e.g. 'successful'/'failed') to add context to logs.
"""
lock = self._lock
if lock is None:
return
self._stop_heartbeat()
# Lock release errors should never mask the real error/exit code.
try:
lock.release()
except LockNotOwnedError:
self._logger.warning(
"DB migration lock not owned on release; ignoring. status=%s log_context=%s",
status,
self._log_context,
exc_info=True,
)
except RedisError:
self._logger.warning(
"Failed to release DB migration lock due to Redis error; ignoring. status=%s log_context=%s",
status,
self._log_context,
exc_info=True,
)
except Exception:
self._logger.warning(
"Unexpected error while releasing DB migration lock; ignoring. status=%s log_context=%s",
status,
self._log_context,
exc_info=True,
)
finally:
self._acquired = False
self._lock = None
def _stop_heartbeat(self) -> None:
if self._stop_event is None:
return
self._stop_event.set()
if self._thread is not None:
# Best-effort join: if Redis calls are blocked, the daemon thread may remain alive.
join_timeout_seconds = max(
MIN_JOIN_TIMEOUT_SECONDS,
min(MAX_JOIN_TIMEOUT_SECONDS, self._renew_interval_seconds * JOIN_TIMEOUT_MULTIPLIER),
)
self._thread.join(timeout=join_timeout_seconds)
if self._thread.is_alive():
self._logger.warning(
"DB migration lock heartbeat thread did not stop within %.2fs; ignoring. log_context=%s",
join_timeout_seconds,
self._log_context,
)
self._stop_event = None
self._thread = None

2
api/libs/helper.py
View File

@ -21,8 +21,8 @@ from pydantic.functional_validators import AfterValidator
from configs import dify_config
from core.app.features.rate_limiting.rate_limit import RateLimitGenerator
from core.file import helpers as file_helpers
from core.model_runtime.utils.encoders import jsonable_encoder
from core.workflow.file import helpers as file_helpers
from extensions.ext_redis import redis_client
if TYPE_CHECKING:

4
api/models/model.py
View File

@ -18,10 +18,10 @@ from sqlalchemy.orm import Mapped, Session, mapped_column
from configs import dify_config
from constants import DEFAULT_FILE_NUMBER_LIMITS
from core.file import FILE_MODEL_IDENTITY, File, FileTransferMethod
from core.file import helpers as file_helpers
from core.tools.signature import sign_tool_file
from core.workflow.enums import WorkflowExecutionStatus
from core.workflow.file import FILE_MODEL_IDENTITY, File, FileTransferMethod
from core.workflow.file import helpers as file_helpers
from libs.helper import generate_string # type: ignore[import-not-found]
from libs.uuid_utils import uuidv7

4
api/models/workflow.py
View File

@ -22,8 +22,6 @@ from sqlalchemy import (
from sqlalchemy.orm import Mapped, declared_attr, mapped_column
from typing_extensions import deprecated
from core.file.constants import maybe_file_object
from core.file.models import File
from core.variables import utils as variable_utils
from core.variables.variables import FloatVariable, IntegerVariable, StringVariable
from core.workflow.constants import (
@ -33,6 +31,8 @@ from core.workflow.constants import (
from core.workflow.entities.graph_config import NodeConfigDict, NodeConfigDictAdapter
from core.workflow.entities.pause_reason import HumanInputRequired, PauseReason, PauseReasonType, SchedulingPause
from core.workflow.enums import NodeType, WorkflowExecutionStatus
from core.workflow.file.constants import maybe_file_object
from core.workflow.file.models import File
from extensions.ext_storage import Storage
from factories.variable_factory import TypeMismatchError, build_segment_with_type
from libs.datetime_utils import naive_utc_now

14
api/pyproject.toml
View File

@ -23,7 +23,7 @@ dependencies = [
"flask-sqlalchemy~=3.1.1",
"gevent~=25.9.1",
"gevent-websocket~=0.10.1",
"gmpy2~=2.2.1",
"gmpy2~=2.3.0",
"google-api-core==2.18.0",
"google-api-python-client==2.189.0",
"google-auth==2.29.0",
@ -69,17 +69,17 @@ dependencies = [
"psycogreen~=1.0.2",
"psycopg2-binary~=2.9.6",
"pycryptodome==3.23.0",
"pydantic~=2.11.4",
"pydantic~=2.12.5",
"pydantic-extra-types~=2.10.3",
"pydantic-settings~=2.12.0",
"pyjwt~=2.10.1",
"pypdfium2==5.2.0",
"python-docx~=1.1.0",
"python-docx~=1.2.0",
"python-dotenv==1.0.1",
"python-socketio~=5.13.0",
"pyyaml~=6.0.1",
"readabilipy~=0.3.0",
"redis[hiredis]~=6.1.0",
"redis[hiredis]~=7.2.0",
"resend~=2.9.0",
"sentry-sdk[flask]~=2.28.0",
# opentelemetry-instrumentation==0.48b0 imports pkg_resources, removed for setuptools>=81.
@ -145,9 +145,9 @@ dev = [
"types-flask-cors~=5.0.0",
"types-flask-migrate~=4.1.0",
"types-gevent~=25.9.0",
"types-greenlet~=3.1.0",
"types-greenlet~=3.3.0",
"types-html5lib~=1.1.11",
"types-markdown~=3.7.0",
"types-markdown~=3.10.2",
"types-oauthlib~=3.2.0",
"types-objgraph~=3.6.0",
"types-olefile~=0.47.0",
@ -220,7 +220,7 @@ vdb = [
"clickzetta-connector-python>=0.8.102",
"couchbase~=4.3.0",
"elasticsearch==8.14.0",
"opensearch-py==2.4.0",
"opensearch-py==3.1.0",
"oracledb==3.3.0",
"pgvecto-rs[sqlalchemy]~=0.2.1",
"pgvector==0.2.5",

2
api/services/dataset_service.py
View File

@ -18,7 +18,6 @@ from werkzeug.exceptions import Forbidden, NotFound
from configs import dify_config
from core.db.session_factory import session_factory
from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
from core.file import helpers as file_helpers
from core.helper.name_generator import generate_incremental_name
from core.model_manager import ModelManager
from core.model_runtime.entities.model_entities import ModelFeature, ModelType
@ -26,6 +25,7 @@ from core.model_runtime.model_providers.__base.text_embedding_model import TextE
from core.rag.index_processor.constant.built_in_field import BuiltInField
from core.rag.index_processor.constant.index_type import IndexStructureType
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from core.workflow.file import helpers as file_helpers
from enums.cloud_plan import CloudPlan
from events.dataset_event import dataset_was_deleted
from events.document_event import document_was_deleted

2
api/services/file_service.py
View File

@ -19,8 +19,8 @@ from constants import (
IMAGE_EXTENSIONS,
VIDEO_EXTENSIONS,
)
from core.file import helpers as file_helpers
from core.rag.extractor.extract_processor import ExtractProcessor
from core.workflow.file import helpers as file_helpers
from extensions.ext_database import db
from extensions.ext_storage import storage
from libs.datetime_utils import naive_utc_now

71
api/services/plugin/plugin_service.py
View File

@ -3,13 +3,15 @@ from collections.abc import Mapping, Sequence
from mimetypes import guess_type
from pydantic import BaseModel
from sqlalchemy import select
from sqlalchemy import delete, select, update
from sqlalchemy.orm import Session
from yarl import URL
from configs import dify_config
from core.helper import marketplace
from core.helper.download import download_with_size_limit
from core.helper.marketplace import download_plugin_pkg
from core.helper.model_provider_cache import ProviderCredentialsCache, ProviderCredentialsCacheType
from core.plugin.entities.bundle import PluginBundleDependency
from core.plugin.entities.plugin import (
PluginDeclaration,
@ -28,7 +30,7 @@ from core.plugin.impl.debugging import PluginDebuggingClient
from core.plugin.impl.plugin import PluginInstaller
from extensions.ext_database import db
from extensions.ext_redis import redis_client
from models.provider import ProviderCredential
from models.provider import Provider, ProviderCredential
from models.provider_ids import GenericProviderID
from services.errors.plugin import PluginInstallationForbiddenError
from services.feature_service import FeatureService, PluginInstallationScope
@ -511,30 +513,55 @@ class PluginService:
manager = PluginInstaller()
# Get plugin info before uninstalling to delete associated credentials
try:
plugins = manager.list_plugins(tenant_id)
plugin = next((p for p in plugins if p.installation_id == plugin_installation_id), None)
plugins = manager.list_plugins(tenant_id)
plugin = next((p for p in plugins if p.installation_id == plugin_installation_id), None)
if plugin:
plugin_id = plugin.plugin_id
logger.info("Deleting credentials for plugin: %s", plugin_id)
if not plugin:
return manager.uninstall(tenant_id, plugin_installation_id)
# Delete provider credentials that match this plugin
credentials = db.session.scalars(
select(ProviderCredential).where(
ProviderCredential.tenant_id == tenant_id,
ProviderCredential.provider_name.like(f"{plugin_id}/%"),
)
).all()
with Session(db.engine) as session, session.begin():
plugin_id = plugin.plugin_id
logger.info("Deleting credentials for plugin: %s", plugin_id)
for cred in credentials:
db.session.delete(cred)
# Delete provider credentials that match this plugin
credential_ids = session.scalars(
select(ProviderCredential.id).where(
ProviderCredential.tenant_id == tenant_id,
ProviderCredential.provider_name.like(f"{plugin_id}/%"),
)
).all()
db.session.commit()
logger.info("Deleted %d credentials for plugin: %s", len(credentials), plugin_id)
except Exception as e:
logger.warning("Failed to delete credentials: %s", e)
# Continue with uninstall even if credential deletion fails
if not credential_ids:
logger.info("No credentials found for plugin: %s", plugin_id)
return manager.uninstall(tenant_id, plugin_installation_id)
provider_ids = session.scalars(
select(Provider.id).where(
Provider.tenant_id == tenant_id,
Provider.provider_name.like(f"{plugin_id}/%"),
Provider.credential_id.in_(credential_ids),
)
).all()
session.execute(update(Provider).where(Provider.id.in_(provider_ids)).values(credential_id=None))
for provider_id in provider_ids:
ProviderCredentialsCache(
tenant_id=tenant_id,
identity_id=provider_id,
cache_type=ProviderCredentialsCacheType.PROVIDER,
).delete()
session.execute(
delete(ProviderCredential).where(
ProviderCredential.id.in_(credential_ids),
)
)
logger.info(
"Completed deleting credentials and cleaning provider associations for plugin: %s",
plugin_id,
)
return manager.uninstall(tenant_id, plugin_installation_id)

2
api/services/trigger/webhook_service.py
View File

@ -15,10 +15,10 @@ from werkzeug.exceptions import RequestEntityTooLarge
from configs import dify_config
from core.app.entities.app_invoke_entities import InvokeFrom
from core.file.models import FileTransferMethod
from core.tools.tool_file_manager import ToolFileManager
from core.variables.types import SegmentType
from core.workflow.enums import NodeType
from core.workflow.file.models import FileTransferMethod
from enums.quota_type import QuotaType
from extensions.ext_database import db
from extensions.ext_redis import redis_client

2
api/services/variable_truncator.py
View File

@ -6,7 +6,6 @@ from collections.abc import Mapping
from typing import Any, Generic, TypeAlias, TypeVar, overload
from configs import dify_config
from core.file.models import File
from core.model_runtime.entities import PromptMessage
from core.variables.segments import (
ArrayFileSegment,
@ -21,6 +20,7 @@ from core.variables.segments import (
StringSegment,
)
from core.variables.utils import dumps_with_segments
from core.workflow.file.models import File
from core.workflow.nodes.variable_assigner.common.helpers import UpdatedVariable
_MAX_DEPTH = 100

2
api/services/workflow/workflow_converter.py
View File

@ -13,12 +13,12 @@ from core.app.app_config.entities import (
from core.app.apps.agent_chat.app_config_manager import AgentChatAppConfigManager
from core.app.apps.chat.app_config_manager import ChatAppConfigManager
from core.app.apps.completion.app_config_manager import CompletionAppConfigManager
from core.file.models import FileUploadConfig
from core.helper import encrypter
from core.model_runtime.entities.llm_entities import LLMMode
from core.model_runtime.utils.encoders import jsonable_encoder
from core.prompt.simple_prompt_transform import SimplePromptTransform
from core.prompt.utils.prompt_template_parser import PromptTemplateParser
from core.workflow.file.models import FileUploadConfig
from core.workflow.nodes import NodeType
from events.app_event import app_was_created
from extensions.ext_database import db

Some files were not shown because too many files have changed in this diff Show More