opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-08-28 18:24:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge new graph engine

Browse Source
This commit is contained in:
jyong 2025-08-27 17:46:46 +08:00
parent 6c8212d509
commit 90d72f5ddf
35 changed files with 552 additions and 617 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/commands.py
View File

@ -14,7 +14,7 @@ from sqlalchemy.exc import SQLAlchemyError
from configs import dify_config
from constants.languages import languages
from core.helper import encrypter
from core.plugin.entities.plugin import DatasourceProviderID, PluginInstallationSource
from core.plugin.entities.plugin import PluginInstallationSource
from core.plugin.impl.plugin import PluginInstaller
from core.rag.datasource.vdb.vector_factory import Vector
from core.rag.datasource.vdb.vector_type import VectorType
@ -35,7 +35,7 @@ from models.dataset import Document as DatasetDocument
from models.model import Account, App, AppAnnotationSetting, AppMode, Conversation, MessageAnnotation
from models.oauth import DatasourceOauthParamConfig, DatasourceProvider
from models.provider import Provider, ProviderModel
from models.provider_ids import ToolProviderID
from models.provider_ids import DatasourceProviderID, ToolProviderID
from models.source import DataSourceApiKeyAuthBinding, DataSourceOauthBinding
from models.tools import ToolOAuthSystemClient
from services.account_service import AccountService, RegisterService, TenantService

2
api/controllers/console/datasets/rag_pipeline/datasource_auth.py
View File

@ -11,10 +11,10 @@ from controllers.console.wraps import (
setup_required,
)
from core.model_runtime.errors.validate import CredentialsValidateFailedError
from core.plugin.entities.plugin import DatasourceProviderID
from core.plugin.impl.oauth import OAuthHandler
from libs.helper import StrLen
from libs.login import login_required
from models.provider_ids import DatasourceProviderID
from services.datasource_provider_service import DatasourceProviderService
from services.plugin.oauth_service import OAuthProxyService

5
api/controllers/console/datasets/rag_pipeline/rag_pipeline_draft_variable.py
View File

@ -17,10 +17,11 @@ from core.variables.segment_group import SegmentGroup
from core.variables.segments import ArrayFileSegment, FileSegment, Segment
from core.variables.types import SegmentType
from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID
from extensions.ext_database import db
from factories.file_factory import build_from_mapping, build_from_mappings
from factories.variable_factory import build_segment_with_type
from libs.login import current_user, login_required
from models import db
from models.account import Account
from models.dataset import Pipeline
from models.workflow import WorkflowDraftVariable
from services.rag_pipeline.rag_pipeline import RagPipelineService
@ -131,7 +132,7 @@ def _api_prerequisite(f):
@account_initialization_required
@get_rag_pipeline
def wrapper(*args, **kwargs):
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
return f(*args, **kwargs)

64
api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py
View File

@ -62,7 +62,7 @@ class DraftRagPipelineApi(Resource):
Get draft rag pipeline's workflow
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
# fetch draft workflow by app_model
@ -84,7 +84,7 @@ class DraftRagPipelineApi(Resource):
Sync draft workflow
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
content_type = request.headers.get("Content-Type", "")
@ -161,7 +161,7 @@ class RagPipelineDraftRunIterationNodeApi(Resource):
Run draft workflow iteration node
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
if not isinstance(current_user, Account):
@ -198,7 +198,7 @@ class RagPipelineDraftRunLoopNodeApi(Resource):
Run draft workflow loop node
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
if not isinstance(current_user, Account):
@ -235,7 +235,7 @@ class DraftRagPipelineRunApi(Resource):
Run draft workflow
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
if not isinstance(current_user, Account):
@ -272,7 +272,7 @@ class PublishedRagPipelineRunApi(Resource):
Run published workflow
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
if not isinstance(current_user, Account):
@ -384,8 +384,6 @@ class PublishedRagPipelineRunApi(Resource):
#
# return result
#
class RagPipelinePublishedDatasourceNodeRunApi(Resource):
@setup_required
@login_required
@ -396,7 +394,7 @@ class RagPipelinePublishedDatasourceNodeRunApi(Resource):
Run rag pipeline datasource
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
if not isinstance(current_user, Account):
@ -441,10 +439,7 @@ class RagPipelineDraftDatasourceNodeRunApi(Resource):
Run rag pipeline datasource
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
raise Forbidden()
if not isinstance(current_user, Account):
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
parser = reqparse.RequestParser()
@ -487,10 +482,7 @@ class RagPipelineDraftNodeRunApi(Resource):
Run draft workflow node
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
raise Forbidden()
if not isinstance(current_user, Account):
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
parser = reqparse.RequestParser()
@ -519,7 +511,7 @@ class RagPipelineTaskStopApi(Resource):
Stop workflow task
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, current_user.id)
@ -538,7 +530,7 @@ class PublishedRagPipelineApi(Resource):
Get published pipeline
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
if not pipeline.is_published:
return None
@ -558,10 +550,7 @@ class PublishedRagPipelineApi(Resource):
Publish workflow
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
raise Forbidden()
if not isinstance(current_user, Account):
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
rag_pipeline_service = RagPipelineService()
@ -595,7 +584,7 @@ class DefaultRagPipelineBlockConfigsApi(Resource):
Get default block config
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
# Get default block configs
@ -613,7 +602,7 @@ class DefaultRagPipelineBlockConfigApi(Resource):
Get default block config
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
if not isinstance(current_user, Account):
@ -659,7 +648,7 @@ class PublishedAllRagPipelineApi(Resource):
"""
Get published workflows
"""
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
parser = reqparse.RequestParser()
@ -708,10 +697,7 @@ class RagPipelineByIdApi(Resource):
Update workflow attributes
"""
# Check permission
if not current_user.is_editor:
raise Forbidden()
if not isinstance(current_user, Account):
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
parser = reqparse.RequestParser()
@ -767,7 +753,7 @@ class PublishedRagPipelineSecondStepApi(Resource):
Get second step parameters of rag pipeline
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
parser = reqparse.RequestParser()
parser.add_argument("node_id", type=str, required=True, location="args")
@ -792,7 +778,7 @@ class PublishedRagPipelineFirstStepApi(Resource):
Get first step parameters of rag pipeline
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
parser = reqparse.RequestParser()
parser.add_argument("node_id", type=str, required=True, location="args")
@ -817,7 +803,7 @@ class DraftRagPipelineFirstStepApi(Resource):
Get first step parameters of rag pipeline
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
parser = reqparse.RequestParser()
parser.add_argument("node_id", type=str, required=True, location="args")
@ -842,7 +828,7 @@ class DraftRagPipelineSecondStepApi(Resource):
Get second step parameters of rag pipeline
"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
parser = reqparse.RequestParser()
parser.add_argument("node_id", type=str, required=True, location="args")
@ -926,8 +912,11 @@ class DatasourceListApi(Resource):
@account_initialization_required
def get(self):
user = current_user
if not isinstance(user, Account):
raise Forbidden()
tenant_id = user.current_tenant_id
if not tenant_id:
raise Forbidden()
return jsonable_encoder(RagPipelineManageService.list_rag_pipeline_datasources(tenant_id))
@ -974,10 +963,7 @@ class RagPipelineDatasourceVariableApi(Resource):
"""
Set datasource variables
"""
if not current_user.is_editor:
raise Forbidden()
if not isinstance(current_user, Account):
if not isinstance(current_user, Account) or not current_user.is_editor:
raise Forbidden()
parser = reqparse.RequestParser()

4
api/controllers/console/datasets/wraps.py
View File

@ -5,6 +5,7 @@ from typing import Optional
from controllers.console.datasets.error import PipelineNotFoundError
from extensions.ext_database import db
from libs.login import current_user
from models.account import Account
from models.dataset import Pipeline
@ -17,6 +18,9 @@ def get_rag_pipeline(
if not kwargs.get("pipeline_id"):
raise ValueError("missing pipeline_id in path parameters")
if not isinstance(current_user, Account):
raise ValueError("current_user is not an account")
pipeline_id = kwargs.get("pipeline_id")
pipeline_id = str(pipeline_id)

3
api/controllers/service_api/dataset/document.py
View File

@ -133,6 +133,9 @@ class DocumentAddByTextApi(DatasetApiResource):
# validate args
DocumentService.document_create_args_validate(knowledge_config)
if not current_user:
raise ValueError("current_user is required")
try:
documents, batch = DocumentService.save_document_with_dataset_id(
dataset=dataset,

4
api/core/agent/base_agent_runner.py
View File

@ -90,7 +90,9 @@ class BaseAgentRunner(AppRunner):
tenant_id=tenant_id,
dataset_ids=app_config.dataset.dataset_ids if app_config.dataset else [],
retrieve_config=app_config.dataset.retrieve_config if app_config.dataset else None,
return_resource=app_config.additional_features.show_retrieve_source,
return_resource=(
app_config.additional_features.show_retrieve_source if app_config.additional_features else False
),
invoke_from=application_generate_entity.invoke_from,
hit_callback=hit_callback,
user_id=user_id,

2
api/core/app/apps/advanced_chat/app_generator.py
View File

@ -154,7 +154,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
if invoke_from == InvokeFrom.DEBUGGER:
# always enable retriever resource in debugger mode
app_config.additional_features.show_retrieve_source = True
app_config.additional_features.show_retrieve_source = True # type: ignore
workflow_run_id = str(uuid.uuid4())
# init application generate entity

4
api/core/app/apps/chat/app_runner.py
View File

@ -162,7 +162,9 @@ class ChatAppRunner(AppRunner):
config=app_config.dataset,
query=query,
invoke_from=application_generate_entity.invoke_from,
show_retrieve_source=app_config.additional_features.show_retrieve_source,
show_retrieve_source=(
app_config.additional_features.show_retrieve_source if app_config.additional_features else False
),
hit_callback=hit_callback,
memory=memory,
message_id=message.id,

2
api/core/app/apps/common/workflow_response_converter.py
View File

@ -36,8 +36,8 @@ from core.app.entities.task_entities import (
WorkflowStartStreamResponse,
)
from core.file import FILE_MODEL_IDENTITY, File
from core.tools.entities.tool_entities import ToolProviderType
from core.plugin.impl.datasource import PluginDatasourceManager
from core.tools.entities.tool_entities import ToolProviderType
from core.tools.tool_manager import ToolManager
from core.variables.segments import ArrayFileSegment, FileSegment, Segment
from core.workflow.entities import WorkflowExecution, WorkflowNodeExecution

63
api/core/app/apps/pipeline/pipeline_runner.py
View File

@ -1,8 +1,7 @@
import logging
from collections.abc import Mapping
from typing import Any, Optional, cast
import time
from typing import Optional, cast
from configs import dify_config
from core.app.apps.base_app_queue_manager import AppQueueManager
from core.app.apps.pipeline.pipeline_config_manager import PipelineConfig
from core.app.apps.workflow_app_runner import WorkflowBasedAppRunner
@ -11,10 +10,12 @@ from core.app.entities.app_invoke_entities import (
RagPipelineGenerateEntity,
)
from core.variables.variables import RAGPipelineVariable, RAGPipelineVariableInput
from core.workflow.callbacks import WorkflowCallback, WorkflowLoggingCallback
from core.workflow.entities.graph_init_params import GraphInitParams
from core.workflow.entities.graph_runtime_state import GraphRuntimeState
from core.workflow.entities.variable_pool import VariablePool
from core.workflow.graph_engine.entities.event import GraphEngineEvent, GraphRunFailedEvent
from core.workflow.graph_engine.entities.graph import Graph
from core.workflow.graph import Graph
from core.workflow.graph_events import GraphEngineEvent, GraphRunFailedEvent
from core.workflow.nodes.node_factory import DifyNodeFactory
from core.workflow.system_variable import SystemVariable
from core.workflow.variable_loader import VariableLoader
from core.workflow.workflow_entry import WorkflowEntry
@ -22,7 +23,7 @@ from extensions.ext_database import db
from models.dataset import Document, Pipeline
from models.enums import UserFrom
from models.model import EndUser
from models.workflow import Workflow, WorkflowType
from models.workflow import Workflow
logger = logging.getLogger(__name__)
@ -84,24 +85,30 @@ class PipelineRunner(WorkflowBasedAppRunner):
db.session.close()
workflow_callbacks: list[WorkflowCallback] = []
if dify_config.DEBUG:
workflow_callbacks.append(WorkflowLoggingCallback())
# if only single iteration run is requested
if self.application_generate_entity.single_iteration_run:
graph_runtime_state = GraphRuntimeState(
variable_pool=VariablePool.empty(),
start_at=time.time(),
)
# if only single iteration run is requested
graph, variable_pool = self._get_graph_and_variable_pool_of_single_iteration(
workflow=workflow,
node_id=self.application_generate_entity.single_iteration_run.node_id,
user_inputs=self.application_generate_entity.single_iteration_run.inputs,
graph_runtime_state=graph_runtime_state,
)
elif self.application_generate_entity.single_loop_run:
graph_runtime_state = GraphRuntimeState(
variable_pool=VariablePool.empty(),
start_at=time.time(),
)
# if only single loop run is requested
graph, variable_pool = self._get_graph_and_variable_pool_of_single_loop(
workflow=workflow,
node_id=self.application_generate_entity.single_loop_run.node_id,
user_inputs=self.application_generate_entity.single_loop_run.inputs,
graph_runtime_state=graph_runtime_state,
)
else:
inputs = self.application_generate_entity.inputs
@ -121,6 +128,7 @@ class PipelineRunner(WorkflowBasedAppRunner):
datasource_info=self.application_generate_entity.datasource_info,
invoke_from=self.application_generate_entity.invoke_from.value,
)
rag_pipeline_variables = []
if workflow.rag_pipeline_variables:
for v in workflow.rag_pipeline_variables:
@ -143,11 +151,13 @@ class PipelineRunner(WorkflowBasedAppRunner):
conversation_variables=[],
rag_pipeline_variables=rag_pipeline_variables,
)
graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())
# init graph
graph = self._init_rag_pipeline_graph(
graph_config=workflow.graph_dict,
graph_runtime_state=graph_runtime_state,
start_node_id=self.application_generate_entity.start_node_id,
workflow=workflow,
)
# RUN WORKFLOW
@ -155,7 +165,6 @@ class PipelineRunner(WorkflowBasedAppRunner):
tenant_id=workflow.tenant_id,
app_id=workflow.app_id,
workflow_id=workflow.id,
workflow_type=WorkflowType.value_of(workflow.type),
graph=graph,
graph_config=workflow.graph_dict,
user_id=self.application_generate_entity.user_id,
@ -166,11 +175,10 @@ class PipelineRunner(WorkflowBasedAppRunner):
),
invoke_from=self.application_generate_entity.invoke_from,
call_depth=self.application_generate_entity.call_depth,
variable_pool=variable_pool,
thread_pool_id=self.workflow_thread_pool_id,
graph_runtime_state=graph_runtime_state,
)
generator = workflow_entry.run(callbacks=workflow_callbacks)
generator = workflow_entry.run()
for event in generator:
self._update_document_status(
@ -194,10 +202,13 @@ class PipelineRunner(WorkflowBasedAppRunner):
# return workflow
return workflow
def _init_rag_pipeline_graph(self, graph_config: Mapping[str, Any], start_node_id: Optional[str] = None) -> Graph:
def _init_rag_pipeline_graph(
self, workflow: Workflow, graph_runtime_state: GraphRuntimeState, start_node_id: Optional[str] = None
) -> Graph:
"""
Init pipeline graph
"""
graph_config = workflow.graph_dict
if "nodes" not in graph_config or "edges" not in graph_config:
raise ValueError("nodes or edges not found in workflow graph")
@ -227,7 +238,23 @@ class PipelineRunner(WorkflowBasedAppRunner):
graph_config["nodes"] = real_run_nodes
graph_config["edges"] = real_edges
# init graph
graph = Graph.init(graph_config=graph_config)
# Create required parameters for Graph.init
graph_init_params = GraphInitParams(
tenant_id=workflow.tenant_id,
app_id=self._app_id,
workflow_id=workflow.id,
graph_config=graph_config,
user_id="",
user_from=UserFrom.ACCOUNT.value,
invoke_from=InvokeFrom.SERVICE_API.value,
call_depth=0,
)
node_factory = DifyNodeFactory(
graph_init_params=graph_init_params,
graph_runtime_state=graph_runtime_state,
)
graph = Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id=start_node_id)
if not graph:
raise ValueError("graph not found in workflow")

2
api/core/plugin/impl/datasource.py
View File

@ -10,13 +10,13 @@ from core.datasource.entities.datasource_entities import (
OnlineDriveDownloadFileRequest,
WebsiteCrawlMessage,
)
from core.plugin.entities.plugin import DatasourceProviderID, GenericProviderID
from core.plugin.entities.plugin_daemon import (
PluginBasicBooleanResponse,
PluginDatasourceProviderEntity,
)
from core.plugin.impl.base import BasePluginClient
from core.schemas.resolver import resolve_dify_schema_refs
from models.provider_ids import DatasourceProviderID, GenericProviderID
from services.tools.tools_transform_service import ToolTransformService

2
api/core/rag/index_processor/index_processor_base.py
View File

@ -2,7 +2,7 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import Any, TYPE_CHECKING, Optional
from typing import TYPE_CHECKING, Any, Optional
from configs import dify_config
from core.rag.extractor.entity.extract_setting import ExtractSetting

8
api/core/schemas/registry.py
View File

@ -85,7 +85,6 @@ class SchemaRegistry:
except (OSError, json.JSONDecodeError) as e:
print(f"Warning: failed to load schema {version}/{schema_name}: {e}")
def get_schema(self, uri: str) -> Optional[Any]:
"""Retrieves a schema by URI with version support"""
version, schema_name = self._parse_uri(uri)
@ -101,6 +100,7 @@ class SchemaRegistry:
def _parse_uri(self, uri: str) -> tuple[str, str]:
"""Parses a schema URI to extract version and schema name"""
from core.schemas.resolver import parse_dify_schema_uri
return parse_dify_schema_uri(uri)
def list_versions(self) -> list[str]:
@ -121,10 +121,6 @@ class SchemaRegistry:
result = []
for schema_name, schema in version_schemas.items():
result.append({
"name": schema_name,
"label": schema.get("title", schema_name),
"schema": schema
})
result.append({"name": schema_name, "label": schema.get("title", schema_name), "schema": schema})
return result

61
api/core/schemas/resolver.py
View File

@ -19,11 +19,13 @@ _DIFY_SCHEMA_PATTERN = re.compile(r"^https://dify\.ai/schemas/(v\d+)/(.+)\.json$
class SchemaResolutionError(Exception):
"""Base exception for schema resolution errors"""
pass
class CircularReferenceError(SchemaResolutionError):
"""Raised when a circular reference is detected"""
def __init__(self, ref_uri: str, ref_path: list[str]):
self.ref_uri = ref_uri
self.ref_path = ref_path
@ -32,6 +34,7 @@ class CircularReferenceError(SchemaResolutionError):
class MaxDepthExceededError(SchemaResolutionError):
"""Raised when maximum resolution depth is exceeded"""
def __init__(self, max_depth: int):
self.max_depth = max_depth
super().__init__(f"Maximum resolution depth ({max_depth}) exceeded")
@ -39,6 +42,7 @@ class MaxDepthExceededError(SchemaResolutionError):
class SchemaNotFoundError(SchemaResolutionError):
"""Raised when a referenced schema cannot be found"""
def __init__(self, ref_uri: str):
self.ref_uri = ref_uri
super().__init__(f"Schema not found: {ref_uri}")
@ -47,6 +51,7 @@ class SchemaNotFoundError(SchemaResolutionError):
@dataclass
class QueueItem:
"""Represents an item in the BFS queue"""
current: Any
parent: Optional[Any]
key: Optional[Union[str, int]]
@ -104,16 +109,11 @@ class SchemaResolver:
# Slow path: schema contains refs, perform full resolution
import copy
result = copy.deepcopy(schema)
# Initialize BFS queue
queue = deque([QueueItem(
current=result,
parent=None,
key=None,
depth=0,
ref_path=set()
)])
queue = deque([QueueItem(current=result, parent=None, key=None, depth=0, ref_path=set())])
while queue:
item = queue.popleft()
@ -144,13 +144,9 @@ class SchemaResolver:
next_depth = item.depth + 1
if next_depth >= self.max_depth:
raise MaxDepthExceededError(self.max_depth)
queue.append(QueueItem(
current=value,
parent=item.current,
key=key,
depth=next_depth,
ref_path=item.ref_path
))
queue.append(
QueueItem(current=value, parent=item.current, key=key, depth=next_depth, ref_path=item.ref_path)
)
def _process_list(self, queue: deque, item: QueueItem) -> None:
"""Process a list item"""
@ -159,13 +155,9 @@ class SchemaResolver:
next_depth = item.depth + 1
if next_depth >= self.max_depth:
raise MaxDepthExceededError(self.max_depth)
queue.append(QueueItem(
current=value,
parent=item.current,
key=idx,
depth=next_depth,
ref_path=item.ref_path
))
queue.append(
QueueItem(current=value, parent=item.current, key=idx, depth=next_depth, ref_path=item.ref_path)
)
def _resolve_ref(self, queue: deque, item: QueueItem, ref_uri: str) -> None:
"""Resolve a $ref reference"""
@ -194,23 +186,21 @@ class SchemaResolver:
# Root level replacement
item.current.clear()
item.current.update(resolved_schema)
queue.append(QueueItem(
current=item.current,
parent=None,
key=None,
depth=next_depth,
ref_path=new_ref_path
))
queue.append(
QueueItem(current=item.current, parent=None, key=None, depth=next_depth, ref_path=new_ref_path)
)
else:
# Update parent container
item.parent[item.key] = resolved_schema.copy()
queue.append(QueueItem(
queue.append(
QueueItem(
current=item.parent[item.key],
parent=item.parent,
key=item.key,
depth=next_depth,
ref_path=new_ref_path
))
ref_path=new_ref_path,
)
)
def _get_resolved_schema(self, ref_uri: str) -> Optional[SchemaDict]:
"""Get resolved schema from cache or registry"""
@ -233,9 +223,7 @@ class SchemaResolver:
def resolve_dify_schema_refs(
schema: SchemaType,
registry: Optional[SchemaRegistry] = None,
max_depth: int = 30
schema: SchemaType, registry: Optional[SchemaRegistry] = None, max_depth: int = 30
) -> SchemaType:
"""
Resolve $ref references in Dify schema to actual schema content
@ -353,14 +341,15 @@ def _has_dify_refs_hybrid(schema: SchemaType) -> bool:
# Phase 1: Fast string-based pre-filtering
try:
import json
schema_str = json.dumps(schema, separators=(',', ':'))
schema_str = json.dumps(schema, separators=(",", ":"))
# Quick elimination: no $ref at all
if '"$ref"' not in schema_str:
return False
# Quick elimination: no Dify schema URLs
if 'https://dify.ai/schemas/' not in schema_str:
if "https://dify.ai/schemas/" not in schema_str:
return False
except (TypeError, ValueError, OverflowError):

5
api/core/schemas/schema_manager.py
View File

@ -37,10 +37,7 @@ class SchemaManager:
schema = self.registry.get_schema(uri)
if schema:
return {
"name": schema_name,
"schema": schema
}
return {"name": schema_name, "schema": schema}
return None
def list_available_schemas(self, version: str = "v1") -> list[str]:

8
api/core/workflow/entities/variable_pool.py
View File

@ -68,10 +68,10 @@ class VariablePool(BaseModel):
# Add rag pipeline variables to the variable pool
if self.rag_pipeline_variables:
rag_pipeline_variables_map = defaultdict(dict)
for var in self.rag_pipeline_variables:
node_id = var.variable.belong_to_node_id
key = var.variable.variable
value = var.value
for rag_var in self.rag_pipeline_variables:
node_id = rag_var.variable.belong_to_node_id
key = rag_var.variable.variable
value = rag_var.value
rag_pipeline_variables_map[node_id][key] = value
for key, value in rag_pipeline_variables_map.items():
self.add((RAG_PIPELINE_VARIABLE_NODE_ID, key), value)

4
api/core/workflow/enums.py
View File

@ -37,12 +37,14 @@ class NodeType(StrEnum):
ANSWER = "answer"
LLM = "llm"
KNOWLEDGE_RETRIEVAL = "knowledge-retrieval"
KNOWLEDGE_INDEX = "knowledge-index"
IF_ELSE = "if-else"
CODE = "code"
TEMPLATE_TRANSFORM = "template-transform"
QUESTION_CLASSIFIER = "question-classifier"
HTTP_REQUEST = "http-request"
TOOL = "tool"
DATASOURCE = "datasource"
VARIABLE_AGGREGATOR = "variable-aggregator"
LEGACY_VARIABLE_AGGREGATOR = "variable-assigner" # TODO: Merge this into VARIABLE_AGGREGATOR in the database.
LOOP = "loop"
@ -83,6 +85,7 @@ class WorkflowType(StrEnum):
WORKFLOW = "workflow"
CHAT = "chat"
RAG_PIPELINE = "rag-pipeline"
class WorkflowExecutionStatus(StrEnum):
@ -116,6 +119,7 @@ class WorkflowNodeExecutionMetadataKey(StrEnum):
LOOP_DURATION_MAP = "loop_duration_map" # single loop duration if loop node runs
ERROR_STRATEGY = "error_strategy" # node in continue on error mode return the field
LOOP_VARIABLE_MAP = "loop_variable_map" # single loop variable output
DATASOURCE_INFO = "datasource_info"
class WorkflowNodeExecutionStatus(StrEnum):

2
api/core/workflow/graph/graph.py
View File

@ -109,7 +109,7 @@ class Graph:
start_node_id = None
for nid in root_candidates:
node_data = node_configs_map[nid].get("data", {})
if node_data.get("type") == NodeType.START.value:
if node_data.get("type") in [NodeType.START, NodeType.DATASOURCE]:
start_node_id = nid
break

69
api/core/workflow/nodes/datasource/datasource_node.py
View File

@ -19,16 +19,14 @@ from core.file.enums import FileTransferMethod, FileType
from core.plugin.impl.exc import PluginDaemonClientSideError
from core.variables.segments import ArrayAnySegment
from core.variables.variables import ArrayAnyVariable
from core.workflow.entities.node_entities import NodeRunResult
from core.workflow.entities.variable_pool import VariablePool, VariableValue
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
from core.workflow.enums import SystemVariableKey
from core.workflow.nodes.base import BaseNode
from core.workflow.enums import ErrorStrategy, NodeType, SystemVariableKey
from core.workflow.node_events import NodeRunResult, StreamChunkEvent, StreamCompletedEvent
from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig
from core.workflow.nodes.enums import ErrorStrategy, NodeType
from core.workflow.nodes.event.event import RunCompletedEvent, RunStreamChunkEvent
from core.workflow.nodes.base.node import Node
from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser
from core.workflow.nodes.tool.exc import ToolFileError
from core.workflow.utils.variable_template_parser import VariableTemplateParser
from extensions.ext_database import db
from factories import file_factory
from models.model import UploadFile
@ -39,7 +37,7 @@ from .entities import DatasourceNodeData
from .exc import DatasourceNodeError, DatasourceParameterError
class DatasourceNode(BaseNode):
class DatasourceNode(Node):
"""
Datasource Node
"""
@ -97,8 +95,8 @@ class DatasourceNode(BaseNode):
datasource_type=DatasourceProviderType.value_of(datasource_type),
)
except DatasourceNodeError as e:
yield RunCompletedEvent(
run_result=NodeRunResult(
yield StreamCompletedEvent(
node_run_result=NodeRunResult(
status=WorkflowNodeExecutionStatus.FAILED,
inputs={},
metadata={WorkflowNodeExecutionMetadataKey.DATASOURCE_INFO: datasource_info},
@ -172,8 +170,8 @@ class DatasourceNode(BaseNode):
datasource_type=datasource_type,
)
case DatasourceProviderType.WEBSITE_CRAWL:
yield RunCompletedEvent(
run_result=NodeRunResult(
yield StreamCompletedEvent(
node_run_result=NodeRunResult(
status=WorkflowNodeExecutionStatus.SUCCEEDED,
inputs=parameters_for_log,
metadata={WorkflowNodeExecutionMetadataKey.DATASOURCE_INFO: datasource_info},
@ -204,10 +202,10 @@ class DatasourceNode(BaseNode):
size=upload_file.size,
storage_key=upload_file.key,
)
variable_pool.add([self.node_id, "file"], file_info)
variable_pool.add([self._node_id, "file"], file_info)
# variable_pool.add([self.node_id, "file"], file_info.to_dict())
yield RunCompletedEvent(
run_result=NodeRunResult(
yield StreamCompletedEvent(
node_run_result=NodeRunResult(
status=WorkflowNodeExecutionStatus.SUCCEEDED,
inputs=parameters_for_log,
metadata={WorkflowNodeExecutionMetadataKey.DATASOURCE_INFO: datasource_info},
@ -220,8 +218,8 @@ class DatasourceNode(BaseNode):
case _:
raise DatasourceNodeError(f"Unsupported datasource provider: {datasource_type}")
except PluginDaemonClientSideError as e:
yield RunCompletedEvent(
run_result=NodeRunResult(
yield StreamCompletedEvent(
node_run_result=NodeRunResult(
status=WorkflowNodeExecutionStatus.FAILED,
inputs=parameters_for_log,
metadata={WorkflowNodeExecutionMetadataKey.DATASOURCE_INFO: datasource_info},
@ -230,8 +228,8 @@ class DatasourceNode(BaseNode):
)
)
except DatasourceNodeError as e:
yield RunCompletedEvent(
run_result=NodeRunResult(
yield StreamCompletedEvent(
node_run_result=NodeRunResult(
status=WorkflowNodeExecutionStatus.FAILED,
inputs=parameters_for_log,
metadata={WorkflowNodeExecutionMetadataKey.DATASOURCE_INFO: datasource_info},
@ -425,8 +423,10 @@ class DatasourceNode(BaseNode):
elif message.type == DatasourceMessage.MessageType.TEXT:
assert isinstance(message.message, DatasourceMessage.TextMessage)
text += message.message.text
yield RunStreamChunkEvent(
chunk_content=message.message.text, from_variable_selector=[self.node_id, "text"]
yield StreamChunkEvent(
selector=[self._node_id, "text"],
chunk=message.message.text,
is_final=False,
)
elif message.type == DatasourceMessage.MessageType.JSON:
assert isinstance(message.message, DatasourceMessage.JsonMessage)
@ -442,7 +442,11 @@ class DatasourceNode(BaseNode):
assert isinstance(message.message, DatasourceMessage.TextMessage)
stream_text = f"Link: {message.message.text}\n"
text += stream_text
yield RunStreamChunkEvent(chunk_content=stream_text, from_variable_selector=[self.node_id, "text"])
yield StreamChunkEvent(
selector=[self._node_id, "text"],
chunk=stream_text,
is_final=False,
)
elif message.type == DatasourceMessage.MessageType.VARIABLE:
assert isinstance(message.message, DatasourceMessage.VariableMessage)
variable_name = message.message.variable_name
@ -454,17 +458,24 @@ class DatasourceNode(BaseNode):
variables[variable_name] = ""
variables[variable_name] += variable_value
yield RunStreamChunkEvent(
chunk_content=variable_value, from_variable_selector=[self.node_id, variable_name]
yield StreamChunkEvent(
selector=[self._node_id, variable_name],
chunk=variable_value,
is_final=False,
)
else:
variables[variable_name] = variable_value
elif message.type == DatasourceMessage.MessageType.FILE:
assert message.meta is not None
files.append(message.meta["file"])
yield RunCompletedEvent(
run_result=NodeRunResult(
# mark the end of the stream
yield StreamChunkEvent(
selector=[self._node_id, "text"],
chunk="",
is_final=True,
)
yield StreamCompletedEvent(
node_run_result=NodeRunResult(
status=WorkflowNodeExecutionStatus.SUCCEEDED,
outputs={"json": json, "files": files, **variables, "text": text},
metadata={
@ -526,9 +537,9 @@ class DatasourceNode(BaseNode):
tenant_id=self.tenant_id,
)
if file:
variable_pool.add([self.node_id, "file"], file)
yield RunCompletedEvent(
run_result=NodeRunResult(
variable_pool.add([self._node_id, "file"], file)
yield StreamCompletedEvent(
node_run_result=NodeRunResult(
status=WorkflowNodeExecutionStatus.SUCCEEDED,
inputs=parameters_for_log,
metadata={WorkflowNodeExecutionMetadataKey.DATASOURCE_INFO: datasource_info},

14
api/core/workflow/nodes/knowledge_index/knowledge_index_node.py
View File

@ -9,16 +9,15 @@ from sqlalchemy import func
from core.app.entities.app_invoke_entities import InvokeFrom
from core.rag.index_processor.index_processor_factory import IndexProcessorFactory
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from core.workflow.entities.node_entities import NodeRunResult
from core.workflow.entities.variable_pool import VariablePool
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
from core.workflow.enums import SystemVariableKey
from core.workflow.enums import ErrorStrategy, NodeType, SystemVariableKey
from core.workflow.node_events import NodeRunResult
from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig
from core.workflow.nodes.enums import ErrorStrategy, NodeType
from core.workflow.nodes.base.node import Node
from extensions.ext_database import db
from models.dataset import Dataset, Document, DocumentSegment
from ..base import BaseNode
from .entities import KnowledgeIndexNodeData
from .exc import (
KnowledgeIndexNodeError,
@ -35,7 +34,7 @@ default_retrieval_model = {
}
class KnowledgeIndexNode(BaseNode):
class KnowledgeIndexNode(Node):
_node_data: KnowledgeIndexNodeData
_node_type = NodeType.KNOWLEDGE_INDEX
@ -93,15 +92,12 @@ class KnowledgeIndexNode(BaseNode):
return NodeRunResult(
status=WorkflowNodeExecutionStatus.SUCCEEDED,
inputs=variables,
process_data=None,
outputs=outputs,
)
results = self._invoke_knowledge_index(
dataset=dataset, node_data=node_data, chunks=chunks, variable_pool=variable_pool
)
return NodeRunResult(
status=WorkflowNodeExecutionStatus.SUCCEEDED, inputs=variables, process_data=None, outputs=results
)
return NodeRunResult(status=WorkflowNodeExecutionStatus.SUCCEEDED, inputs=variables, outputs=results)
except KnowledgeIndexNodeError as e:
logger.warning("Error when running knowledge index node")

6
api/models/dataset.py
View File

@ -172,7 +172,7 @@ class Dataset(Base):
)
@property
def doc_form(self):
def doc_form(self) -> Optional[str]:
if self.chunk_structure:
return self.chunk_structure
document = db.session.query(Document).filter(Document.dataset_id == self.id).first()
@ -424,7 +424,7 @@ class Document(Base):
return status
@property
def data_source_info_dict(self):
def data_source_info_dict(self) -> dict[str, Any]:
if self.data_source_info:
try:
data_source_info_dict = json.loads(self.data_source_info)
@ -432,7 +432,7 @@ class Document(Base):
data_source_info_dict = {}
return data_source_info_dict
return None
return {}
@property
def data_source_detail_dict(self):

5
api/models/provider_ids.py
View File

@ -52,3 +52,8 @@ class ToolProviderID(GenericProviderID):
if self.organization == "langgenius":
if self.provider_name in ["jina", "siliconflow", "stepfun", "gitee_ai"]:
self.plugin_name = f"{self.provider_name}_tool"
class DatasourceProviderID(GenericProviderID):
def __init__(self, value: str, is_hardcoded: bool = False) -> None:
super().__init__(value, is_hardcoded)

22
api/services/dataset_service.py
View File

@ -718,9 +718,9 @@ class DatasetService:
model_manager = ModelManager()
embedding_model = model_manager.get_model_instance(
tenant_id=current_user.current_tenant_id,
provider=knowledge_configuration.embedding_model_provider,
provider=knowledge_configuration.embedding_model_provider or "",
model_type=ModelType.TEXT_EMBEDDING,
model=knowledge_configuration.embedding_model,
model=knowledge_configuration.embedding_model or "",
)
dataset.embedding_model = embedding_model.model
dataset.embedding_model_provider = embedding_model.provider
@ -1159,7 +1159,7 @@ class DocumentService:
return
documents = db.session.query(Document).where(Document.id.in_(document_ids)).all()
file_ids = [
document.data_source_info_dict["upload_file_id"]
document.data_source_info_dict.get("upload_file_id", "")
for document in documents
if document.data_source_type == "upload_file"
]
@ -1281,7 +1281,7 @@ class DocumentService:
account: Account | Any,
dataset_process_rule: Optional[DatasetProcessRule] = None,
created_from: str = "web",
):
) -> tuple[list[Document], str]:
# check doc_form
DatasetService.check_doc_form(dataset, knowledge_config.doc_form)
# check document limit
@ -1386,7 +1386,7 @@ class DocumentService:
"Invalid process rule mode: %s, can not find dataset process rule",
process_rule.mode,
)
return
return [], ""
db.session.add(dataset_process_rule)
db.session.flush()
lock_name = f"add_document_lock_dataset_id_{dataset.id}"
@ -2595,7 +2595,9 @@ class SegmentService:
return segment_data_list
@classmethod
def update_segment(cls, args: SegmentUpdateArgs, segment: DocumentSegment, document: Document, dataset: Dataset):
def update_segment(
cls, args: SegmentUpdateArgs, segment: DocumentSegment, document: Document, dataset: Dataset
) -> DocumentSegment:
indexing_cache_key = f"segment_{segment.id}_indexing"
cache_result = redis_client.get(indexing_cache_key)
if cache_result is not None:
@ -2764,6 +2766,8 @@ class SegmentService:
segment.error = str(e)
db.session.commit()
new_segment = db.session.query(DocumentSegment).where(DocumentSegment.id == segment.id).first()
if not new_segment:
raise ValueError("new_segment is not found")
return new_segment
@classmethod
@ -2804,7 +2808,11 @@ class SegmentService:
index_node_ids = [seg.index_node_id for seg in segments]
total_words = sum(seg.word_count for seg in segments)
document.word_count -= total_words
if document.word_count is None:
document.word_count = 0
else:
document.word_count = max(0, document.word_count - total_words)
db.session.add(document)
delete_segment_from_index_task.delay(index_node_ids, dataset.id, document.id)

6
api/services/datasource_provider_service.py
View File

@ -11,7 +11,6 @@ from core.helper import encrypter
from core.helper.name_generator import generate_incremental_name
from core.helper.provider_cache import NoOpProviderCredentialCache
from core.model_runtime.entities.provider_entities import FormType
from core.plugin.entities.plugin import DatasourceProviderID
from core.plugin.impl.datasource import PluginDatasourceManager
from core.plugin.impl.oauth import OAuthHandler
from core.tools.entities.tool_entities import CredentialType
@ -19,6 +18,7 @@ from core.tools.utils.encryption import ProviderConfigCache, ProviderConfigEncry
from extensions.ext_database import db
from extensions.ext_redis import redis_client
from models.oauth import DatasourceOauthParamConfig, DatasourceOauthTenantParamConfig, DatasourceProvider
from models.provider_ids import DatasourceProviderID
from services.plugin.plugin_service import PluginService
logger = logging.getLogger(__name__)
@ -809,9 +809,7 @@ class DatasourceProviderService:
credentials = self.list_datasource_credentials(
tenant_id=tenant_id, provider=datasource.provider, plugin_id=datasource.plugin_id
)
redirect_uri = (
f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{datasource_provider_id}/datasource/callback"
)
redirect_uri = f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{datasource_provider_id}/datasource/callback"
datasource_credentials.append(
{
"provider": datasource.provider,

20
api/services/entities/knowledge_entities/rag_pipeline_entities.py
View File

@ -1,6 +1,6 @@
from typing import Literal, Optional
from pydantic import BaseModel
from pydantic import BaseModel, field_validator
class IconInfo(BaseModel):
@ -110,7 +110,21 @@ class KnowledgeConfiguration(BaseModel):
chunk_structure: str
indexing_technique: Literal["high_quality", "economy"]
embedding_model_provider: Optional[str] = ""
embedding_model: Optional[str] = ""
embedding_model_provider: str = ""
embedding_model: str = ""
keyword_number: Optional[int] = 10
retrieval_model: RetrievalSetting
@field_validator("embedding_model_provider", mode="before")
@classmethod
def validate_embedding_model_provider(cls, v):
if v is None:
return ""
return v
@field_validator("embedding_model", mode="before")
@classmethod
def validate_embedding_model(cls, v):
if v is None:
return ""
return v

50
api/services/rag_pipeline/rag_pipeline.py
View File

@ -28,26 +28,23 @@ from core.datasource.online_document.online_document_plugin import OnlineDocumen
from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin
from core.rag.entities.event import (
BaseDatasourceEvent,
DatasourceCompletedEvent,
DatasourceErrorEvent,
DatasourceProcessingEvent,
)
from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository
from core.variables.variables import Variable
from core.workflow.entities.node_entities import NodeRunResult
from core.workflow.entities.variable_pool import VariablePool
from core.workflow.entities.workflow_node_execution import (
WorkflowNodeExecution,
WorkflowNodeExecutionStatus,
)
from core.workflow.enums import SystemVariableKey
from core.workflow.enums import ErrorStrategy, NodeType, SystemVariableKey
from core.workflow.errors import WorkflowNodeRunFailedError
from core.workflow.graph_engine.entities.event import InNodeEvent
from core.workflow.nodes.base.node import BaseNode
from core.workflow.nodes.enums import ErrorStrategy, NodeType
from core.workflow.nodes.event.event import RunCompletedEvent
from core.workflow.nodes.event.types import NodeEvent
from core.workflow.graph_events.base import GraphNodeEventBase
from core.workflow.node_events.base import NodeRunResult
from core.workflow.node_events.node import StreamCompletedEvent
from core.workflow.nodes.base.node import Node
from core.workflow.nodes.node_mapping import LATEST_VERSION, NODE_TYPE_CLASSES_MAPPING
from core.workflow.repositories.workflow_node_execution_repository import OrderConfig
from core.workflow.system_variable import SystemVariable
@ -105,12 +102,13 @@ class RagPipelineService:
if type == "built-in":
mode = dify_config.HOSTED_FETCH_PIPELINE_TEMPLATES_MODE
retrieval_instance = PipelineTemplateRetrievalFactory.get_pipeline_template_factory(mode)()
result: Optional[dict] = retrieval_instance.get_pipeline_template_detail(template_id)
built_in_result: Optional[dict] = retrieval_instance.get_pipeline_template_detail(template_id)
return built_in_result
else:
mode = "customized"
retrieval_instance = PipelineTemplateRetrievalFactory.get_pipeline_template_factory(mode)()
result: Optional[dict] = retrieval_instance.get_pipeline_template_detail(template_id)
return result
customized_result: Optional[dict] = retrieval_instance.get_pipeline_template_detail(template_id)
return customized_result
@classmethod
def update_customized_pipeline_template(cls, template_id: str, template_info: PipelineTemplateInfoEntity):
@ -471,7 +469,7 @@ class RagPipelineService:
datasource_type: str,
is_published: bool,
credential_id: Optional[str] = None,
) -> Generator[BaseDatasourceEvent, None, None]:
) -> Generator[Mapping[str, Any], None, None]:
"""
Run published workflow datasource
"""
@ -563,9 +561,9 @@ class RagPipelineService:
user_id=account.id,
request=OnlineDriveBrowseFilesRequest(
bucket=user_inputs.get("bucket"),
prefix=user_inputs.get("prefix"),
prefix=user_inputs.get("prefix", ""),
max_keys=user_inputs.get("max_keys", 20),
start_after=user_inputs.get("start_after"),
next_page_parameters=user_inputs.get("next_page_parameters"),
),
provider_type=datasource_runtime.datasource_provider_type(),
)
@ -600,7 +598,7 @@ class RagPipelineService:
end_time = time.time()
if message.result.status == "completed":
crawl_event = DatasourceCompletedEvent(
data=message.result.web_info_list,
data=message.result.web_info_list or [],
total=message.result.total,
completed=message.result.completed,
time_consuming=round(end_time - start_time, 2),
@ -681,9 +679,9 @@ class RagPipelineService:
datasource_runtime.get_online_document_page_content(
user_id=account.id,
datasource_parameters=GetOnlineDocumentPageContentRequest(
workspace_id=user_inputs.get("workspace_id"),
page_id=user_inputs.get("page_id"),
type=user_inputs.get("type"),
workspace_id=user_inputs.get("workspace_id", ""),
page_id=user_inputs.get("page_id", ""),
type=user_inputs.get("type", ""),
),
provider_type=datasource_type,
)
@ -740,7 +738,7 @@ class RagPipelineService:
def _handle_node_run_result(
self,
getter: Callable[[], tuple[BaseNode, Generator[NodeEvent | InNodeEvent, None, None]]],
getter: Callable[[], tuple[Node, Generator[GraphNodeEventBase, None, None]]],
start_at: float,
tenant_id: str,
node_id: str,
@ -758,17 +756,16 @@ class RagPipelineService:
node_run_result: NodeRunResult | None = None
for event in generator:
if isinstance(event, RunCompletedEvent):
node_run_result = event.run_result
if isinstance(event, StreamCompletedEvent):
node_run_result = event.node_run_result
# sign output files
node_run_result.outputs = WorkflowEntry.handle_special_values(node_run_result.outputs)
node_run_result.outputs = WorkflowEntry.handle_special_values(node_run_result.outputs) or {}
break
if not node_run_result:
raise ValueError("Node run failed with no run result")
# single step debug mode error handling return
if node_run_result.status == WorkflowNodeExecutionStatus.FAILED and node_instance.continue_on_error:
if node_run_result.status == WorkflowNodeExecutionStatus.FAILED and node_instance.error_strategy:
node_error_args: dict[str, Any] = {
"status": WorkflowNodeExecutionStatus.EXCEPTION,
"error": node_run_result.error,
@ -808,7 +805,7 @@ class RagPipelineService:
workflow_id=node_instance.workflow_id,
index=1,
node_id=node_id,
node_type=node_instance.type_,
node_type=node_instance.node_type,
title=node_instance.title,
elapsed_time=time.perf_counter() - start_at,
finished_at=datetime.now(UTC).replace(tzinfo=None),
@ -1208,6 +1205,3 @@ class RagPipelineService:
)
session.commit()
return workflow_node_execution_db_model

2
api/services/rag_pipeline/rag_pipeline_dsl_service.py
View File

@ -23,8 +23,8 @@ from core.helper import ssrf_proxy
from core.helper.name_generator import generate_incremental_name
from core.model_runtime.utils.encoders import jsonable_encoder
from core.plugin.entities.plugin import PluginDependency
from core.workflow.enums import NodeType
from core.workflow.nodes.datasource.entities import DatasourceNodeData
from core.workflow.nodes.enums import NodeType
from core.workflow.nodes.knowledge_retrieval.entities import KnowledgeRetrievalNodeData
from core.workflow.nodes.llm.entities import LLMNodeData
from core.workflow.nodes.parameter_extractor.entities import ParameterExtractorNodeData

5
api/services/rag_pipeline/rag_pipeline_transform_service.py
View File

@ -1,6 +1,7 @@
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Optional
from uuid import uuid4
import yaml
@ -87,7 +88,7 @@ class RagPipelineTransformService:
"status": "success",
}
def _get_transform_yaml(self, doc_form: str, datasource_type: str, indexing_technique: str):
def _get_transform_yaml(self, doc_form: str, datasource_type: str, indexing_technique: Optional[str]):
if doc_form == "text_model":
match datasource_type:
case "upload_file":
@ -148,7 +149,7 @@ class RagPipelineTransformService:
return node
def _deal_knowledge_index(
self, dataset: Dataset, doc_form: str, indexing_technique: str, retrieval_model: dict, node: dict
self, dataset: Dataset, doc_form: str, indexing_technique: Optional[str], retrieval_model: dict, node: dict
):
knowledge_configuration_dict = node.get("data", {})
knowledge_configuration = KnowledgeConfiguration(**knowledge_configuration_dict)

5
api/tasks/batch_clean_document_task.py
View File

@ -1,5 +1,6 @@
import logging
import time
from typing import Optional
import click
from celery import shared_task
@ -15,7 +16,7 @@ logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def batch_clean_document_task(document_ids: list[str], dataset_id: str, doc_form: str, file_ids: list[str]):
def batch_clean_document_task(document_ids: list[str], dataset_id: str, doc_form: Optional[str], file_ids: list[str]):
"""
Clean document when document deleted.
:param document_ids: document ids
@ -29,6 +30,8 @@ def batch_clean_document_task(document_ids: list[str], dataset_id: str, doc_form
start_at = time.perf_counter()
try:
if not doc_form:
raise ValueError("doc_form is required")
dataset = db.session.query(Dataset).where(Dataset.id == dataset_id).first()
if not dataset:

7
api/tasks/rag_pipeline/rag_pipeline_run_task.py
View File

@ -21,14 +21,16 @@ from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom
@shared_task(queue="dataset")
def rag_pipeline_run_task(pipeline_id: str,
def rag_pipeline_run_task(
pipeline_id: str,
application_generate_entity: dict,
user_id: str,
tenant_id: str,
workflow_id: str,
streaming: bool,
workflow_execution_id: str | None = None,
workflow_thread_pool_id: str | None = None):
workflow_thread_pool_id: str | None = None,
):
"""
Async Run rag pipeline
:param pipeline_id: Pipeline ID
@ -106,6 +108,7 @@ def rag_pipeline_run_task(pipeline_id: str,
# Don't create a new app context here - let _generate handle it
# Just ensure the user is available in contextvars
from core.app.apps.pipeline.pipeline_generator import PipelineGenerator
pipeline_generator = PipelineGenerator()
pipeline_generator._generate(
flask_app=flask_app,

269
api/tests/unit_tests/core/schemas/test_resolver.py
View File

@ -33,9 +33,7 @@ class TestSchemaResolver:
def test_simple_ref_resolution(self):
"""Test resolving a simple $ref to a complete schema"""
schema_with_ref = {
"$ref": "https://dify.ai/schemas/v1/qa_structure.json"
}
schema_with_ref = {"$ref": "https://dify.ai/schemas/v1/qa_structure.json"}
resolved = resolve_dify_schema_refs(schema_with_ref)
@ -55,14 +53,9 @@ class TestSchemaResolver:
nested_schema = {
"type": "object",
"properties": {
"file_data": {
"$ref": "https://dify.ai/schemas/v1/file.json"
"file_data": {"$ref": "https://dify.ai/schemas/v1/file.json"},
"metadata": {"type": "string", "description": "Additional metadata"},
},
"metadata": {
"type": "string",
"description": "Additional metadata"
}
}
}
resolved = resolve_dify_schema_refs(nested_schema)
@ -87,10 +80,8 @@ class TestSchemaResolver:
"""Test resolving $refs in array items"""
array_schema = {
"type": "array",
"items": {
"$ref": "https://dify.ai/schemas/v1/general_structure.json"
},
"description": "Array of general structures"
"items": {"$ref": "https://dify.ai/schemas/v1/general_structure.json"},
"description": "Array of general structures",
}
resolved = resolve_dify_schema_refs(array_schema)
@ -109,13 +100,9 @@ class TestSchemaResolver:
external_ref_schema = {
"type": "object",
"properties": {
"external_data": {
"$ref": "https://example.com/external-schema.json"
"external_data": {"$ref": "https://example.com/external-schema.json"},
"dify_data": {"$ref": "https://dify.ai/schemas/v1/file.json"},
},
"dify_data": {
"$ref": "https://dify.ai/schemas/v1/file.json"
}
}
}
resolved = resolve_dify_schema_refs(external_ref_schema)
@ -132,18 +119,10 @@ class TestSchemaResolver:
simple_schema = {
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "Name field"
"name": {"type": "string", "description": "Name field"},
"items": {"type": "array", "items": {"type": "number"}},
},
"items": {
"type": "array",
"items": {
"type": "number"
}
}
},
"required": ["name"]
"required": ["name"],
}
resolved = resolve_dify_schema_refs(simple_schema)
@ -162,12 +141,7 @@ class TestSchemaResolver:
# Wrap it in fewer layers to make the test more reasonable
for _ in range(2):
deep_schema = {
"type": "object",
"properties": {
"nested": deep_schema
}
}
deep_schema = {"type": "object", "properties": {"nested": deep_schema}}
# Should handle normal cases fine with reasonable depth
resolved = resolve_dify_schema_refs(deep_schema, max_depth=25)
@ -185,7 +159,7 @@ class TestSchemaResolver:
mock_registry = MagicMock()
mock_registry.get_schema.side_effect = lambda uri: {
"$ref": "https://dify.ai/schemas/v1/circular.json",
"type": "object"
"type": "object",
}
schema = {"$ref": "https://dify.ai/schemas/v1/circular.json"}
@ -244,10 +218,7 @@ class TestSchemaResolver:
"""Test that the resolver is thread-safe"""
schema = {
"type": "object",
"properties": {
f"prop_{i}": {"$ref": "https://dify.ai/schemas/v1/file.json"}
for i in range(10)
}
"properties": {f"prop_{i}": {"$ref": "https://dify.ai/schemas/v1/file.json"} for i in range(10)},
}
results = []
@ -276,10 +247,7 @@ class TestSchemaResolver:
complex_schema = {
"type": "object",
"properties": {
"files": {
"type": "array",
"items": {"$ref": "https://dify.ai/schemas/v1/file.json"}
},
"files": {"type": "array", "items": {"$ref": "https://dify.ai/schemas/v1/file.json"}},
"nested": {
"type": "object",
"properties": {
@ -290,12 +258,12 @@ class TestSchemaResolver:
"type": "object",
"properties": {
"general": {"$ref": "https://dify.ai/schemas/v1/general_structure.json"}
}
}
}
}
}
}
},
},
},
},
},
},
}
resolved = resolve_dify_schema_refs(complex_schema, max_depth=20)
@ -335,49 +303,34 @@ class TestUtilityFunctions:
"""Test _has_dify_refs function"""
# Schemas with Dify refs
assert _has_dify_refs({"$ref": "https://dify.ai/schemas/v1/file.json"})
assert _has_dify_refs({
"type": "object",
"properties": {
"data": {"$ref": "https://dify.ai/schemas/v1/file.json"}
}
})
assert _has_dify_refs([
{"type": "string"},
{"$ref": "https://dify.ai/schemas/v1/file.json"}
])
assert _has_dify_refs({
assert _has_dify_refs(
{"type": "object", "properties": {"data": {"$ref": "https://dify.ai/schemas/v1/file.json"}}}
)
assert _has_dify_refs([{"type": "string"}, {"$ref": "https://dify.ai/schemas/v1/file.json"}])
assert _has_dify_refs(
{
"type": "array",
"items": {
"type": "object",
"properties": {
"nested": {"$ref": "https://dify.ai/schemas/v1/qa_structure.json"}
"properties": {"nested": {"$ref": "https://dify.ai/schemas/v1/qa_structure.json"}},
},
}
}
})
)
# Schemas without Dify refs
assert not _has_dify_refs({"type": "string"})
assert not _has_dify_refs({
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "number"}
}
})
assert not _has_dify_refs([
{"type": "string"},
{"type": "number"},
{"type": "object", "properties": {"name": {"type": "string"}}}
])
assert not _has_dify_refs(
{"type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "number"}}}
)
assert not _has_dify_refs(
[{"type": "string"}, {"type": "number"}, {"type": "object", "properties": {"name": {"type": "string"}}}]
)
# Schemas with non-Dify refs (should return False)
assert not _has_dify_refs({"$ref": "https://example.com/schema.json"})
assert not _has_dify_refs({
"type": "object",
"properties": {
"external": {"$ref": "https://example.com/external.json"}
}
})
assert not _has_dify_refs(
{"type": "object", "properties": {"external": {"$ref": "https://example.com/external.json"}}}
)
# Primitive types
assert not _has_dify_refs("string")
@ -392,29 +345,13 @@ class TestUtilityFunctions:
{"type": "string"},
{"type": "object", "properties": {"name": {"type": "string"}}},
[{"type": "string"}, {"type": "number"}],
# With Dify refs
{"$ref": "https://dify.ai/schemas/v1/file.json"},
{
"type": "object",
"properties": {
"data": {"$ref": "https://dify.ai/schemas/v1/file.json"}
}
},
[
{"type": "string"},
{"$ref": "https://dify.ai/schemas/v1/qa_structure.json"}
],
{"type": "object", "properties": {"data": {"$ref": "https://dify.ai/schemas/v1/file.json"}}},
[{"type": "string"}, {"$ref": "https://dify.ai/schemas/v1/qa_structure.json"}],
# With non-Dify refs
{"$ref": "https://example.com/schema.json"},
{
"type": "object",
"properties": {
"external": {"$ref": "https://example.com/external.json"}
}
},
{"type": "object", "properties": {"external": {"$ref": "https://example.com/external.json"}}},
# Complex nested
{
"type": "object",
@ -422,21 +359,20 @@ class TestUtilityFunctions:
"level1": {
"type": "object",
"properties": {
"level2": {
"type": "array",
"items": {"$ref": "https://dify.ai/schemas/v1/file.json"}
}
}
}
"level2": {"type": "array", "items": {"$ref": "https://dify.ai/schemas/v1/file.json"}}
},
}
},
},
# Edge cases
{"description": "This mentions $ref but is not a reference"},
{"$ref": "not-a-url"},
# Primitive types
"string", 123, True, None, []
"string",
123,
True,
None,
[],
]
for schema in test_schemas:
@ -465,7 +401,7 @@ class TestUtilityFunctions:
"version": "should be removed",
"type": "object",
"title": "should remain",
"properties": {}
"properties": {},
}
cleaned = _remove_metadata_fields(schema)
@ -521,7 +457,7 @@ class TestSchemaResolverClass:
list_schema = [
{"$ref": "https://dify.ai/schemas/v1/file.json"},
{"type": "string"},
{"$ref": "https://dify.ai/schemas/v1/qa_structure.json"}
{"$ref": "https://dify.ai/schemas/v1/qa_structure.json"},
]
resolver = SchemaResolver()
@ -545,7 +481,7 @@ class TestSchemaResolverClass:
"properties": {
f"prop_{i}": {"$ref": "https://dify.ai/schemas/v1/file.json"}
for i in range(50) # Reduced to avoid depth issues
}
},
}
# First run (no cache) - run multiple times to warm up
@ -585,14 +521,11 @@ class TestSchemaResolverClass:
"properties": {
"name": {"type": "string"},
"value": {"type": "number"},
"items": {
"type": "array",
"items": {"type": "string"}
}
}
"items": {"type": "array", "items": {"type": "string"}},
},
}
for i in range(50)
}
},
}
# Measure fast path (no refs) performance
@ -614,7 +547,7 @@ class TestSchemaResolverClass:
"properties": {
f"property_{i}": {"$ref": "https://dify.ai/schemas/v1/file.json"}
for i in range(20) # Fewer to avoid depth issues but still comparable
}
},
}
# Measure slow path (with refs) performance
@ -645,10 +578,7 @@ class TestSchemaResolverClass:
schemas_without_refs = [
{
"type": "object",
"properties": {
f"field_{j}": {"type": "string" if j % 2 else "number"}
for j in range(10)
}
"properties": {f"field_{j}": {"type": "string" if j % 2 else "number"} for j in range(10)},
}
for i in range(100)
]
@ -669,18 +599,12 @@ class TestSchemaResolverClass:
def test_has_dify_refs_performance(self):
"""Test that _has_dify_refs is fast for large schemas without refs"""
# Create a very large schema without refs
large_schema = {
"type": "object",
"properties": {}
}
large_schema = {"type": "object", "properties": {}}
# Add many nested properties
current = large_schema
for i in range(100):
current["properties"][f"level_{i}"] = {
"type": "object",
"properties": {}
}
current["properties"][f"level_{i}"] = {"type": "object", "properties": {}}
current = current["properties"][f"level_{i}"]
# _has_dify_refs should be fast even for large schemas
@ -704,16 +628,9 @@ class TestSchemaResolverClass:
# Case 1: Small schema without refs (most common case)
{
"name": "small_no_refs",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"value": {"type": "number"}
}
"schema": {"type": "object", "properties": {"name": {"type": "string"}, "value": {"type": "number"}}},
"expected": False,
},
"expected": False
},
# Case 2: Medium schema without refs
{
"name": "medium_no_refs",
@ -725,28 +642,16 @@ class TestSchemaResolverClass:
"properties": {
"name": {"type": "string"},
"value": {"type": "number"},
"items": {
"type": "array",
"items": {"type": "string"}
}
}
"items": {"type": "array", "items": {"type": "string"}},
},
}
for i in range(20)
}
},
"expected": False
},
"expected": False,
},
# Case 3: Large schema without refs
{
"name": "large_no_refs",
"schema": {
"type": "object",
"properties": {}
},
"expected": False
},
{"name": "large_no_refs", "schema": {"type": "object", "properties": {}}, "expected": False},
# Case 4: Schema with Dify refs
{
"name": "with_dify_refs",
@ -754,33 +659,26 @@ class TestSchemaResolverClass:
"type": "object",
"properties": {
"file": {"$ref": "https://dify.ai/schemas/v1/file.json"},
"data": {"type": "string"}
}
"data": {"type": "string"},
},
"expected": True
},
"expected": True,
},
# Case 5: Schema with non-Dify refs
{
"name": "with_external_refs",
"schema": {
"type": "object",
"properties": {
"external": {"$ref": "https://example.com/schema.json"},
"data": {"type": "string"}
}
"properties": {"external": {"$ref": "https://example.com/schema.json"}, "data": {"type": "string"}},
},
"expected": False,
},
"expected": False
}
]
# Add deep nesting to large schema
current = test_cases[2]["schema"]
for i in range(50):
current["properties"][f"level_{i}"] = {
"type": "object",
"properties": {}
}
current["properties"][f"level_{i}"] = {"type": "object", "properties": {}}
current = current["properties"][f"level_{i}"]
# Performance comparison
@ -829,11 +727,8 @@ class TestSchemaResolverClass:
schema_false_positive = {
"type": "object",
"properties": {
"description": {
"type": "string",
"description": "This field explains how $ref works in JSON Schema"
}
}
"description": {"type": "string", "description": "This field explains how $ref works in JSON Schema"}
},
}
# Both methods should return False
@ -847,16 +742,11 @@ class TestSchemaResolverClass:
"config": {
"type": "object",
"properties": {
"dify_url": {
"type": "string",
"default": "https://dify.ai/schemas/info"
"dify_url": {"type": "string", "default": "https://dify.ai/schemas/info"},
"actual_ref": {"$ref": "https://dify.ai/schemas/v1/file.json"},
},
"actual_ref": {
"$ref": "https://dify.ai/schemas/v1/file.json"
}
}
}
}
},
}
# Both methods should return True (due to actual_ref)
@ -865,10 +755,11 @@ class TestSchemaResolverClass:
# Case 3: Non-JSON serializable objects (should fall back to recursive)
import datetime
non_serializable = {
"type": "object",
"timestamp": datetime.datetime.now(),
"data": {"$ref": "https://dify.ai/schemas/v1/file.json"}
"data": {"$ref": "https://dify.ai/schemas/v1/file.json"},
}
# Hybrid should fall back to recursive and still work