From a15bf8e8fe468cee6d5b95e45ebac4eda05ee2d8 Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Fri, 23 May 2025 17:35:26 +0800 Subject: [PATCH 1/2] remove output schema --- api/core/datasource/entities/datasource_entities.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/api/core/datasource/entities/datasource_entities.py b/api/core/datasource/entities/datasource_entities.py index 22ec5c3d23..6a9fc5d9f9 100644 --- a/api/core/datasource/entities/datasource_entities.py +++ b/api/core/datasource/entities/datasource_entities.py @@ -15,7 +15,7 @@ from core.plugin.entities.parameters import ( init_frontend_parameter, ) from core.tools.entities.common_entities import I18nObject -from core.tools.entities.tool_entities import ToolLabelEnum, ToolProviderEntity +from core.tools.entities.tool_entities import ToolLabelEnum class DatasourceProviderType(enum.StrEnum): @@ -124,13 +124,13 @@ class DatasourceEntity(BaseModel): identity: DatasourceIdentity parameters: list[DatasourceParameter] = Field(default_factory=list) description: I18nObject = Field(..., description="The label of the datasource") - output_schema: Optional[dict] = None @field_validator("parameters", mode="before") @classmethod def set_parameters(cls, v, validation_info: ValidationInfo) -> list[DatasourceParameter]: return v or [] + class DatasourceProviderIdentity(BaseModel): author: str = Field(..., description="The author of the tool") name: str = Field(..., description="The name of the tool") @@ -142,11 +142,12 @@ class DatasourceProviderIdentity(BaseModel): description="The tags of the tool", ) - + class DatasourceProviderEntity(BaseModel): """ Datasource provider entity """ + identity: DatasourceProviderIdentity credentials_schema: list[ProviderConfig] = Field(default_factory=list) oauth_schema: Optional[OAuthSchema] = None @@ -212,7 +213,6 @@ class GetOnlineDocumentPagesRequest(BaseModel): """ - class OnlineDocumentPageIcon(BaseModel): """ Online document page icon @@ -284,6 +284,7 @@ class GetWebsiteCrawlRequest(BaseModel): """ Get website crawl request """ + crawl_parameters: dict = Field(..., description="The crawl parameters") From 6123f1ab2162418e21957d4dcaefde0d025036ef Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Fri, 23 May 2025 19:22:50 +0800 Subject: [PATCH 2/2] refactor: reorganize imports and fix datasource endpoint URL --- .../console/datasets/rag_pipeline/rag_pipeline_workflow.py | 2 +- api/core/plugin/impl/datasource.py | 5 +++-- api/services/rag_pipeline/rag_pipeline.py | 6 +++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py b/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py index bdd40fcabe..b25a362674 100644 --- a/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py +++ b/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py @@ -8,7 +8,6 @@ from flask_restful.inputs import int_range # type: ignore from sqlalchemy.orm import Session from werkzeug.exceptions import Forbidden, InternalServerError, NotFound -from models.model import EndUser import services from configs import dify_config from controllers.console import api @@ -40,6 +39,7 @@ from libs.helper import TimestampField, uuid_value from libs.login import current_user, login_required from models.account import Account from models.dataset import Pipeline +from models.model import EndUser from services.errors.app import WorkflowHashNotEqualError from services.errors.llm import InvokeRateLimitError from services.rag_pipeline.pipeline_generate_service import PipelineGenerateService diff --git a/api/core/plugin/impl/datasource.py b/api/core/plugin/impl/datasource.py index 430a9a6c01..aa8b1ad4d6 100644 --- a/api/core/plugin/impl/datasource.py +++ b/api/core/plugin/impl/datasource.py @@ -1,4 +1,5 @@ -from typing import Any, Mapping +from collections.abc import Mapping +from typing import Any from core.datasource.entities.api_entities import DatasourceProviderApiEntity from core.datasource.entities.datasource_entities import ( @@ -63,7 +64,7 @@ class PluginDatasourceManager(BasePluginClient): response = self._request_with_plugin_daemon_response( "GET", - f"plugin/{tenant_id}/management/datasources", + f"plugin/{tenant_id}/management/datasource", PluginDatasourceProviderEntity, params={"provider": tool_provider_id.provider_name, "plugin_id": tool_provider_id.plugin_id}, transformer=transformer, diff --git a/api/services/rag_pipeline/rag_pipeline.py b/api/services/rag_pipeline/rag_pipeline.py index 3bee0538ab..461e694a1b 100644 --- a/api/services/rag_pipeline/rag_pipeline.py +++ b/api/services/rag_pipeline/rag_pipeline.py @@ -12,7 +12,11 @@ from sqlalchemy.orm import Session import contexts from configs import dify_config -from core.datasource.entities.datasource_entities import DatasourceProviderType, GetOnlineDocumentPagesRequest, GetOnlineDocumentPagesResponse, GetWebsiteCrawlRequest, GetWebsiteCrawlResponse +from core.datasource.entities.datasource_entities import ( + DatasourceProviderType, + GetOnlineDocumentPagesResponse, + GetWebsiteCrawlResponse, +) from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin from core.model_runtime.utils.encoders import jsonable_encoder